YGGverse/YGGo
1
0
Fork 0
mirror of https://github.com/YGGverse/YGGo.git synced 2025-03-13 05:41:02 +00:00
Code Issues Projects Releases Wiki Activity

add distributed hosts crawling using yggo nodes manifest

Browse Source
This commit is contained in:
ghost 2023-05-05 05:26:53 +03:00
parent f0b2eb1613
commit 5999fb3a73
6 changed files with 542 additions and 230 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
README.md
View File

@ -150,7 +150,7 @@ GET m=SphinxQL
* [ ] Implement database auto backup on crawl process completing
* [x] Add transactions to prevent data loss on DB crashes
* [x] JSON API
* [ ] Distributed index data sharing between the nodes trough service API
* [x] Distributed index data sharing between the nodes trough service API
* [x] An idea to make unique gravatars for sites without favicons, because simpler to ident, comparing to ipv6
* [ ] An idea to make some visitors counters, like in good old times?

44
config/app.php.txt
View File

@ -121,8 +121,10 @@ define('CRAWL_STOP_DISK_QUOTA_MB_LEFT', 500);
* Usually up to 20 pages per minute,
* to prevent websites overload by sending GET crawling requests
*
* Set 0 to disable
*
*/
define('CRAWL_PAGE_LIMIT', 10);
define('CRAWL_PAGE_LIMIT', 20);
/*
* Images (URI) processing limit in the crawler.php queue
@ -133,8 +135,27 @@ define('CRAWL_PAGE_LIMIT', 10);
* Usually up to 20 pages per minute,
* to prevent websites overload by sending GET crawling requests
*
* Set 0 to disable
*
*/
define('CRAWL_IMAGE_LIMIT', 20);
define('CRAWL_IMAGE_LIMIT', 10);
/*
* Manifest (URI) processing limit in the crawler.php queue
*
* Used to collect distributed data index
* that match CRAWL_URL_REGEXP & CRAWL_MANIFEST_API_VERSION
*
* This option related to CRAWL_MANIFEST_SECONDS_OFFSET value
* and the crontab task frequency (https://github.com/YGGverse/YGGo#crontab)
*
* Usually up to 20 pages per minute,
* to prevent websites overload by sending GET crawling requests
*
* Set 0 to disable
*
*/
define('CRAWL_MANIFEST_LIMIT', 10);
/*
* Renew page index by timing offset provided
@ -162,6 +183,19 @@ define('CRAWL_PAGE_SECONDS_OFFSET', 60*60*24*30*12);
*/
define('CRAWL_IMAGE_SECONDS_OFFSET', 60*60*24*30*12);
/*
* Renew manifests index by timing offset provided
*
* This option works with CRAWL_MANIFEST_LIMIT step queue
*
* Pay attention, that CRAWL_MANIFEST_LIMIT + CRAWL_MANIFEST_SECONDS_OFFSET pair
* must have enough value to crawl all manifests collected in the DB index
*
* or the crawler can stuck in queue
*
*/
define('CRAWL_MANIFEST_SECONDS_OFFSET', 60*60*24*30);
/*
* Only URL addresses match this rule will be auto-crawled
*
@ -242,6 +276,12 @@ define('CRAWL_ROBOTS_POSTFIX_RULES', null); // string|null
*/
define('CRAWL_MANIFEST', true);
/*
* Manifest API version compatibility
*
*/
define('CRAWL_MANIFEST_API_VERSION', 0.4);
/*
* Set default auto-crawl status for new manifest added
*

101
crontab/cleaner.php
View File

@ -22,29 +22,31 @@ $db = new MySQL(DB_HOST, DB_PORT, DB_NAME, DB_USERNAME, DB_PASSWORD);
$timeStart = microtime(true);
$hostsTotal = $db->getTotalHosts();
$manifestsTotal = $db->getTotalManifests();
$hostsUpdated = 0;
$hostsPagesDeleted = 0;
$hostsImagesDeleted = 0;
$manifestsDeleted = 0;
// Get host queue
foreach ($db->getCleanerQueue(CLEAN_HOST_LIMIT, time() - CLEAN_HOST_SECONDS_OFFSET) as $host) {
// Begin update
$db->beginTransaction();
// Parse host info
$hostURL = $host->scheme . '://' . $host->name . ($host->port ? ':' . $host->port : false);
try {
// Get robots.txt if exists
$curl = new Curl($hostURL . '/robots.txt', CRAWL_CURLOPT_USERAGENT);
// Get cleaner queue
foreach ($db->getCleanerQueue(CLEAN_HOST_LIMIT, time() - CLEAN_HOST_SECONDS_OFFSET) as $host) {
if (200 == $curl->getCode() && false !== stripos($curl->getContent(), 'user-agent:')) {
$hostRobots = $curl->getContent();
} else {
$hostRobots = null;
}
// Parse host info
$hostURL = $host->scheme . '://' . $host->name . ($host->port ? ':' . $host->port : false);
// Begin update
$db->beginTransaction();
// Get robots.txt if exists
$curl = new Curl($hostURL . '/robots.txt', CRAWL_CURLOPT_USERAGENT);
try {
if (200 == $curl->getCode() && false !== stripos($curl->getContent(), 'user-agent:')) {
$hostRobots = $curl->getContent();
} else {
$hostRobots = null;
}
// Update host data
$hostsUpdated += $db->updateHostRobots($host->hostId, $hostRobots, time());
@ -118,15 +120,66 @@ foreach ($db->getCleanerQueue(CLEAN_HOST_LIMIT, time() - CLEAN_HOST_SECONDS_OFFS
// Delete host image
$hostsImagesDeleted += $db->deleteHostImage($hostImage->hostImageId);
}
$db->commit();
} catch(Exception $e){
var_dump($e);
$db->rollBack();
}
// Clean up deprecated manifests
foreach ($db->getManifests() as $manifest) {
$delete = false;
$curl = new Curl($manifest->url);
// Skip processing non 200 code
if (200 != $curl->getCode()) {
continue; // Wait for reconnect
}
// Skip processing without returned data
if (!$remoteManifest = $curl->getContent()) {
$delete = true;
}
// Skip processing on json encoding error
if (!$remoteManifest = @json_decode($remoteManifest)) {
$delete = true;
}
// Skip processing on required fields missed
if (empty($remoteManifest->status) ||
empty($remoteManifest->result->config->crawlUrlRegexp) ||
empty($remoteManifest->result->api->version)) {
$delete = true;
}
// Skip processing on API version not compatible
if ($remoteManifest->result->api->version !== CRAWL_MANIFEST_API_VERSION) {
$delete = true;
}
// Skip processing on crawlUrlRegexp does not match CRAWL_URL_REGEXP condition
if ($remoteManifest->result->config->crawlUrlRegexp !== CRAWL_URL_REGEXP) {
$delete = true;
}
if ($delete) {
$manifestsDeleted += $db->deleteManifest($manifest->manifestId);
}
}
$db->commit();
} catch(Exception $e){
var_dump($e);
$db->rollBack();
}
// Debug
@ -134,4 +187,6 @@ echo 'Hosts total: ' . $hostsTotal . PHP_EOL;
echo 'Hosts updated: ' . $hostsUpdated . PHP_EOL;
echo 'Hosts pages deleted: ' . $hostsPagesDeleted . PHP_EOL;
echo 'Hosts images deleted: ' . $hostsImagesDeleted . PHP_EOL;
echo 'Execution time: ' . microtime(true) - $timeStart . PHP_EOL;
echo 'Manifests total: ' . $manifestsTotal . PHP_EOL;
echo 'Manifests deleted: ' . $manifestsDeleted . PHP_EOL;
echo 'Execution time: ' . microtime(true) - $timeStart . PHP_EOL . PHP_EOL;

574
crontab/crawler.php
View File

@ -29,8 +29,10 @@ $timeStart = microtime(true);
$hostPagesProcessed = 0;
$hostImagesProcessed = 0;
$manifestsProcessed = 0;
$hostPagesIndexed = 0;
$hostImagesIndexed = 0;
$manifestsIndexed = 0;
$hostPagesAdded = 0;
$hostImagesAdded = 0;
$hostsAdded = 0;
@ -38,175 +40,350 @@ $hostsAdded = 0;
// Connect database
$db = new MySQL(DB_HOST, DB_PORT, DB_NAME, DB_USERNAME, DB_PASSWORD);
// Process images crawl queue
foreach ($db->getHostImageCrawlQueue(CRAWL_IMAGE_LIMIT, time() - CRAWL_IMAGE_SECONDS_OFFSET) as $queueHostImage) {
$db->beginTransaction();
// Build URL from the DB
$queueHostImageURL = $queueHostImage->scheme . '://' . $queueHostImage->name . ($queueHostImage->port ? ':' . $queueHostImage->port : false) . $queueHostImage->uri;
try {
$curl = new Curl($queueHostImageURL, CRAWL_CURLOPT_USERAGENT);
// Process manifests crawl queue
foreach ($db->getManifestCrawlQueue(CRAWL_MANIFEST_LIMIT, time() - CRAWL_MANIFEST_SECONDS_OFFSET) as $queueManifest) {
// Update image index anyway, with the current time and http code
$hostImagesProcessed += $db->updateHostImageCrawlQueue($queueHostImage->hostImageId, time(), $curl->getCode());
$curl = new Curl($queueManifest->url);
// Skip next image processing non 200 code
if (200 != $curl->getCode()) {
// Update manifest index anyway, with the current time and http code
$manifestsProcessed += $db->updateManifestCrawlQueue($queueManifest->manifestId, time(), $curl->getCode());
continue;
// Skip processing non 200 code
if (200 != $curl->getCode()) {
continue;
}
// Skip processing without returned data
if (!$remoteManifest = $curl->getContent()) {
continue;
}
// Skip processing on json encoding error
if (!$remoteManifest = @json_decode($remoteManifest)) {
continue;
}
// Skip processing on required fields missed
if (empty($remoteManifest->status) ||
empty($remoteManifest->result->config->crawlUrlRegexp) ||
empty($remoteManifest->result->api->version) ||
empty($remoteManifest->result->api->hosts)) {
continue;
}
// Skip processing on API version not compatible
if ($remoteManifest->result->api->version !== CRAWL_MANIFEST_API_VERSION) {
continue;
}
// Skip processing on host API not available
if (!$remoteManifest->result->api->hosts) {
continue;
}
// Skip processing on crawlUrlRegexp does not match CRAWL_URL_REGEXP condition
if ($remoteManifest->result->config->crawlUrlRegexp !== CRAWL_URL_REGEXP) {
continue;
}
// Skip processing on host link does not match condition
if (false === preg_match(CRAWL_URL_REGEXP, $remoteManifest->result->api->hosts)) {
continue;
}
// Begin hosts collection
$curl = new Curl($remoteManifest->result->api->hosts);
// Skip processing non 200 code
if (200 != $curl->getCode()) {
continue;
}
// Skip processing without returned data
if (!$remoteManifestHosts = $curl->getContent()) {
continue;
}
// Skip processing on json encoding error
if (!$remoteManifestHosts = @json_decode($remoteManifestHosts)) {
continue;
}
// Skip processing on required fields missed
if (empty($remoteManifestHosts->status) ||
empty($remoteManifestHosts->result)) {
continue;
}
// Begin hosts processing
foreach ($remoteManifestHosts->result as $remoteManifestHost) {
// Skip processing on required fields missed
if (empty($remoteManifestHost->scheme) ||
empty($remoteManifestHost->name)) {
continue;
}
$hostURL = $remoteManifestHost->scheme . '://' .
$remoteManifestHost->name .
(!empty($remoteManifestHost->port) ? ':' . $remoteManifestHost->port : false);
// Validate formatted link
if (filter_var($hostURL, FILTER_VALIDATE_URL) && preg_match(CRAWL_URL_REGEXP, $hostURL)) {
// Host exists
if ($host = $db->getHost(crc32($hostURL))) {
$hostStatus = $host->status;
$hostPageLimit = $host->crawlPageLimit;
$hostImageLimit = $host->crawlImageLimit;
$hostId = $host->hostId;
$hostRobots = $host->robots;
$hostRobotsPostfix = $host->robotsPostfix;
// Register new host
} else {
// Get robots.txt if exists
$curl = new Curl($hostURL . '/robots.txt', CRAWL_CURLOPT_USERAGENT);
if (200 == $curl->getCode() && false !== stripos($curl->getContent(), 'user-agent:')) {
$hostRobots = $curl->getContent();
} else {
$hostRobots = CRAWL_ROBOTS_DEFAULT_RULES;
}
$hostRobotsPostfix = CRAWL_ROBOTS_POSTFIX_RULES;
$hostStatus = CRAWL_HOST_DEFAULT_STATUS;
$hostPageLimit = CRAWL_HOST_DEFAULT_PAGES_LIMIT;
$hostImageLimit= CRAWL_HOST_DEFAULT_IMAGES_LIMIT;
$hostId = $db->addHost($remoteManifestHosts->result->scheme,
$remoteManifestHosts->result->name,
$remoteManifestHosts->result->port,
crc32($hostURL),
time(),
null,
$hostPageLimit,
$hostImageLimit,
(string) CRAWL_HOST_DEFAULT_META_ONLY,
(string) $hostStatus,
$hostRobots,
$hostRobotsPostfix);
if ($hostId) {
$hostsAdded++;
} else {
continue;
}
}
// Init robots parser
$robots = new Robots(($hostRobots ? (string) $hostRobots : (string) CRAWL_ROBOTS_DEFAULT_RULES) . PHP_EOL . ($hostRobotsPostfix ? (string) $hostRobotsPostfix : (string) CRAWL_ROBOTS_POSTFIX_RULES));
// Save home page info
// Until page API not implemented, save at least home page to have ability to crawl
// @TODO
if ($hostStatus && // host enabled
$robots->uriAllowed('/') && // page allowed by robots.txt rules
$hostPageLimit > $db->getTotalHostPages($hostId) && // pages quantity not reached host limit
!$db->getHostPage($hostId, crc32('/'))) { // page not exists
if ($db->addHostPage($hostId, crc32('/'), '/', time())) {
$hostPagesAdded++;
}
}
}
}
}
// Save image content on data settings enabled
if (!CRAWL_HOST_DEFAULT_META_ONLY) {
// Process images crawl queue
foreach ($db->getHostImageCrawlQueue(CRAWL_IMAGE_LIMIT, time() - CRAWL_IMAGE_SECONDS_OFFSET) as $queueHostImage) {
// Skip next image processing images without returned data
// Build URL from the DB
$queueHostImageURL = $queueHostImage->scheme . '://' . $queueHostImage->name . ($queueHostImage->port ? ':' . $queueHostImage->port : false) . $queueHostImage->uri;
$curl = new Curl($queueHostImageURL, CRAWL_CURLOPT_USERAGENT);
// Update image index anyway, with the current time and http code
$hostImagesProcessed += $db->updateHostImageCrawlQueue($queueHostImage->hostImageId, time(), $curl->getCode());
// Skip next image processing non 200 code
if (200 != $curl->getCode()) {
continue;
}
// Save image content on data settings enabled
if (!CRAWL_HOST_DEFAULT_META_ONLY) {
// Skip next image processing images without returned data
if (!$content = $curl->getContent()) {
continue;
}
// Convert remote image data to base64 string to prevent direct URL call
if (!$hostImageType = @pathinfo($queueHostImageURL, PATHINFO_EXTENSION)) {
continue;
}
if (!$hostImageBase64 = @base64_encode($curl->getContent())) {
continue;
}
$hostImagesIndexed += $db->updateHostImageData($hostImage->hostImageId, (string) 'data:image/' . $hostImageType . ';base64,' . $hostImageBase64, time());
}
}
// Process pages crawl queue
foreach ($db->getHostPageCrawlQueue(CRAWL_PAGE_LIMIT, time() - CRAWL_PAGE_SECONDS_OFFSET) as $queueHostPage) {
// Build URL from the DB
$queueHostPageURL = $queueHostPage->scheme . '://' . $queueHostPage->name . ($queueHostPage->port ? ':' . $queueHostPage->port : false) . $queueHostPage->uri;
$curl = new Curl($queueHostPageURL, CRAWL_CURLOPT_USERAGENT);
// Update page index anyway, with the current time and http code
$hostPagesProcessed += $db->updateHostPageCrawlQueue($queueHostPage->hostPageId, time(), $curl->getCode());
// Skip next page processing non 200 code
if (200 != $curl->getCode()) {
continue;
}
// Skip next page processing pages without returned data
if (!$content = $curl->getContent()) {
continue;
}
// Convert remote image data to base64 string to prevent direct URL call
if (!$hostImageType = @pathinfo($queueHostImageURL, PATHINFO_EXTENSION)) {
// Grab page content
$dom = new DomDocument();
@$dom->loadHTML($content);
// Skip index page links without titles
$title = @$dom->getElementsByTagName('title');
if ($title->length == 0) {
continue;
}
// Get optional page meta data
$metaDescription = '';
$metaKeywords = '';
$metaRobots = '';
$metaYggoManifest = '';
foreach (@$dom->getElementsByTagName('meta') as $meta) {
if (@$meta->getAttribute('name') == 'description') {
$metaDescription = @$meta->getAttribute('content');
}
if (@$meta->getAttribute('name') == 'keywords') {
$metaKeywords = @$meta->getAttribute('content');
}
if (@$meta->getAttribute('name') == 'robots') {
$metaRobots = @$meta->getAttribute('content');
}
if (@$meta->getAttribute('name') == 'yggo:manifest') {
$metaYggoManifest = Filter::url(@$meta->getAttribute('content'));
}
}
// Update queued page data
$hostPagesIndexed += $db->updateHostPage($queueHostPage->hostPageId,
Filter::pageTitle($title->item(0)->nodeValue),
Filter::pageDescription($metaDescription),
Filter::pageKeywords($metaKeywords),
CRAWL_HOST_DEFAULT_META_ONLY ? null : Filter::pageData($content));
// Update manifest registry
if (CRAWL_MANIFEST && !empty($metaYggoManifest) && filter_var($metaYggoManifest, FILTER_VALIDATE_URL) && preg_match(CRAWL_URL_REGEXP, $metaYggoManifest)) {
$metaYggoManifestCRC32 = crc32($metaYggoManifest);
if (!$db->getManifest($metaYggoManifestCRC32)) {
$db->addManifest($metaYggoManifestCRC32,
$metaYggoManifest,
(string) CRAWL_MANIFEST_DEFAULT_STATUS,
time());
}
}
// Append page with meta robots:noindex value to the robotsPostfix disallow list
if (false !== stripos($metaRobots, 'noindex')) {
continue;
}
if (!$hostImageBase64 = @base64_encode($curl->getContent())) {
// Skip page links following by robots:nofollow attribute detected
if (false !== stripos($metaRobots, 'nofollow')) {
continue;
}
$hostImagesIndexed += $db->updateHostImageData($hostImage->hostImageId, (string) 'data:image/' . $hostImageType . ';base64,' . $hostImageBase64, time());
}
}
// Collect page images
if (CRAWL_HOST_DEFAULT_IMAGES_LIMIT > 0) {
// Process pages crawl queue
foreach ($db->getHostPageCrawlQueue(CRAWL_PAGE_LIMIT, time() - CRAWL_PAGE_SECONDS_OFFSET) as $queueHostPage) {
foreach (@$dom->getElementsByTagName('img') as $img) {
// Build URL from the DB
$queueHostPageURL = $queueHostPage->scheme . '://' . $queueHostPage->name . ($queueHostPage->port ? ':' . $queueHostPage->port : false) . $queueHostPage->uri;
// Skip images without src attribute
if (!$imageSrc = @$img->getAttribute('src')) {
$curl = new Curl($queueHostPageURL, CRAWL_CURLOPT_USERAGENT);
continue;
}
// Update page index anyway, with the current time and http code
$hostPagesProcessed += $db->updateHostPageCrawlQueue($queueHostPage->hostPageId, time(), $curl->getCode());
// Skip images without alt attribute
if (!$imageAlt = @$img->getAttribute('alt')) {
// Skip next page processing non 200 code
if (200 != $curl->getCode()) {
continue;
}
continue;
}
if (!$imageTitle = @$img->getAttribute('title')) {
$imageTitle = null;
}
// Skip next page processing pages without returned data
if (!$content = $curl->getContent()) {
// Add domain to the relative src links
if (!parse_url($imageSrc, PHP_URL_HOST)) {
continue;
}
$imageSrc = $queueHostPage->scheme . '://' .
$queueHostPage->name .
($queueHostPage->port ? ':' . $queueHostPage->port : '') .
'/' . trim(ltrim(str_replace(['./', '../'], '', $imageSrc), '/'), '.');
}
// Grab page content
$dom = new DomDocument();
@$dom->loadHTML($content);
// Skip index page links without titles
$title = @$dom->getElementsByTagName('title');
if ($title->length == 0) {
continue;
}
// Get optional page meta data
$metaDescription = '';
$metaKeywords = '';
$metaRobots = '';
$metaYggoManifest = '';
foreach (@$dom->getElementsByTagName('meta') as $meta) {
if (@$meta->getAttribute('name') == 'description') {
$metaDescription = @$meta->getAttribute('content');
}
if (@$meta->getAttribute('name') == 'keywords') {
$metaKeywords = @$meta->getAttribute('content');
}
if (@$meta->getAttribute('name') == 'robots') {
$metaRobots = @$meta->getAttribute('content');
}
if (@$meta->getAttribute('name') == 'yggo:manifest') {
$metaYggoManifest = Filter::url(@$meta->getAttribute('content'));
}
}
// Update queued page data
$hostPagesIndexed += $db->updateHostPage($queueHostPage->hostPageId,
Filter::pageTitle($title->item(0)->nodeValue),
Filter::pageDescription($metaDescription),
Filter::pageKeywords($metaKeywords),
CRAWL_HOST_DEFAULT_META_ONLY ? null : Filter::pageData($content));
// Update manifest registry
if (CRAWL_MANIFEST && !empty($metaYggoManifest) && filter_var($metaYggoManifest, FILTER_VALIDATE_URL) && preg_match(CRAWL_URL_REGEXP, $metaYggoManifest)) {
$metaYggoManifestCRC32 = crc32($metaYggoManifest);
if (!$db->getManifest($metaYggoManifestCRC32)) {
$db->addManifest($metaYggoManifestCRC32,
$metaYggoManifest,
(string) CRAWL_MANIFEST_DEFAULT_STATUS,
time());
}
}
// Append page with meta robots:noindex value to the robotsPostfix disallow list
if (false !== stripos($metaRobots, 'noindex')) {
continue;
}
// Skip page links following by robots:nofollow attribute detected
if (false !== stripos($metaRobots, 'nofollow')) {
continue;
}
// Collect page images
if (CRAWL_HOST_DEFAULT_IMAGES_LIMIT > 0) {
foreach (@$dom->getElementsByTagName('img') as $img) {
// Skip images without src attribute
if (!$imageSrc = @$img->getAttribute('src')) {
continue;
}
// Skip images without alt attribute
if (!$imageAlt = @$img->getAttribute('alt')) {
continue;
}
if (!$imageTitle = @$img->getAttribute('title')) {
$imageTitle = null;
}
// Add domain to the relative src links
if (!parse_url($imageSrc, PHP_URL_HOST)) {
$imageSrc = $queueHostPage->scheme . '://' .
$queueHostPage->name .
($queueHostPage->port ? ':' . $queueHostPage->port : '') .
'/' . trim(ltrim(str_replace(['./', '../'], '', $imageSrc), '/'), '.');
}
// Validate formatted src link
if (filter_var($imageSrc, FILTER_VALIDATE_URL) && preg_match(CRAWL_URL_REGEXP, $imageSrc)) {
$db->beginTransaction();
try {
// Validate formatted src link
if (filter_var($imageSrc, FILTER_VALIDATE_URL) && preg_match(CRAWL_URL_REGEXP, $imageSrc)) {
// Parse formatted src link
$hostImageURL = Parser::hostURL($imageSrc);
@ -269,9 +446,9 @@ foreach ($db->getHostPageCrawlQueue(CRAWL_PAGE_LIMIT, time() - CRAWL_PAGE_SECOND
$hostImageId = $db->getHostImageId($hostId, crc32($hostImageURI->string));
if (!$hostImageId && // image not exists
$hostStatus && // host enabled
$robots->uriAllowed($hostImageURI->string) && // src allowed by robots.txt rules
$hostImageLimit > $db->getTotalHostImages($hostId)) { // images quantity not reached host limit
$hostStatus && // host enabled
$robots->uriAllowed($hostImageURI->string) && // src allowed by robots.txt rules
$hostImageLimit > $db->getTotalHostImages($hostId)) { // images quantity not reached host limit
// Add host image
if ($hostImageId = $db->addHostImage($hostId, crc32($hostImageURI->string), $hostImageURI->string, time(), null, 200)) {
@ -289,11 +466,11 @@ foreach ($db->getHostPageCrawlQueue(CRAWL_PAGE_LIMIT, time() - CRAWL_PAGE_SECOND
// Add/update host image description
$db->setHostImageDescription($hostImageId,
crc32(md5((string) $imageAlt . (string) $imageTitle)),
Filter::imageAlt($imageAlt),
Filter::imageTitle($imageTitle),
time(),
time());
crc32(md5((string) $imageAlt . (string) $imageTitle)),
Filter::imageAlt($imageAlt),
Filter::imageTitle($imageTitle),
time(),
time());
// Relate host image with host page was found
$db->setHostImageToHostPage($hostImageId, $queueHostPage->hostPageId, time(), time(), 1);
@ -302,77 +479,64 @@ foreach ($db->getHostPageCrawlQueue(CRAWL_PAGE_LIMIT, time() - CRAWL_PAGE_SECOND
// Increase image rank when link does not match the current host
if ($hostImageURL->scheme . '://' .
$hostImageURL->name .
($hostImageURL->port ? ':' . $hostImageURL->port : '')
($hostImageURL->port ? ':' . $hostImageURL->port : '')
!=
$queueHostPage->scheme . '://' .
$queueHostPage->name .
($queueHostPage->port ? ':' . $queueHostPage->port : '')) {
($queueHostPage->port ? ':' . $queueHostPage->port : '')) {
$db->updateHostImageRank($hostId, crc32($hostImageURI->string), 1);
}
$db->commit();
} catch(Exception $e) {
var_dump($e);
$db->rollBack();
}
}
}
}
// Collect internal links from page content
foreach(@$dom->getElementsByTagName('a') as $a) {
// Collect internal links from page content
foreach(@$dom->getElementsByTagName('a') as $a) {
// Skip links without required attribute
if (!$href = @$a->getAttribute('href')) {
// Skip links without required attribute
if (!$href = @$a->getAttribute('href')) {
continue;
}
continue;
}
// Skip anchor links
if (false !== strpos($href, '#')) {
// Skip anchor links
if (false !== strpos($href, '#')) {
continue;
}
continue;
}
// Skip javascript links
if (false !== strpos($href, 'javascript:')) {
// Skip javascript links
if (false !== strpos($href, 'javascript:')) {
continue;
}
continue;
}
// Skip mailto links
if (false !== strpos($href, 'mailto:')) {
// Skip mailto links
if (false !== strpos($href, 'mailto:')) {
continue;
}
continue;
}
// Skip x-raw-image links
if (false !== strpos($href, 'x-raw-image:')) {
// Skip x-raw-image links
if (false !== strpos($href, 'x-raw-image:')) {
continue;
}
continue;
}
// @TODO skip other apps
// @TODO skip other apps
// Add absolute URL prefixes to the relative links found
if (!parse_url($href, PHP_URL_HOST)) {
// Add absolute URL prefixes to the relative links found
if (!parse_url($href, PHP_URL_HOST)) {
$href = $queueHostPage->scheme . '://' .
$queueHostPage->name .
($queueHostPage->port ? ':' . $queueHostPage->port : '') .
'/' . trim(ltrim(str_replace(['./', '../'], '', $href), '/'), '.');
}
$href = $queueHostPage->scheme . '://' .
$queueHostPage->name .
($queueHostPage->port ? ':' . $queueHostPage->port : '') .
'/' . trim(ltrim(str_replace(['./', '../'], '', $href), '/'), '.');
}
// Validate formatted link
if (filter_var($href, FILTER_VALIDATE_URL) && preg_match(CRAWL_URL_REGEXP, $href)) {
$db->beginTransaction();
try {
// Validate formatted link
if (filter_var($href, FILTER_VALIDATE_URL) && preg_match(CRAWL_URL_REGEXP, $href)) {
// Parse formatted link
$hostURL = Parser::hostURL($href);
@ -435,7 +599,7 @@ foreach ($db->getHostPageCrawlQueue(CRAWL_PAGE_LIMIT, time() - CRAWL_PAGE_SECOND
if ($hostStatus && // host enabled
$robots->uriAllowed($hostPageURI->string) && // page allowed by robots.txt rules
$hostPageLimit > $db->getTotalHostPages($hostId) && // pages quantity not reached host limit
!$db->getHostPage($hostId, crc32($hostPageURI->string))) { // page not exists
!$db->getHostPage($hostId, crc32($hostPageURI->string))) { // page not exists
if ($db->addHostPage($hostId, crc32($hostPageURI->string), $hostPageURI->string, time())) {
@ -454,17 +618,17 @@ foreach ($db->getHostPageCrawlQueue(CRAWL_PAGE_LIMIT, time() - CRAWL_PAGE_SECOND
$db->updateHostPageRank($hostId, crc32($hostPageURI->string), 1);
}
$db->commit();
} catch(Exception $e){
var_dump($e);
$db->rollBack();
}
}
}
$db->commit();
} catch(Exception $e) {
var_dump($e);
$db->rollBack();
}
// Debug
@ -474,5 +638,7 @@ echo 'Pages added: ' . $hostPagesAdded . PHP_EOL;
echo 'Images processed: ' . $hostImagesProcessed . PHP_EOL;
echo 'Images indexed: ' . $hostImagesIndexed . PHP_EOL;
echo 'Images added: ' . $hostImagesAdded . PHP_EOL;
echo 'Manifests processed: ' . $manifestsProcessed . PHP_EOL;
echo 'Manifests indexed: ' . $manifestsIndexed . PHP_EOL;
echo 'Hosts added: ' . $hostsAdded . PHP_EOL;
echo 'Total time: ' . microtime(true) - $timeStart . PHP_EOL;
echo 'Total time: ' . microtime(true) - $timeStart . PHP_EOL . PHP_EOL;

BIN
database/yggo.mwb
View File

Binary file not shown.

51
library/mysql.php
View File

@ -29,6 +29,24 @@ class MySQL {
}
// Manifest
public function getTotalManifests() {
$query = $this->_db->prepare('SELECT COUNT(*) AS `total` FROM `manifest`');
$query->execute();
return $query->fetch()->total;
}
public function getManifests() {
$query = $this->_db->prepare('SELECT * FROM `manifest`');
$query->execute();
return $query->fetchAll();
}
public function getManifest(int $crc32url) {
$query = $this->_db->prepare('SELECT * FROM `manifest` WHERE `crc32url` = ? LIMIT 1');
@ -47,6 +65,15 @@ class MySQL {
return $this->_db->lastInsertId();
}
public function deleteManifest(int $manifestId) {
$query = $this->_db->prepare('DELETE FROM `manifest` WHERE `manifestId` = ? LIMIT 1');
$query->execute([$manifestId]);
return $query->rowCount();
}
// Host
public function getAPIHosts(string $apiHostFields) {
@ -570,4 +597,28 @@ class MySQL {
return $query->rowCount();
}
public function getManifestCrawlQueue(int $limit, int $timeFrom) {
$query = $this->_db->prepare('SELECT * FROM `manifest`
WHERE (`timeUpdated` IS NULL OR `timeUpdated` < ? ) AND `status` <> 0
ORDER BY RAND()
LIMIT ' . (int) $limit);
$query->execute([$timeFrom]);
return $query->fetchAll();
}
public function updateManifestCrawlQueue(int $manifestId, int $timeUpdated, int $httpCode) {
$query = $this->_db->prepare('UPDATE `manifest` SET `timeUpdated` = ?, `httpCode` = ? WHERE `manifestId` = ? LIMIT 1');
$query->execute([$timeUpdated, $httpCode, $manifestId]);
return $query->rowCount();
}
}