YGGverse
/
YGGo
mirror of https://github.com/YGGverse/YGGo.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity
YGGo! Distributed Web Search Engine
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
529 Commits
2 Branches
0 Tags
4.3 MiB
 
Tree: 7bb1eb5b61
Branches Tags
${ item.name }
Create tag ${ searchTerm }
Create branch ${ searchTerm }
from '7bb1eb5b61'
${ noResults }
YGGo/crontab/crawler.php

1132 lines
32 KiB
Raw Blame History

<?php
// Stop crawler on cli running
$semaphore = sem_get(crc32('cli.yggo'), 1);
if (false === sem_acquire($semaphore, true)) {
echo 'cli.yggo process running in another thread.' . PHP_EOL;
exit;
}
// Lock multi-thread execution
$semaphore = sem_get(crc32('crontab.crawler'), 1);
if (false === sem_acquire($semaphore, true)) {
echo 'process locked by another thread.' . PHP_EOL;
exit;
}
// Begin debug output
echo '-- ' . date('c') . ' --' . PHP_EOL . PHP_EOL;
// Load system dependencies
require_once(__DIR__ . '/../config/app.php');
require_once(__DIR__ . '/../library/ftp.php');
require_once(__DIR__ . '/../library/curl.php');
require_once(__DIR__ . '/../library/robots.php');
require_once(__DIR__ . '/../library/sitemap.php');
require_once(__DIR__ . '/../library/url.php');
require_once(__DIR__ . '/../library/filter.php');
require_once(__DIR__ . '/../library/mysql.php');
require_once(__DIR__ . '/../library/helper.php');
require_once(__DIR__ . '/../library/yggstate.php');
require_once(__DIR__ . '/../library/vendor/simple_html_dom.php');
// Check disk quota
if (CRAWL_STOP_DISK_QUOTA_MB_LEFT > disk_free_space('/') / 1000000) {
echo 'Disk quota reached.' . PHP_EOL;
exit;
}
// Debug
$timeStart = microtime(true);
$httpRequestsTotal = 0;
$httpRequestsSizeTotal = 0;
$httpDownloadSizeTotal = 0;
$httpRequestsTimeTotal = 0;
$hostsProcessed = 0;
$hostsAdded = 0;
$hostPagesProcessed = 0;
$hostPagesBanned = 0;
$hostPagesSnapAdded = 0;
$hostPagesAdded = 0;
$manifestsProcessed = 0;
$sitemapsProcessed = 0;
// Connect database
try {
$db = new MySQL(DB_HOST, DB_PORT, DB_NAME, DB_USERNAME, DB_PASSWORD);
} catch(Exception $e) {
var_dump($e);
exit;
}
// Connect memcached
try {
$memcached = new Memcached();
$memcached->addServer(MEMCACHED_HOST, MEMCACHED_PORT);
} catch(Exception $e) {
var_dump($e);
exit;
}
// Check YGGstate connections to discover new hosts
if (CRAWL_YGGSTATE) {
foreach (json_decode(CRAWL_YGGSTATE) as $server => $nodes) {
foreach ($nodes as $i => $node) {
switch ($server) {
case 'db':
try {
if (!$memcached->get(sprintf('Crontab.crawler.YGGstate.%s.%s.timeout', $server, $i))) {
$yggStateDB = new YGGstate($node->host, $node->port, $node->database, $node->username, $node->password);
foreach ($yggStatePeers = $yggStateDB->getPeersByMinLastUptime($node->peer_min_last_uptime) as $yggStatePeer) {
// Register new host
if ($linkToDBresult = Helper::addLinkToDB($db, $memcached, sprintf('http://[%s]/', $yggStatePeer->address))) {
$hostsAdded += count($linkToDBresult->new->hostId);
$hostPagesAdded += count($linkToDBresult->new->hostPageId);
}
}
$memcached->set(sprintf('Crontab.crawler.YGGstate.%s.%s.timeout', $server, $i), true, time() + $node->timeout);
}
} catch(Exception $e) {
var_dump($e);
continue 2;
}
break;
}
}
}
}
// Process hosts crawl queue
foreach ($db->getHostCrawlQueue(CRAWL_HOST_LIMIT, time() - CRAWL_HOST_SECONDS_OFFSET) as $queueHost) {
try {
$db->beginTransaction();
// Update host crawl queue
$hostsProcessed += $db->updateHostCrawlQueue($queueHost->hostId, time());
// Update host robots.txt settings from remote host
if (CRAWL_ROBOTS) {
$curl = new Curl($queueHost->url . '/robots.txt', CRAWL_CURLOPT_USERAGENT);
// Update curl stats
$httpRequestsTotal++;
$httpRequestsSizeTotal += $curl->getSizeRequest();
$httpDownloadSizeTotal += $curl->getSizeDownload();
$httpRequestsTimeTotal += $curl->getTotalTime();
// Update robots.txt rules
if (200 == $curl->getCode() && false !== stripos(trim(mb_strtolower((string) $curl->getContentType())), 'text/plain')) {
Helper::setHostSetting($db, $memcached, $queueHost->hostId, 'ROBOTS_TXT', (string) $curl->getContent());
}
}
// Process sitemaps when enabled
if (CRAWL_SITEMAPS) {
// Look for custom sitemap URL served in robots.txt
$robots = new Robots(
Helper::getHostSetting($db, $memcached, $queueHost->hostId, 'ROBOTS_TXT', NULL) . PHP_EOL .
Helper::getHostSetting($db, $memcached, $queueHost->hostId, 'ROBOTS_TXT_POSTFIX', DEFAULT_HOST_ROBOTS_TXT_POSTFIX)
);
if ($sitemapLink = $robots->getSitemap()) {
// Replace relative paths
$sitemapURL = sprintf('%s/%s', $queueHost->url, trim(str_ireplace($queueHost->url, '', $sitemapLink), '/'));
// Set default path
} else {
$sitemapURL = sprintf('%s/sitemap.xml', $queueHost->url);
}
// Init sitemap
$sitemap = new Sitemap($sitemapURL);
if ($sitemapLinks = $sitemap->getLinks()) {
$sitemapsProcessed++;
// Process collected sitemap links
foreach ($sitemapLinks as $loc => $attributes) {
// Replace relative paths
$loc = sprintf('%s/%s', $queueHost->url, trim(str_ireplace($queueHost->url, '', $loc), '/'));
// Validate link
if (!$link = URL::parse($loc)) {
continue;
}
// Collect this host links only
if ($link->host->url != $queueHost->url) {
continue;
}
// Register new link
if ($linkToDBresult = Helper::addLinkToDB($db, $memcached, $loc)) {
$hostsAdded += count($linkToDBresult->new->hostId);
$hostPagesAdded += count($linkToDBresult->new->hostPageId);
}
}
}
}
// Update manifests
if (CRAWL_MANIFEST) {
// Host have manifest provided
if ($manifestURL = Helper::getHostSetting($db, $memcached, $queueHost->hostId, 'MANIFEST_URL', NULL)) {
// Get remote manifest
$curl = new Curl($manifestURL, CRAWL_CURLOPT_USERAGENT);
// Update curl stats
$httpRequestsTotal++;
$httpRequestsSizeTotal += $curl->getSizeRequest();
$httpDownloadSizeTotal += $curl->getSizeDownload();
$httpRequestsTimeTotal += $curl->getTotalTime();
// Skip processing non 200 code
if (200 != $curl->getCode()) {
continue;
}
// Skip processing without returned data
if (!$remoteManifest = $curl->getContent()) {
continue;
}
// Skip processing on json encoding error
if (!$remoteManifest = @json_decode($remoteManifest)) {
continue;
}
// Skip processing on required fields missed
if (empty($remoteManifest->status) ||
empty($remoteManifest->result->config->DEFAULT_HOST_URL_REGEXP) ||
empty($remoteManifest->result->api->version) ||
empty($remoteManifest->result->api->hosts)) {
continue;
}
// Skip processing on API version not compatible
if ($remoteManifest->result->api->version !== CRAWL_MANIFEST_API_VERSION) {
continue;
}
// Skip processing on host API not available
if (!$remoteManifest->result->api->hosts) {
continue;
}
// Skip processing on remote host URL does not match local condition
if ($remoteManifest->result->config->DEFAULT_HOST_URL_REGEXP !=
Helper::getHostSetting($db, $memcached, $queueHost->hostId, 'URL_REGEXP', DEFAULT_HOST_URL_REGEXP)) {
continue;
}
// Skip processing on remote host link does not match local condition
if (false === preg_match(Helper::getHostSetting($db, $memcached, $queueHost->hostId, 'URL_REGEXP', DEFAULT_HOST_URL_REGEXP),
$remoteManifest->result->api->hosts)) {
continue;
}
// Grab host URLs
$curl = new Curl($remoteManifest->result->api->hosts, CRAWL_CURLOPT_USERAGENT);
// Update curl stats
$httpRequestsTotal++;
$httpRequestsSizeTotal += $curl->getSizeRequest();
$httpDownloadSizeTotal += $curl->getSizeDownload();
$httpRequestsTimeTotal += $curl->getTotalTime();
// Skip processing non 200 code
if (200 != $curl->getCode()) {
continue;
}
// Skip processing without returned data
if (!$remoteManifest = $curl->getContent()) {
continue;
}
// Skip processing on json encoding error
if (!$remoteManifestHosts = @json_decode($remoteManifest)) {
continue;
}
// Skip processing on required fields missed
if (empty($remoteManifestHosts->result)) {
continue;
}
// Begin hosts processing
foreach ($remoteManifestHosts->result as $remoteManifestHost) {
// Skip processing on required fields missed
if (empty($remoteManifestHost->url)) {
continue;
}
// Register new link
if ($linkToDBresult = Helper::addLinkToDB($db, $memcached, $remoteManifestHost->url)) {
$hostsAdded += count($linkToDBresult->new->hostId);
$hostPagesAdded += count($linkToDBresult->new->hostPageId);
}
}
}
}
$db->commit();
// Process update errors
} catch (Exception $e) {
// Debug std
var_dump($e);
// Skip item
$db->rollBack();
continue;
}
}
// Process pages crawl queue
foreach ($db->getHostPageCrawlQueue(CRAWL_HOST_PAGE_QUEUE_LIMIT, time() - CRAWL_HOST_PAGE_QUEUE_SECONDS_OFFSET) as $queueHostPage) {
$db->beginTransaction();
try {
// Init page request
$curl = new Curl($queueHostPage->hostPageURL, CRAWL_CURLOPT_USERAGENT);
// Update curl stats
$httpRequestsTotal++;
$httpRequestsSizeTotal += $curl->getSizeRequest();
$httpDownloadSizeTotal += $curl->getSizeDownload();
$httpRequestsTimeTotal += $curl->getTotalTime();
// Update page rank
if (CRAWL_HOST_PAGE_RANK_UPDATE) {
$hostPageRank = 0;
// Get referrers
foreach ($db->getHostPagesToHostPageByHostPageIdTarget($queueHostPage->hostPageId) as $hostPageToHostPageByHostPageIdTarget) {
// Get source page details
if ($hostPageSource = $db->getHostPage($hostPageToHostPageByHostPageIdTarget->hostPageIdSource)) {
// Increase PR on external referrer only
if ($hostPageSource->hostId != $queueHostPage->hostId) {
$hostPageRank++;
}
// Delegate page rank value from redirected pages
if (false !== strpos($hostPageSource->httpCode, '30')) {
$hostPageRank += $hostPageSource->rank;
}
}
}
// Update registry
$db->updateHostPageRank($queueHostPage->hostPageId, $hostPageRank);
}
// Update page index anyway, with the current time and http code
$hostPagesProcessed += $db->updateHostPageCrawlQueue($queueHostPage->hostPageId, time(), $curl->getCode(), $curl->getSizeDownload());
// This page not available
if (200 != $curl->getCode()) {
// Ban this page
$hostPagesBanned += $db->updateHostPageTimeBanned($queueHostPage->hostPageId, time());
// Try to receive target page location on page redirect available by following location
$curl = new Curl($queueHostPage->hostPageURL, CRAWL_CURLOPT_USERAGENT, 10, true, true);
// Update curl stats
$httpRequestsTotal++;
$httpRequestsSizeTotal += $curl->getSizeRequest();
$httpDownloadSizeTotal += $curl->getSizeDownload();
$httpRequestsTimeTotal += $curl->getTotalTime();
if (200 == $curl->getCode()) {
if (preg_match('~Location: (.*)~i', $curl->getContent(), $match)) {
if (empty($match[1])) {
$db->commit();
continue;
}
$url = trim($match[1]);
//Make relative links absolute
if (!parse_url($url, PHP_URL_HOST)) {
$url = $queueHostPage->hostURL . '/' . trim(ltrim(str_replace(['./', '../'], '', $url), '/'), '.');
}
// Register new link
if ($linkToDBresult = Helper::addLinkToDB($db, $memcached, $url)) {
$hostsAdded += count($linkToDBresult->new->hostId);
$hostPagesAdded += count($linkToDBresult->new->hostPageId);
// Register referrer
if ($linkToDBresult->old->hostPageId) {
foreach ($linkToDBresult->old->hostPageId as $hostPageIdTarget) {
$db->setHostPageToHostPage($queueHostPage->hostPageId, $hostPageIdTarget);
}
}
if ($linkToDBresult->new->hostPageId) {
foreach ($linkToDBresult->new->hostPageId as $hostPageIdTarget) {
$db->setHostPageToHostPage($queueHostPage->hostPageId, $hostPageIdTarget);
}
}
}
}
}
// Skip other this page actions
$db->commit();
continue;
}
// Validate MIME content type
if ($contentType = $curl->getContentType()) {
$db->updateHostPageMime($queueHostPage->hostPageId, Filter::mime($contentType), time());
// Ban page if not available
} else {
$hostPagesBanned += $db->updateHostPageTimeBanned($queueHostPage->hostPageId, time());
$db->commit();
continue;
}
// Check for MIME
$hostPageInMime = false;
foreach ((array) explode(',', Helper::getHostSetting($db, $memcached, $queueHostPage->hostId, 'PAGES_MIME', DEFAULT_HOST_PAGES_MIME)) as $mime) {
// Ban page on MIME type not allowed in settings
if (false !== stripos(Filter::mime($contentType), Filter::mime($mime))) {
$hostPageInMime = true;
break;
}
}
// Ban page not in MIME list
if (!$hostPageInMime) {
$hostPagesBanned += $db->updateHostPageTimeBanned($queueHostPage->hostPageId, time());
$db->commit();
continue;
}
// Skip page processing without returned data
if (!$content = $curl->getContent()) {
// Prevent page ban when it MIME in the whitelist, skip steps below only
// This case possible for multimedia/streaming resources index
// $hostPagesBanned += $db->updateHostPageTimeBanned($queueHostPage->hostPageId, time());
$db->commit();
continue;
}
// Begin snaps
if (SNAP_STORAGE) {
// Register snap in DB
if ($hostPageSnapId = $db->addHostPageSnap($queueHostPage->hostPageId, time())) {
// Default storage success
$snapFilesExists = false;
// Prepare filenames
$hostPageSnapPath = 'hps/' . substr(trim(chunk_split($hostPageSnapId, 1, '/'), '/'), 0, -1);
$hostPageSnapFile = $hostPageSnapPath . substr($hostPageSnapId, -1) . '.zip';
$hostPageSnapFilenameTmp = __DIR__ . '/../storage/tmp/' . md5($hostPageSnapFile);
// Create ZIP container
$zip = new ZipArchive();
if (true === $zip->open($hostPageSnapFilenameTmp, ZipArchive::CREATE)) {
// Insert compressed snap data into the tmp storage
if (true === $zip->addFromString('DATA', $content) &&
true === $zip->addFromString('META', sprintf('MIME: %s', Filter::mime($contentType)) . PHP_EOL .
sprintf('SOURCE: %s', Filter::url($queueHostPage->hostPageURL)) . PHP_EOL .
sprintf('TIMESTAMP: %s', time()))) {
}
}
$zip->close();
// Temporarily snap file exists
if (file_exists($hostPageSnapFilenameTmp)) {
// Copy files to each storage
foreach (json_decode(SNAP_STORAGE) as $node => $storages) {
foreach ($storages as $location => $storage) {
// Generate storage id
$crc32name = crc32(sprintf('%s.%s', $node, $location));
switch ($node) {
case 'localhost':
// Validate mime
if (!$storage->quota->mime) continue 2;
$snapMimeValid = false;
foreach ((array) explode(',', $storage->quota->mime) as $mime) {
if (false !== stripos(Filter::mime($contentType), Filter::mime($mime))) {
$snapMimeValid = true;
break;
}
}
if (!$snapMimeValid) continue 2;
// Copy tmp snap file to the permanent storage
@mkdir($storage->directory . $hostPageSnapPath, 0755, true);
if (copy($hostPageSnapFilenameTmp, $storage->directory . $hostPageSnapFile)) {
// Register storage name
if ($db->addHostPageSnapStorage($hostPageSnapId, $crc32name, time())) {
$snapFilesExists = true;
}
}
break;
case 'ftp':
// Validate mime
if (!$storage->quota->mime) continue 2;
$snapMimeValid = false;
foreach ((array) explode(',', $storage->quota->mime) as $mime) {
if (false !== stripos(Filter::mime($contentType), Filter::mime($mime))) {
$snapMimeValid = true;
break;
}
}
if (!$snapMimeValid) continue 2;
$attempt = 1;
do {
// Copy tmp snap file to the permanent storage
$ftp = new Ftp();
// Remote host connected well...
if ($connection = $ftp->connect($storage->host, $storage->port, $storage->username, $storage->password, $storage->directory, $storage->timeout, $storage->passive)) {
$ftp->mkdir($hostPageSnapPath, true);
if ($ftp->copy($hostPageSnapFilenameTmp, $hostPageSnapFile)) {
// Register storage name
if ($db->addHostPageSnapStorage($hostPageSnapId, $crc32name, time())) {
$snapFilesExists = true;
}
}
$ftp->close();
// On remote connection lost, repeat attempt after 60 seconds...
} else {
echo sprintf(_('[attempt: %s] wait for remote storage %s id %s connection...'), $attempt++, $node, $location) . PHP_EOL;
sleep(60);
}
} while ($connection === false);
break;
}
}
}
}
// At least one file have been stored
if ($snapFilesExists) {
$hostPagesSnapAdded++;
} else {
$db->deleteHostPageSnap($hostPageSnapId);
}
// Delete tmp snap
unlink($hostPageSnapFilenameTmp);
}
}
// Is HTML document
if (false !== stripos(Filter::mime($contentType), 'text/html')) {
// Define variables
$metaDescription = null;
$metaKeywords = null;
$metaYggoManifestURL = null;
// Parse page content
$dom = new DomDocument();
if ($encoding = mb_detect_encoding($content)) {
@$dom->loadHTML(sprintf('<?xml encoding="%s" ?>', $encoding) . $content);
} else {
$hostPagesBanned += $db->updateHostPageTimeBanned($queueHostPage->hostPageId, time());
$db->commit();
continue;
}
// Skip index page links without title tag
$title = @$dom->getElementsByTagName('title');
if ($title->length == 0) {
$metaTitle = null;
/*
$hostPagesBanned += $db->updateHostPageTimeBanned($queueHostPage->hostPageId, time());
$db->commit();
continue;
*/
} else {
$metaTitle = $title->item(0)->nodeValue;
}
// Get optional page meta data
foreach (@$dom->getElementsByTagName('meta') as $meta) {
if (@$meta->getAttribute('name') == 'description') {
$metaDescription = @$meta->getAttribute('content');
}
if (@$meta->getAttribute('name') == 'keywords') {
$metaKeywords = @$meta->getAttribute('content');
}
if (@$meta->getAttribute('name') == 'robots') {
$metaRobots = @$meta->getAttribute('content');
// Ban page with meta robots:noindex attribute
if (false !== stripos($metaRobots, 'noindex')) {
$hostPagesBanned += $db->updateHostPageTimeBanned($queueHostPage->hostPageId, time());
continue;
}
}
// Grab meta yggo:manifest link when available
if (@$meta->getAttribute('name') == 'yggo:manifest') {
$metaYggoManifestURL = Filter::url(@$meta->getAttribute('content'));
}
}
// Add queued page description if not exists
$db->addHostPageDescription($queueHostPage->hostPageId,
$metaTitle,
$metaDescription ? Filter::pageDescription($metaDescription) : null,
$metaKeywords ? Filter::pageKeywords($metaKeywords) : null,
$content ? (Helper::getHostSetting($db, $memcached, $queueHostPage->hostId, 'PAGES_DATA', DEFAULT_HOST_PAGES_DATA) ? base64_encode($content) : null) : null,
time());
// Collect page DOM elements data on enabled
if ($hostPageDomSelectors = Helper::getHostSetting($db, $memcached, $queueHostPage->hostId, 'PAGES_DOM_SELECTORS', DEFAULT_HOST_PAGES_DOM_SELECTORS)) {
// Begin selectors extraction
$html = str_get_html($content);
foreach ((array) explode(';', $hostPageDomSelectors) as $selector) {
foreach($html->find($selector) as $element) {
if (!empty($element->innertext)) {
$db->addHostPageDom($queueHostPage->hostPageId,
time(),
$selector,
trim(Helper::getHostSetting($db, $memcached, $queueHostPage->hostId, 'PAGE_DOM_STRIP_TAGS', DEFAULT_HOST_PAGE_DOM_STRIP_TAGS) ? strip_tags( preg_replace('/[\s]+/',
' ',
str_replace(['<br />', '<br/>', '<br>', '</'],
[' ', ' ', ' ', ' </'],
$element->innertext))) : $element->innertext));
}
}
}
}
// Skip page links following with meta robots:nofollow attribute
foreach (@$dom->getElementsByTagName('meta') as $meta) {
if (@$meta->getAttribute('name') == 'robots') {
if (false !== stripos($metaRobots, 'nofollow')) {
$db->commit();
continue 2;
}
}
}
// Update manifest registry
if (CRAWL_MANIFEST &&
!empty($metaYggoManifestURL) &&
filter_var($metaYggoManifestURL, FILTER_VALIDATE_URL) &&
preg_match(DEFAULT_HOST_URL_REGEXP, $metaYggoManifestURL)) {
$manifestsProcessed += $db->setHostSetting($queueHostPage->hostId, 'MANIFEST_URL', $metaYggoManifestURL);
}
// Begin page links collection
$links = [];
// Collect image links
foreach (@$dom->getElementsByTagName('img') as $img) {
// Skip images without src attribute
if (!$src = @$img->getAttribute('src')) {
continue;
}
// Skip images without alt attribute
if (!$alt = @$img->getAttribute('alt')) {
continue;
}
if (!$title = @$img->getAttribute('title')) {
$title = null;
}
// Skip encoded content
if (false !== stripos($src, 'data:')) {
continue;
}
// Add link to queue
$links[] = [
'title' => null,
'description' => null,
'keywords' => Filter::pageKeywords($alt . ($title ? ',' . $title : '')),
'data' => null,
'mime' => null,
'href' => $src,
];
}
// Collect media links
foreach (@$dom->getElementsByTagName('source') as $source) {
// Skip images without src attribute
if (!$src = @$source->getAttribute('src')) {
continue;
}
// Skip media without type attribute
if (!$type = @$source->getAttribute('type')) {
continue;
}
// Skip encoded content
if (false !== stripos($src, 'data:')) {
continue;
}
// Add link to queue
$links[] = [
'title' => null,
'description' => null,
'keywords' => null,
'data' => null,
'mime' => Filter::mime($type),
'href' => $src,
];
}
foreach (@$dom->getElementsByTagName('video') as $video) {
// Skip images without src attribute
if (!$src = @$video->getAttribute('src')) {
continue;
}
// Skip media without type attribute
if (!$type = @$video->getAttribute('type')) {
$type = 'video/*';
}
// Skip encoded content
if (false !== stripos($src, 'data:')) {
continue;
}
// Add link to queue
$links[] = [
'title' => null,
'description' => null,
'keywords' => null,
'data' => null,
'mime' => Filter::mime($type),
'href' => $src,
];
}
foreach (@$dom->getElementsByTagName('audio') as $audio) {
// Skip images without src attribute
if (!$src = @$audio->getAttribute('src')) {
continue;
}
// Skip media without type attribute
if (!$type = @$audio->getAttribute('type')) {
$type = 'audio/*';
}
// Skip encoded content
if (false !== stripos($src, 'data:')) {
continue;
}
// Add link to queue
$links[] = [
'title' => null,
'description' => null,
'keywords' => null,
'data' => null,
'mime' => Filter::mime($type),
'href' => $src,
];
}
foreach (@$dom->getElementsByTagName('script') as $script) {
// Skip nodes without href attribute
if (!$src = @$script->getAttribute('src')) {
continue;
}
// Add link to queue
$links[] = [
'title' => null,
'description' => null,
'keywords' => null,
'data' => null,
'mime' => null,
'href' => $src,
];
}
foreach (@$dom->getElementsByTagName('link') as $link) {
// Skip nodes without href attribute
if (!$href = @$link->getAttribute('href')) {
continue;
}
// Add link to queue
$links[] = [
'title' => null,
'description' => null,
'keywords' => null,
'data' => null,
'mime' => null,
'href' => $href,
];
}
// Collect internal links from page content
foreach(@$dom->getElementsByTagName('a') as $a) {
// Skip links without required attribute
if (!$href = @$a->getAttribute('href')) {
continue;
}
// Get title attribute if available
if (!$title = @$a->getAttribute('title')) {
$title = null;
}
// Skip anchor links
if (false !== stripos($href, '#')) {
continue;
}
// Skip javascript links
if (false !== stripos($href, 'javascript:')) {
continue;
}
// Skip mailto links
if (false !== stripos($href, 'mailto:')) {
continue;
}
// Skip magnet links
if (false !== stripos($href, 'magnet:')) {
continue;
}
// Skip xmpp links
if (false !== stripos($href, 'xmpp:')) {
continue;
}
// Skip x-raw-image links
/*
if (false !== stripos($href, 'x-raw-image:')) {
continue;
}
*/
// Add link to queue
$links[] = [
'title' => null,
'description' => null,
'keywords' => Filter::pageKeywords($title),
'data' => null,
'mime' => null,
'href' => $href,
];
}
// Process links collected
foreach ($links as $link) {
// Make relative links absolute
if (!parse_url($link['href'], PHP_URL_HOST)) {
$link['href'] = $queueHostPage->hostURL . '/' . trim(ltrim(str_replace(['./', '../'], '', $link['href']), '/'), '.');
}
// Register new link
if ($linkToDBresult = Helper::addLinkToDB($db, $memcached, $link['href'])) {
// Increase new hosts counters
if ($linkToDBresult->new->hostId) {
$hostsAdded += count($linkToDBresult->new->hostId);
}
if ($linkToDBresult->new->hostPageId) {
$hostPagesAdded += count($linkToDBresult->new->hostPageId);
}
// Register referrer
if ($linkToDBresult->old->hostPageId) {
foreach ($linkToDBresult->old->hostPageId as $hostPageIdTarget) {
$db->setHostPageToHostPage($queueHostPage->hostPageId, $hostPageIdTarget);
}
}
if ($linkToDBresult->new->hostPageId) {
foreach ($linkToDBresult->new->hostPageId as $hostPageIdTarget) {
$db->setHostPageToHostPage($queueHostPage->hostPageId, $hostPageIdTarget);
}
}
}
}
}
// Apply changes
$db->commit();
// Process update errors
} catch (Exception $e) {
// Debug std
var_dump($e);
// Ban page that throws the data type error and stuck the crawl queue
if (!empty($queueHostPage->hostPageId) &&
!empty($e->errorInfo[0]) && in_array($e->errorInfo[0], ['HY000']) &&
!empty($e->errorInfo[1]) && in_array($e->errorInfo[1], [1366])) { // @TODO change DB
$hostPagesBanned += $db->updateHostPageTimeBanned($queueHostPage->hostPageId, time());
$hostPagesProcessed++;
// Apply changes
$db->commit();
} else {
// Skip item
$db->rollBack();
}
continue;
}
}
// Debug
$executionTimeTotal = microtime(true) - $timeStart;
$httpRequestsTimeTotal = $httpRequestsTimeTotal / 1000000;
// Debug output
echo 'Crawl queue completed:' . PHP_EOL;
echo '[hosts]' . PHP_EOL;
echo ' processed: ' . $hostsProcessed . PHP_EOL;
echo ' added: ' . $hostsAdded . PHP_EOL . PHP_EOL;
echo '[hosts pages]' . PHP_EOL;
echo ' processed: ' . $hostPagesProcessed . PHP_EOL;
echo ' added: ' . $hostPagesAdded . PHP_EOL;
echo ' banned: ' . $hostPagesBanned . PHP_EOL . PHP_EOL;
echo '[host page snaps]' . PHP_EOL;
echo ' added: ' . $hostPagesSnapAdded . PHP_EOL . PHP_EOL;
echo '[sitemaps]' . PHP_EOL;
echo ' processed: ' . $sitemapsProcessed . PHP_EOL . PHP_EOL;
echo '[manifests]' . PHP_EOL;
echo ' processed: ' . $manifestsProcessed . PHP_EOL . PHP_EOL;
echo '[HTTP]' . PHP_EOL;
echo ' requests total:' . $httpRequestsTotal . PHP_EOL;
echo ' requests size: ' . $httpRequestsSizeTotal . PHP_EOL;
echo ' download size: ' . $httpDownloadSizeTotal . PHP_EOL;
echo ' requests time: ' . $httpRequestsTimeTotal . PHP_EOL . PHP_EOL;
echo '[MySQL]' . PHP_EOL;
echo 'queries:' . PHP_EOL;
echo ' select: ' . $db->getDebug()->query->select->total . PHP_EOL;
echo ' insert: ' . $db->getDebug()->query->insert->total . PHP_EOL;
echo ' update: ' . $db->getDebug()->query->update->total . PHP_EOL;
echo ' delete: ' . $db->getDebug()->query->delete->total . PHP_EOL . PHP_EOL;
echo '-- completed in ' . $executionTimeTotal . ' seconds --' . PHP_EOL . PHP_EOL;