diff --git a/config/app.php.txt b/config/app.php.txt index a01f713..3e8a1d6 100644 --- a/config/app.php.txt +++ b/config/app.php.txt @@ -321,6 +321,16 @@ define('CRAWL_HOST_DEFAULT_NSFW', false); */ define('CRAWL_SITEMAPS', true); +/* + * Re-calculate page rank on page update + * + * When enabled, may enlarge execution time + * + * true|false + * + */ +define('CRAWL_PAGE_RANK_UPDATE', true); + /* * Renew robots.txt index by timing offset provided * diff --git a/crontab/crawler.php b/crontab/crawler.php index b326b65..f302bb4 100644 --- a/crontab/crawler.php +++ b/crontab/crawler.php @@ -357,26 +357,29 @@ foreach ($db->getHostPageCrawlQueue(CRAWL_PAGE_LIMIT, time() - CRAWL_PAGE_SECOND $httpRequestsTimeTotal += $curl->getTotalTime(); // Update page rank - // @TODO add common method + if (CRAWL_PAGE_RANK_UPDATE) { - $hostPageRank = 0; + // @TODO add common method - // Get referrers - foreach ($db->getHostPagesToHostPageByHostPageIdTarget($queueHostPage->hostPageId) as $hostPageToHostPageByHostPageIdTarget) { + $hostPageRank = 0; - // Get source page details - if ($hostPageSource = $db->getHostPage($hostPageToHostPageByHostPageIdTarget->hostPageIdSource)) { + // Get referrers + foreach ($db->getHostPagesToHostPageByHostPageIdTarget($queueHostPage->hostPageId) as $hostPageToHostPageByHostPageIdTarget) { - // Increase PR on external referrer only - if ($hostPageSource->hostId != $queueHostPage->hostId) { + // Get source page details + if ($hostPageSource = $db->getHostPage($hostPageToHostPageByHostPageIdTarget->hostPageIdSource)) { - $hostPageRank++; - } + // Increase PR on external referrer only + if ($hostPageSource->hostId != $queueHostPage->hostId) { + + $hostPageRank++; + } - // Delegate page rank value from redirected pages - if (false !== strpos($hostPageSource->httpCode, '30')) { + // Delegate page rank value from redirected pages + if (false !== strpos($hostPageSource->httpCode, '30')) { - $hostPageRank += $hostPageSource->rank; + $hostPageRank += $hostPageSource->rank; + } } } }