YGGverse/YGGo
1
0
Fork 0
mirror of https://github.com/YGGverse/YGGo.git synced 2025-01-08 22:07:56 +00:00
Code Issues Projects Releases Wiki Activity

add page rank update optional in the crawl queue

Browse Source
This commit is contained in:
ghost 2023-08-02 21:21:23 +03:00
parent a5a2ec233e
commit 5df59661d8
2 changed files with 26 additions and 13 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
config/app.php.txt
View File

@ -321,6 +321,16 @@ define('CRAWL_HOST_DEFAULT_NSFW', false);
*/ */
define('CRAWL_SITEMAPS', true); define('CRAWL_SITEMAPS', true);
/*
* Re-calculate page rank on page update
*
* When enabled, may enlarge execution time
*
* true|false
*
*/
define('CRAWL_PAGE_RANK_UPDATE', true);
/* /*
* Renew robots.txt index by timing offset provided * Renew robots.txt index by timing offset provided
* *

3
crontab/crawler.php
View File

@ -357,6 +357,8 @@ foreach ($db->getHostPageCrawlQueue(CRAWL_PAGE_LIMIT, time() - CRAWL_PAGE_SECOND
$httpRequestsTimeTotal += $curl->getTotalTime(); $httpRequestsTimeTotal += $curl->getTotalTime();
// Update page rank // Update page rank
if (CRAWL_PAGE_RANK_UPDATE) {
// @TODO add common method // @TODO add common method
$hostPageRank = 0; $hostPageRank = 0;
@ -380,6 +382,7 @@ foreach ($db->getHostPageCrawlQueue(CRAWL_PAGE_LIMIT, time() - CRAWL_PAGE_SECOND
} }
} }
} }
}
// Update registry // Update registry
$db->updateHostPageRank($queueHostPage->hostPageId, $hostPageRank); $db->updateHostPageRank($queueHostPage->hostPageId, $hostPageRank);