YGGverse/YGGo
1
0
Fork 0
mirror of https://github.com/YGGverse/YGGo.git synced 2025-02-03 18:35:04 +00:00
Code Issues Projects Releases Wiki Activity

reduce quantity of http requests for each page in queue by CRAWL_HOST_PAGE_SECONDS_DELAY setting

Browse Source
This commit is contained in:
ghost 2023-08-17 18:56:29 +03:00
parent 468ef50ee3
commit f9cf414901
2 changed files with 14 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
src/config/app.php.example
View File

@ -264,6 +264,14 @@ define('CRAWL_HOST_PAGE_QUEUE_LIMIT', 10);
*/
define('CRAWL_HOST_PAGE_QUEUE_SECONDS_OFFSET', 60*60*24*30*12);
/*
* Reduce quantity of http requests for each page in queue
*
* int|false
*
*/
define('CRAWL_HOST_PAGE_SECONDS_DELAY', 1);
/*
* Re-calculate page rank on page update
*

6
src/crontab/crawler.php
View File

@ -1106,6 +1106,12 @@ foreach ($db->getHostPageCrawlQueue(CRAWL_HOST_PAGE_QUEUE_LIMIT, time() - CRAWL_
// Apply changes
$db->commit();
// Reduce quantity of http requests for each page in queue
if (CRAWL_HOST_PAGE_SECONDS_DELAY) {
sleep((int) CRAWL_HOST_PAGE_SECONDS_DELAY);
}
// Process update errors
} catch (Exception $e) {