YGGverse/YGGo
1
0
Fork 0
mirror of https://github.com/YGGverse/YGGo.git synced 2025-09-10 13:22:18 +00:00
Code Issues Projects Releases Wiki Activity

prevent page ban when it MIME in the whitelist, skip steps below only (make multimedia/streaming resources visible in search results)

Browse Source
This commit is contained in:
ghost 2023-06-04 17:44:09 +03:00
parent 4fa33afe40
commit c5e25d17fb
1 changed files with 3 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
crontab/crawler.php
View File

@ -424,7 +424,9 @@ try {
// Skip page processing without returned data // Skip page processing without returned data
if (!$content = $curl->getContent()) { if (!$content = $curl->getContent()) {
$hostPagesBanned += $db->updateHostPageTimeBanned($queueHostPage->hostPageId, time()); // Prevent page ban when it MIME in the whitelist, skip steps below only
// This case possible for multimedia/streaming resources index
// $hostPagesBanned += $db->updateHostPageTimeBanned($queueHostPage->hostPageId, time());
continue; continue;
} }