Browse Source

prevent page ban when it MIME in the whitelist, skip steps below only (make multimedia/streaming resources visible in search results)

main
ghost 2 years ago
parent
commit
c5e25d17fb
  1. 4
      crontab/crawler.php

4
crontab/crawler.php

@ -424,7 +424,9 @@ try {
// Skip page processing without returned data // Skip page processing without returned data
if (!$content = $curl->getContent()) { if (!$content = $curl->getContent()) {
$hostPagesBanned += $db->updateHostPageTimeBanned($queueHostPage->hostPageId, time()); // Prevent page ban when it MIME in the whitelist, skip steps below only
// This case possible for multimedia/streaming resources index
// $hostPagesBanned += $db->updateHostPageTimeBanned($queueHostPage->hostPageId, time());
continue; continue;
} }

Loading…
Cancel
Save