From c5e25d17fb04d09777449cf79c5509b1935b3e22 Mon Sep 17 00:00:00 2001 From: ghost <noreply@localhost> Date: Sun, 4 Jun 2023 17:44:09 +0300 Subject: [PATCH] prevent page ban when it MIME in the whitelist, skip steps below only (make multimedia/streaming resources visible in search results) --- crontab/crawler.php | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/crontab/crawler.php b/crontab/crawler.php index a3d8265..bceec9c 100644 --- a/crontab/crawler.php +++ b/crontab/crawler.php @@ -424,7 +424,9 @@ try { // Skip page processing without returned data if (!$content = $curl->getContent()) { - $hostPagesBanned += $db->updateHostPageTimeBanned($queueHostPage->hostPageId, time()); + // Prevent page ban when it MIME in the whitelist, skip steps below only + // This case possible for multimedia/streaming resources index + // $hostPagesBanned += $db->updateHostPageTimeBanned($queueHostPage->hostPageId, time()); continue; }