From c5e25d17fb04d09777449cf79c5509b1935b3e22 Mon Sep 17 00:00:00 2001
From: ghost <noreply@localhost>
Date: Sun, 4 Jun 2023 17:44:09 +0300
Subject: [PATCH] prevent page ban when it MIME in the whitelist, skip steps
 below only (make multimedia/streaming resources visible in search results)

---
 crontab/crawler.php | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/crontab/crawler.php b/crontab/crawler.php
index a3d8265..bceec9c 100644
--- a/crontab/crawler.php
+++ b/crontab/crawler.php
@@ -424,7 +424,9 @@ try {
     // Skip page processing without returned data
     if (!$content = $curl->getContent()) {
 
-      $hostPagesBanned += $db->updateHostPageTimeBanned($queueHostPage->hostPageId, time());
+      // Prevent page ban when it MIME in the whitelist, skip steps below only
+      // This case possible for multimedia/streaming resources index
+      // $hostPagesBanned += $db->updateHostPageTimeBanned($queueHostPage->hostPageId, time());
 
       continue;
     }