From 06c136f05c835d6925275d9aa5509c5217ced7d2 Mon Sep 17 00:00:00 2001
From: ghost <localhost>
Date: Wed, 2 Aug 2023 10:56:25 +0300
Subject: [PATCH] fix meta/nofollow attribute processing

---
 crontab/crawler.php | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/crontab/crawler.php b/crontab/crawler.php
index 798b905..135da2b 100644
--- a/crontab/crawler.php
+++ b/crontab/crawler.php
@@ -799,7 +799,9 @@ foreach ($db->getHostPageCrawlQueue(CRAWL_PAGE_LIMIT, time() - CRAWL_PAGE_SECOND
 
           if (false !== stripos($metaRobots, 'nofollow')) {
 
-            continue;
+            $db->commit();
+
+            continue 2;
           }
         }
       }