diff --git a/crontab/crawler.php b/crontab/crawler.php index 798b905..135da2b 100644 --- a/crontab/crawler.php +++ b/crontab/crawler.php @@ -799,7 +799,9 @@ foreach ($db->getHostPageCrawlQueue(CRAWL_PAGE_LIMIT, time() - CRAWL_PAGE_SECOND if (false !== stripos($metaRobots, 'nofollow')) { - continue; + $db->commit(); + + continue 2; } } }