YGGverse/YGGo
1
0
Fork 0
mirror of https://github.com/YGGverse/YGGo.git synced 2025-01-24 13:34:25 +00:00
Code Issues Projects Releases Wiki Activity

ban page that throws the error and stuck the crawl queue

Browse Source
This commit is contained in:
ghost 2023-06-04 12:04:41 +03:00
parent 5a47c66e55
commit 512bd56056
1 changed files with 20 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

21
crontab/crawler.php
View File

@ -796,13 +796,32 @@ try {
}
}
// Apply changes
$db->commit();
// Process update errors
} catch(Exception $e) {
// Decline DB changes
$db->rollBack();
// Debug std
var_dump($e);
$db->rollBack();
// Ban page that throws the error and stuck the crawl queue
if (!empty($queueHostPage->hostPageId)) {
$hostPagesBanned = $db->updateHostPageTimeBanned($queueHostPage->hostPageId, time());
// Reset counters
$hostPagesProcessed = $hostPagesBanned;
$manifestsProcessed = 0;
$hostPagesIndexed = 0;
$manifestsAdded = 0;
$hostPagesAdded = 0;
$hostsAdded = 0;
$hostPagesSnapAdded = 0;
}
}
// Debug