diff --git a/crontab/crawler.php b/crontab/crawler.php index e36dc12..090afba 100644 --- a/crontab/crawler.php +++ b/crontab/crawler.php @@ -37,7 +37,7 @@ $hostImagesProcessed = 0; $manifestsProcessed = 0; $hostPagesIndexed = 0; $hostImagesIndexed = 0; -$manifestsAdded = 0; +$manifestsAdded = 0; $hostPagesAdded = 0; $hostImagesAdded = 0; $hostsAdded = 0; @@ -265,7 +265,9 @@ try { // Skip image processing non 200 code if (200 != $curl->getCode()) { - $hostImagesBanned += $db->updateHostImageTimeBanned($queueHostImage->hostImageId, time()); + $db->updateHostImageHttpCode($queueHostImage->hostImageId, $curl->getCode(), time()); + + $hostImagesBanned += $db->updateHostImageTimeBanned($queueHostImage->hostImageId, $curl->getCode(), time()); continue; } @@ -291,6 +293,8 @@ try { if ($hostImageBanned) { + $db->updateHostImageMime($queueHostImage->hostImageId, $hostImageContentType, time()); + $hostImagesBanned += $db->updateHostImageTimeBanned($queueHostImage->hostImageId, time()); continue; @@ -355,6 +359,8 @@ try { // Skip page processing non 200 code if (200 != $curl->getCode()) { + $db->updateHostPageHttpCode($queueHostPage->hostPageId, $curl->getCode(), time()); + $hostPagesBanned += $db->updateHostPageTimeBanned($queueHostPage->hostPageId, time()); continue; @@ -381,6 +387,8 @@ try { if ($hostPageBanned) { + $db->updateHostPageMime($queueHostPage->hostPageId, $contentType, time()); + $hostPagesBanned += $db->updateHostPageTimeBanned($queueHostPage->hostPageId, time()); continue; diff --git a/library/mysql.php b/library/mysql.php index f5b3bdf..76d5d34 100644 --- a/library/mysql.php +++ b/library/mysql.php @@ -228,6 +228,17 @@ class MySQL { return $query->rowCount(); } + public function updateHostImageMime(int $hostImageId, + string $mime, + int $timeUpdated) { + + $query = $this->_db->prepare('UPDATE `hostImage` SET `mime` = ?, `timeUpdated` = ? WHERE `hostImageId` = ? LIMIT 1'); + + $query->execute([$mime, $timeUpdated, $hostImageId]); + + return $query->rowCount(); + } + public function updateHostImage(int $hostImageId, string $mime, mixed $data, @@ -510,6 +521,24 @@ class MySQL { return $query->rowCount(); } + public function updateHostPageHttpCode(int $hostPageId, int $httpCode) { + + $query = $this->_db->prepare('UPDATE `hostPage` SET `httpCode` = ? WHERE `hostPageId` = ? LIMIT 1'); + + $query->execute([$httpCode, $hostPageId]); + + return $query->rowCount(); + } + + public function updateHostPageMime(int $hostPageId, string $mime) { + + $query = $this->_db->prepare('UPDATE `hostPage` SET `mime` = ? WHERE `hostPageId` = ? LIMIT 1'); + + $query->execute([$mime, $hostPageId]); + + return $query->rowCount(); + } + public function deleteHostPage(int $hostPageId) { $query = $this->_db->prepare('DELETE FROM `hostPage` WHERE `hostPageId` = ? LIMIT 1'); diff --git a/public/search.php b/public/search.php index c16aea8..aa2d1bf 100644 --- a/public/search.php +++ b/public/search.php @@ -355,6 +355,8 @@ if (!empty($q)) { if (200 != $hostImageHttpCode) { + $db->updateHostImageHttpCode($hostImage->hostImageId, $hostImageHttpCode, time()); + $db->updateHostImageTimeBanned($hostImage->hostImageId, time()); continue; @@ -381,6 +383,8 @@ if (!empty($q)) { if ($hostImageBanned) { + $db->updateHostImageMime($hostImage->hostImageId, $hostImageContentType, time()); + $hostImagesBanned += $db->updateHostImageTimeBanned($hostImage->hostImageId, time()); continue;