update http code and mime type on page/image ban event

crontab/crawler.php

@@ -37,7 +37,7 @@ $hostImagesProcessed   = 0;
 $manifestsProcessed    = 0;
 $hostPagesIndexed      = 0;
 $hostImagesIndexed     = 0;
-$manifestsAdded         = 0;
+$manifestsAdded        = 0;
 $hostPagesAdded        = 0;
 $hostImagesAdded       = 0;
 $hostsAdded            = 0;
@@ -265,7 +265,9 @@ try {
     // Skip image processing non 200 code
     if (200 != $curl->getCode()) {
 
-      $hostImagesBanned += $db->updateHostImageTimeBanned($queueHostImage->hostImageId, time());
+      $db->updateHostImageHttpCode($queueHostImage->hostImageId, $curl->getCode(), time());
+
+      $hostImagesBanned += $db->updateHostImageTimeBanned($queueHostImage->hostImageId, $curl->getCode(), time());
 
       continue;
     }
@@ -291,6 +293,8 @@ try {
 
     if ($hostImageBanned) {
 
+      $db->updateHostImageMime($queueHostImage->hostImageId, $hostImageContentType, time());
+
       $hostImagesBanned += $db->updateHostImageTimeBanned($queueHostImage->hostImageId, time());
 
       continue;
@@ -355,6 +359,8 @@ try {
     // Skip page processing non 200 code
     if (200 != $curl->getCode()) {
 
+      $db->updateHostPageHttpCode($queueHostPage->hostPageId, $curl->getCode(), time());
+
       $hostPagesBanned += $db->updateHostPageTimeBanned($queueHostPage->hostPageId, time());
 
       continue;
@@ -381,6 +387,8 @@ try {
 
     if ($hostPageBanned) {
 
+      $db->updateHostPageMime($queueHostPage->hostPageId, $contentType, time());
+
       $hostPagesBanned += $db->updateHostPageTimeBanned($queueHostPage->hostPageId, time());
 
       continue;

library/mysql.php

@@ -228,6 +228,17 @@ class MySQL {
     return $query->rowCount();
   }
 
+  public function updateHostImageMime(int $hostImageId,
+                                      string $mime,
+                                      int $timeUpdated) {
+
+    $query = $this->_db->prepare('UPDATE `hostImage` SET `mime` = ?, `timeUpdated` = ? WHERE `hostImageId` = ? LIMIT 1');
+
+    $query->execute([$mime, $timeUpdated, $hostImageId]);
+
+    return $query->rowCount();
+  }
+
   public function updateHostImage(int $hostImageId,
                                   string $mime,
                                   mixed $data,
@@ -510,6 +521,24 @@ class MySQL {
     return $query->rowCount();
   }
 
+  public function updateHostPageHttpCode(int $hostPageId, int $httpCode) {
+
+    $query = $this->_db->prepare('UPDATE `hostPage` SET `httpCode` = ? WHERE `hostPageId` = ? LIMIT 1');
+
+    $query->execute([$httpCode, $hostPageId]);
+
+    return $query->rowCount();
+  }
+
+  public function updateHostPageMime(int $hostPageId, string $mime) {
+
+    $query = $this->_db->prepare('UPDATE `hostPage` SET `mime` = ? WHERE `hostPageId` = ? LIMIT 1');
+
+    $query->execute([$mime, $hostPageId]);
+
+    return $query->rowCount();
+  }
+
   public function deleteHostPage(int $hostPageId) {
 
     $query = $this->_db->prepare('DELETE FROM `hostPage` WHERE `hostPageId` = ? LIMIT 1');

public/search.php

@@ -355,6 +355,8 @@ if (!empty($q)) {
 
                 if (200 != $hostImageHttpCode) {
 
+                  $db->updateHostImageHttpCode($hostImage->hostImageId, $hostImageHttpCode, time());
+
                   $db->updateHostImageTimeBanned($hostImage->hostImageId, time());
 
                   continue;
@@ -381,6 +383,8 @@ if (!empty($q)) {
 
                 if ($hostImageBanned) {
 
+                  $db->updateHostImageMime($hostImage->hostImageId, $hostImageContentType, time());
+
                   $hostImagesBanned += $db->updateHostImageTimeBanned($hostImage->hostImageId, time());
 
                   continue;