Browse Source

add hostPage.size collection

main
ghost 2 years ago
parent
commit
ab78e17ca8
  1. 2
      crontab/crawler.php
  2. BIN
      database/yggo.mwb
  3. 6
      library/mysql.php
  4. BIN
      media/db-prototype.png

2
crontab/crawler.php

@ -282,7 +282,7 @@ foreach ($db->getHostPageCrawlQueue(CRAWL_PAGE_LIMIT, time() - CRAWL_PAGE_SECOND @@ -282,7 +282,7 @@ foreach ($db->getHostPageCrawlQueue(CRAWL_PAGE_LIMIT, time() - CRAWL_PAGE_SECOND
$httpRequestsTimeTotal += $curl->getTotalTime();
// Update page index anyway, with the current time and http code
$hostPagesProcessed += $db->updateHostPageCrawlQueue($queueHostPage->hostPageId, time(), $curl->getCode());
$hostPagesProcessed += $db->updateHostPageCrawlQueue($queueHostPage->hostPageId, time(), $curl->getCode(), $curl->getSizeDownload());
// This page has on 200 code
if (200 != $curl->getCode()) {

BIN
database/yggo.mwb

Binary file not shown.

6
library/mysql.php

@ -616,11 +616,11 @@ class MySQL { @@ -616,11 +616,11 @@ class MySQL {
return $query->fetchAll();
}
public function updateHostPageCrawlQueue(int $hostPageId, int $timeUpdated, int $httpCode) {
public function updateHostPageCrawlQueue(int $hostPageId, int $timeUpdated, int $httpCode, int $size) {
$query = $this->_db->prepare('UPDATE `hostPage` SET `timeUpdated` = ?, `httpCode` = ? WHERE `hostPageId` = ? LIMIT 1');
$query = $this->_db->prepare('UPDATE `hostPage` SET `timeUpdated` = ?, `httpCode` = ?, `size` = ? WHERE `hostPageId` = ? LIMIT 1');
$query->execute([$timeUpdated, $httpCode, $hostPageId]);
$query->execute([$timeUpdated, $httpCode, $size, $hostPageId]);
return $query->rowCount();
}

BIN
media/db-prototype.png

Binary file not shown.

Before

Width:  |  Height:  |  Size: 181 KiB

After

Width:  |  Height:  |  Size: 181 KiB

Loading…
Cancel
Save