@ -102,267 +102,53 @@ class MySQL {
return $query->fetch()->total; return $query->fetch()->total;
} }
public function addHost(string $scheme, string $name, mixed $port, int $crc32url, int $timeAdded, mixed $timeUpdated, int $crawlPageLimit, int $crawlImageLimit, string $crawlMetaOnly, string $status, string $nsfw, mixed $robots, mixed $robotsPostfix) { public function addHost(string $scheme,
string $name,
$query = $this->_db->prepare('INSERT INTO `host` (`scheme`, `name`, `port`, `crc32url`, `timeAdded`, `timeUpdated`, `crawlPageLimit`, `crawlImageLimit`, `crawlMetaOnly`, `status`, `nsfw`, `robots`, `robotsPostfix`) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)'); mixed $port,
int $crc32url,
$query->execute([$scheme, $name, $port, $crc32url, $timeAdded, $timeUpdated, $crawlPageLimit, $crawlImageLimit, $crawlMetaOnly, $status, $nsfw, $robots, $robotsPostfix]);
return $this->_db->lastInsertId();
}
public function updateHostRobots(int $hostId, mixed $robots, int $timeUpdated) {
$query = $this->_db->prepare('UPDATE `host` SET `robots` = ?, `timeUpdated` = ? WHERE `hostId` = ? LIMIT 1');
$query->execute([$robots, $timeUpdated, $hostId]);
return $query->rowCount();
}
// Images
public function getTotalHostImages(int $hostId) {
$query = $this->_db->prepare('SELECT COUNT(*) AS `total` FROM `hostImage` WHERE `hostId` = ?');
$query->execute([$hostId]);
return $query->fetch()->total;
}
public function getHostImageId(int $hostId, int $crc32uri) {
$query = $this->_db->prepare('SELECT `hostImageId` FROM `hostImage` WHERE `hostId` = ? AND `crc32uri` = ? LIMIT 1');
$query->execute([$hostId, $crc32uri]);
return $query->rowCount() ? $query->fetch()->hostImageId : 0;
}
public function getHostImages(int $hostId) {
$query = $this->_db->prepare('SELECT * FROM `hostImage` WHERE `hostId` = ?');
$query->execute([$hostId]);
return $query->fetchAll();
}
public function getUnrelatedHostImages() {
$query = $this->_db->prepare('SELECT * FROM `hostImage`
WHERE `hostImage`.`hostImageId` NOT IN (SELECT `hostImageToHostPage`.`hostImageId`
FROM `hostImageToHostPage`
WHERE `hostImageToHostPage`.`hostImageId` = `hostImage`.`hostImageId`)');
$query->execute();
return $query->fetchAll();
}
public function getHostImagesByLimit(int $hostId, int $limit) {
$query = $this->_db->prepare('SELECT * FROM `hostImage` WHERE `hostId` = ? ORDER BY hostImageId DESC LIMIT ' . (int) $limit);
$query->execute([$hostId]);
return $query->fetchAll();
}
public function addHostImage(int $hostId,
int $crc32uri,
string $uri,
int $timeAdded, int $timeAdded,
mixed $timeUpdated = null, mixed $timeUpdated,
mixed $timeBanned = null, int $crawlPageLimit,
mixed $httpCode = null, string $crawlMetaOnly,
mixed $mime = null, string $status,
mixed $rank = null) { string $nsfw,
mixed $robots,
$query = $this->_db->prepare('INSERT INTO `hostImage` ( `hostId`, mixed $robotsPostfix) {
`crc32uri`,
`uri`, $query = $this->_db->prepare('INSERT INTO `host` (`scheme`,
`name`,
`port`,
`crc32url`,
`timeAdded`, `timeAdded`,
`timeUpdated`, `timeUpdated`,
`timeBanned`, `crawlPageLimit`,
`httpCode`, `crawlMetaOnly`,
`mime`, `status`,
`rank`) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)'); `nsfw`,
`robots`,
$query->execute([$hostId, $crc32uri, $uri, $timeAdded, $timeUpdated, $timeBanned, $httpCode, $mime, $rank]); `robotsPostfix`) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)');
return $this->_db->lastInsertId(); $query->execute([ $scheme,
} $name,
$port,
public function updateHostImageRank(int $hostId, $crc32url,
int $crc32uri, $timeAdded,
int $increment) { $timeUpdated,
$crawlPageLimit,
$query = $this->_db->prepare('UPDATE `hostImage` SET `rank` = `rank` + ' . (int) $increment . ' WHERE `hostId` = ? AND `crc32uri` = ? LIMIT 1'); $crawlMetaOnly,
$status,
$query->execute([$hostId, $crc32uri]); $nsfw,
$robots,
return $query->rowCount(); $robotsPostfix]);
}
public function updateHostImageTimeBanned(int $hostImageId, int $timeBanned) {
$query = $this->_db->prepare('UPDATE `hostImage` SET `timeBanned` = ? WHERE `hostImageId` = ? LIMIT 1');
$query->execute([$timeBanned, $hostImageId]);
return $query->rowCount();
}
public function updateHostImageHttpCode(int $hostImageId,
int $httpCode,
int $timeUpdated) {
$query = $this->_db->prepare('UPDATE `hostImage` SET `httpCode` = ?, `timeUpdated` = ? WHERE `hostImageId` = ? LIMIT 1');
$query->execute([$httpCode, $timeUpdated, $hostImageId]);
return $query->rowCount();
}
public function updateHostImageMime(int $hostImageId,
string $mime,
int $timeUpdated) {
$query = $this->_db->prepare('UPDATE `hostImage` SET `mime` = ?, `timeUpdated` = ? WHERE `hostImageId` = ? LIMIT 1');
$query->execute([$mime, $timeUpdated, $hostImageId]);
return $query->rowCount();
}
public function updateHostImage(int $hostImageId,
string $mime,
int $timeUpdated,
mixed $timeBanned = null) {
$query = $this->_db->prepare('UPDATE `hostImage` SET `mime` = ?, `timeUpdated` = ?, `timeBanned` = ? WHERE `hostImageId` = ? LIMIT 1');
$query->execute([$mime, $timeUpdated, $timeBanned, $hostImageId]);
return $query->rowCount();
}
public function deleteHostImage(int $hostImageId) {
$query = $this->_db->prepare('DELETE FROM `hostImage` WHERE `hostImageId` = ? LIMIT 1');
$query->execute([$hostImageId]);
return $query->rowCount();
}
public function setHostImageDescription(int $hostImageId,
int $crc32id,
string $alt,
string $title,
mixed $data,
int $timeAdded,
mixed $timeUpdated) {
$query = $this->_db->prepare('INSERT INTO `hostImageDescription` (`hostImageId`,
`crc32id`,
`alt`,
`title`,
`timeAdded`) VALUES (?, ?, ?, ?, ?)
ON DUPLICATE KEY UPDATE `alt` = ?,
`title` = ?,
`timeUpdated` = ?');
$query->execute([$hostImageId, $crc32id, $alt, $title, $timeAdded, $alt, $title, $timeUpdated]);
return $this->_db->lastInsertId();
}
public function setHostImageDescriptionData(int $hostImageId,
int $crc32id,
mixed $data,
int $timeAdded,
mixed $timeUpdated) {
$query = $this->_db->prepare('INSERT INTO `hostImageDescription` (`hostImageId`,
`crc32id`,
`data`,
`timeAdded`) VALUES (?, ?, ?, ?)
ON DUPLICATE KEY UPDATE `timeUpdated` = ?');
$query->execute([$hostImageId, $crc32id, $data, $timeAdded, $timeUpdated]);
return $this->_db->lastInsertId(); return $this->_db->lastInsertId();
} }
public function deleteHostImageDescription(int $hostImageId) { public function updateHostRobots(int $hostId, mixed $robots, int $timeUpdated) {
$query = $this->_db->prepare('DELETE FROM `hostImageDescription` WHERE `hostImageId` = ?');
$query->execute([$hostImageId]);
return $query->rowCount();
}
public function getLastHostImageDescription(int $hostImageId) {
$query = $this->_db->prepare('SELECT * FROM `hostImageDescription` WHERE `hostImageId` = ? ORDER BY `timeUpdated` DESC, `timeAdded` DESC LIMIT 1');
$query->execute([$hostImageId]);
return $query->fetch();
}
public function getHostImageHostPages(int $hostImageId, int $limit = 5) {
$query = $this->_db->prepare('SELECT * FROM `hostImageToHostPage`
JOIN `hostPage` ON (`hostPage`.`hostPageId` = `hostImageToHostPage`.`hostPageId`)
WHERE `hostImageId` = ?
ORDER BY `hostPage`.`rank` DESC, RAND(`hostPage`.`hostId`)
LIMIT ' . (int) $limit);
$query->execute([$hostImageId]);
return $query->fetchAll();
}
public function getHostImageHostPagesTotal(int $hostImageId) {
$query = $this->_db->prepare('SELECT COUNT(*) AS `total` FROM `hostImageToHostPage` WHERE `hostImageId` = ?');
$query->execute([$hostImageId]);
return $query->fetch()->total;
}
public function setHostImageToHostPage(int $hostImageId, int $hostPageId, int $time, int $quantity) {
$query = $this->_db->prepare('INSERT INTO `hostImageToHostPage` (`hostImageId`,
`hostPageId`,
`timeAdded`,
`timeUpdated`,
`quantity`) VALUES (?, ?, ?, ?, ?)
ON DUPLICATE KEY UPDATE `timeUpdated` = ?,
`quantity` = `quantity` + ' . (int) $quantity);
$query->execute([$hostImageId, $hostPageId, $time, null, $quantity, $time]);
return $query->rowCount(); // no primary key
}
public function deleteHostImageToHostPage(int $hostImageId) {
$query = $this->_db->prepare('DELETE FROM `hostImageToHostPage` WHERE `hostImageId` = ? '); $query = $this->_db->prepare('UPDATE `host` SET `robots` = ?, `timeUpdated` = ? WHERE `hostId` = ? LIMIT 1');
$query->execute([$hostImage Id]); $query->execute([$robots, $timeUpdated, $hostId]);
return $query->rowCount(); return $query->rowCount();
} }
@ -421,18 +207,9 @@ class MySQL {
return $query->fetchAll(); return $query->fetchAll();
} }
public function getHostPageDescription(int $hostPageId, int $crc32data) {
$query = $this->_db->prepare('SELECT * FROM `hostPageDescription` WHERE `hostPageId` = ? AND `crc32data` = ? LIMIT 1');
$query->execute([$hostPageId, $crc32data]);
return $query->fetch();
}
public function getLastPageDescription(int $hostPageId) { public function getLastPageDescription(int $hostPageId) {
$query = $this->_db->prepare('SELECT * FROM `hostPageDescription` WHERE `hostPageId` = ? ORDER BY `timeUpdated` DESC, `time Added` DESC LIMIT 1'); $query = $this->_db->prepare('SELECT * FROM `hostPageDescription` WHERE `hostPageId` = ? ORDER BY `timeAdded` DESC LIMIT 1');
$query->execute([$hostPageId]); $query->execute([$hostPageId]);
@ -442,7 +219,6 @@ class MySQL {
public function getFoundHostPage(int $hostPageId) { public function getFoundHostPage(int $hostPageId) {
$query = $this->_db->prepare('SELECT `hostPage`.`uri`, $query = $this->_db->prepare('SELECT `hostPage`.`uri`,
`hostPage`.`rank`,
`host`.`scheme`, `host`.`scheme`,
`host`.`name`, `host`.`name`,
`host`.`port` `host`.`port`
@ -459,28 +235,6 @@ class MySQL {
return $query->fetch(); return $query->fetch();
} }
public function getFoundHostImage(int $hostImageId) {
$query = $this->_db->prepare('SELECT `hostImage`.`hostImageId`,
`hostImage`.`uri`,
`hostImage`.`rank`,
`host`.`scheme`,
`host`.`name`,
`host`.`port`,
`host`.`crawlMetaOnly`
FROM `hostImage`
JOIN `host` ON (`host`.`hostId` = `hostImage`.`hostId`)
WHERE `hostImage`.`hostImageId` = ?
LIMIT 1');
$query->execute([$hostImageId]);
return $query->fetch();
}
public function addHostPage(int $hostId, public function addHostPage(int $hostId,
int $crc32uri, int $crc32uri,
string $uri, string $uri,
@ -488,8 +242,7 @@ class MySQL {
mixed $timeUpdated = null, mixed $timeUpdated = null,
mixed $timeBanned = null, mixed $timeBanned = null,
mixed $httpCode = null, mixed $httpCode = null,
mixed $mime = null, mixed $mime = null) {
mixed $rank = null) {
$query = $this->_db->prepare('INSERT INTO `hostPage` (`hostId`, $query = $this->_db->prepare('INSERT INTO `hostPage` (`hostId`,
`crc32uri`, `crc32uri`,
@ -498,10 +251,9 @@ class MySQL {
`timeUpdated`, `timeUpdated`,
`timeBanned`, `timeBanned`,
`httpCode`, `httpCode`,
`mime`, `mime`) VALUES (?, ?, ?, ?, ?, ?, ?, ?)');
`rank`) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)');
$query->execute([$hostId, $crc32uri, $uri, $timeAdded, $timeUpdated, $timeBanned, $httpCode, $mime, $rank ]); $query->execute([$hostId, $crc32uri, $uri, $timeAdded, $timeUpdated, $timeBanned, $httpCode, $mime]);
return $this->_db->lastInsertId(); return $this->_db->lastInsertId();
} }
@ -515,22 +267,6 @@ class MySQL {
return $query->rowCount(); return $query->rowCount();
} }
public function updateHostPageRank(int $hostId,
int $crc32uri,
int $increment) {
$query = $this->_db->prepare('UPDATE `hostPage` SET `rank` = `rank` + ' . (int) $increment . '
WHERE `hostId` = ?
AND `crc32uri` = ?
LIMIT 1');
$query->execute([$hostId, $crc32uri]);
return $query->rowCount();
}
public function updateHostPageTimeBanned(int $hostPageId, int $timeBanned) { public function updateHostPageTimeBanned(int $hostPageId, int $timeBanned) {
$query = $this->_db->prepare('UPDATE `hostPage` SET `timeBanned` = ? WHERE `hostPageId` = ? LIMIT 1'); $query = $this->_db->prepare('UPDATE `hostPage` SET `timeBanned` = ? WHERE `hostPageId` = ? LIMIT 1');
@ -576,48 +312,52 @@ class MySQL {
return $query->rowCount(); return $query->rowCount();
} }
public function deleteHostPageToHostImage(int $hostPageId) { public function addHostPageDescription(int $hostPageId,
mixed $title,
$query = $this->_db->prepare('DELETE FROM `hostImageToHostPage` WHERE `hostPageId` = ?'); mixed $description,
mixed $keywords,
$query->execute([$hostPageId]);
return $query->rowCount();
}
public function setHostPageDescription(int $hostPageId,
int $crc32data,
mixed $metaTitle,
mixed $metaDescription,
mixed $metaKeywords,
mixed $data, mixed $data,
int $time) { int $timeAdded) {
$query = $this->_db->prepare('INSERT INTO `hostPageDescription` ( `hostPageId`, $query = $this->_db->prepare('INSERT INTO `hostPageDescription` ( `hostPageId`,
`crc32data`, `title`,
`metaTitle`, `description`,
`metaDescription`, `keywords`,
`metaKeywords`,
`data`, `data`,
`timeAdded` `timeAdded`
) VALUES (?, ?, ?, ?, ?, ?, ?) ) VALUES (?, ?, ?, ?, ?, ?)');
ON DUPLICATE KEY UPDATE `timeUpdated` = ?');
$query->execute([ $query->execute([
$hostPageId, $hostPageId,
$crc32data, $title,
$metaTitle, $description,
$metaDescription, $keywords,
$metaKeywords,
$data, $data,
$time, $timeAdded,
$time
]); ]);
return $query->rowCount(); return $query->rowCount();
} }
public function addHostPageToHostPage(int $hostPageIdSource, int $hostPageIdTarget) {
$query = $this->_db->prepare('INSERT INTO `hostPageToHostPage` (`hostPageIdSource`, `hostPageIdTarget`, `quantity`) VALUES (?, ?, 0)
ON DUPLICATE KEY UPDATE `quantity` = `quantity` + 1');
$query->execute([$hostPageIdSource, $hostPageIdTarget]);
}
public function deleteHostPageToHostPage(int $hostPageId) {
$query = $this->_db->prepare('DELETE FROM `hostPageToHostPage` WHERE `hostPageIdSource` = ? OR `hostPageIdTarget` = ?');
$query->execute([$hostPageId, $hostPageId]);
return $query->rowCount();
}
// Cleaner tools // Cleaner tools
public function getCleanerQueue(int $limit, int $timeFrom) { public function getCleanerQueue(int $limit, int $timeFrom) {
@ -652,33 +392,12 @@ class MySQL {
return $query->rowCount(); return $query->rowCount();
} }
public function resetBannedHostImages(int $timeOffset) {
$query = $this->_db->prepare('UPDATE `hostImage` SET `timeBanned` = NULL WHERE `timeBanned` IS NOT NULL AND `timeBanned` < ' . (int) $timeOffset);
$query->execute();
return $query->rowCount();
}
public function deleteHostImageDescriptionsByTimeAdded(int $timeOffset) {
$query = $this->_db->prepare('DELETE FROM `hostImageDescription` WHERE `timeAdded` < ' . (int) $timeOffset);
$query->execute();
return $query->rowCount();
}
public function addCleanerLog(int $timeAdded, public function addCleanerLog(int $timeAdded,
int $hostsTotal, int $hostsTotal,
int $hostsUpdated, int $hostsUpdated,
int $hostPagesDeleted, int $hostPagesDeleted,
int $hostPageDescriptionsDeleted, int $hostPageDescriptionsDeleted,
int $hostPagesBansRemoved, int $hostPagesBansRemoved,
int $hostImagesDeleted,
int $hostImageDescriptionsDeleted,
int $hostImagesBansRemoved,
int $manifestsTotal, int $manifestsTotal,
int $manifestsDeleted, int $manifestsDeleted,
int $logsCleanerDeleted, int $logsCleanerDeleted,
@ -695,9 +414,6 @@ class MySQL {
`hostPagesDeleted`, `hostPagesDeleted`,
`hostPageDescriptionsDeleted`, `hostPageDescriptionsDeleted`,
`hostPagesBansRemoved`, `hostPagesBansRemoved`,
`hostImagesDeleted`,
`hostImageDescriptionsDeleted`,
`hostImagesBansRemoved`,
`manifestsTotal`, `manifestsTotal`,
`manifestsDeleted`, `manifestsDeleted`,
`logsCleanerDeleted`, `logsCleanerDeleted`,
@ -706,7 +422,7 @@ class MySQL {
`httpRequestsSizeTotal`, `httpRequestsSizeTotal`,
`httpDownloadSizeTotal`, `httpDownloadSizeTotal`,
`httpRequestsTimeTotal`, `httpRequestsTimeTotal`,
`executionTimeTotal`) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ? )'); `executionTimeTotal`) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)');
$query->execute([ $query->execute([
$timeAdded, $timeAdded,
@ -715,9 +431,6 @@ class MySQL {
$hostPagesDeleted, $hostPagesDeleted,
$hostPageDescriptionsDeleted, $hostPageDescriptionsDeleted,
$hostPagesBansRemoved, $hostPagesBansRemoved,
$hostImagesDeleted,
$hostImageDescriptionsDeleted,
$hostImagesBansRemoved,
$manifestsTotal, $manifestsTotal,
$manifestsDeleted, $manifestsDeleted,
$logsCleanerDeleted, $logsCleanerDeleted,
@ -751,7 +464,6 @@ class MySQL {
`host`.`name`, `host`.`name`,
`host`.`port`, `host`.`port`,
`host`.`crawlPageLimit`, `host`.`crawlPageLimit`,
`host`.`crawlImageLimit`,
`host`.`crawlMetaOnly`, `host`.`crawlMetaOnly`,
`host`.`robots`, `host`.`robots`,
`host`.`robotsPostfix` `host`.`robotsPostfix`
@ -762,7 +474,7 @@ class MySQL {
WHERE (`hostPage`.`timeUpdated` IS NULL OR `hostPage`.`timeUpdated` < ? ) AND `host`.`status` < > 0 WHERE (`hostPage`.`timeUpdated` IS NULL OR `hostPage`.`timeUpdated` < ? ) AND `host`.`status` < > 0
AND `hostPage`.`timeBanned` IS NULL AND `hostPage`.`timeBanned` IS NULL
ORDER BY `hostPage`.`rank` DESC, RAND() ORDER BY RAND()
LIMIT ' . (int) $limit); LIMIT ' . (int) $limit);
@ -780,40 +492,6 @@ class MySQL {
return $query->rowCount(); return $query->rowCount();
} }
public function getHostImageCrawlQueue(int $limit, int $timeFrom) {
$query = $this->_db->prepare('SELECT `hostImage`.`hostId`,
`hostImage`.`hostImageId`,
`hostImage`.`uri`,
`host`.`scheme`,
`host`.`name`,
`host`.`port`,
`host`.`crawlMetaOnly`
FROM `hostImage`
JOIN `host` ON (`host`.`hostId` = `hostImage`.`hostId`)
WHERE (`hostImage`.`timeUpdated` IS NULL OR `hostImage`.`timeUpdated` < ? ) AND `host`.`status` < > 0
AND `hostImage`.`timeBanned` IS NULL
ORDER BY `hostImage`.`rank` DESC, RAND()
LIMIT ' . (int) $limit);
$query->execute([$timeFrom]);
return $query->fetchAll();
}
public function updateHostImageCrawlQueue(int $hostImageId, int $timeUpdated, int $httpCode) {
$query = $this->_db->prepare('UPDATE `hostImage` SET `timeUpdated` = ?, `httpCode` = ? WHERE `hostImageId` = ? LIMIT 1');
$query->execute([$timeUpdated, $httpCode, $hostImageId]);
return $query->rowCount();
}
public function getManifestCrawlQueue(int $limit, int $timeFrom) { public function getManifestCrawlQueue(int $limit, int $timeFrom) {
$query = $this->_db->prepare('SELECT * FROM `manifest` $query = $this->_db->prepare('SELECT * FROM `manifest`
@ -844,10 +522,6 @@ class MySQL {
int $hostPagesIndexed, int $hostPagesIndexed,
int $hostPagesAdded, int $hostPagesAdded,
int $hostPagesBanned, int $hostPagesBanned,
int $hostImagesIndexed,
int $hostImagesProcessed,
int $hostImagesAdded,
int $hostImagesBanned,
int $manifestsProcessed, int $manifestsProcessed,
int $manifestsAdded, int $manifestsAdded,
int $httpRequestsTotal, int $httpRequestsTotal,
@ -862,17 +536,13 @@ class MySQL {
`hostPagesIndexed`, `hostPagesIndexed`,
`hostPagesAdded`, `hostPagesAdded`,
`hostPagesBanned`, `hostPagesBanned`,
`hostImagesIndexed`,
`hostImagesProcessed`,
`hostImagesAdded`,
`hostImagesBanned`,
`manifestsProcessed`, `manifestsProcessed`,
`manifestsAdded`, `manifestsAdded`,
`httpRequestsTotal`, `httpRequestsTotal`,
`httpRequestsSizeTotal`, `httpRequestsSizeTotal`,
`httpDownloadSizeTotal`, `httpDownloadSizeTotal`,
`httpRequestsTimeTotal`, `httpRequestsTimeTotal`,
`executionTimeTotal`) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ? )'); `executionTimeTotal`) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)');
$query->execute([ $query->execute([
$timeAdded, $timeAdded,
@ -881,10 +551,6 @@ class MySQL {
$hostPagesIndexed, $hostPagesIndexed,
$hostPagesAdded, $hostPagesAdded,
$hostPagesBanned, $hostPagesBanned,
$hostImagesIndexed,
$hostImagesProcessed,
$hostImagesAdded,
$hostImagesBanned,
$manifestsProcessed, $manifestsProcessed,
$manifestsAdded, $manifestsAdded,
$httpRequestsTotal, $httpRequestsTotal,
ghost
2 years ago
