YGGverse
/
YGGo
mirror of https://github.com/YGGverse/YGGo.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity
Browse Source

memory usage optimization, rename methods, remove memchached dependency from the model

main
ghost 1 year ago
parent
cbabea595b
commit
11e02da66d
4 changed files with 149 additions and 91 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Unified View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 2
      crontab/cleaner.php
  2. 10
      crontab/crawler.php
  3. 222
      library/mysql.php
  4. 6
      public/top.php

2
crontab/cleaner.php
Unescape View File

@ -60,7 +60,7 @@ try {
foreach ($db->getCleanerQueue(CLEAN_HOST_LIMIT, time() - CLEAN_HOST_SECONDS_OFFSET) as $host) { foreach ($db->getCleanerQueue(CLEAN_HOST_LIMIT, time() - CLEAN_HOST_SECONDS_OFFSET) as $host) {
// Get robots.txt if exists // Get robots.txt if exists
$curl = new Curl($host->hostURL . '/robots.txt', CRAWL_CURLOPT_USERAGENT); $curl = new Curl($host->url . '/robots.txt', CRAWL_CURLOPT_USERAGENT);
// Update curl stats // Update curl stats
$httpRequestsTotal++; $httpRequestsTotal++;

10
crontab/crawler.php
Unescape View File

@ -277,7 +277,7 @@ foreach ($db->getManifestCrawlQueue(CRAWL_MANIFEST_LIMIT, time() - CRAWL_MANIFES
foreach ($db->getHostRobotsCrawlQueue(CRAWL_ROBOTS_LIMIT, time() - CRAWL_ROBOTS_SECONDS_OFFSET) as $host) { foreach ($db->getHostRobotsCrawlQueue(CRAWL_ROBOTS_LIMIT, time() - CRAWL_ROBOTS_SECONDS_OFFSET) as $host) {
// Get robots.txt // Get robots.txt
$curl = new Curl($host->hostURL . '/robots.txt', CRAWL_CURLOPT_USERAGENT); $curl = new Curl($host->url . '/robots.txt', CRAWL_CURLOPT_USERAGENT);
// Update curl stats // Update curl stats
$httpRequestsTotal++; $httpRequestsTotal++;
@ -308,13 +308,13 @@ foreach ($db->getHostRobotsCrawlQueue(CRAWL_ROBOTS_LIMIT, time() - CRAWL_ROBOTS_
// Replace relative paths // Replace relative paths
$hostSitemapPath = trim($hostSitemapPath, '/'); $hostSitemapPath = trim($hostSitemapPath, '/');
$hostSitemapPath = str_replace($host->hostURL, '', $hostSitemapPath); $hostSitemapPath = str_replace($host->url, '', $hostSitemapPath);
$hostSitemapPath = sprintf('%s%s', $host->hostURL, $hostSitemapPath); $hostSitemapPath = sprintf('%s%s', $host->url, $hostSitemapPath);
// Set default path when not exists // Set default path when not exists
} else { } else {
$hostSitemapPath = sprintf('%s/sitemap.xml', $host->hostURL); $hostSitemapPath = sprintf('%s/sitemap.xml', $host->url);
} }
// Init sitemap data // Init sitemap data
@ -329,7 +329,7 @@ foreach ($db->getHostRobotsCrawlQueue(CRAWL_ROBOTS_LIMIT, time() - CRAWL_ROBOTS_
// Add host page // Add host page
if (filter_var($link, FILTER_VALIDATE_URL) && preg_match(CRAWL_URL_REGEXP, $link) && // validate link format if (filter_var($link, FILTER_VALIDATE_URL) && preg_match(CRAWL_URL_REGEXP, $link) && // validate link format
$linkHostURL->string == $host->hostURL && // this host links only $linkHostURL->string == $host->url && // this host links only
$robots->uriAllowed($linkURI->string) && // page allowed by robots.txt rules $robots->uriAllowed($linkURI->string) && // page allowed by robots.txt rules
$host->crawlPageLimit > $db->getTotalHostPages($host->hostId) && // pages quantity not reached host limit $host->crawlPageLimit > $db->getTotalHostPages($host->hostId) && // pages quantity not reached host limit
!$db->findHostPageByCRC32URI($host->hostId, crc32($linkURI->string))) { // page does not exists !$db->findHostPageByCRC32URI($host->hostId, crc32($linkURI->string))) { // page does not exists

222
library/mysql.php
Unescape View File

@ -3,18 +3,13 @@
class MySQL { class MySQL {
private PDO $_db; private PDO $_db;
private Memcached $_memcached;
public function __construct(string $host, int $port, string $database, string $username, string $password, Memcached $memcached = null) { public function __construct(string $host, int $port, string $database, string $username, string $password) {
$this->_db = new PDO('mysql:dbname=' . $database . ';host=' . $host . ';port=' . $port . ';charset=utf8', $username, $password, [PDO::MYSQL_ATTR_INIT_COMMAND => 'SET NAMES utf8']); $this->_db = new PDO('mysql:dbname=' . $database . ';host=' . $host . ';port=' . $port . ';charset=utf8', $username, $password, [PDO::MYSQL_ATTR_INIT_COMMAND => 'SET NAMES utf8']);
$this->_db->setAttribute(PDO::ATTR_ERRMODE, PDO::ERRMODE_EXCEPTION); $this->_db->setAttribute(PDO::ATTR_ERRMODE, PDO::ERRMODE_EXCEPTION);
$this->_db->setAttribute(PDO::ATTR_DEFAULT_FETCH_MODE, PDO::FETCH_OBJ); $this->_db->setAttribute(PDO::ATTR_DEFAULT_FETCH_MODE, PDO::FETCH_OBJ);
$this->_db->setAttribute(PDO::ATTR_TIMEOUT, 600); $this->_db->setAttribute(PDO::ATTR_TIMEOUT, 600);
if ($memcached) {
$this->_memcached = $memcached;
}
} }
// System // System
@ -96,7 +91,22 @@ class MySQL {
return $query->fetchAll(); return $query->fetchAll();
} }
public function getHost(int $crc32url) { public function getHost(int $hostId) {
$query = $this->_db->prepare("SELECT *,
IF (`port` IS NOT NULL,
CONCAT(`scheme`, '://', `name`, ':', `port`),
CONCAT(`scheme`, '://', `name`)
) AS `url`
FROM `host` WHERE `hostId` = ? LIMIT 1");
$query->execute([$hostId]);
return $query->fetch();
}
public function getHostByCRC32URL(int $crc32url) {
$query = $this->_db->prepare('SELECT * FROM `host` WHERE `crc32url` = ? LIMIT 1'); $query = $this->_db->prepare('SELECT * FROM `host` WHERE `crc32url` = ? LIMIT 1');
@ -204,53 +214,57 @@ class MySQL {
public function getTopHostPages(int $limit = 100) { public function getTopHostPages(int $limit = 100) {
if ($this->_memcached) { // Get ID (to prevent memory over usage)
$query = $this->_db->query("SELECT `hostPage`.`hostPageId`
if ($result = $this->_memcached->get(sprintf('MySQL.getTopHostPages.%s', $limit))) { FROM `hostPage`
JOIN `host` ON (`hostPage`.`hostId` = `host`.`hostId`)
return $result; WHERE `host`.`status` = '1'
} AND `hostPage`.`httpCode` = 200
} AND `hostPage`.`rank` > 0
AND `hostPage`.`timeBanned` IS NULL
AND `hostPage`.`mime` IS NOT NULL
ORDER BY `rank` DESC
$query = $this->_db->query(" SELECT LIMIT " . (int) $limit);
`hostPage`.`hostId`, // Get required page details
`hostPage`.`hostPageId`, foreach ($query->fetchAll() as $top) {
`hostPage`.`uri`,
`hostPage`.`rank`,
`host`.`scheme`, $query = $this->_db->prepare("SELECT `hostPage`.`hostId`,
`host`.`name`, `hostPage`.`hostPageId`,
`host`.`port`, `hostPage`.`uri`,
`hostPage`.`rank`,
IF (`host`.`port` IS NOT NULL, `host`.`scheme`,
CONCAT(`host`.`scheme`, '://', `host`.`name`, ':', `host`.`port`), `host`.`name`,
CONCAT(`host`.`scheme`, '://', `host`.`name`) `host`.`port`,
) AS `hostURL`,
IF (`host`.`port` IS NOT NULL, IF (`host`.`port` IS NOT NULL,
CONCAT(`host`.`scheme`, '://', `host`.`name`, ':', `host`.`port`, `hostPage`.`uri`), CONCAT(`host`.`scheme`, '://', `host`.`name`, ':', `host`.`port`),
CONCAT(`host`.`scheme`, '://', `host`.`name`, `hostPage`.`uri`) CONCAT(`host`.`scheme`, '://', `host`.`name`)
) AS `hostPageURL` ) AS `hostURL`,
FROM `hostPage` IF (`host`.`port` IS NOT NULL,
JOIN `host` ON (`hostPage`.`hostId` = `host`.`hostId`) CONCAT(`host`.`scheme`, '://', `host`.`name`, ':', `host`.`port`, `hostPage`.`uri`),
CONCAT(`host`.`scheme`, '://', `host`.`name`, `hostPage`.`uri`)
) AS `hostPageURL`
WHERE `host`.`status` = '1' FROM `hostPage`
AND `hostPage`.`httpCode` = 200 JOIN `host` ON (`hostPage`.`hostId` = `host`.`hostId`)
AND `hostPage`.`rank` > 0
AND `hostPage`.`timeBanned` IS NULL
AND `hostPage`.`mime` IS NOT NULL
ORDER BY `rank` DESC WHERE `hostPage`.`hostPageId` = ?
LIMIT " . (int) $limit); LIMIT 1");
$result = $query->fetchAll(); $query->execute([$top->hostPageId]);
if ($this->_memcached) { if ($query->rowCount()) {
$this->_memcached->set(sprintf('MySQL.getTopHostPages.%s', $limit), $result, time() + 3600); $result[] = $query->fetch();
}
} }
return $result; return $result;
@ -582,20 +596,28 @@ class MySQL {
// Cleaner tools // Cleaner tools
public function getCleanerQueue(int $limit, int $timeFrom) { public function getCleanerQueue(int $limit, int $timeFrom) {
$query = $this->_db->prepare("SELECT *, IF (`port` IS NOT NULL, $result = [];
CONCAT(`scheme`, '://', `name`, ':', `port`),
CONCAT(`scheme`, '://', `name`) // Get ID (to prevent memory over usage)
) AS `hostURL` FROM `host` $query = $this->_db->prepare("SELECT `hostId`
WHERE (`timeUpdated` IS NULL OR `timeUpdated` < ? ) AND `host`.`status` <> ? FROM `host`
ORDER BY `hostId` WHERE (`timeUpdated` IS NULL OR `timeUpdated` < ? ) AND `host`.`status` <> ?
LIMIT " . (int) $limit); ORDER BY `hostId`
LIMIT " . (int) $limit);
$query->execute([$timeFrom, 0]); $query->execute([$timeFrom, 0]);
return $query->fetchAll(); // Get required page details
foreach ($query->fetchAll() as $host) {
$result[] = $this->getHost($host->hostId);
}
return (object) $result;
} }
public function getHostPagesBanned() { public function getHostPagesBanned() {
@ -702,7 +724,13 @@ class MySQL {
FROM `hostPage` FROM `hostPage`
JOIN `host` ON (`host`.`hostId` = `hostPage`.`hostId`) JOIN `host` ON (`host`.`hostId` = `hostPage`.`hostId`)
WHERE (`hostPage`.`timeUpdated` IS NULL OR `hostPage`.`timeUpdated` < ? OR (`hostPage`.`uri` = '/' AND `hostPage`.`timeUpdated` < ?)) WHERE (
`hostPage`.`timeUpdated` IS NULL OR
`hostPage`.`timeUpdated` < ? OR (
`hostPage`.`uri` = '/' AND
`hostPage`.`timeUpdated` < ?
)
)
AND `host`.`status` <> ? AND `host`.`status` <> ?
AND `hostPage`.`timeBanned` IS NULL"); AND `hostPage`.`timeBanned` IS NULL");
@ -714,32 +742,22 @@ class MySQL {
public function getHostPageCrawlQueue(int $limit, int $hostPageTimeFrom, int $hostPageHomeTimeFrom) { public function getHostPageCrawlQueue(int $limit, int $hostPageTimeFrom, int $hostPageHomeTimeFrom) {
$query = $this->_db->prepare("SELECT `hostPage`.`hostId`, $result = [];
`hostPage`.`hostPageId`,
`hostPage`.`uri`,
`host`.`scheme`,
`host`.`name`,
`host`.`port`,
`host`.`crawlPageLimit`,
`host`.`crawlMetaOnly`,
`host`.`robots`,
`host`.`robotsPostfix`,
IF (`host`.`port` IS NOT NULL,
CONCAT(`host`.`scheme`, '://', `host`.`name`, ':', `host`.`port`),
CONCAT(`host`.`scheme`, '://', `host`.`name`)
) AS `hostURL`,
IF (`host`.`port` IS NOT NULL, // Get ID (to prevent memory over usage)
CONCAT(`host`.`scheme`, '://', `host`.`name`, ':', `host`.`port`, `hostPage`.`uri`), $query = $this->_db->prepare("SELECT `hostPage`.`hostPageId`
CONCAT(`host`.`scheme`, '://', `host`.`name`, `hostPage`.`uri`)
) AS `hostPageURL`
FROM `hostPage` FROM `hostPage`
JOIN `host` ON (`host`.`hostId` = `hostPage`.`hostId`) JOIN `host` ON (`host`.`hostId` = `hostPage`.`hostId`)
WHERE (`hostPage`.`timeUpdated` IS NULL OR `hostPage`.`timeUpdated` < ? OR (`hostPage`.`uri` = '/' AND `hostPage`.`timeUpdated` < ?)) WHERE (
`hostPage`.`timeUpdated` IS NULL OR
`hostPage`.`timeUpdated` < ?
OR (
`hostPage`.`uri` = '/' AND
`hostPage`.`timeUpdated` < ?
)
)
AND `host`.`status` <> ? AND `host`.`status` <> ?
AND `hostPage`.`timeBanned` IS NULL AND `hostPage`.`timeBanned` IS NULL
@ -750,7 +768,45 @@ class MySQL {
$query->execute([$hostPageTimeFrom, $hostPageHomeTimeFrom, 0]); $query->execute([$hostPageTimeFrom, $hostPageHomeTimeFrom, 0]);
return $query->fetchAll(); // Get required page details
foreach ($query->fetchAll() as $queue) {
$query = $this->_db->prepare("SELECT `hostPage`.`hostId`,
`hostPage`.`hostPageId`,
`hostPage`.`uri`,
`host`.`scheme`,
`host`.`name`,
`host`.`port`,
`host`.`crawlPageLimit`,
`host`.`crawlMetaOnly`,
`host`.`robots`,
`host`.`robotsPostfix`,
IF (`host`.`port` IS NOT NULL,
CONCAT(`host`.`scheme`, '://', `host`.`name`, ':', `host`.`port`),
CONCAT(`host`.`scheme`, '://', `host`.`name`)
) AS `hostURL`,
IF (`host`.`port` IS NOT NULL,
CONCAT(`host`.`scheme`, '://', `host`.`name`, ':', `host`.`port`, `hostPage`.`uri`),
CONCAT(`host`.`scheme`, '://', `host`.`name`, `hostPage`.`uri`)
) AS `hostPageURL`
FROM `hostPage`
JOIN `host` ON (`host`.`hostId` = `hostPage`.`hostId`)
WHERE `hostPage`.`hostPageId` = ? LIMIT 1");
$query->execute([$queue->hostPageId]);
if ($query->rowCount()) {
$result[] = $query->fetch();
}
}
return (object) $result;
} }
public function updateHostPageCrawlQueue(int $hostPageId, int $timeUpdated, int $httpCode, int $size) { public function updateHostPageCrawlQueue(int $hostPageId, int $timeUpdated, int $httpCode, int $size) {
@ -764,22 +820,28 @@ class MySQL {
public function getHostRobotsCrawlQueue(int $limit, int $timeFrom) { public function getHostRobotsCrawlQueue(int $limit, int $timeFrom) {
$query = $this->_db->prepare("SELECT *, IF (`port` IS NOT NULL, $result = [];
CONCAT(`scheme`, '://', `name`, ':', `port`),
CONCAT(`scheme`, '://', `name`) // Get ID (to prevent memory over usage)
) AS `hostURL` $query = $this->_db->prepare("SELECT `hostId`
FROM `host` FROM `host`
WHERE (`timeUpdated` IS NULL OR `timeUpdated` < ? ) AND `status` <> ? WHERE (`timeUpdated` IS NULL OR `timeUpdated` < ? ) AND `status` <> ?
ORDER BY RAND() ORDER BY RAND()
LIMIT " . (int) $limit); LIMIT " . (int) $limit);
$query->execute([$timeFrom, 0]); $query->execute([$timeFrom, 0]);
return $query->fetchAll(); // Get required page details
foreach ($query->fetchAll() as $host) {
$result[] = $this->getHost($host->hostId);
}
return (object) $result;
} }
public function getManifestCrawlQueue(int $limit, int $timeFrom) { public function getManifestCrawlQueue(int $limit, int $timeFrom) {

6
public/top.php
Unescape View File

@ -9,12 +9,8 @@ require_once(__DIR__ . '/../library/sphinxql.php');
// Connect Sphinx search server // Connect Sphinx search server
$sphinx = new SphinxQL(SPHINX_HOST, SPHINX_PORT); $sphinx = new SphinxQL(SPHINX_HOST, SPHINX_PORT);
// Connect Memcached
$memcached = new Memcached();
$memcached->addServer(MEMCACHED_HOST, MEMCACHED_PORT);
// Connect database // Connect database
$db = new MySQL(DB_HOST, DB_PORT, DB_NAME, DB_USERNAME, DB_PASSWORD, $memcached); $db = new MySQL(DB_HOST, DB_PORT, DB_NAME, DB_USERNAME, DB_PASSWORD);
// Define page basics // Define page basics
$totalPages = $sphinx->getHostPagesTotal(); $totalPages = $sphinx->getHostPagesTotal();

Write Preview
Loading…
Cancel
Save