YGGverse/YGGo
1
0
Fork 0
mirror of https://github.com/YGGverse/YGGo.git synced 2025-01-24 05:24:32 +00:00
Code Issues Projects Releases Wiki Activity

add disk quota validation

Browse Source
This commit is contained in:
ghost 2023-04-23 04:05:00 +03:00
parent 7bee0ebb4d
commit 8dbb4a06af
3 changed files with 39 additions and 22 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
config/app.php.txt
View File

@ -40,6 +40,12 @@ define('SPHINX_PORT', 9306);
// Crawler settings // Crawler settings
/*
* Stop crawler on disk quota reached (Mb)
*
*/
define('CRAWL_STOP_DISK_QUOTA_MB_LEFT', 500);
/* /*
* Pages (URI) processing limit in the crawler.php queue * Pages (URI) processing limit in the crawler.php queue
* *

7
crontab/crawler.php
View File

@ -9,6 +9,13 @@ if (false === sem_acquire($semaphore, true)) {
exit; exit;
} }
// Check disk quota
if (CRAWL_STOP_DISK_QUOTA_MB_LEFT > disk_free_space('/') / 1000000) {
echo 'Disk quota reached.' . PHP_EOL;
exit;
}
// Load system dependencies // Load system dependencies
require_once('../config/app.php'); require_once('../config/app.php');
require_once('../library/curl.php'); require_once('../library/curl.php');

48
public/search.php
View File

@ -49,30 +49,34 @@ if (filter_var($q, FILTER_VALIDATE_URL) && preg_match(CRAWL_URL_REGEXP, $q)) {
// Register new host // Register new host
} else { } else {
// Get robots.txt if exists // Disk quota not reached
$curl = new Curl($hostURL->string . '/robots.txt'); if (CRAWL_STOP_DISK_QUOTA_MB_LEFT < disk_free_space('/') / 1000000) {
if (200 == $curl->getCode() && false !== stripos($curl->getContent(), 'user-agent:')) { // Get robots.txt if exists
$hostRobots = $curl->getContent(); $curl = new Curl($hostURL->string . '/robots.txt');
} else {
$hostRobots = null; if (200 == $curl->getCode() && false !== stripos($curl->getContent(), 'user-agent:')) {
$hostRobots = $curl->getContent();
} else {
$hostRobots = null;
}
$hostRobotsPostfix = CRAWL_ROBOTS_POSTFIX_RULES;
$hostStatus = CRAWL_HOST_DEFAULT_STATUS;
$hostPageLimit = CRAWL_HOST_DEFAULT_PAGES_LIMIT;
$hostId = $db->addHost($hostURL->scheme,
$hostURL->name,
$hostURL->port,
crc32($hostURL->string),
time(),
null,
$hostPageLimit,
(string) CRAWL_HOST_DEFAULT_META_ONLY,
(string) $hostStatus,
$hostRobots,
$hostRobotsPostfix);
} }
$hostRobotsPostfix = CRAWL_ROBOTS_POSTFIX_RULES;
$hostStatus = CRAWL_HOST_DEFAULT_STATUS;
$hostPageLimit = CRAWL_HOST_DEFAULT_PAGES_LIMIT;
$hostId = $db->addHost($hostURL->scheme,
$hostURL->name,
$hostURL->port,
crc32($hostURL->string),
time(),
null,
$hostPageLimit,
(string) CRAWL_HOST_DEFAULT_META_ONLY,
(string) $hostStatus,
$hostRobots,
$hostRobotsPostfix);
} }
// Parse page URI // Parse page URI