YGGverse/YGGo
1
0
Fork 0
mirror of https://github.com/YGGverse/YGGo.git synced 2025-08-31 16:21:56 +00:00
Code Issues Projects Releases Wiki Activity

add disk quota validation

Browse Source
This commit is contained in:
ghost 2023-04-23 04:05:00 +03:00
parent 7bee0ebb4d
commit 8dbb4a06af
3 changed files with 39 additions and 22 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
config/app.php.txt
View File

@ -40,6 +40,12 @@ define('SPHINX_PORT', 9306);
// Crawler settings
/*
* Stop crawler on disk quota reached (Mb)
*
*/
define('CRAWL_STOP_DISK_QUOTA_MB_LEFT', 500);
/*
* Pages (URI) processing limit in the crawler.php queue
*

7
crontab/crawler.php
View File

@ -9,6 +9,13 @@ if (false === sem_acquire($semaphore, true)) {
exit;
}
// Check disk quota
if (CRAWL_STOP_DISK_QUOTA_MB_LEFT > disk_free_space('/') / 1000000) {
echo 'Disk quota reached.' . PHP_EOL;
exit;
}
// Load system dependencies
require_once('../config/app.php');
require_once('../library/curl.php');

4
public/search.php
View File

@ -49,6 +49,9 @@ if (filter_var($q, FILTER_VALIDATE_URL) && preg_match(CRAWL_URL_REGEXP, $q)) {
// Register new host
} else {
// Disk quota not reached
if (CRAWL_STOP_DISK_QUOTA_MB_LEFT < disk_free_space('/') / 1000000) {
// Get robots.txt if exists
$curl = new Curl($hostURL->string . '/robots.txt');
@ -74,6 +77,7 @@ if (filter_var($q, FILTER_VALIDATE_URL) && preg_match(CRAWL_URL_REGEXP, $q)) {
$hostRobots,
$hostRobotsPostfix);
}
}
// Parse page URI
$hostPageURI = Parser::uri($q);