YGGverse/Yo
1
0
Fork 0
mirror of https://github.com/YGGverse/Yo.git synced 2025-03-13 05:41:18 +00:00
Code Issues Projects Releases Wiki Activity

use yo-tools-php library

Browse Source
This commit is contained in:
yggverse 2024-04-03 18:12:32 +03:00
parent 17c9eacb4c
commit dc60f0376f
2 changed files with 5 additions and 69 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
composer.json
View File

@ -17,6 +17,7 @@
"manticoresoftware/manticoresearch-php": "^3.1", "manticoresoftware/manticoresearch-php": "^3.1",
"yggverse/ftp": "^1.0", "yggverse/ftp": "^1.0",
"yggverse/net": "^1.2", "yggverse/net": "^1.2",
"yggverse/gemini": "^0.4.0" "yggverse/gemini": "^0.4.0",
"yggverse/yo-tools": "^0.1.0"
} }
} }

71
src/cli/document/crawl.php
View File

@ -6,71 +6,6 @@ $microtime = microtime(true);
// Load dependencies // Load dependencies
require_once __DIR__ . '/../../../vendor/autoload.php'; require_once __DIR__ . '/../../../vendor/autoload.php';
// Define helpers @TODO move to separated library (yo-php)
function getLastSnapTime(array $files): int
{
$time = [];
foreach ($files as $file)
{
if (!str_ends_with($file, '.tar.gz'))
{
continue;
}
$time[] = preg_replace(
'/^([\d]+)\.tar\.gz$/',
'$1',
basename(
$file
)
);
}
if ($time)
{
return max(
$time
);
}
return 0;
}
function relative2absolute(
string $source, // current document url to grab the base
string $target, // relative or absolute link
?string &$scheme = null,
?string &$host = null,
?int &$port = null
) {
if (!parse_url($target, PHP_URL_HOST))
{
$scheme = parse_url($base, PHP_URL_SCHEME);
$host = parse_url($base, PHP_URL_HOST);
$port = parse_url($base, PHP_URL_PORT);
return $scheme . '://' . $host . ($port ? ':' . $port : null) .
'/' .
trim(
ltrim(
str_replace(
[
'./',
'../'
],
'',
$target
),
'/'
),
'.'
);
}
return $target;
}
// Init config // Init config
$config = json_decode( $config = json_decode(
file_get_contents( file_get_contents(
@ -378,7 +313,7 @@ foreach($index->search('')
if ($url = $link->getAddress()) if ($url = $link->getAddress())
{ {
//Make relative links absolute //Make relative links absolute
$url = relative2absolute( $url = \Yggverse\YoTools\Link::relative2absolute(
$document->get('url'), $document->get('url'),
$url, $url,
$scheme, $scheme,
@ -667,7 +602,7 @@ foreach($index->search('')
@mkdir($filepath, 0755, true); @mkdir($filepath, 0755, true);
// Check latest snap older than defined in settings // Check latest snap older than defined in settings
if (time() - getLastSnapTime((array) scandir($filepath)) > $config->cli->document->crawl->snap->timeout) if (time() - \Yggverse\YoTools\Snap::getTimeLast((array) scandir($filepath)) > $config->cli->document->crawl->snap->timeout)
{ {
$filename = sprintf( $filename = sprintf(
'%s/%s', '%s/%s',
@ -836,7 +771,7 @@ foreach($index->search('')
); );
// Check latest snap older than defined in settings // Check latest snap older than defined in settings
if (time() - getLastSnapTime((array) $remote->nlist($filepath)) > $config->cli->document->crawl->snap->timeout) if (time() - \Yggverse\YoTools\Snap::getTimeLast((array) $remote->nlist($filepath)) > $config->cli->document->crawl->snap->timeout)
{ {
if ($remote->copy($tmp, $filename)) if ($remote->copy($tmp, $filename))
{ {