YGGverse/Yo
1
0
Fork 0
mirror of https://github.com/YGGverse/Yo.git synced 2025-01-14 08:48:08 +00:00
Code Issues Projects Releases Wiki Activity

add snaps update by timeout feature

Browse Source
This commit is contained in:
yggverse 2024-03-23 00:47:08 +02:00
parent 36c0460fe4
commit 3a9efeabc5
2 changed files with 112 additions and 24 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
example/config.json
View File

@ -256,7 +256,8 @@
}, },
"snap": "snap":
{ {
"enabled":true "enabled":true,
"timeout":2592000
} }
} }
} }

133
src/cli/document/crawl.php
View File

@ -6,6 +6,37 @@ $microtime = microtime(true);
// Load dependencies // Load dependencies
require_once __DIR__ . '/../../../vendor/autoload.php'; require_once __DIR__ . '/../../../vendor/autoload.php';
// Define helpers
function getLastSnapTime(array $files): int
{
$time = [];
foreach ($files as $file)
{
if (in_array($file, ['.', '..']))
{
continue;
}
$time[] = preg_replace(
'/\D/',
'',
basename(
$file
)
);
}
if ($time)
{
return max(
$time
);
}
return 0;
}
// Init config // Init config
$config = json_decode( $config = json_decode(
file_get_contents( file_get_contents(
@ -702,24 +733,51 @@ foreach($index->search('')
@mkdir($filepath, 0755, true); @mkdir($filepath, 0755, true);
$filename = sprintf( // Check latest snap older than defined in settings
'%s/%s', if (time() - getLastSnapTime((array) scandir($filepath)) > $config->cli->document->crawl->snap->timeout)
$filepath,
sprintf(
'%s.tar.gz',
$time
)
);
if (!copy($tmp, $filename))
{ {
if ($config->cli->document->crawl->debug->level->error) $filename = sprintf(
'%s/%s',
$filepath,
sprintf(
'%s.tar.gz',
$time
)
);
if (copy($tmp, $filename))
{
if ($config->cli->document->crawl->debug->level->notice)
{
echo sprintf(
_('[%s] [notice] save snap to "%s" on local storage') . PHP_EOL,
date('c'),
$filename
);
}
}
else
{
if ($config->cli->document->crawl->debug->level->error)
{
echo sprintf(
_('[%s] [error] could not copy "%s" to "%s" on local storage') . PHP_EOL,
date('c'),
$tmp,
$filename
);
}
}
}
else
{
if ($config->cli->document->crawl->debug->level->notice)
{ {
echo sprintf( echo sprintf(
_('[%s] [error] could not copy "%" to "%" on local storage') . PHP_EOL, _('[%s] [notice] local snap is up to date by timeout settings') . PHP_EOL,
date('c'), date('c')
$tmp,
$filename
); );
} }
} }
@ -824,16 +882,45 @@ foreach($index->search('')
true true
); );
if (!$remote->copy($tmp, $filename)) // Check latest snap older than defined in settings
if (time() - getLastSnapTime((array) $remote->nlist($filepath)) > $config->cli->document->crawl->snap->timeout)
{ {
if ($config->cli->document->crawl->debug->level->error) if ($remote->copy($tmp, $filename))
{
if ($config->cli->document->crawl->debug->level->notice)
{
echo sprintf(
_('[%s] [notice] save snap to "%s" on remote host "%s"') . PHP_EOL,
date('c'),
$filename,
$ftp->connection->host
);
}
}
else
{
if ($config->cli->document->crawl->debug->level->error)
{
echo sprintf(
_('[%s] [error] could not copy snap "%s" to "%s" on destination "%s"') . PHP_EOL,
date('c'),
$tmp,
$filename,
$ftp->connection->host
);
}
}
}
else
{
if ($config->cli->document->crawl->debug->level->notice)
{ {
echo sprintf( echo sprintf(
_('[%s] [error] could not copy "%" to "%" on destination "%s"') . PHP_EOL, _('[%s] [notice] remote snap on destination "%s" is up to date by timeout settings') . PHP_EOL,
date('c'), date('c'),
$tmp, $ftp->connection->host
$filename,
$ftp->connection->host,
); );
} }
} }
@ -887,7 +974,7 @@ foreach($index->search('')
if ($config->cli->document->crawl->debug->level->notice) if ($config->cli->document->crawl->debug->level->notice)
{ {
echo sprintf( echo sprintf(
_('[%s] [notice] remove tmp snap file %s') . PHP_EOL, _('[%s] [notice] remove tmp snap file "%s"') . PHP_EOL,
date('c'), date('c'),
$tmp $tmp
); );
@ -899,7 +986,7 @@ foreach($index->search('')
if ($config->cli->document->crawl->debug->level->error) if ($config->cli->document->crawl->debug->level->error)
{ {
echo sprintf( echo sprintf(
_('[%s] [error] could not remove tmp snap file %s') . PHP_EOL, _('[%s] [error] could not remove tmp snap file "%s"') . PHP_EOL,
date('c'), date('c'),
$tmp $tmp
); );