use yo-tools-php library

composer.json

@@ -17,6 +17,7 @@
         "manticoresoftware/manticoresearch-php": "^3.1",
         "yggverse/ftp": "^1.0",
         "yggverse/net": "^1.2",
-        "yggverse/gemini": "^0.4.0"
+        "yggverse/gemini": "^0.4.0",
+        "yggverse/yo-tools": "^0.1.0"
     }
 }

src/cli/document/crawl.php

@@ -6,71 +6,6 @@ $microtime = microtime(true);
 // Load dependencies
 require_once __DIR__ . '/../../../vendor/autoload.php';
 
-// Define helpers @TODO move to separated library (yo-php)
-function getLastSnapTime(array $files): int
-{
-    $time = [];
-
-    foreach ($files as $file)
-    {
-        if (!str_ends_with($file, '.tar.gz'))
-        {
-            continue;
-        }
-
-        $time[] = preg_replace(
-            '/^([\d]+)\.tar\.gz$/',
-            '$1',
-            basename(
-                $file
-            )
-        );
-    }
-
-    if ($time)
-    {
-        return max(
-            $time
-        );
-    }
-
-    return 0;
-}
-
-function relative2absolute(
-    string $source, // current document url to grab the base
-    string $target, // relative or absolute link
-    ?string &$scheme = null,
-    ?string &$host = null,
-    ?int &$port = null
-) {
-    if (!parse_url($target, PHP_URL_HOST))
-    {
-        $scheme = parse_url($base, PHP_URL_SCHEME);
-        $host   = parse_url($base, PHP_URL_HOST);
-        $port   = parse_url($base, PHP_URL_PORT);
-
-        return $scheme . '://' . $host . ($port ? ':' . $port : null) .
-        '/' .
-        trim(
-            ltrim(
-                str_replace(
-                    [
-                        './',
-                        '../'
-                    ],
-                    '',
-                    $target
-                ),
-                '/'
-            ),
-            '.'
-        );
-    }
-
-    return $target;
-}
-
 // Init config
 $config = json_decode(
     file_get_contents(
@@ -378,7 +313,7 @@ foreach($index->search('')
                 if ($url = $link->getAddress())
                 {
                     //Make relative links absolute
-                    $url = relative2absolute(
+                    $url = \Yggverse\YoTools\Link::relative2absolute(
                         $document->get('url'),
                         $url,
                         $scheme,
@@ -667,7 +602,7 @@ foreach($index->search('')
                         @mkdir($filepath, 0755, true);
 
                         // Check latest snap older than defined in settings
-                        if (time() - getLastSnapTime((array) scandir($filepath)) > $config->cli->document->crawl->snap->timeout)
+                        if (time() - \Yggverse\YoTools\Snap::getTimeLast((array) scandir($filepath)) > $config->cli->document->crawl->snap->timeout)
                         {
                             $filename = sprintf(
                                 '%s/%s',
@@ -836,7 +771,7 @@ foreach($index->search('')
                             );
 
                             // Check latest snap older than defined in settings
-                            if (time() - getLastSnapTime((array) $remote->nlist($filepath)) > $config->cli->document->crawl->snap->timeout)
+                            if (time() - \Yggverse\YoTools\Snap::getTimeLast((array) $remote->nlist($filepath)) > $config->cli->document->crawl->snap->timeout)
                             {
                                 if ($remote->copy($tmp, $filename))
                                 {