freeze crawl on remote storage connection lost, infinitely repeat new attempt after 60 seconds until storage connected again

crontab/crawler.php

@@ -18,6 +18,9 @@ if (false === sem_acquire($semaphore, true)) {
   exit;
 }
 
+// Begin debug output
+echo '-- ' . date('c') . ' --' . PHP_EOL . PHP_EOL;
+
 // Load system dependencies
 require_once(__DIR__ . '/../config/app.php');
 require_once(__DIR__ . '/../library/ftp.php');
@@ -553,10 +556,15 @@ foreach ($db->getHostPageCrawlQueue(CRAWL_HOST_PAGE_QUEUE_LIMIT, time() - CRAWL_
 
                   if (!$snapMimeValid) continue 2;
 
+                  $attempt = 1;
+
+                  do {
+
                     // Copy tmp snap file to the permanent storage
                     $ftp = new Ftp();
 
-                  if ($ftp->connect($storage->host, $storage->port, $storage->username, $storage->password, $storage->directory, $storage->timeout, $storage->passive)) {
+                    // Remote host connected well...
+                    if ($connection = $ftp->connect($storage->host, $storage->port, $storage->username, $storage->password, $storage->directory, $storage->timeout, $storage->passive)) {
 
                       $ftp->mkdir($hostPageSnapPath, true);
 
@@ -570,8 +578,17 @@ foreach ($db->getHostPageCrawlQueue(CRAWL_HOST_PAGE_QUEUE_LIMIT, time() - CRAWL_
                       }
 
                       $ftp->close();
+
+                    // On remote connection lost, repeat attempt after 60 seconds...
+                    } else {
+
+                      echo sprintf(_('[attempt: %s] wait for remote storage %s id %s connection...'), $attempt++, $node, $location) . PHP_EOL;
+
+                      sleep(60);
                     }
 
+                  } while ($connection === false);
+
                 break;
               }
             }
@@ -1035,9 +1052,7 @@ $executionTimeTotal    = microtime(true) - $timeStart;
 $httpRequestsTimeTotal = $httpRequestsTimeTotal / 1000000;
 
 // Debug output
-echo PHP_EOL;
+echo 'Crawl queue completed:' . PHP_EOL;
-
-echo '-- ' . date('c') . ' --' . PHP_EOL . PHP_EOL;
 
 echo '[hosts]' . PHP_EOL;
 echo '  processed: ' . $hostsProcessed . PHP_EOL;