generate hostPageDom target value based on source selector

src/cli/yggo.php

@@ -473,7 +473,7 @@ if (!empty($argv[1])) {
 
       switch ($argv[2]) {
 
-        case 'generate':
+        case 'parse':
 
           // Validate hostId
           if (empty($argv[3])) {
@@ -488,10 +488,17 @@ if (!empty($argv[1])) {
             exit;
           }
 
-          // Validate selector
+          // Validate selector source
           if (empty($argv[4])) {
 
-            CLI::danger(_('CSS selector required'));
+            CLI::danger(_('CSS selector source required'));
+            exit;
+          }
+
+          // Validate selector target
+          if (empty($argv[5])) {
+
+            CLI::danger(_('CSS selector target required'));
             exit;
           }
 
@@ -500,50 +507,27 @@ if (!empty($argv[1])) {
           $hostPagesSkippedTotal   = 0;
           $hostPageDomAddedTotal   = 0;
 
-          // Begin selectors extraction
-          foreach ($db->getHostPages($argv[3]) as $hostPage) {
-
-            $hostPagesProcessedTotal++;
-
-            if (false === stripos(Filter::mime($hostPage->mime), 'text/html')) {
-
-              CLI::warning(sprintf(_('not supported MIME type for hostPageId "%s", skipped'), $hostPage->hostPageId));
-
-              $hostPagesSkippedTotal++;
-
-              continue;
-            }
-
-            if (!$hostPageDescription = $db->getLastPageDescription($hostPage->hostPageId)) {
-
-              CLI::warning(sprintf(_('last hostPageId "%s" description empty, skipped'), $hostPage->hostPageId));
-
-              $hostPagesSkippedTotal++;
-
-              continue;
-            }
-
-            if (empty($hostPageDescription->data)) {
+          try {
 
-              CLI::warning(sprintf(_('empty hostPageDescription.data value for hostPageId "%s", skipped'), $hostPage->hostPageId));
+            $db->beginTransaction();
 
-              $hostPagesSkippedTotal++;
+            // Begin selectors values processing by hostId
+            foreach ($db->getHostPages($argv[3]) as $hostPage) {
 
-              continue;
-            }
+              $hostPagesProcessedTotal++;
 
-            if (!$html = base64_decode($hostPageDescription->data)) {
+              if (!$hostPageDomSelectorSource = $db->findLastHostPageDomBySelector($hostPage->hostPageId, $argv[4])) {
 
-              CLI::danger(sprintf(_('could not decode base64 for hostPageDescription.data value for hostPageId "%s", skipped'), $hostPage->hostPageId));
+                CLI::warning(sprintf(_('[selector source "%s"] not found for hostPageId "%s", skipped'), $argv[4], $hostPage->hostPageId));
 
                 $hostPagesSkippedTotal++;
 
                 continue;
               }
 
-            if (empty($html)) {
+              if (empty($hostPageDomSelectorSource->value)) {
 
-              CLI::warning(sprintf(_('empty decoded hostPageDescription.data value for hostPageId "%s", skipped'), $hostPage->hostPageId));
+                CLI::warning(sprintf(_('[selector source "%s"] value is empty for hostPageId "%s", skipped'), $argv[4], $hostPage->hostPageId));
 
                 $hostPagesSkippedTotal++;
 
@@ -552,40 +536,54 @@ if (!empty($argv[1])) {
 
               // Init crawler
               $crawler = new Symfony\Component\DomCrawler\Crawler();
-            $crawler->addHtmlContent($html);
+              $crawler->addHtmlContent($hostPageDomSelectorSource->value);
 
-            $selector = trim($argv[4]);
+              // Extract target selector data
+              foreach ($crawler->filter($argv[5]) as $selectorTarget) {
 
-            if ($elements = $crawler->filter($selector)) {
+                foreach ($selectorTarget->childNodes as $node) {
 
-              foreach ($elements as $element) {
-
-                $value = trim($element->nodeValue);
-                $value = strip_tags($value, empty($argv[5]) ? null : $argv[5]);
+                  $value = trim($element->ownerDocument->saveHtml());
 
                   if (empty($value)) {
 
+                    CLI::warning(sprintf(_('[selector target "%s"] value is empty for hostPageId "%s", skipped'), $argv[5], $hostPage->hostPageId));
+
+                    $hostPagesSkippedTotal++;
+
                     continue;
                   }
 
                   // Save selector value
                   $db->addHostPageDom(
                     $hostPage->hostPageId,
-                  $selector,
+                    $argv[5],
                     $value,
                     time()
                   );
 
                   $hostPageDomAddedTotal++;
                 }
+                exit;
               }
             }
 
+            $db->commit();
+
             CLI::success(sprintf(_('Host pages processed: %s'), $hostPagesProcessedTotal));
             CLI::success(sprintf(_('Host pages skipped: %s'), $hostPagesSkippedTotal));
             CLI::success(sprintf(_('Host page DOM elements added: %s'), $hostPageDomAddedTotal));
             exit;
 
+          } catch(Exception $e) {
+
+            $db->rollBack();
+
+            var_dump($e);
+
+            exit;
+          }
+
         break;
         case 'truncate':
 
@@ -638,7 +636,7 @@ CLI::default('      fs                         - check all storages for snap fil
 CLI::default('    reindex                      - search for host pages without snap records, add found pages to the crawl queue');
 CLI::break();
 CLI::default('  hostPageDom                                            ');
-CLI::default('    generate [hostId] [selector] [allowed tags] - generate hostPageDom values based on indexed hostPage.data field');
+CLI::default('    parse [hostId] [selector source] [selector target] - parse new hostPageDom.selector target based on hostPageDom.selector source');
 CLI::default('    truncate                                           - flush hostPageDom table');
 
 CLI::break();