From 2495a2bbc79ff4cd5eb62acafe389bd825f0e566 Mon Sep 17 00:00:00 2001
From: ghost <noreply@localhost>
Date: Fri, 7 Apr 2023 04:04:24 +0300
Subject: [PATCH] implement MySQL/Sphinx data model #3, add basical robots.txt
 support #2

---
 .gitignore             |   3 +
 README.md              |  15 +++-
 config/app.php.txt     |  16 +++-
 config/sphinx.conf.txt |  22 +++++
 crontab/crawler.php    | 182 +++++++++++++++++++++++++-------------
 database/yggo.mwb      | Bin 0 -> 9355 bytes
 library/filter.php     |  11 +--
 library/mysql.php      | 196 +++++++++++++++++++++++++++++++++++++++++
 library/parser.php     |  73 +++++++++++++++
 library/robots.php     |  14 +--
 library/sphinxql.php   |  31 +++++++
 library/sqlite.php     | 170 -----------------------------------
 public/index.php       |   4 +-
 public/search.php      | 102 +++++++++++++++++----
 14 files changed, 561 insertions(+), 278 deletions(-)
 create mode 100644 config/sphinx.conf.txt
 create mode 100644 database/yggo.mwb
 create mode 100644 library/mysql.php
 create mode 100644 library/parser.php
 create mode 100644 library/sphinxql.php
 delete mode 100644 library/sqlite.php

diff --git a/.gitignore b/.gitignore
index b3e2ff2..3d95099 100644
--- a/.gitignore
+++ b/.gitignore
@@ -2,5 +2,8 @@
 .ftpignore
 
 config/app.php
+config/sphinx.conf
+
+database/yggo.mwb.bak
 
 storage
diff --git a/README.md b/README.md
index f8dc048..791185a 100644
--- a/README.md
+++ b/README.md
@@ -7,7 +7,7 @@ The project goal - simple interface, clear architecture and lightweight server r
 
 #### Online examples
 
-* [http://[201:23b4:991a:634d:8359:4521:5576:15b7]/yggo](http://[201:23b4:991a:634d:8359:4521:5576:15b7]/yggo)  
+* [http://[201:23b4:991a:634d:8359:4521:5576:15b7]/yggo](http://[201:23b4:991a:634d:8359:4521:5576:15b7]/yggo)
 * [http://94.140.114.241/yggo/](http://94.140.114.241/yggo)
 
 #### Screenshots
@@ -28,7 +28,8 @@ php-dom
 php-pdo
 php-curl
 php-gd
-sqlite / fts5
+php-mysql
+sphinx search server
 ```
 
 #### Installation
@@ -39,12 +40,16 @@ sqlite / fts5
 * Set up the `/crontab/crawler.php` script for execution every the minute, but it mostly related of the configs and targetal network volume, there is no debug implemented yet, so let's silentize it by `/dev/null`
 * Script has no MVC model, because of super simple. It's is just 2 files, and everything else stored incapsulated in `/library` classes.
 
+#### Configuration
+
+todo
+
 #### Roadmap / ideas
 
 * [x] Web pages full text ranking search
 * [x] Make search results pagination
 * [ ] Blacklist domains (useful for some mirrors)
-* [ ] Add robots.txt support (Issue #2)
+* [x] Add robots.txt support (Issue #2)
 * [ ] Improve yggdrasil links detection, add .ygg domain zone support
 * [ ] Make page description visible - based on the cached content dump, when website description tag not available, add condition highlights
 * [ ] Images search (basically implemented but requires testing and some performance optimization)
@@ -57,7 +62,7 @@ sqlite / fts5
 * [x] An idea to make unique gravatars for sites without favicons, because simpler to ident, comparing to ipv6
 * [ ] An idea to make some visitors counters, like in good old times?
 
-#### Contributions 
+#### Contributions
 
 Please make a new master branch for each patch in your fork before create PR
 
@@ -66,6 +71,8 @@ git checkout master
 git checkout -b my-pr-branch-name
 ```
 
+See also: [SQLite tree](https://github.com/YGGverse/YGGo/tree/sqliteway)
+
 #### Donate to contributors
 
 * @d47081: [BTC](https://www.blockchain.com/explorer/addresses/btc/bc1qngdf2kwty6djjqpk0ynkpq9wmlrmtm7e0c534y) | [DOGE](https://dogechain.info/address/D5Sez493ibLqTpyB3xwQUspZvJ1cxEdRNQ)
diff --git a/config/app.php.txt b/config/app.php.txt
index 9880af4..169c388 100644
--- a/config/app.php.txt
+++ b/config/app.php.txt
@@ -11,16 +11,24 @@ define('WEBSITE_PAGINATION_SEARCH_RESULTS_LIMIT', 100);
 define('WEBSITE_IDENTICON_IMAGE_CACHE', true);
 
 // Database
-define('DB_NAME', 'database.sqlite');
+define('DB_HOST', 'localhost');
+define('DB_PORT', 3306);
+define('DB_NAME', '');
 define('DB_USERNAME', '');
 define('DB_PASSWORD', '');
 
-// Crawl settings
-define('CRAWL_IMAGE', false); // @TODO
+// Sphinx
+define('SPHINX_HOST', '127.0.0.1');
+define('SPHINX_PORT', 9306);
 
+// Crawl settings
 define('CRAWL_PAGE_LIMIT', 10);
 define('CRAWL_PAGE_SECONDS_OFFSET', 3600);
 
 define('CRAWL_URL_REGEXP', '/^.*$/ui'); // ipv6 only '/^http:\/\/\[[\w:]+\].*$/ui'
 
-define('CRAWL_META_ONLY', false);
+define('CRAWL_HOST_DEFAULT_PAGES_LIMIT', 1000);
+define('CRAWL_HOST_DEFAULT_STATUS', true);
+define('CRAWL_HOST_DEFAULT_META_ONLY', false);
+
+define('CRAWL_ROBOTS_DEFAULT_RULES', "");
diff --git a/config/sphinx.conf.txt b/config/sphinx.conf.txt
new file mode 100644
index 0000000..f0f12a9
--- /dev/null
+++ b/config/sphinx.conf.txt
@@ -0,0 +1,22 @@
+source hostPage
+{
+  type        = mysql
+
+  sql_host    = localhost
+  sql_user    =
+  sql_pass    =
+  sql_db      =
+  sql_port    = 3306 # optional, default is 3306
+
+  sql_query   = \
+    SELECT hostPageId, metaTitle, metaDescription, metaKeywords, data, uri  \
+    FROM hostPage
+
+  sql_attr_uint = hostPageId
+}
+
+index hostPage
+{
+  source = hostPage
+  path   =
+}
\ No newline at end of file
diff --git a/crontab/crawler.php b/crontab/crawler.php
index 59879c7..4c1aece 100644
--- a/crontab/crawler.php
+++ b/crontab/crawler.php
@@ -11,31 +11,46 @@ if (false === sem_acquire($semaphore, true)) {
 // Load system dependencies
 require_once('../config/app.php');
 require_once('../library/curl.php');
+require_once('../library/robots.php');
 require_once('../library/filter.php');
-require_once('../library/sqlite.php');
+require_once('../library/parser.php');
+require_once('../library/mysql.php');
+
+// Debug
+$timeStart = microtime(true);
+
+$hostPagesProcessed = 0;
+$hostPagesIndexed   = 0;
+$hostPagesAdded     = 0;
+$hostsAdded         = 0;
 
 // Connect database
-$db = new SQLite(DB_NAME, DB_USERNAME, DB_PASSWORD);
+$db = new MySQL(DB_HOST, DB_PORT, DB_NAME, DB_USERNAME, DB_PASSWORD);
 
 // Process crawl queue
-foreach ($db->getPageQueue(CRAWL_PAGE_LIMIT, time() - CRAWL_PAGE_SECONDS_OFFSET) as $queue) {
+foreach ($db->getCrawlQueue(CRAWL_PAGE_LIMIT, time() - CRAWL_PAGE_SECONDS_OFFSET) as $queueHostPage) {
+
+  // Build URL from the DB
+  $queueHostPageURL = $queueHostPage->scheme . '://' . $queueHostPage->name . ($queueHostPage->port ? ':' . $queueHostPage->port : false) . $queueHostPage->uri;
 
-  $url = new Curl($queue->url);
+  $curl = new Curl($queueHostPageURL);
 
-  $db->updatePageQueue($queue->pageId, time(), $url->getCode());
+  // Update page index anyway, with the current time and http code
+  $hostPagesProcessed += $db->updateCrawlQueue($queueHostPage->hostPageId, time(), $curl->getCode());
 
-  // Skip processing non 200 code
-  if (200 != $url->getCode()) {
+  // Skip next page processing non 200 code
+  if (200 != $curl->getCode()) {
 
     continue;
   }
 
-  // Skip processing pages without returned data
-  if (!$content = $url->getContent()) {
+  // Skip next page processing pages without returned data
+  if (!$content = $curl->getContent()) {
 
     continue;
   }
 
+  // Grab page content
   $dom = new DomDocument();
 
   @$dom->loadHTML($content);
@@ -62,48 +77,12 @@ foreach ($db->getPageQueue(CRAWL_PAGE_LIMIT, time() - CRAWL_PAGE_SECONDS_OFFSET)
     }
   }
 
-  // Index page data
-  $db->updatePage($queue->pageId,
-                  Filter::pageTitle($title->item(0)->nodeValue),
-                  Filter::pageDescription($description),
-                  Filter::pageKeywords($keywords),
-                  CRAWL_META_ONLY ? '' : Filter::pageData($content),
-                  time());
-
-  // Update images
-  $db->deleteImages($queue->pageId);
-
-  if (CRAWL_IMAGE) {
-
-    foreach (@$dom->getElementsByTagName('img') as $image) {
-
-      // Skip images without required attributes
-      if (!$src = @$image->getAttribute('src')) {
-
-        continue;
-      }
-
-      if (!$alt = @$image->getAttribute('alt')) {
-
-        continue;
-      }
-
-      // Add domain to the relative links
-      if (!parse_url($src, PHP_URL_HOST)) {
-
-        $src = parse_url($queue->url, PHP_URL_SCHEME) . '://' .
-               parse_url($queue->url, PHP_URL_HOST) .
-               parse_url($queue->url, PHP_URL_PORT) .
-               $src; // @TODO sometimes wrong URL prefix available
-      }
-
-      // Add page images
-      $db->addImage($queue->pageId,
-                    Filter::url($src),
-                    crc32($src),
-                    Filter::imageAlt($alt));
-    }
-  }
+  // Update queued page data
+  $hostPagesIndexed += $db->updateHostPage($queueHostPage->hostPageId,
+                                           Filter::pageTitle($title->item(0)->nodeValue),
+                                           Filter::pageDescription($description),
+                                           Filter::pageKeywords($keywords),
+                                           CRAWL_HOST_DEFAULT_META_ONLY ? null : Filter::pageData($content));
 
   // Collect internal links from page content
   foreach(@$dom->getElementsByTagName('a') as $a) {
@@ -120,22 +99,101 @@ foreach ($db->getPageQueue(CRAWL_PAGE_LIMIT, time() - CRAWL_PAGE_SECONDS_OFFSET)
       continue;
     }
 
-    // Add absolute prefixes to the relative links
+    // Add absolute URL prefixes to the relative links found
     if (!parse_url($href, PHP_URL_HOST)) {
 
-      $href = parse_url($queue->url, PHP_URL_SCHEME) . '://' .
-              parse_url($queue->url, PHP_URL_HOST) .
-              parse_url($queue->url, PHP_URL_PORT) .
-              $href;
+      $href = $queueHostPage->scheme . '://' .
+              $queueHostPage->name .
+             ($queueHostPage->port ? ':' . $queueHostPage->port : '') .
+             '/' . ltrim($href, '/');
     }
 
-    // Filter href URL
-    $href = Filter::url($href);
-
-    // Save valid internal links to the index queue
+    // Validate formatted link
     if (filter_var($href, FILTER_VALIDATE_URL) && preg_match(CRAWL_URL_REGEXP, $href)) {
 
-      $db->initPage($href, crc32($href), time());
+      $db->beginTransaction();
+
+      try {
+
+        // Parse formatted link
+        $hostURL     = Parser::hostURL($href);
+        $hostPageURI = Parser::uri($href);
+
+        // Host exists
+        if ($host = $db->getHost(crc32($hostURL->string))) {
+
+          $hostStatus    = $host->status;
+          $hostPageLimit = $host->crawlPageLimit;
+          $hostId        = $host->hostId;
+          $hostRobots    = $host->robots;
+
+        // Register new host
+        } else {
+
+          // Get robots.txt if exists
+          $curl = new Curl($hostURL->string . '/robots.txt');
+
+          if (200 == $curl->getCode() && false !== stripos($curl->getContent(), 'user-agent:')) {
+            $hostRobots = $curl->getContent();
+          } else {
+            $hostRobots = null;
+          }
+
+          $hostStatus    = CRAWL_HOST_DEFAULT_STATUS;
+          $hostPageLimit = CRAWL_HOST_DEFAULT_PAGES_LIMIT;
+          $hostId        = $db->addHost($hostURL->scheme,
+                                        $hostURL->name,
+                                        $hostURL->port,
+                                        crc32($hostURL->string),
+                                        time(),
+                                        null,
+                                        $hostPageLimit,
+                                        (string) CRAWL_HOST_DEFAULT_META_ONLY,
+                                        (string) $hostStatus,
+                                        $hostRobots);
+
+          if ($hostId) {
+
+            echo 'hostmane ' . $hostURL->string . PHP_EOL;
+
+            $hostsAdded++;
+
+          } else {
+
+            continue;
+          }
+        }
+
+        // Init robots parser
+        $robots = new Robots(!$hostRobots ? (string) $hostRobots : (string) CRAWL_ROBOTS_DEFAULT_RULES);
+
+        // Save page info
+        if ($hostStatus && // host enabled
+            $robots->uriAllowed($hostPageURI->string) && // page allowed by robots.txt rules
+            $hostPageLimit > $db->getTotalHostPages($hostId) && // pages quantity not reached host limit
+            !$db->getHostPage($hostId, crc32($hostPageURI->string))) {  // page not exists
+
+            if ($db->addHostPage($hostId, crc32($hostPageURI->string), $hostPageURI->string, time())) {
+
+              $hostPagesAdded++;
+            }
+        }
+
+        $db->commit();
+
+      } catch(Exception $e){
+
+        var_dump($e);
+
+        $db->rollBack();
+      }
     }
   }
-}
\ No newline at end of file
+}
+
+// Debug
+echo 'Pages processed: ' . $hostPagesProcessed . PHP_EOL;
+echo 'Pages indexed: ' . $hostPagesIndexed . PHP_EOL;
+echo 'Pages added: ' . $hostPagesAdded . PHP_EOL;
+echo 'Hosts added: ' . $hostsAdded . PHP_EOL;
+echo 'Total time: ' . microtime(true) - $timeStart . PHP_EOL;
diff --git a/database/yggo.mwb b/database/yggo.mwb
new file mode 100644
index 0000000000000000000000000000000000000000..14fad6b51497020d325a5517885a78497b498f92
GIT binary patch
literal 9355
zcmZ{K18^ovvvzFTPBt6cPTpW++qP|Iqm6Czjcwc3#<sn=d%jck|F>@4J5y8LgRZHX
zs(GgS=~0jdhrj><1Azg#CJI+uWYuzhCjtQp#Q+0A0RaIqu{U<LHMMhLuyr?L@UXSH
z)YW$0Z%*(N?CC%B9la_i41b<b*=r!$VBL?rbflA)yGsHCRWOr@=b>P4zCXv#qt>hk
zpn>ehOBN~}N-o|QH-J8@#7IBOe|e=1dT%?VNLb$8cl+gCex2VC3z8}XbUCVh;FhdG
zU<h90AB6KK^{+vE8_FHp$N1{>zvX&+^ts-)bi>|(_<f~Y1T*qpX|NpF<p1)kt1GuV
z&?D^msHM119|!B*+cSR3;FbXt-RagZ(8)oG-^8C$MDV&JuoAjGze_abuyesR`nPAG
zb@#o@7I-@pAuU3R!VT6yhY!cAr@ps?t1`04He>G0H{q;Ua+tl~GhemA+SVZ$`hi)Q
zRO)La!9Sh)!ZY#53G}A#&RQn^^WHWatiiL+zL~!{vAJWN$e3gjgUXJ!-mK|>&7IRy
z5W~1>DhYY7Ek0i(YRpv*-=QK;z@<UBCZb%NM0nu9Lt=)uO+=Di9YE`Ovs7ntlJqr9
z+%IJ7OBaG3W>cSMzDPa2lA<g9f0(Wcew>cn5z~En5>UF!W}hmn`IG}^h&^&qc9Au`
z?&SrQZYoxQ0&e%bZz!Jm(RO#C(Hr)t-92EimygbM>NWIBs)aTKNWQp1GWNvxOSkGj
z>RUDRCZ0K)wI^!*VmnxXwpWg};W^hNTiQSR$>8CTsD8cr?*$vPSdG@|B=+s5^s$1l
z(2r;$jKV!g6}&lug6Vdar_?y+#0rWt|4AxtI86-H*3-h(pQ;Z*Gr>ijk`bZ===qVR
zSU$e$&^HEnTahdr*pZ0B`@rK3vofElqH2UesOpq9ko{QBpww=AsM5?hygeLiV1WCD
z(M(ksyz?~~L8uWCudd&A5SJgL=;@kbDAtzvyDiS*bpWw#lsuIjAoxovgO6b|*ym4-
zG@N=61}8}bHr)4<<By@8UAz%YBEsRE{Y*S3kY}c5n~{11LYugvF=vqN;g!=*Zn@#1
zOL*i0EV3UdNNSY*fAW|8?xT(Zpps?@MlAU<Dc}Rg8DGWOZ|3wve`pV_*kGJ69N8s<
zco1>8XpqZQYxPx0&Y-33vW#*q;=62YcE1>A>}G}W#8hvPjN_o!pkUyzb2>KQNXIG0
zn;AsEhG+%{wUB_<_O8KTr)GZW9Zf@bxON+_Pmc_WGP2zG2z53}A~2>5Gv_e~&S=~A
z3MUT<_@)h`Mn}>rgeee$IKr4Y-r|4iHey#BWgVtAaG71nG2$a}#~`*)`QQXQ>hXSX
z^gQjZOw6s7PAnvHfQG7|9;R9{uLLlJp{O6dW$%0rQuJEbp^K3^p%ogWY-|K6u*8Ya
znc(do$BKK#^WeURGi9rYbBp#-)Ce%Xy$Ykm{4}T}{k~nBIT=%R^O7d9-D0ViMQRww
zCZ0;b6;#I`<vmw4Dqbfxw#4K!b<0PF{(Ap8rmB2EldBVNOY3w=-bTi&a!@hqIx>oB
zHp^zFxtXn}84o2^JIcf#((iPohI*j@gSE>EsQvm3ej50?`po;f5gf?Zy0}%k{1u7R
zA|1|^F54^{#vN~(;$AAkkp+rs7!Xfn)vF}NMM5@6nChZZ)JFL>00ffB0;;90wgQ<N
zBB)*>JeJW)9Mkim25V-UUE1fQZ_ndi*cUN+o)Yv$nNa+eFnW-}^?T${n;e{QQc!Iq
znXAxWO7KC3>cIC_XCE;~p5l!~XD((ifQB29FtDh8;JAcs8&sliExhpcc?duM$k0Sz
ze}BVA`t;1m@2P8=+$cz!Ka_d<^5K%sD53;!Dk@K5drf6KmEwT484Y8l5I|rXK(2*@
zh8_x62o3%$5`5qv5N4<6iL;!u%)s1g`vqw#85J|Sk_UTgWSo~~4>AeuO5kCamsVr{
zS0lzZ`>UBxs_%qXE9<8~S&|wu(&A4CoXmh4TBKoZO~BCu{K4}C!MQme7V~3?(DMz$
z!yp$C$}&JPxprZ$_Gyj685}^nDQ*j+3uCk5zzZZc9O&g2NMvd9P8?D03rHYRGO#aq
zJE<*OQw}^LF_t<l(7)wKB$mZVM#y+{jD%I4zUHutC!ml@^$@6TG>00{I6F)lPi)Mk
zAzD<&NA~SAzT9qWsu=VUgp_<Ih29-u8<^uFEugDfy%UL{uS|uOj60?ielaw}L&p%N
zBc(Kgd`m)|V{K(6OX%>K3Af1gCkXb)N^tAWj^IUMNGR5hElMQQ@=N!f-=F!qz=<t~
zq_T;t6sR-*-b==8fqy%(8$2ACBbt>5OmN^@sp9KcMr9$`i#EFeEFFrbEap$~P4p+R
z{4Nfg(Gd6AfS2SWX?(gvC8X&QYt)b6);X97=L&T%Jh9)T)GENyANTnzNbn&Cl23tc
zlCGTd*Tqn{j9OOygtjK@-q2bU>K`VFEZnP$5}8QJelJ&FF~^oz@sG$M>liP99VP0Z
zb`u!p8Do|%_?oP}8CPAR!SPLZ&Cm(JrDEoXh2w$P1x{&M1{;3_WVNS^rAap*YtckG
z%>HD6!erJEJxeNBak$N7Q`{MQkl%xH!Repy2rnKP&+yT{MbyP0gKsPL`uOt_26~Zi
zkCTyMY4B^>`Ay?B?DGy$6Yc)8(SDR=KH0v^XF&ZPafU%&&j0!X*J_)ebc2L&pjRo@
zP>?`04Cx)tFaLTDk}q3u?p^J(zt{Dh%baVaX{Orx5c{e(i&PX4vX#SXf^#~~sN%O<
z`5LOvIOF%})$Tf?Kw<kSy`pg4WsSp??}Yfii^t-sx#o?rEL(vx2*IaQ`rLN1#f87$
z5R%qc0N=k-G?l>#A|@4%EqNpA$hI9s9l->71Kg}8sYNt~t0yavam$)scN1K`@~#Dg
z^?O<pNkAtMmwG*Ii<a+UyPjMvD}>umn)+s=j6L@}ryNiK+pG=AVJ;vX&DSH}V3ry&
zROj3FYQl4@)XrS8ujJJAf*Ub=dr5a{B$$HZ2_1kW8&~{HykG++c8QF5>TCrHWQJhA
z2yzN-$U^Al_yx2>CJ)5{ZJ&v=YntGGdxRUML#h3qsb>LMwvnd`NMN>8zvX$ztpCI0
znlD#S1wUxXQgT>>Jd;4d|4I_89Qzqt9}R=ed}<8-bCep_KM}Cc?6^)7f3&SJYQ!Fa
zgV`WdcAcnAJa&=^^V>%>tB=hZQ9q+=49x&yp?nr7^E4Vvf5yZ%;8Uu3&1Notzpb4j
zGy=nf^;2lI@K<s4RZdA)i;iD4vp5-@t7J#5hce1O6p-ko<a=gsI#`<BJ#uuix;5&V
zKT5?tk(L$?Pxp|oKzX?9Z*bXb58$KslIJ(sf;9%WW>+hjCC^kpE4o<(c}$qI-E!uP
zRXI!;rEl|Yb<@0vHl|Bn<^yOt5yp<J-QL_<ERF1p)Tt$eu58{o6xMk8WNQKmRhDm&
zCYWPaBxB=btDFE}*zs!YmFShPex#0jnofMA!MhYt15o?77o$^2FA(22B{GLkrZL5-
zJHuKVsRu1qn^IY@qK8&>_)2r>G9&2#jl%>!mK~C{fItmyB*1^8V-sp>VM%+04|Ts5
z21oBb=F?m*B0W^WtG@LKL#o-dm7ez(y;pnXV7ip3)Njv)UN9?0>vKOM!|E9-3YI}{
z&sV+|V96m3ummAS7zdud2ioz_gb|{Q@BkwXy~pZeODos3hk`dVgCmW+vdCZ$_x?US
zg8@1A5J2@38x;gD3@8^@Ow*?XP~8Fb(0H%mfDQ!?_3^9hmL^B&9VQ2#yc9-gfR93`
z-zSAYk%%C`=VOqw53==Vq=v}N2m(6^8bUOR2n0JI2%JoKMsZG`QEJwC=}Hw4WJKW+
z<eUaDeY4<J83pB{5OIk{fduBmI0}WFoYT-;P?EldMZ#f{MxyR9zQmU>O&`{h(yqsq
z&BEIdTUwbqD5&6+t=8tMz^K}+a?(UA7sL|giP9*z<CzGr@xP(VkYrJaNzS86EFIa9
zrdfRu-{V}nv6)3_S|=L`oil#$M#0f#^l&D{^*kGA>{`eq9y3lZ-vx#6byuFyT0-$(
zi6H`^I#ION{jinmQ*K^0(_3CuJuw6I*syDbIOmV6lwXvH!|9}mwDshoIRmSuzq3>J
z@ZsWyxAv~Eu0N&fKgGt{Pj~D5FsNR#qBVtWEPe_`TCW*XL`D;V|4z?9GP(khKQM(B
z<%&90jU+EMxwy#e>lu9KIVJgfm%C<ge-Eiyq6{?mpc~~v%39P)jmTh^K&j@+FVsD^
zva>rk)0&xy$B4iqo1ieY@fU$+v3>4^GDICjv;KEl9m-4SO{w=54Yjm1@UrLHDu-h0
z!D?$&HIH}GP|XVM3l~kplz8AcTxETKjhfoa+1th5$=k1K*U-mUBJpqBSf-&(IjNXb
z1b8p@T~)P>9*4;ja19R{no(M`-K5eao!;GsJB37VVaY(!7GSb{tM7L0yU$jN@4%I^
z-Z-a9w;}ZQ&Bfip!}D}bK9%d$DFT0|D>3tp?IHQofW*OhN6ij>Mv8qrlCdMU{8~i|
zjBZc(Z=+)COzo9%?al6%Q6*E0(k?(s{O>vK3l2UX@*7_TDsVss9uZ9%-ujI~UlLr&
zS#Wtxs(Fte|CSdUrPB9PQ<UOofrua=FsRHALQzIn?di{&Ya$H<Gbr-PsV{>U=k)Lk
z9jRIENze_uWy297h~oS%B-)1p!PLwE44DjQXXbU$;Wil`q$a8&6|bDRt%Rc$C%mD$
z2#!}%+yO+mqLQB%!~Oyzy97~&i;7`Z{cDKr8QGhFSL8N(pIc?`(d4&?_~VCGt^3Q#
zjTCiv8`)!>A1$ixoBoQ7eo$mCKD^THtG_74e}m?UqMMw4j<98dD846R=Z%&_whp5V
z*20ftPCK!HZ1A<WgR^58L7e1c^I)brKb&L6X_6Qyb9BwZDYOW8L{b=U0{2``I3c%!
zWpv9Dsn5Y2GolMa7(^+gyzYbvN(tTxO>t2fjOIte2uoSiy6x*N_^9NxDd#`%;WewE
zPz#N)U><#)&dG6o(fcy(BUsbSs&}OuI5okd4R|nlm<NeGYXO8|W@*=K=h;UWr&r#(
z*rzx|DE|+A%{o^Q|AW5OYt@zPaSSrQF-BYe2^=jWZ9lJZ23<0P4aQEkO1j>NSIw;}
zoylEtmK{)<eIQynyEK}OHN%jeUiC11@Dr(xnE|v+jkKtPK0XkpY38x2RDCK?CFfws
ztJBSqZ09t|fgdVRtHCtM!o<qB2HT3oG^!aPrWqk5c-JDB@#D%|nX`guG**saa12v*
zTm;)aPLxh`j0i1&78>VS+F4r*tw_6f$(H<|m|HaMTdM8JV~zGdP3YC{^HB}15ECO!
z+|x|rBQ4*h>NSQM|Mp@`08f>!tKw{QLF~1MILjX&6_qe<C=iv_Igv*1lC?#<wH488
ztZ17t<7su#<7v0}WC9EGOiXM5A5k)aErobf#-H+>*?&-S|N6tPF!#Ry`on@K(vqCx
zAS@Jj;Iq*AN7Q%tXs@K0kB^BTC|B}J=m0-nfL~}<{p=8JN{Al-OvP1RFaHYmY?OSX
z%v;HJxSv=1`C#b>-ANdsUB?ydgj(xLnRfwxY~fi>()u&I?h^lQwfB5-dIk^G@9g$(
zTgp3??cWgEzD>~2=58R@Gq;<N^*Yh!lVCjLq5W{ZfPJ9ZXtE1YBfiI;c8yVKJ}08G
zFy<z1-GE)k5pVx7TY}bD{)1Z934v;&$<E+q8NE>|${<V=Hk6X>H~;qRU~71{VX?L7
zYZ(6nV#mym?HcEW0Bnhm=pnw_4WfHB+W2}=Jk3uL$tA^UBf%^{pWFxBpF&9fEf)G3
z^(I9ZcyIm(Vp`qz3XUJhpAsp~>%5nO_0bR2%i@8n?uHg%s`!9V^<V-bnogzN<cgU2
zM2qqMjL4EVRN3-%>I#l=HLF4r1XbH4!xiz<W^&Obam6M@6<Z8d{Ni*m1XX;~ohC&D
z((8Ws9&1|+>+>eX@gWpd5&#91k+R_wvTG`*o6HO%0d9x8U)0T~lM>9feXZ)99Mun=
zWQ|Ich7IPR1$z(fBBsyH5(K@DCCiecr|OvIvpV`Ek7Wq%Z^Ne<Y*2JcUXM_5tBqER
zBez_bXKf!Gvu2<N_5QZaHK&K|nY;tLobH4&*{$iuOpZ+%>kf_!#Tj-8dE|W=F%D0N
z=Eq{$@cN#fyK4r(Wbm=LLO$@c)ajE*cP%}GWN;k9L>oTbLqu4i97do$&vXknGbmup
zQP{bDJ=2Rp5UqbUF&LsWr+O@~J^M$(DML)B3=+oSuvlt@>1f~hbsa4<>;Y5cfzZ^S
zNGGY9Z-}Q@(74^H!Kz|y#*$|j9#GdTo?E_D&{hN|Ic8S%y$H&wi`yf(!|wHTDR!&#
zdbe@~wzZ})C$NMI3DKujFs7t^n{nbivE-IFJ0n6FI_MyzDNxD9=y9XQmZC=Tu9Qds
zU9uE%ROxwMC`GV(38^EfQqRC5PrGhQN%DrCl0==HzI2EbHm0q3s3s|WK2Yo%&m;E3
zNieB2P^0lSDEnK8DsuG|h;Jm;O*k$aV7}P`#ZF|z`XQ!t_}s@hF4t!48$0Y>8^c|h
zR3SiH>tTV7D&@S5*@Va6Hyn|3uCI53^@2pMejln>zEMVe!}8}-_t&r82A^l<Wwi8r
z_bm~~sIxYMntd51hzLTl$IgoP!Vfe4L?6m@>YI40KIM)sAFmI45&nb#cMldu&SpUN
zY`Dbnas4xPlbr|V*2y@F%K5S|&t(5qZxTCh7wEI++jQoup8Z*?jRL#-B#Zkbk3HQF
zpPk2_Gq<-KKW9MgY9bhM5N@{!e7OWMclgy;^fI5tOOH!OIDHERZm`N{ZnHrZ_1h|3
zg&G5uf(p4Du(?pW1)$r3EwV^1{9Wn<5g5=$wXR52ZPyo0E?)Ew+f$EgrShdKzT~<u
zSW+vlO?~lpDn#5$NgeP>QTn+~{2QxOhhA>8wBv}|e1~52=nug~o9#x|>fa<dOFy2o
z<FIWy?Z15|>H$PP(=l;HeiKqGD>&25^HBEojeO=^PP&x5?Y4aWxhLz(gyH{rcs{jO
z3^VpF;Afj4`1$$pVtp-H>)DBv4B<O~hXQMMS`~rj6F|(Ag{g6yCqs0=lfF=Wr7a3N
zIRv;}pq^k!FRJ8AxLUDw-$J9$4d6<FmT1>!%N!j@495^N_B1fe0@KtOgd0xCQ{aI&
zqJu{muppCdY##@LDf|?x3d?99OBf4HtPA&ilr7#<WL4RL)+`hvTik?_f{0!)p}SZ8
zc@)HjX!3~Wso*>%1yPid2=lC-Lsege@Aw#L5kxugiTeX0X?dH9CBLX^k37Zfd%2D$
zD3$V>$Go0f^a3nHk><7~%}hF}`%c?0dfC<&M>^Xg1~%vgt{iOKRMSf{N7OMD6Ig29
z%;Yo{RXnlz&NH?c<~T}hIWu%kM?{8pLOhYJX<eTw$0s_ptZE&jshEK3X{9PYetWJ{
z70%)t{3(}ID(k3NhUe%ac#v8qt+G9B$aN;ePqqQHNiwtt^qvqolg@>W<978*)30Q+
zC~bZqZ{fkIT+Hz_LLzGX{!mx@j5F&L0@p-(POBkVxmD%VvzxZ{eirv059$gV{^Ha4
zQaUG)H}fctr(*0#aUp7ivz%J$=)gT4%OD4Z(CpNgsKin+{JuZyrFLU{#DwoVQfz8L
zXPoU+_aYo9kH+#lTL-)Ym_2ey%P&s|VIK#yKu0L5DAVxQmPgNKH_??4X~f*pbRu!m
zuRquWY<Y(1SefO)jv2;_EFb!lVKqb!hsze+{#%=6F%1n>XF9%MOVn`SviD$r`d9ow
z96)@7X*aMTR51f@u4Vvd?RTQSdoS!vJW?%4kMF5WHZq&*PG7`2&(0+`mEIh3vyKwC
zxg<QDpnlXW4hZWsyo=+tP8}~%RRN7Xa08kbzbwNFnPq*n`AL1+9W-3s<tb{MxA0}i
zLX7zOjo0x{@OMPSPzR4kBZ2EWkh|wNv;YPc2f>VC#kq3IBdAMbwj9%!@)@;lWd0FO
z_5%o!$`Y_&43G659u}5{CohXqH&rX1FFg2a)3ET~%&r2!cA6mTJv!;2f+FQ5Hue??
z9@Fl{L(U<ANZ)Sgca^33TBF>{`WbhU0I#R=@pA}~>>b$#<YN;A`ifM5>`^gC0{kQ}
zepBl`{196TRVV$-6;vN;2%&rN=<{deCT2*VfOD{VsFFN!35hV9$g*enb0W8n@Gb|M
zVh~wG$EdIh6R73`r@+rUO9Ue8QZ`LQ|24YGG51tChB&T0EKH(S*XK$12xTMGKNHUa
zK78DJs)R>NZ*ZcU3Bx^<C0h><z~N5<J_hrp{k!}A!O(2XZZgN{fgn8FRCU?LA8885
zIRPe6dx{V4ZNw@1BLJQ)!A6;@_bYO-sQk0;?IugFK|wZ%7bMhK_`<2ZAnL*{$U<HA
zXvzb8q)m%=x0>i?+e~ei^^CMR4no61*+l2=?}fgq`Zq_ah@8?dB?!-02=1x%P2ilH
zraZ?)xOOYeZ>FuTQ&`QL&0kRT?#f*(eQmjjpL(49mwCgM<+<;my2pNc(aG4QMjMMR
zUS3`5O*FYT4#u`Xm-f0aCgbP!JpT^+2Go+BAKA3;f-eoWCz35BwLzw5=Z$FtcLXW4
zQ>{YfZOxXW>rZShCjO~EN2k1bDH-R-bKI_(5~sHiYOPs)#^=`>twNbUH`UI6Eo3z*
zRt&$7uO?YS-D^5@&!IJ=R1Uw^PjJ(%pz!y+(`3<%mx)Y#%SxK_Tv=ZipCnKp0mtLM
zmV#{f&To95zvho_d)59<<U#x}>OHV+$+Nau#n|$*{#r15<GQq~RJ!@2&JNt}lV}l%
z>QY%P8V*g1jBG4O3+>L)cmKO6j{22JR|zyU)}W5xZ=Z^tq3*4Aby2gH`+Z7X#&+k|
zP+JN5^)xhUf0-TCl(4ybD}0$($sdR@IV4AZ3OghV4ZiI$h(bhy)DBy-59L|KHrOgP
zU#0k0f#QD_wlhnPK632|Pq1D~w4uX_dpM(?uPqU$Kh3=N!orM&Wk-ICV<(xpzfh98
zJsf~<J!Cs`>1Eir=a{oyWvmKL3OS9~iaqu^*I~|6B<JYD%l^tIBt;8$-fx(cVuAuZ
z&fE}e%zfSwcCMTEB4iBHMZN6S9pb1d^mjDfG|(N1bI>D#i&_gqIg$okXlpU$4){YR
zrvc&l40s8uy~4zD=jYvlz!`>E;Mb1Cw@ze{DN8j!R7__7UyV9b;o<@MQlD54E)$KN
z>On1tD|d~aEsc0PeZDf8&wLqTAN-5|;czeW<9120TS4^kSaMbb!i!&2oSDzOEdJsD
z4zHkku)O>dBJs0K&&BTdZ224iAu-0c4DJD5u{CQEM>%%bTD=65ef)*EI*SA^VY}9`
zBOE0!vR&7p6DtoZLC_`FiF#LToTs*mGQXsH`;mVQ^xg_br81EkKZuX&L5ahV25IsB
z8C%dn`%;!!cm9z(S79HVrB;pZ0XYJrxPy0Da|7>3p$I<?NrZ|b!2~jOr#zBQy#5bi
zK~EJ(WOWpd^q1duh+14=+vhX^756Fwt;E&Z^bBn-CTCn+zwJq3?(9X6s}XLvcoF_Y
zuV<5l|MMpZVL_DIMeD5tN3br7yZ{<WKwWOf9JQo^Xs_ix9n4255Sq{4A&jUsccA{*
z_mR7}<D<<qu;<c}w9q<skVvRZ=EQaIAv_z!OD(gIK>$wEjj!pttt=dwaLTQ20E%&z
zVhLR69z27OyaaZ^DkvC-Zj-C;Ry*Wc-T{Yl?g-E_43XwiVxIG7eiq?A+pKG0lYh#i
zX5*Uuw+RH~=x}S=FYPQ0MVvWqo96Gt`4UsKzo%y&R>CmASl1_6OBhQJYUeCdoNdiI
zK=zI2+4k9@TFvj&YSonB@HfxXB%R}+n;dfG%Zbgnr@|Ajb!V(T7OvAB^0lMfG=U_4
zj{6Wj&YFz7anEI?MIBO14}_zlM6AUgx$$hCKH8f|0+(C^kHJjbCyG0z{$3t&J{X)9
zatH0u*w18g-TU>zm7c|}=2)!^+n-&YvKM=f59hB^4)J6xI@|vJVopXLv5{aX+5zHA
z34>9X->DiI6U$f@Rgg#{=5*P*a=uA3{IpZHBht(31Rs=blD+00EE%>sVK(65=aSlu
zCMx2S?suVINf@)O2nK1*!Sa}pO)_k6+gdrsGaI7cGDCwr6-^Z2$aUx*eD*$cBRtYB
zpWvElk6HFby6>{R&rk9kR9DR6pYpfYyr4$pH9fn|Fgx|(g>mJ3ns=ya9r9iRn8le=
z(R~UTcU#(J0P9^M8-iXSX6!%y1}u4*%JSYJ@GLhl;6OL$KV;AhLBR8ls{(elc+Luk
z#|sum{K5lKEsR5>Hdg~~*>QizW6!pxA#NsH;0Ew`!?N|dBcE4k{Yp#T5uh_<Ql}yR
z&6DNEs4lN5a}q!Lp)RwO!foO3Il1ZXeJ8I|&+Iq&GAmvsZv!IeR)CT7(_8lW`4kI-
z;GYNb>|x%IkuPwCzln0VzbSDL5RbwzHMdfKcj&+RKMmq<v#~d}=44}KX5|F=XW_pY
z^)AS8wP)1iNBn<QFoA%;{}n<eMvNwgE`|&yMwjWb_V@st5%VrNB?L8lF;xa<5h<1M
zwZA3Z!7`$tf{fZSA?4s;vJ&rmY}949LnsI4jnoT8R_otWf+<fW3IOM3YH+g<)iOuH
z#+%VM_FlQqT$-K~uzz~Gd<+f@Y~DDJ0G$*jq=0P&B?U2<>-r7O*mR(GN_!03su5*y
zT8OQDm6s#%e0*KvU(!=Lvw(d(_dxOL`ZS$=@G=k>7apdVLoN%DMi<f6G;@g>W3jK}
zGbR&_M57f=dplcRAD_nC)O_{uMZ5|}SPQ>@zw~`kcv$40>p*(urfHn~!Rj|Ft#?Fp
zFnLcGQkrmFjM3LAmL`opji0SaAxT{Wmtbk8$TUMqRnH{+(Vv=2$=cxTJho77Rp`#S
zk*SSeS@>%`{Swa?HZw*I{%Vu=WWWkWg`XTsT=W%k^CYvd1Nt*?)o+K`BpRjiWC^uk
z>zzf+<okehH@cZg+s=gMn$099e(n4K_<fl#1<!EqMb+h%j^Je*qgTO#fXm94`ON<1
z#`%qkxmRP|rPXFz&edlhzL!eN0<%Q$ijJ-G`En_(w)ah96aULIVvo~AsXjd$K7951
z!^OA`7*cGb=jPYPEWM;QAC61q{S%P2-u>)%S(mV!w{lJedlzY`AlOPlK><VPQDVBa
z!4jd1n7@ISud~G>2!qJRx!cYtGeQSm^5++nPE=%M1_b-O_1NqOI`Nk<KZBIYB1J`d
zPJmW5veoB9c*UN7wU2F1&Z6Dx+lMCS@^NaLfP3dK_rm(TLJm_1YW|YfoyOOf{`uFg
zQFFsw;1_6~-qP0&^3FP1t9M<G<WncIhf~vfc#vaJQm@@MuIE5^I1{V6uh3|mvp|3u
z-?u7^edz4Fk%iOf+)>WCbb$F2aaQ_1yiPl*WGKIfoj>zY1dG2f7MuQM6{)(PuL+EN
zXcN~rzt5%6=LVGfud*`O>94|134m<l=eMTJJhz4?^cMpJ57QUL_vz9pedMpg76?sg
zZrs$j5GM8Mk||Qr>EbCn2$~<Fq6@VMm}sfZ4?d2h*HnL;RT*`%Yk%_U)zANRTkiDd
zyj11+w>&1`MN{o*EoJOHsISj>Gnp-eYI2kDSkt`)in?jxl@8WbkO8>8tfnT@wQzgJ
ze~2;JPj3ZZAzQIrJs%G~PiX`M@sAkO+5`HVaIk|#(uUVw2v`nue5NOqm=tGcJx+oQ
z<*KQ_C}%`GgV`x3*A<BHMVRAh*aVIta03;&7%{N1<`i<0B53(0B{UR1<a?icTUs<m
zTq?Y}d$uB8Ujdy%GKKl^VK>pnnz47sD0&B36Lh70oplLLs6Jn4+Qlx9-X1vebxZYw
zeSC?KE7Ts{5yzCqt08ajwv-|=cMkq0WvX;sUn9|2z#~P&r!Q;;X;3gh@c;W?&R-z;
zPiYSPNBqxUI{#Y#-(dBxEeJ?+;CJGGq3gfz_rIam|F=;3Yw_POOhFnF>YqvQzqS0Y
zOd$sW;Q*2KR92KBQMY%pHZrv{wjhzUH!-y#F?2GvuyiveVP;^0v;Aj;-r3l~)Yj0M
Mhs4v|+#c?~02aODegFUf

literal 0
HcmV?d00001

diff --git a/library/filter.php b/library/filter.php
index eb925c8..cb3b545 100644
--- a/library/filter.php
+++ b/library/filter.php
@@ -4,16 +4,7 @@ class Filter {
 
   static public function url(string $url) {
 
-    return trim($url);
-  }
-
-  static public function imageAlt(string $alt) {
-
-    $alt = preg_replace('/[\s]+/', ' ', $alt);
-
-    $alt = trim($alt);
-
-    return $alt;
+    return trim(urldecode($url));
   }
 
   static public function pageTitle(string $title) {
diff --git a/library/mysql.php b/library/mysql.php
new file mode 100644
index 0000000..7be3c48
--- /dev/null
+++ b/library/mysql.php
@@ -0,0 +1,196 @@
+<?php
+
+class MySQL {
+
+  private PDO $_db;
+
+  public function __construct(string $host, int $port, string $database, string $username, string $password) {
+
+    $this->_db = new PDO('mysql:dbname=' . $database . ';host=' . $host . ';port=' . $port . ';charset=utf8', $username, $password, [PDO::MYSQL_ATTR_INIT_COMMAND => 'SET NAMES utf8']);
+    $this->_db->setAttribute(PDO::ATTR_ERRMODE, PDO::ERRMODE_EXCEPTION);
+    $this->_db->setAttribute(PDO::ATTR_DEFAULT_FETCH_MODE, PDO::FETCH_OBJ);
+    $this->_db->setAttribute(PDO::ATTR_TIMEOUT, 600);
+  }
+
+  // System
+  public function beginTransaction() {
+
+    $this->_db->beginTransaction();
+  }
+
+  public function commit() {
+
+    $this->_db->commit();
+  }
+
+  public function rollBack() {
+
+    $this->_db->rollBack();
+  }
+
+  // Host
+  public function getHost(int $crc32url) {
+
+    $query = $this->_db->prepare('SELECT * FROM `host` WHERE `crc32url` = ? LIMIT 1');
+
+    $query->execute([$crc32url]);
+
+    return $query->fetch();
+  }
+
+  public function addHost(string $scheme, string $name, mixed $port, int $crc32url, int $timeAdded, mixed $timeUpdated, int $crawlPageLimit, string $crawlPageMetaOnly, string $status, mixed $robots) {
+
+    $query = $this->_db->prepare('INSERT INTO `host` (`scheme`, `name`, `port`, `crc32url`, `timeAdded`, `timeUpdated`, `crawlPageLimit`, `crawlPageMetaOnly`, `status`, `robots`) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)');
+
+    $query->execute([$scheme, $name, $port, $crc32url, $timeAdded, $timeUpdated, $crawlPageLimit, $crawlPageMetaOnly, $status, $robots]);
+
+    return $this->_db->lastInsertId();
+  }
+
+  // Pages
+  public function getTotalHostPages(int $hostId) {
+
+    $query = $this->_db->prepare('SELECT COUNT(*) AS `total` FROM `hostPage` WHERE `hostId` = ?');
+
+    $query->execute([$hostId]);
+
+    return $query->fetch()->total;
+  }
+
+  public function getTotalPagesByHttpCode(mixed $httpCode) {
+
+    if (is_null($httpCode)) {
+
+      $query = $this->_db->query('SELECT COUNT(*) AS `total` FROM `hostPage` WHERE `httpCode` IS NULL');
+
+    } else {
+
+      $query = $this->_db->prepare('SELECT COUNT(*) AS `total` FROM `hostPage` WHERE `httpCode` = ?');
+
+      $query->execute([$httpCode]);
+
+    }
+
+    return $query->fetch()->total;
+  }
+
+  public function getTotalPages() {
+
+    $query = $this->_db->prepare('SELECT COUNT(*) AS `total` FROM `hostPage`');
+
+    $query->execute();
+
+    return $query->fetch()->total;
+  }
+
+  public function getHostPage(int $hostId, int $crc32uri) {
+
+    $query = $this->_db->prepare('SELECT * FROM `hostPage` WHERE `hostId` = ? AND `crc32uri` = ? LIMIT 1');
+
+    $query->execute([$hostId, $crc32uri]);
+
+    return $query->fetch();
+  }
+
+  public function getFoundHostPage(int $hostPageId) {
+
+    $query = $this->_db->prepare('SELECT `hostPage`.`metaTitle`,
+                                         `hostPage`.`metaDescription`,
+                                         `hostPage`.`data`,
+                                         `hostPage`.`uri`,
+                                         `host`.`scheme`,
+                                         `host`.`name`,
+                                         `host`.`port`
+
+                                          FROM `hostPage`
+                                          JOIN `host` ON (`host`.`hostId` = `hostPage`.`hostId`)
+
+                                          WHERE `hostPage`.`hostPageId` = ?
+
+                                          LIMIT 1');
+
+    $query->execute([$hostPageId]);
+
+    return $query->fetch();
+  }
+
+  public function addHostPage(int $hostId,
+                              int $crc32uri,
+                              string $uri,
+                              int $timeAdded,
+                              mixed $timeUpdated = null,
+                              mixed $httpCode = null,
+                              mixed $rank = null,
+                              mixed $metaTitle = null,
+                              mixed $metaDescription = null,
+                              mixed $metaKeywords = null,
+                              mixed $data = null) {
+
+    $query = $this->_db->prepare('INSERT INTO `hostPage` (`hostId`,
+                                                          `crc32uri`,
+                                                          `uri`,
+                                                          `timeAdded`,
+                                                          `timeUpdated`,
+                                                          `httpCode`,
+                                                          `rank`,
+                                                          `metaTitle`,
+                                                          `metaDescription`,
+                                                          `metaKeywords`,
+                                                          `data`) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)');
+
+    $query->execute([$hostId, $crc32uri, $uri, $timeAdded, $timeUpdated, $httpCode, $rank, $metaTitle, $metaDescription, $metaKeywords, $data]);
+
+    return $this->_db->lastInsertId();
+  }
+
+  public function updateHostPage( int $hostPageId,
+                                  mixed $metaTitle,
+                                  mixed $metaDescription,
+                                  mixed $metaKeywords,
+                                  mixed $data) {
+
+    $query = $this->_db->prepare('UPDATE `hostPage` SET `metaTitle`       = ?,
+                                                        `metaDescription` = ?,
+                                                        `metaKeywords`    = ?,
+                                                        `data`            = ? WHERE `hostPageId` = ? LIMIT 1');
+
+    $query->execute([$metaTitle, $metaDescription, $metaKeywords, $data, $hostPageId]);
+
+    return $query->rowCount();
+  }
+
+  // Crawl tools
+  public function getCrawlQueue(int $limit, int $timeFrom) {
+
+    $query = $this->_db->prepare('SELECT `hostPage`.`hostPageId`,
+                                         `hostPage`.`uri`,
+                                         `host`.`scheme`,
+                                         `host`.`name`,
+                                         `host`.`port`,
+                                         `host`.`crawlPageLimit`,
+                                         `host`.`crawlPageMetaOnly`,
+                                         `host`.`robots`
+
+                                          FROM `hostPage`
+                                          JOIN `host` ON (`host`.`hostId` = `hostPage`.`hostId`)
+
+                                          WHERE (`hostPage`.`timeUpdated` IS NULL OR `hostPage`.`timeUpdated` < ? ) AND `host`.`status` <> 0
+
+                                          ORDER BY `hostPage`.`hostPageId`
+
+                                          LIMIT ' . (int) $limit);
+
+    $query->execute([$timeFrom]);
+
+    return $query->fetchAll();
+  }
+
+  public function updateCrawlQueue(string $hostPageId, int $timeUpdated, int $httpCode) {
+
+    $query = $this->_db->prepare('UPDATE `hostPage` SET `timeUpdated` = ?, `httpCode` = ? WHERE `hostPageId` = ? LIMIT 1');
+
+    $query->execute([$timeUpdated, $httpCode, $hostPageId]);
+
+    return $query->rowCount();
+  }
+}
diff --git a/library/parser.php b/library/parser.php
new file mode 100644
index 0000000..5149427
--- /dev/null
+++ b/library/parser.php
@@ -0,0 +1,73 @@
+<?php
+
+class Parser {
+
+  static public function hostURL(string $string) {
+
+    $result = [
+      'string' => null,
+      'scheme' => null,
+      'name'   => null,
+      'port'   => null,
+    ];
+
+    if ($hostScheme = parse_url($string, PHP_URL_SCHEME)) {
+
+      $result['string'] = $hostScheme . '://';
+
+      $result['scheme'] = $hostScheme;
+
+    } else {
+
+      return false;
+    }
+
+    if ($hostName = parse_url($string, PHP_URL_HOST)) {
+
+      $result['string'] .= $hostName;
+
+      $result['name'] = $hostName;
+
+    } else {
+
+      return false;
+    }
+
+    if ($hostPort = parse_url($string, PHP_URL_PORT)) {
+
+      $result['string'] .= ':' . $hostPort;
+
+      $result['port'] = $hostPort;
+
+    }
+
+    return (object) $result;
+  }
+
+  static public function uri(string $string) {
+
+    $result = [
+      'string' => '/',
+      'path'   => '/',
+      'query'  => null,
+    ];
+
+    if ($path = parse_url($string, PHP_URL_PATH)) {
+
+      $result['string'] = $path;
+
+      $result['path'] = $path;
+
+    }
+
+    if ($query = parse_url($string, PHP_URL_QUERY)) {
+
+      $result['string'] .= '?' . $query;
+
+      $result['query'] = '?' . $query;
+
+    }
+
+    return (object) $result;
+  }
+}
\ No newline at end of file
diff --git a/library/robots.php b/library/robots.php
index 49beb10..40949cb 100644
--- a/library/robots.php
+++ b/library/robots.php
@@ -1,7 +1,5 @@
 <?php
 
-// @TODO #2
-
 class Robots {
 
   private $_rule = [];
@@ -42,16 +40,10 @@ class Robots {
     }
   }
 
-  public function indexURL(string $url) {
+  public function uriAllowed(string $uri) {
 
     // Unify case match
-    $url = strtolower(trim($url));
-
-    // Convert to URI
-    $url = str_replace(parse_url($url, PHP_URL_SCHEME) . '://' .
-                       parse_url($url, PHP_URL_HOST) .
-                       parse_url($url, PHP_URL_PORT),
-                       '', $url);
+    $uri = strtolower(trim($uri));
 
     // Index by default
     $result = true;
@@ -59,7 +51,7 @@ class Robots {
     // Begin index rules by ASC priority
     foreach ($this->_rule as $rule => $value) {
 
-      if (preg_match('!^' . $rule . '!', $url)) {
+      if (preg_match('!^' . $rule . '!', $uri)) {
 
         $result = $value;
       }
diff --git a/library/sphinxql.php b/library/sphinxql.php
new file mode 100644
index 0000000..1780b8b
--- /dev/null
+++ b/library/sphinxql.php
@@ -0,0 +1,31 @@
+<?php
+
+class SphinxQL {
+
+  private $_sphinx;
+
+  public function __construct(string $host, int $port) {
+
+    $this->_sphinx = new PDO('mysql:host=' . $host . ';port=' . $port . ';charset=utf8', false, false, [PDO::MYSQL_ATTR_INIT_COMMAND => 'SET NAMES utf8']);
+    $this->_sphinx->setAttribute(PDO::ATTR_ERRMODE, PDO::ERRMODE_EXCEPTION);
+    $this->_sphinx->setAttribute(PDO::ATTR_DEFAULT_FETCH_MODE, PDO::FETCH_OBJ);
+  }
+
+  public function searchHostPages(string $keyword, int $start, int $limit) {
+
+    $query = $this->_sphinx->prepare('SELECT * FROM `hostPage` WHERE MATCH(?) LIMIT ' . (int) $start . ',' . (int) $limit);
+
+    $query->execute([$keyword]);
+
+    return $query->fetchAll();
+  }
+
+  public function searchHostPagesTotal(string $keyword) {
+
+    $query = $this->_sphinx->prepare('SELECT COUNT(*) AS `total` FROM `hostPage` WHERE MATCH(?)');
+
+    $query->execute([$keyword]);
+
+    return $query->fetch()->total;
+  }
+}
diff --git a/library/sqlite.php b/library/sqlite.php
deleted file mode 100644
index 8878986..0000000
--- a/library/sqlite.php
+++ /dev/null
@@ -1,170 +0,0 @@
-<?php
-
-class SQLite {
-
-  private PDO $_db;
-
-  public function __construct(string $database, string $username, string $password) {
-
-    $this->_db = new PDO('sqlite:' . $database, $username, $password);
-    $this->_db->setAttribute(PDO::ATTR_ERRMODE, PDO::ERRMODE_EXCEPTION);
-    $this->_db->setAttribute(PDO::ATTR_DEFAULT_FETCH_MODE, PDO::FETCH_OBJ);
-    $this->_db->setAttribute(PDO::ATTR_TIMEOUT, 600);
-
-    $this->_db->query('
-      CREATE TABLE IF NOT EXISTS "page" (
-        "pageId"	INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT,
-        "crc32url"	INTEGER NOT NULL UNIQUE,
-        "httpCode"	INTEGER,
-        "timeAdded"	INTEGER NOT NULL,
-        "timeUpdated"	INTEGER,
-        "title"	TEXT,
-        "data"	TEXT,
-        "description"	TEXT,
-        "keywords"	TEXT,
-        "url"	TEXT NOT NULL
-      )
-    ');
-
-    $this->_db->query('
-      CREATE TABLE IF NOT EXISTS "image" (
-        "imageId"	INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT,
-        "crc32src"	INTEGER NOT NULL UNIQUE,
-        "pageId"	INTEGER NOT NULL,
-        "alt"	TEXT NOT NULL,
-        "src"	TEXT NOT NULL
-      )
-    ');
-
-    // FTS5
-    $this->_db->query('
-      CREATE VIRTUAL TABLE IF NOT EXISTS `ftsPage` USING fts5(`url`, `title`, `description`, `keywords`, `data`, tokenize=`unicode61`, content=`page`, content_rowid=`pageId`)
-    ');
-
-    $this->_db->query('
-      CREATE TRIGGER IF NOT EXISTS `pageInsert` AFTER INSERT ON `page` BEGIN
-        INSERT INTO ftsPage(`rowid`, `url`, `title`, `description`, `keywords`, `data`) VALUES (`new`.`pageId`, `new`.`url`, `new`.`title`, `new`.`description`, `new`.`keywords`, `new`.`data`);
-      END
-    ');
-
-    $this->_db->query('
-      CREATE TRIGGER IF NOT EXISTS `pageDelete` AFTER DELETE ON `page` BEGIN
-        INSERT INTO ftsPage(`ftsPage`, `rowid`, `url`, `title`, `description`, `keywords`, `data`) VALUES ("delete", `old`.`pageId`, `old`.`url`, `old`.`title`, `old`.`description`, `old`.`keywords`, `old`.`data`);
-      END
-    ');
-
-    $this->_db->query('
-      CREATE TRIGGER IF NOT EXISTS `pageUpdate` UPDATE ON `page` BEGIN
-        INSERT INTO ftsPage(`ftsPage`, `rowid`, `url`, `title`, `description`, `keywords`, `data`) VALUES ("delete", `old`.`pageId`, `old`.`url`, `old`.`title`, `old`.`description`, `old`.`keywords`, `old`.`data`);
-        INSERT INTO ftsPage(`rowid`, `url`, `title`, `description`, `keywords`, `data`) VALUES (`new`.`pageId`, `new`.`url`, `new`.`title`, `new`.`description`, `new`.`keywords`, `new`.`data`);
-      END
-    ');
-  }
-
-  public function getTotalPagesByHttpCode(mixed $httpCode) {
-
-    if (is_null($httpCode)) {
-
-      $query = $this->_db->query('SELECT COUNT(*) AS `total` FROM `page` WHERE `httpCode` IS NULL');
-
-    } else {
-
-      $query = $this->_db->prepare('SELECT COUNT(*) AS `total` FROM `page` WHERE `httpCode` = ?');
-
-      $query->execute([$httpCode]);
-
-    }
-
-    return $query->fetch()->total;
-  }
-
-  public function getTotalPages() {
-
-    $query = $this->_db->prepare('SELECT COUNT(*) AS `total` FROM `page`');
-
-    $query->execute();
-
-    return $query->fetch()->total;
-  }
-
-  public function updatePage(int $pageId, string $title, string $description, string $keywords, string $data, int $timeUpdated) {
-
-    $query = $this->_db->prepare('UPDATE `page` SET `title` = ?, `description` = ?, `data` = ?, `timeUpdated` = ? WHERE `pageId` = ?');
-
-    $query->execute([$title, $description, $data, $timeUpdated, $pageId]);
-
-    return $query->rowCount();
-  }
-
-  public function addPage(string $title, string $description, string $keywords, string $data, int $timeAdded) {
-
-    $query = $this->_db->prepare('INSERT INTO `page` (`title`, `description`, `data`, `timeAdded`) VALUES (?, ?, ?, ?)');
-
-    $query->execute([$title, $description, $data, $timeAdded]);
-
-    return $this->_db->lastInsertId();
-  }
-
-  public function initPage(string $url, int $crc32url, int $timeAdded) {
-
-    $query = $this->_db->prepare('INSERT OR IGNORE INTO `page` (`url`, `crc32url`, `timeAdded`) VALUES (?, ?, ?)');
-
-    $query->execute([$url, $crc32url, $timeAdded]);
-
-    return $this->_db->lastInsertId();
-  }
-
-  public function addImage(int $pageId, string $src, int $crc32src, string $alt) {
-
-    $query = $this->_db->prepare('INSERT OR IGNORE INTO `image` (`pageId`, `src`, `crc32src`, `alt`) VALUES (?, ?, ?, ?)');
-
-    $query->execute([$pageId, $src, $crc32src, $alt]);
-
-    return $this->_db->lastInsertId();
-  }
-
-  public function deleteImages(int $pageId) {
-
-    $query = $this->_db->prepare('DELETE FROM `image` WHERE `pageId` = ?');
-
-    $query->execute([$pageId]);
-
-    return $query->rowCount();
-  }
-
-  public function getPageQueue(int $limit, int $timeFrom) {
-
-    $query = $this->_db->prepare('SELECT * FROM `page` WHERE `timeUpdated` IS NULL OR `timeUpdated` < ? ORDER BY `pageId` LIMIT ' . (int) $limit);
-
-    $query->execute([$timeFrom]);
-
-    return $query->fetchAll();
-  }
-
-  public function updatePageQueue(string $pageId, int $timeUpdated, int $httpCode) {
-
-    $query = $this->_db->prepare('UPDATE `page` SET `timeUpdated` = ?, `httpCode` = ? WHERE `pageId` = ? LIMIT 1');
-
-    $query->execute([$timeUpdated, $httpCode, $pageId]);
-
-    return $query->rowCount();
-  }
-
-  public function searchPages(string $q, int $start = 0, int $limit = 100) {
-
-    $query = $this->_db->prepare('SELECT `title`, `description`, `url` FROM `ftsPage` WHERE `data` MATCH ? ORDER BY `rank` LIMIT ' . (int) $start . ',' . (int) $limit);
-
-    $query->execute([$q]);
-
-    return $query->fetchAll();
-  }
-
-  public function searchPagesTotal(string $q) {
-
-    $query = $this->_db->prepare('SELECT COUNT(*) AS `total` FROM `ftsPage` WHERE `data` MATCH ?');
-
-    $query->execute([$q]);
-
-    return $query->fetch()->total;
-  }
-}
diff --git a/public/index.php b/public/index.php
index cf485eb..4c789eb 100644
--- a/public/index.php
+++ b/public/index.php
@@ -3,10 +3,10 @@
 // Load system dependencies
 require_once('../config/app.php');
 require_once('../library/filter.php');
-require_once('../library/sqlite.php');
+require_once('../library/mysql.php');
 
 // Connect database
-$db = new SQLite(DB_NAME, DB_USERNAME, DB_PASSWORD);
+$db = new MySQL(DB_HOST, DB_PORT, DB_NAME, DB_USERNAME, DB_PASSWORD);
 
 $totalPages = $db->getTotalPages();
 
diff --git a/public/search.php b/public/search.php
index e8ad6e9..6fcee6a 100644
--- a/public/search.php
+++ b/public/search.php
@@ -2,11 +2,18 @@
 
 // Load system dependencies
 require_once('../config/app.php');
+require_once('../library/curl.php');
+require_once('../library/robots.php');
 require_once('../library/filter.php');
-require_once('../library/sqlite.php');
+require_once('../library/parser.php');
+require_once('../library/mysql.php');
+require_once('../library/sphinxql.php');
+
+// Connect Sphinx search server
+$sphinx = new SphinxQL(SPHINX_HOST, SPHINX_PORT);
 
 // Connect database
-$db = new SQLite(DB_NAME, DB_USERNAME, DB_PASSWORD);
+$db = new MySQL(DB_HOST, DB_PORT, DB_NAME, DB_USERNAME, DB_PASSWORD);
 
 // Define page basics
 $totalPages = $db->getTotalPages();
@@ -23,14 +30,76 @@ $p = !empty($_GET['p']) ? (int) $_GET['p'] : 1;
 // Crawl request
 if (filter_var($q, FILTER_VALIDATE_URL) && preg_match(CRAWL_URL_REGEXP, $q)) {
 
-  $db->initPage($q, crc32($q), time());
+  $db->beginTransaction();
+
+  try {
+
+    // Parse host info
+    if ($hostURL = Parser::hostURL($q)) {
+
+      // Host exists
+      if ($host = $db->getHost(crc32($hostURL->string))) {
+
+        $hostStatus    = $host->status;
+        $hostPageLimit = $host->crawlPageLimit;
+        $hostId        = $host->hostId;
+        $hostRobots    = $host->robots;
+
+      // Register new host
+      } else {
+
+        // Get robots.txt if exists
+        $curl = new Curl($hostURL->string . '/robots.txt');
+
+        if (200 == $curl->getCode() && false !== stripos($curl->getContent(), 'user-agent:')) {
+          $hostRobots = $curl->getContent();
+        } else {
+          $hostRobots = null;
+        }
+
+        $hostStatus    = CRAWL_HOST_DEFAULT_STATUS;
+        $hostPageLimit = CRAWL_HOST_DEFAULT_PAGES_LIMIT;
+        $hostId        = $db->addHost($hostURL->scheme,
+                                      $hostURL->name,
+                                      $hostURL->port,
+                                      crc32($hostURL->string),
+                                      time(),
+                                      null,
+                                      $hostPageLimit,
+                                      (string) CRAWL_HOST_DEFAULT_META_ONLY,
+                                      (string) $hostStatus,
+                                      $hostRobots);
+      }
+
+      // Parse page URI
+      $hostPageURI = Parser::uri($q);
+
+      // Init robots parser
+      $robots = new Robots(!$hostRobots ? (string) $hostRobots : (string) CRAWL_ROBOTS_DEFAULT_RULES);
+
+      // Save page info
+      if ($hostStatus && // host enabled
+          $robots->uriAllowed($hostPageURI->string) && // page allowed by robots.txt rules
+          $hostPageLimit > $db->getTotalHostPages($hostId) && // pages quantity not reached host limit
+         !$db->getHostPage($hostId, crc32($hostPageURI->string))) {  // page not exists
+
+          $db->addHostPage($hostId, crc32($hostPageURI->string), $hostPageURI->string, time());
+      }
+    }
+
+    $db->commit();
+
+  } catch(Exception $e){
+
+    $db->rollBack();
+  }
 }
 
 // Search request
 if (!empty($q)) {
 
-  $results      = $db->searchPages('"' . $q . '"', $p * WEBSITE_PAGINATION_SEARCH_RESULTS_LIMIT - WEBSITE_PAGINATION_SEARCH_RESULTS_LIMIT, WEBSITE_PAGINATION_SEARCH_RESULTS_LIMIT);
-  $resultsTotal = $db->searchPagesTotal('"' . $q . '"');
+  $results      = $sphinx->searchHostPages('"' . $q . '"', $p * WEBSITE_PAGINATION_SEARCH_RESULTS_LIMIT - WEBSITE_PAGINATION_SEARCH_RESULTS_LIMIT, WEBSITE_PAGINATION_SEARCH_RESULTS_LIMIT);
+  $resultsTotal = $sphinx->searchHostPagesTotal('"' . $q . '"');
 
 } else {
 
@@ -196,16 +265,19 @@ if (!empty($q)) {
           <?php } ?>
         </div>
         <?php foreach ($results as $result) { ?>
-          <div>
-            <h2><?php echo $result->title ?></h2>
-            <?php if (!empty($result->description)) { ?>
-            <span><?php echo $result->description ?></span>
-            <?php } ?>
-            <a href="<?php echo $result->url ?>">
-              <img src="<?php echo WEBSITE_DOMAIN; ?>/image.php?q=<?php echo urlencode(parse_url($result->url, PHP_URL_HOST)) ?>" alt="favicon" width="16" height="16" />
-              <?php echo $result->url ?>
-            </a>
-          </div>
+          <?php if ($hostPage = $db->getFoundHostPage($result->id)) { ?>
+          <?php $hostPageURL = $hostPage->scheme . '://' . $hostPage->name . ($hostPage->port ? ':' . $hostPage->port : false) . $hostPage->uri ?>
+            <div>
+              <h2><?php echo $hostPage->metaTitle ?></h2>
+              <?php if (!empty($hostPage->metaDescription)) { ?>
+              <span><?php echo $hostPage->metaDescription ?></span>
+              <?php } ?>
+              <a href="<?php echo $hostPageURL ?>">
+                <img src="<?php echo WEBSITE_DOMAIN; ?>/image.php?q=<?php echo urlencode($hostPage->name) ?>" alt="favicon" width="16" height="16" />
+                <?php echo $hostPageURL ?>
+              </a>
+            </div>
+          <?php } ?>
         <?php } ?>
         <?php if ($p * WEBSITE_PAGINATION_SEARCH_RESULTS_LIMIT <= $resultsTotal) { ?>
           <div>