From feda96c33d069b880ed8127c9dbb491071b3a8c0 Mon Sep 17 00:00:00 2001
From: ghost <ghost@ghost.ghost>
Date: Sat, 7 Aug 2021 13:42:01 +0300
Subject: [PATCH] initial commit

---
 LICENSE                 |  21 ++++++
 README.md               |   1 +
 config-default.php      |  24 +++++++
 crawler.php             |  98 ++++++++++++++++++++++++++++
 library/base58.php      |  16 +++++
 library/base58check.php |  51 +++++++++++++++
 library/crypto.php      |  88 +++++++++++++++++++++++++
 library/hash.php        |  19 ++++++
 library/keva.php        | 110 ++++++++++++++++++++++++++++++++
 library/mysql.php       | 138 ++++++++++++++++++++++++++++++++++++++++
 10 files changed, 566 insertions(+)
 create mode 100644 LICENSE
 create mode 100644 README.md
 create mode 100644 config-default.php
 create mode 100644 crawler.php
 create mode 100644 library/base58.php
 create mode 100644 library/base58check.php
 create mode 100644 library/crypto.php
 create mode 100644 library/hash.php
 create mode 100644 library/keva.php
 create mode 100644 library/mysql.php

diff --git a/LICENSE b/LICENSE
new file mode 100644
index 0000000..d48824e
--- /dev/null
+++ b/LICENSE
@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2021 kvazar-network
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
diff --git a/README.md b/README.md
new file mode 100644
index 0000000..80cb8ee
--- /dev/null
+++ b/README.md
@@ -0,0 +1 @@
+# crawler-full-node
\ No newline at end of file
diff --git a/config-default.php b/config-default.php
new file mode 100644
index 0000000..d8515c8
--- /dev/null
+++ b/config-default.php
@@ -0,0 +1,24 @@
+<?php
+
+// Debug
+ini_set('display_errors', '1');
+ini_set('display_startup_errors', '1');
+error_reporting(E_ALL);
+
+// Application
+define('KEVA_NS', '');           // KEVA_NS or empty to collect all the blockchain
+define('STEP_BLOCK_LIMIT', 50);  // Blocks per query
+define('CRAWLER_DEBUG', true);   // Debug output
+
+// Kevacoin wallet
+define('KEVA_HOST', '127.0.0.1');
+define('KEVA_PORT', '9992');
+define('KEVA_USERNAME', '');
+define('KEVA_PASSWORD', '');
+
+// Database
+define('DB_HOST', '127.0.0.1');
+define('DB_PORT', '3306');
+define('DB_NAME', '');
+define('DB_USERNAME', '');
+define('DB_PASSWORD', '');
diff --git a/crawler.php b/crawler.php
new file mode 100644
index 0000000..620e6e2
--- /dev/null
+++ b/crawler.php
@@ -0,0 +1,98 @@
+<?php
+
+require_once('config.php');
+require_once('library/mysql.php');
+require_once('library/keva.php');
+require_once('library/hash.php');
+require_once('library/base58.php');
+require_once('library/base58check.php');
+require_once('library/crypto.php');
+
+$db   = new MySQL();
+$node = new Keva();
+
+$node->host     = KEVA_HOST;
+$node->username = KEVA_USERNAME;
+$node->password = KEVA_PASSWORD;
+$node->port     = KEVA_PORT;
+
+$blockLast  = $db->getLastBlock();
+$blockTotal = $node->getblockcount();
+
+$response = [];
+
+if ($blockTotal > $blockLast) {
+
+  for ($blockCurrent = $blockLast; $blockCurrent <= $blockLast + STEP_BLOCK_LIMIT; $blockCurrent++) {
+
+    if ($blockHash = $node->getblockhash($blockCurrent)) {
+
+      $blockData = $node->getblock($blockHash);
+
+      if (!$blockId = $db->getBlock($blockCurrent)) {
+           $blockId = $db->addBlock($blockCurrent);
+      }
+
+      foreach ($blockData['tx'] as $transaction) {
+
+        $transactionRaw = $node->getrawtransaction($transaction, 1);
+
+        foreach($transactionRaw['vout'] as $vout) {
+
+          $asmArray = explode(' ', $vout['scriptPubKey']['asm']);
+
+          if($asmArray[0] == 'OP_KEVA_NAMESPACE' || $asmArray[0] == 'OP_KEVA_PUT') { // OP_KEVA_DELETE
+
+            $hash      = Base58Check::encode($asmArray[1], false , 0 , false);
+            $nameSpace = $node->keva_get($hash, '_KEVA_NS_');
+
+            $nameSpaceValue = strip_tags(html_entity_decode($nameSpace['value'], ENT_QUOTES, 'UTF-8'));
+
+            if ((empty(KEVA_NS) || (!empty(KEVA_NS) && KEVA_NS == $nameSpaceValue))) {
+
+               if (!$nameSpaceId = $db->getNameSpace($hash)) {
+                    $nameSpaceId = $db->addNameSpace($hash, $nameSpaceValue);
+               }
+
+               if (!$db->getData($blockId, $nameSpaceId)) {
+                    $db->addData($blockId,
+                                 $nameSpaceId,
+                                 $transactionRaw['time'],
+                                 $transactionRaw['size'],
+                                 $transactionRaw['txid'],
+                                 strip_tags(html_entity_decode(@hex2bin($asmArray[2]), ENT_QUOTES, 'UTF-8')),
+                                 strip_tags(html_entity_decode(@hex2bin($asmArray[3]), ENT_QUOTES, 'UTF-8')));
+               }
+
+               if (CRAWLER_DEBUG) {
+                 $response[] = [
+                   'blocktotal'=> $blockTotal,
+                   'block'     => $blockCurrent,
+                   'blockhash' => $transactionRaw['blockhash'],
+                   'txid'      => $transactionRaw['txid'],
+                   'version'   => $transactionRaw['version'],
+                   'size'      => $transactionRaw['size'],
+                   'time'      => $transactionRaw['time'],
+                   'blocktime' => $transactionRaw['blocktime'],
+                   'namehash'  => $hash,
+                   'title'     => $nameSpaceValue,
+                   'key'       => strip_tags(html_entity_decode(@hex2bin($asmArray[2]), ENT_QUOTES, 'UTF-8')),
+                   'vale'      => strip_tags(html_entity_decode(@hex2bin($asmArray[3]), ENT_QUOTES, 'UTF-8'))
+                 ];
+               }
+            }
+          }
+        }
+      }
+
+    } else {
+
+      // @TODO block not found
+    }
+  }
+}
+
+// Debug
+if (CRAWLER_DEBUG) {
+  print_r($response);
+}
diff --git a/library/base58.php b/library/base58.php
new file mode 100644
index 0000000..221d78d
--- /dev/null
+++ b/library/base58.php
@@ -0,0 +1,16 @@
+<?php
+
+class Base58 {
+
+  const AVAILABLE_CHARS = '123456789ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz';
+
+  public static function encode($num, $length = 58): string {
+
+    return Crypto::dec2base($num, $length, self::AVAILABLE_CHARS);
+  }
+
+  public static function decode(string $addr, int $length = 58): string {
+
+    return Crypto::base2dec($addr, $length, self::AVAILABLE_CHARS);
+  }
+}
diff --git a/library/base58check.php b/library/base58check.php
new file mode 100644
index 0000000..9c59059
--- /dev/null
+++ b/library/base58check.php
@@ -0,0 +1,51 @@
+<?php
+
+class Base58Check {
+
+  public static function encode(string $string, int $prefix = 128, bool $compressed = true) {
+
+    $string = hex2bin($string);
+
+    if ($prefix) {
+      $string = chr($prefix) . $string;
+    }
+
+    if ($compressed) {
+      $string .= chr(0x01);
+    }
+
+    $string = $string . substr(Hash::SHA256(Hash::SHA256($string)), 0, 4);
+
+    $base58 = Base58::encode(Crypto::bin2bc($string));
+
+    for ($i = 0; $i < strlen($string); $i++) {
+
+      if ($string[$i] != '\x00') {
+        break;
+      }
+
+      $base58 = '1' . $base58;
+    }
+
+    return $base58;
+  }
+
+  public static function decode(string $string, int $removeLeadingBytes = 1, int $removeTrailingBytes = 4, bool $removeCompression = true) {
+
+    $string = bin2hex(Crypto::bc2bin(Base58::decode($string)));
+
+    if ($removeLeadingBytes) {
+      $string = substr($string, $removeLeadingBytes * 2);
+    }
+
+    if ($removeTrailingBytes) {
+      $string = substr($string, 0, -($removeTrailingBytes * 2));
+    }
+
+    if ($removeCompression) {
+      $string = substr($string, 0, -2);
+    }
+
+    return $string;
+  }
+}
diff --git a/library/crypto.php b/library/crypto.php
new file mode 100644
index 0000000..da25747
--- /dev/null
+++ b/library/crypto.php
@@ -0,0 +1,88 @@
+<?php
+
+
+
+class Crypto {
+
+  public static function bc2bin($num) {
+
+    return self::dec2base($num, 256);
+  }
+
+  public static function dec2base($dec, $base, $digits = false) {
+
+    if ($base < 2 || $base > 256) {
+      die('Invalid Base: ' . $base);
+    }
+
+    bcscale(0);
+
+    $value = '';
+
+    if (!$digits) {
+      $digits = self::digits($base);
+    }
+
+    while ($dec > $base - 1) {
+
+      $rest  = bcmod($dec, $base);
+      $dec   = bcdiv($dec, $base);
+      $value = $digits[$rest] . $value;
+    }
+
+    $value   = $digits[intval($dec)] . $value;
+
+    return (string) $value;
+  }
+
+  public static function base2dec($value, $base, $digits = false) {
+
+    if ($base < 2 || $base > 256) {
+      die('Invalid Base: ' . $base);
+    }
+
+    bcscale(0);
+
+    if ($base < 37) {
+      $value = strtolower($value);
+    }
+    if (!$digits) {
+      $digits = self::digits($base);
+    }
+
+    $size = strlen($value);
+    $dec = '0';
+
+    for ($loop = 0; $loop < $size; $loop++) {
+      $element = strpos($digits, $value[$loop]);
+      $power   = bcpow($base, $size - $loop - 1);
+      $dec     = bcadd($dec, bcmul($element, $power));
+    }
+
+    return (string) $dec;
+  }
+
+  public static function digits($base) {
+
+    if ($base > 64) {
+
+      $digits = '';
+
+      for ($loop = 0; $loop < 256; $loop++) {
+        $digits .= chr($loop);
+      }
+    } else {
+      $digits = '0123456789abcdefghijklmnopqrstuvwxyz';
+      $digits .= 'ABCDEFGHIJKLMNOPQRSTUVWXYZ-_';
+    }
+
+    $digits = substr($digits, 0, $base);
+
+    return (string)$digits;
+  }
+
+  public static function bin2bc($num) {
+
+    return self::base2dec($num, 256);
+  }
+}
diff --git a/library/hash.php b/library/hash.php
new file mode 100644
index 0000000..6ccd511
--- /dev/null
+++ b/library/hash.php
@@ -0,0 +1,19 @@
+<?php
+
+class Hash {
+
+  public static function SHA256(string $data, $raw = true): string {
+
+    return hash('sha256', $data, $raw);
+  }
+
+  public static function sha256d(string $data): string {
+
+    return hash('sha256', hash('sha256', $data, true), true);
+  }
+
+  public static function RIPEMD160(string $data, $raw = true): string {
+
+    return hash('ripemd160', $data, $raw);
+  }
+}
diff --git a/library/keva.php b/library/keva.php
new file mode 100644
index 0000000..949689f
--- /dev/null
+++ b/library/keva.php
@@ -0,0 +1,110 @@
+<?php
+
+class Keva {
+
+  public $username;
+  public $password;
+
+  public $host;
+  public $port;
+  public $url;
+
+  public $proto = 'http';
+  public $CACertificate = null;
+
+  public $status;
+  public $error;
+  public $rawResponse;
+  public $response;
+
+  private $id = 0;
+
+  public function setSSL($certificate = null) {
+    $this->proto         = 'https';
+    $this->CACertificate = $certificate;
+  }
+
+  public function __call($method, $params) {
+
+    $this->status       = null;
+    $this->error        = null;
+    $this->rawResponse  = null;
+    $this->response     = null;
+
+    $params = array_values($params);
+
+    $this->id++;
+
+    $request = json_encode(array(
+      'method' => $method,
+      'params' => $params,
+      'id'     => $this->id
+    ));
+
+    $curl    = curl_init("{$this->proto}://{$this->host}:{$this->port}/{$this->url}");
+    $options = array(
+        CURLOPT_HTTPAUTH       => CURLAUTH_BASIC,
+        CURLOPT_USERPWD        => $this->username . ':' . $this->password,
+        CURLOPT_RETURNTRANSFER => true,
+        CURLOPT_FOLLOWLOCATION => true,
+        CURLOPT_MAXREDIRS      => 10,
+        CURLOPT_HTTPHEADER     => array('Content-type: text/plain'),
+        CURLOPT_POST           => true,
+        CURLOPT_POSTFIELDS     => $request
+    );
+
+    if (ini_get('open_basedir')) {
+      unset($options[CURLOPT_FOLLOWLOCATION]);
+    }
+
+    if ($this->proto == 'https') {
+      if (!empty($this->CACertificate)) {
+        $options[CURLOPT_CAINFO] = $this->CACertificate;
+        $options[CURLOPT_CAPATH] = DIRNAME($this->CACertificate);
+      } else {
+        $options[CURLOPT_SSL_VERIFYPEER] = false;
+      }
+    }
+
+    curl_setopt_array($curl, $options);
+
+    $this->rawResponse = curl_exec($curl);
+    $this->response     = json_decode($this->rawResponse, true);
+
+    $this->status = curl_getinfo($curl, CURLINFO_HTTP_CODE);
+
+    $curl_error = curl_error($curl);
+
+    curl_close($curl);
+
+    if (!empty($curl_error)) {
+        $this->error = $curl_error;
+    }
+
+    if ($this->response['error']) {
+        $this->error = $this->response['error']['message'];
+    } elseif ($this->status != 200) {
+
+      switch ($this->status) {
+        case 400:
+          $this->error = 'HTTP_BAD_REQUEST';
+          break;
+        case 401:
+          $this->error = 'HTTP_UNAUTHORIZED';
+          break;
+        case 403:
+          $this->error = 'HTTP_FORBIDDEN';
+          break;
+        case 404:
+          $this->error = 'HTTP_NOT_FOUND';
+          break;
+      }
+    }
+
+    if ($this->error) {
+      return false;
+    }
+
+    return $this->response['result'];
+  }
+}
diff --git a/library/mysql.php b/library/mysql.php
new file mode 100644
index 0000000..c332bfa
--- /dev/null
+++ b/library/mysql.php
@@ -0,0 +1,138 @@
+<?php
+
+class MySQL {
+
+  public function __construct() {
+
+    try {
+
+      $this->_db = new PDO('mysql:dbname=' . DB_NAME . ';host=' . DB_HOST . ';port=' . DB_PORT . ';charset=utf8', DB_USERNAME, DB_PASSWORD, [PDO::MYSQL_ATTR_INIT_COMMAND => 'SET NAMES utf8']);
+      $this->_db->setAttribute(PDO::ATTR_ERRMODE, PDO::ERRMODE_EXCEPTION);
+      $this->_db->setAttribute(PDO::ATTR_DEFAULT_FETCH_MODE, PDO::FETCH_ASSOC);
+      $this->_db->setAttribute(PDO::ATTR_TIMEOUT, 600);
+
+    } catch(PDOException $e) {
+      trigger_error($e->getMessage());
+    }
+  }
+
+  public function getLastBlock() {
+
+    try {
+
+      $query = $this->_db->query('SELECT MAX(`blockId`) AS `lastBlock` FROM `block`')->fetch();
+
+      return (int) $query['lastBlock'] ? $query['lastBlock'] : 1;
+
+    } catch(PDOException $e) {
+      trigger_error($e->getMessage());
+      return false;
+    }
+  }
+
+  public function getBlock($blockId) {
+
+    try {
+
+      $query = $this->_db->prepare('SELECT `blockId` FROM `block` WHERE `blockId` = ? LIMIT 1');
+
+      $query->execute([$blockId]);
+
+      return $query->rowCount() ? $query->fetch()['blockId'] : false;
+
+    } catch(PDOException $e) {
+      trigger_error($e->getMessage());
+      return false;
+    }
+  }
+
+  public function getNameSpace($hash) {
+
+    try {
+
+      $query = $this->_db->prepare('SELECT `nameSpaceId` FROM `namespace` WHERE `hash` = ? LIMIT 1');
+
+      $query->execute([$hash]);
+
+      return $query->rowCount() ? $query->fetch()['nameSpaceId'] : false;
+
+    } catch(PDOException $e) {
+      trigger_error($e->getMessage());
+      return false;
+    }
+  }
+
+  public function getData($blockId, $nameSpaceId) {
+
+    try {
+
+      $query = $this->_db->prepare('SELECT `dataId` FROM `data` WHERE `blockId` = ? AND `nameSpaceId` = ? LIMIT 1');
+
+      $query->execute([$blockId, $nameSpaceId]);
+
+      return $query->rowCount() ? $query->fetch()['dataId'] : false;
+
+    } catch(PDOException $e) {
+      trigger_error($e->getMessage());
+      return false;
+    }
+  }
+
+  public function addBlock($blockId) {
+
+    try {
+
+      $query = $this->_db->prepare('INSERT INTO `block` SET `blockId` = ?,
+                                                            `timeIndexed` = UNIX_TIMESTAMP()');
+
+      $query->execute([$blockId]);
+
+      return $blockId;
+
+    } catch(PDOException $e) {
+      trigger_error($e->getMessage());
+      return false;
+    }
+  }
+
+  public function addNameSpace($hash, $value) {
+
+    try {
+
+      $query = $this->_db->prepare('INSERT INTO `namespace` SET `hash`  = ?,
+                                                                `value` = ?,
+                                                                `timeIndexed` = UNIX_TIMESTAMP()');
+
+      $query->execute([$hash, $value]);
+
+      return $this->_db->lastInsertId();
+
+    } catch(PDOException $e) {
+      trigger_error($e->getMessage());
+      return false;
+    }
+  }
+
+  public function addData($blockId, $nameSpaceId, $time, $size, $txid, $key, $value) {
+
+    try {
+
+      $query = $this->_db->prepare('INSERT INTO `data` SET `blockId`     = ?,
+                                                           `nameSpaceId` = ?,
+                                                           `time`        = ?,
+                                                           `size`        = ?,
+                                                           `txid`        = ?,
+                                                           `key`         = ?,
+                                                           `value`       = ?,
+                                                           `timeIndexed` = UNIX_TIMESTAMP()');
+
+      $query->execute([$blockId, $nameSpaceId, $time, $size, $txid, $key, $value]);
+
+      return $this->_db->lastInsertId();
+
+    } catch(PDOException $e) {
+      trigger_error($e->getMessage());
+      return false;
+    }
+  }
+}