From 7ece21e5e8e6753f43e12eab86f9fd4ccd4fb321 Mon Sep 17 00:00:00 2001
From: ghost <ghost@ghost.ghost>
Date: Sat, 7 Aug 2021 13:43:53 +0300
Subject: [PATCH] initial-commit

---
 LICENSE                 |  21 +++++
 README.md               |   1 +
 config-default.php      |  17 ++++
 crawler.php             | 189 ++++++++++++++++++++++++++++++++++++++++
 library/api.php         |  53 +++++++++++
 library/base58.php      |  16 ++++
 library/base58check.php |  51 +++++++++++
 library/crypto.php      |  88 +++++++++++++++++++
 library/hash.php        |  19 ++++
 library/helper.php      |  15 ++++
 library/mysql.php       | 187 +++++++++++++++++++++++++++++++++++++++
 11 files changed, 657 insertions(+)
 create mode 100644 LICENSE
 create mode 100644 README.md
 create mode 100644 config-default.php
 create mode 100644 crawler.php
 create mode 100644 library/api.php
 create mode 100644 library/base58.php
 create mode 100644 library/base58check.php
 create mode 100644 library/crypto.php
 create mode 100644 library/hash.php
 create mode 100644 library/helper.php
 create mode 100644 library/mysql.php

diff --git a/LICENSE b/LICENSE
new file mode 100644
index 0000000..d48824e
--- /dev/null
+++ b/LICENSE
@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2021 kvazar-network
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
diff --git a/README.md b/README.md
new file mode 100644
index 0000000..d5e8654
--- /dev/null
+++ b/README.md
@@ -0,0 +1 @@
+# crawler-api-node
\ No newline at end of file
diff --git a/config-default.php b/config-default.php
new file mode 100644
index 0000000..e77877a
--- /dev/null
+++ b/config-default.php
@@ -0,0 +1,17 @@
+<?php
+
+// Debug
+ini_set('display_errors', '1');
+ini_set('display_startup_errors', '1');
+error_reporting(E_ALL);
+
+// Application
+define('STEP_BLOCK_LIMIT', 50);  // Blocks per query
+define('CRAWLER_DEBUG', true);   // Debug output
+
+// Database
+define('DB_HOST', '127.0.0.1');
+define('DB_PORT', '3306');
+define('DB_NAME', '');
+define('DB_USERNAME', '');
+define('DB_PASSWORD', '');
diff --git a/crawler.php b/crawler.php
new file mode 100644
index 0000000..c9bfda8
--- /dev/null
+++ b/crawler.php
@@ -0,0 +1,189 @@
+<?php
+
+$semaphore = sem_get(1);
+
+if (false !== sem_acquire($semaphore, 1)) {
+
+  require_once('config.php');
+  require_once('library/mysql.php');
+  require_once('library/api.php');
+  require_once('library/hash.php');
+  require_once('library/base58.php');
+  require_once('library/base58check.php');
+  require_once('library/crypto.php');
+  require_once('library/helper.php');
+
+  $db  = new MySQL();
+  $api = new API();
+
+  $blockLast  = $db->getLastBlock();
+  $blockTotal = $api->getblockcount();
+
+  if (!$blockTotal) {
+    echo "API connection error.\n";
+    exit;
+  }
+
+  $response = [];
+
+  if (CRAWLER_DEBUG) {
+    echo "scanning blockhain...\n";
+  }
+
+  for ($blockCurrent = ($blockLast + 1); $blockCurrent <= $blockLast + STEP_BLOCK_LIMIT; $blockCurrent++) {
+
+    if ($blockCurrent > $blockTotal) {
+
+      if (CRAWLER_DEBUG) {
+        echo "database is up to date.\n";
+      }
+
+      break;
+    }
+
+    if (CRAWLER_DEBUG) {
+      echo sprintf("reading block %s\n", $blockCurrent);
+    }
+
+    if (!$blockHash = $api->getblockhash($blockCurrent)) {
+
+      if (CRAWLER_DEBUG) {
+        echo "could not read the block hash. waiting for reconnect.\n";
+      }
+
+      break;
+    }
+
+    if (!$blockData = $api->getblock($blockHash)) {
+
+      if (CRAWLER_DEBUG) {
+        echo "could not read the block data. waiting for reconnect.\n";
+      }
+
+      break;
+    }
+
+    if (!$blockId = $db->getBlock($blockCurrent)) {
+         $blockId = $db->addBlock($blockCurrent);
+
+         if (CRAWLER_DEBUG) {
+           echo sprintf("add block %s\n", $blockCurrent);
+         }
+    }
+
+    $lostTransactions = 0;
+
+    foreach ($blockData['tx'] as $transaction) {
+
+      if (!$transactionRaw = $api->getrawtransaction($transaction)) {
+
+        $lostTransactions++;
+
+        $db->setLostTransactions($blockId, $lostTransactions);
+
+        if (CRAWLER_DEBUG) {
+          echo sprintf("could not read the transaction %s. skipped.\n", $transaction);
+        }
+
+        break;
+      }
+
+      foreach($transactionRaw['vout'] as $vout) {
+
+        $asmArray = explode(' ', $vout['scriptPubKey']['asm']);
+
+        if (in_array($asmArray[0], ['OP_KEVA_NAMESPACE', 'OP_KEVA_PUT', 'OP_KEVA_DELETE'])) {
+
+          $hash = Base58Check::encode($asmArray[1], false , 0 , false);
+
+          switch ($asmArray[0]) {
+
+            case 'OP_KEVA_DELETE':
+
+              $key   = filterString(decodeString($asmArray[2]));
+              $value = '';
+
+            break;
+
+            case 'OP_KEVA_NAMESPACE':
+
+              $key   = '_KEVA_NS_';
+              $value = filterString(decodeString($asmArray[2]));
+
+            break;
+
+            default:
+
+              $key   = filterString(decodeString($asmArray[2]));
+              $value = filterString(decodeString($asmArray[3]));
+          }
+
+          if (!$nameSpaceId = $db->getNameSpace($hash)) {
+               $nameSpaceId = $db->addNameSpace($hash);
+
+               if (CRAWLER_DEBUG) {
+                 echo sprintf("add namespace %s\n", $hash);
+               }
+          }
+
+          if (!$dataId = $db->getData($transactionRaw['txid'])) {
+               $dataId = $db->addData($blockId,
+                                      $nameSpaceId,
+                                      $transactionRaw['time'],
+                                      $transactionRaw['size'],
+                                      $transactionRaw['txid'],
+                                      $key,
+                                      $value,
+                                      ($key == '_KEVA_NS_'),
+                                      empty($value));
+
+            if ($value) {
+
+              $db->setDataKeyDeleted($nameSpaceId, $key, false);
+
+              if (CRAWLER_DEBUG) {
+                echo sprintf("add new key/value %s\n", $transactionRaw['txid']);
+              }
+
+            } else {
+
+              $db->setDataKeyDeleted($nameSpaceId, $key, true);
+
+              if (CRAWLER_DEBUG) {
+                echo sprintf("delete key %s from namespace %s\n", $key, $hash);
+              }
+            }
+          }
+
+          if (CRAWLER_DEBUG) {
+            $response[] = [
+              'blocktotal'=> $blockTotal,
+              'block'     => $blockCurrent,
+              'blockhash' => $transactionRaw['blockhash'],
+              'txid'      => $transactionRaw['txid'],
+              'version'   => $transactionRaw['version'],
+              'size'      => $transactionRaw['size'],
+              'time'      => $transactionRaw['time'],
+              'blocktime' => $transactionRaw['blocktime'],
+              'namehash'  => $hash,
+              'key'       => $key,
+              'value'     => $value
+            ];
+          }
+        }
+      }
+    }
+  }
+
+
+  // Debug
+  if (CRAWLER_DEBUG) {
+    echo "scanning completed.\n";
+    # print_r($response);
+  }
+
+  sem_release($semaphore);
+
+} else {
+  echo "database locked by the another process...\n";
+}
diff --git a/library/api.php b/library/api.php
new file mode 100644
index 0000000..0d7f5c1
--- /dev/null
+++ b/library/api.php
@@ -0,0 +1,53 @@
+<?php
+
+class API {
+
+  private $_url = 'https://explorer.kevacoin.org/api/';
+
+  public function getblockcount() {
+
+    // API return: int
+    if (false !== $response = file_get_contents($this->_url . 'getblockcount')) {
+
+      return (int) $response;
+    }
+
+    return false;
+  }
+
+  public function getblockhash($block) {
+
+    if (false !== $response = file_get_contents($this->_url . 'getblockhash?index=' . $block)) {
+
+      // API return: string
+      if (false !== json_decode($response)) {
+        return str_replace('"', '', $response);
+      }
+    }
+
+    return false;
+  }
+
+  public function getblock($hash) {
+
+    // API return: json
+    if (false !== $response = json_decode(file_get_contents($this->_url . 'getblock?hash=' . $hash), true)) {
+      if (isset($response['hash'])) {
+        return $response;
+      }
+    }
+
+    return false;
+  }
+
+  public function getrawtransaction($txid) {
+
+    if (false !== $response = json_decode(file_get_contents($this->_url . 'getrawtransaction?txid=' . $txid . '&decrypt=1'), true)) {
+      if (isset($response['txid'])) {
+        return $response;
+      }
+    }
+
+    return false;
+  }
+}
diff --git a/library/base58.php b/library/base58.php
new file mode 100644
index 0000000..221d78d
--- /dev/null
+++ b/library/base58.php
@@ -0,0 +1,16 @@
+<?php
+
+class Base58 {
+
+  const AVAILABLE_CHARS = '123456789ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz';
+
+  public static function encode($num, $length = 58): string {
+
+    return Crypto::dec2base($num, $length, self::AVAILABLE_CHARS);
+  }
+
+  public static function decode(string $addr, int $length = 58): string {
+
+    return Crypto::base2dec($addr, $length, self::AVAILABLE_CHARS);
+  }
+}
diff --git a/library/base58check.php b/library/base58check.php
new file mode 100644
index 0000000..9c59059
--- /dev/null
+++ b/library/base58check.php
@@ -0,0 +1,51 @@
+<?php
+
+class Base58Check {
+
+  public static function encode(string $string, int $prefix = 128, bool $compressed = true) {
+
+    $string = hex2bin($string);
+
+    if ($prefix) {
+      $string = chr($prefix) . $string;
+    }
+
+    if ($compressed) {
+      $string .= chr(0x01);
+    }
+
+    $string = $string . substr(Hash::SHA256(Hash::SHA256($string)), 0, 4);
+
+    $base58 = Base58::encode(Crypto::bin2bc($string));
+
+    for ($i = 0; $i < strlen($string); $i++) {
+
+      if ($string[$i] != '\x00') {
+        break;
+      }
+
+      $base58 = '1' . $base58;
+    }
+
+    return $base58;
+  }
+
+  public static function decode(string $string, int $removeLeadingBytes = 1, int $removeTrailingBytes = 4, bool $removeCompression = true) {
+
+    $string = bin2hex(Crypto::bc2bin(Base58::decode($string)));
+
+    if ($removeLeadingBytes) {
+      $string = substr($string, $removeLeadingBytes * 2);
+    }
+
+    if ($removeTrailingBytes) {
+      $string = substr($string, 0, -($removeTrailingBytes * 2));
+    }
+
+    if ($removeCompression) {
+      $string = substr($string, 0, -2);
+    }
+
+    return $string;
+  }
+}
diff --git a/library/crypto.php b/library/crypto.php
new file mode 100644
index 0000000..da25747
--- /dev/null
+++ b/library/crypto.php
@@ -0,0 +1,88 @@
+<?php
+
+
+
+class Crypto {
+
+  public static function bc2bin($num) {
+
+    return self::dec2base($num, 256);
+  }
+
+  public static function dec2base($dec, $base, $digits = false) {
+
+    if ($base < 2 || $base > 256) {
+      die('Invalid Base: ' . $base);
+    }
+
+    bcscale(0);
+
+    $value = '';
+
+    if (!$digits) {
+      $digits = self::digits($base);
+    }
+
+    while ($dec > $base - 1) {
+
+      $rest  = bcmod($dec, $base);
+      $dec   = bcdiv($dec, $base);
+      $value = $digits[$rest] . $value;
+    }
+
+    $value   = $digits[intval($dec)] . $value;
+
+    return (string) $value;
+  }
+
+  public static function base2dec($value, $base, $digits = false) {
+
+    if ($base < 2 || $base > 256) {
+      die('Invalid Base: ' . $base);
+    }
+
+    bcscale(0);
+
+    if ($base < 37) {
+      $value = strtolower($value);
+    }
+    if (!$digits) {
+      $digits = self::digits($base);
+    }
+
+    $size = strlen($value);
+    $dec = '0';
+
+    for ($loop = 0; $loop < $size; $loop++) {
+      $element = strpos($digits, $value[$loop]);
+      $power   = bcpow($base, $size - $loop - 1);
+      $dec     = bcadd($dec, bcmul($element, $power));
+    }
+
+    return (string) $dec;
+  }
+
+  public static function digits($base) {
+
+    if ($base > 64) {
+
+      $digits = '';
+
+      for ($loop = 0; $loop < 256; $loop++) {
+        $digits .= chr($loop);
+      }
+    } else {
+      $digits = '0123456789abcdefghijklmnopqrstuvwxyz';
+      $digits .= 'ABCDEFGHIJKLMNOPQRSTUVWXYZ-_';
+    }
+
+    $digits = substr($digits, 0, $base);
+
+    return (string)$digits;
+  }
+
+  public static function bin2bc($num) {
+
+    return self::base2dec($num, 256);
+  }
+}
diff --git a/library/hash.php b/library/hash.php
new file mode 100644
index 0000000..6ccd511
--- /dev/null
+++ b/library/hash.php
@@ -0,0 +1,19 @@
+<?php
+
+class Hash {
+
+  public static function SHA256(string $data, $raw = true): string {
+
+    return hash('sha256', $data, $raw);
+  }
+
+  public static function sha256d(string $data): string {
+
+    return hash('sha256', hash('sha256', $data, true), true);
+  }
+
+  public static function RIPEMD160(string $data, $raw = true): string {
+
+    return hash('ripemd160', $data, $raw);
+  }
+}
diff --git a/library/helper.php b/library/helper.php
new file mode 100644
index 0000000..6db5eda
--- /dev/null
+++ b/library/helper.php
@@ -0,0 +1,15 @@
+<?php
+
+function decodeString($string) {
+
+  if (is_numeric($string) && $string < 0xFFFFFFFF) {
+    return mb_chr($string, 'ASCII');
+  } else {
+    return hex2bin($string);
+  }
+}
+
+function filterString($string) {
+
+  return strip_tags(html_entity_decode($string, ENT_QUOTES, 'UTF-8'));
+}
diff --git a/library/mysql.php b/library/mysql.php
new file mode 100644
index 0000000..8ebc5e9
--- /dev/null
+++ b/library/mysql.php
@@ -0,0 +1,187 @@
+<?php
+
+class MySQL {
+
+  public function __construct() {
+
+    try {
+
+      $this->_db = new PDO('mysql:dbname=' . DB_NAME . ';host=' . DB_HOST . ';port=' . DB_PORT . ';charset=utf8', DB_USERNAME, DB_PASSWORD, [PDO::MYSQL_ATTR_INIT_COMMAND => 'SET NAMES utf8']);
+      $this->_db->setAttribute(PDO::ATTR_ERRMODE, PDO::ERRMODE_EXCEPTION);
+      $this->_db->setAttribute(PDO::ATTR_DEFAULT_FETCH_MODE, PDO::FETCH_ASSOC);
+      $this->_db->setAttribute(PDO::ATTR_TIMEOUT, 600);
+
+    } catch(PDOException $e) {
+      trigger_error($e->getMessage());
+    }
+  }
+
+  public function getLastBlock() {
+
+    try {
+
+      $query = $this->_db->query('SELECT MAX(`blockId`) AS `lastBlock` FROM `block`')->fetch();
+
+      return (int) $query['lastBlock'];
+
+    } catch(PDOException $e) {
+      trigger_error($e->getMessage());
+      return false;
+    }
+  }
+
+  public function getBlock($blockId) {
+
+    try {
+
+      $query = $this->_db->prepare('SELECT `blockId` FROM `block` WHERE `blockId` = ? LIMIT 1');
+
+      $query->execute([$blockId]);
+
+      return $query->rowCount() ? $query->fetch()['blockId'] : false;
+
+    } catch(PDOException $e) {
+      trigger_error($e->getMessage());
+      return false;
+    }
+  }
+
+  public function getNameSpace($hash) {
+
+    try {
+
+      $query = $this->_db->prepare('SELECT `nameSpaceId` FROM `namespace` WHERE `hash` = ? LIMIT 1');
+
+      $query->execute([$hash]);
+
+      return $query->rowCount() ? $query->fetch()['nameSpaceId'] : false;
+
+    } catch(PDOException $e) {
+      trigger_error($e->getMessage());
+      return false;
+    }
+  }
+
+  public function getData($txId) {
+
+    try {
+
+      $query = $this->_db->prepare('SELECT `dataId` FROM `data` WHERE `txId` = ? LIMIT 1');
+
+      $query->execute([$txId]);
+
+      return $query->rowCount() ? $query->fetch()['dataId'] : false;
+
+    } catch(PDOException $e) {
+      trigger_error($e->getMessage());
+      return false;
+    }
+  }
+
+  public function addBlock($blockId) {
+
+    try {
+
+      $query = $this->_db->prepare('INSERT INTO `block` SET `blockId`          = ?,
+                                                            `lostTransactions` = 0,
+                                                            `timeIndexed`      = UNIX_TIMESTAMP()');
+
+      $query->execute([$blockId]);
+
+      return $blockId;
+
+    } catch(PDOException $e) {
+      trigger_error($e->getMessage());
+      return false;
+    }
+  }
+
+  public function setLostTransactions($blockId, $lostTransactions) {
+
+    try {
+
+      $query = $this->_db->prepare('UPDATE `block` SET `lostTransactions` = ? WHERE `blockId` = ? LIMIT 1');
+
+      $query->execute([$lostTransactions, $blockId]);
+
+      return $blockId;
+
+    } catch(PDOException $e) {
+      trigger_error($e->getMessage());
+      return false;
+    }
+  }
+
+  public function addNameSpace($hash) {
+
+    try {
+
+      $query = $this->_db->prepare('INSERT INTO `namespace` SET `hash`  = ?,
+                                                                `timeIndexed` = UNIX_TIMESTAMP()');
+
+      $query->execute([$hash]);
+
+      return $this->_db->lastInsertId();
+
+    } catch(PDOException $e) {
+      trigger_error($e->getMessage());
+      return false;
+    }
+  }
+
+  public function addData($blockId, $nameSpaceId, $time, $size, $txid, $key, $value, $ns, $deleted = false) {
+
+    try {
+
+      $query = $this->_db->prepare('INSERT INTO `data` SET `blockId`     = :blockId,
+                                                           `nameSpaceId` = :nameSpaceId,
+                                                           `time`        = :time,
+                                                           `size`        = :size,
+                                                           `txid`        = :txid,
+                                                           `key`         = :key,
+                                                           `value`       = :value,
+                                                           `deleted`     = :deleted,
+                                                           `ns`          = :ns,
+                                                           `timeIndexed` = UNIX_TIMESTAMP()');
+
+      $query->bindValue(':blockId', $blockId, PDO::PARAM_INT);
+      $query->bindValue(':nameSpaceId', $nameSpaceId, PDO::PARAM_INT);
+      $query->bindValue(':time', $time, PDO::PARAM_INT);
+      $query->bindValue(':size', $size, PDO::PARAM_INT);
+      $query->bindValue(':txid', $txid, PDO::PARAM_STR);
+      $query->bindValue(':key', $key, PDO::PARAM_STR);
+      $query->bindValue(':value', $value, PDO::PARAM_STR);
+      $query->bindValue(':deleted', (int) $deleted, PDO::PARAM_STR);
+      $query->bindValue(':ns', (int) $ns, PDO::PARAM_STR);
+
+      $query->execute();
+
+      return $this->_db->lastInsertId();
+
+    } catch(PDOException $e) {
+      trigger_error($e->getMessage());
+      return false;
+    }
+  }
+
+  public function setDataKeyDeleted($nameSpaceId, $key, $deleted) {
+
+    try {
+
+      $query = $this->_db->prepare('UPDATE `data` SET   `deleted`     = :deleted
+                                                  WHERE `nameSpaceId` = :nameSpaceId AND `key` LIKE :key');
+
+      $query->bindValue(':nameSpaceId', $nameSpaceId, PDO::PARAM_INT);
+      $query->bindValue(':key', $key, PDO::PARAM_STR);
+      $query->bindValue(':deleted', (int) $deleted, PDO::PARAM_STR);
+
+      $query->execute();
+
+      return $query->rowCount();
+
+    } catch(PDOException $e) {
+      trigger_error($e->getMessage());
+      return false;
+    }
+  }
+}