kvazar
/
crawler-api-node
mirror of https://github.com/kvazar-network/crawler-api-node.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity
Browse Source

initial-commit

mysql
ghost 3 years ago
commit
7ece21e5e8
11 changed files with 657 additions and 0 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Unified View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 21
      LICENSE
  2. 1
      README.md
  3. 17
      config-default.php
  4. 189
      crawler.php
  5. 53
      library/api.php
  6. 16
      library/base58.php
  7. 51
      library/base58check.php
  8. 88
      library/crypto.php
  9. 19
      library/hash.php
  10. 15
      library/helper.php
  11. 187
      library/mysql.php

21
LICENSE
Unescape View File

@ -0,0 +1,21 @@
MIT License
Copyright (c) 2021 kvazar-network
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.

1
README.md
Unescape View File

@ -0,0 +1 @@
# crawler-api-node

17
config-default.php
Unescape View File

@ -0,0 +1,17 @@
<?php
// Debug
ini_set('display_errors', '1');
ini_set('display_startup_errors', '1');
error_reporting(E_ALL);
// Application
define('STEP_BLOCK_LIMIT', 50); // Blocks per query
define('CRAWLER_DEBUG', true); // Debug output
// Database
define('DB_HOST', '127.0.0.1');
define('DB_PORT', '3306');
define('DB_NAME', '');
define('DB_USERNAME', '');
define('DB_PASSWORD', '');

189
crawler.php
Unescape View File

@ -0,0 +1,189 @@
<?php
$semaphore = sem_get(1);
if (false !== sem_acquire($semaphore, 1)) {
require_once('config.php');
require_once('library/mysql.php');
require_once('library/api.php');
require_once('library/hash.php');
require_once('library/base58.php');
require_once('library/base58check.php');
require_once('library/crypto.php');
require_once('library/helper.php');
$db = new MySQL();
$api = new API();
$blockLast = $db->getLastBlock();
$blockTotal = $api->getblockcount();
if (!$blockTotal) {
echo "API connection error.\n";
exit;
}
$response = [];
if (CRAWLER_DEBUG) {
echo "scanning blockhain...\n";
}
for ($blockCurrent = ($blockLast + 1); $blockCurrent <= $blockLast + STEP_BLOCK_LIMIT; $blockCurrent++) {
if ($blockCurrent > $blockTotal) {
if (CRAWLER_DEBUG) {
echo "database is up to date.\n";
}
break;
}
if (CRAWLER_DEBUG) {
echo sprintf("reading block %s\n", $blockCurrent);
}
if (!$blockHash = $api->getblockhash($blockCurrent)) {
if (CRAWLER_DEBUG) {
echo "could not read the block hash. waiting for reconnect.\n";
}
break;
}
if (!$blockData = $api->getblock($blockHash)) {
if (CRAWLER_DEBUG) {
echo "could not read the block data. waiting for reconnect.\n";
}
break;
}
if (!$blockId = $db->getBlock($blockCurrent)) {
$blockId = $db->addBlock($blockCurrent);
if (CRAWLER_DEBUG) {
echo sprintf("add block %s\n", $blockCurrent);
}
}
$lostTransactions = 0;
foreach ($blockData['tx'] as $transaction) {
if (!$transactionRaw = $api->getrawtransaction($transaction)) {
$lostTransactions++;
$db->setLostTransactions($blockId, $lostTransactions);
if (CRAWLER_DEBUG) {
echo sprintf("could not read the transaction %s. skipped.\n", $transaction);
}
break;
}
foreach($transactionRaw['vout'] as $vout) {
$asmArray = explode(' ', $vout['scriptPubKey']['asm']);
if (in_array($asmArray[0], ['OP_KEVA_NAMESPACE', 'OP_KEVA_PUT', 'OP_KEVA_DELETE'])) {
$hash = Base58Check::encode($asmArray[1], false , 0 , false);
switch ($asmArray[0]) {
case 'OP_KEVA_DELETE':
$key = filterString(decodeString($asmArray[2]));
$value = '';
break;
case 'OP_KEVA_NAMESPACE':
$key = '_KEVA_NS_';
$value = filterString(decodeString($asmArray[2]));
break;
default:
$key = filterString(decodeString($asmArray[2]));
$value = filterString(decodeString($asmArray[3]));
}
if (!$nameSpaceId = $db->getNameSpace($hash)) {
$nameSpaceId = $db->addNameSpace($hash);
if (CRAWLER_DEBUG) {
echo sprintf("add namespace %s\n", $hash);
}
}
if (!$dataId = $db->getData($transactionRaw['txid'])) {
$dataId = $db->addData($blockId,
$nameSpaceId,
$transactionRaw['time'],
$transactionRaw['size'],
$transactionRaw['txid'],
$key,
$value,
($key == '_KEVA_NS_'),
empty($value));
if ($value) {
$db->setDataKeyDeleted($nameSpaceId, $key, false);
if (CRAWLER_DEBUG) {
echo sprintf("add new key/value %s\n", $transactionRaw['txid']);
}
} else {
$db->setDataKeyDeleted($nameSpaceId, $key, true);
if (CRAWLER_DEBUG) {
echo sprintf("delete key %s from namespace %s\n", $key, $hash);
}
}
}
if (CRAWLER_DEBUG) {
$response[] = [
'blocktotal'=> $blockTotal,
'block' => $blockCurrent,
'blockhash' => $transactionRaw['blockhash'],
'txid' => $transactionRaw['txid'],
'version' => $transactionRaw['version'],
'size' => $transactionRaw['size'],
'time' => $transactionRaw['time'],
'blocktime' => $transactionRaw['blocktime'],
'namehash' => $hash,
'key' => $key,
'value' => $value
];
}
}
}
}
}
// Debug
if (CRAWLER_DEBUG) {
echo "scanning completed.\n";
# print_r($response);
}
sem_release($semaphore);
} else {
echo "database locked by the another process...\n";
}

53
library/api.php
Unescape View File

@ -0,0 +1,53 @@
<?php
class API {
private $_url = 'https://explorer.kevacoin.org/api/';
public function getblockcount() {
// API return: int
if (false !== $response = file_get_contents($this->_url . 'getblockcount')) {
return (int) $response;
}
return false;
}
public function getblockhash($block) {
if (false !== $response = file_get_contents($this->_url . 'getblockhash?index=' . $block)) {
// API return: string
if (false !== json_decode($response)) {
return str_replace('"', '', $response);
}
}
return false;
}
public function getblock($hash) {
// API return: json
if (false !== $response = json_decode(file_get_contents($this->_url . 'getblock?hash=' . $hash), true)) {
if (isset($response['hash'])) {
return $response;
}
}
return false;
}
public function getrawtransaction($txid) {
if (false !== $response = json_decode(file_get_contents($this->_url . 'getrawtransaction?txid=' . $txid . '&decrypt=1'), true)) {
if (isset($response['txid'])) {
return $response;
}
}
return false;
}
}

16
library/base58.php
Unescape View File

@ -0,0 +1,16 @@
<?php
class Base58 {
const AVAILABLE_CHARS = '123456789ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz';
public static function encode($num, $length = 58): string {
return Crypto::dec2base($num, $length, self::AVAILABLE_CHARS);
}
public static function decode(string $addr, int $length = 58): string {
return Crypto::base2dec($addr, $length, self::AVAILABLE_CHARS);
}
}

51
library/base58check.php
Unescape View File

@ -0,0 +1,51 @@
<?php
class Base58Check {
public static function encode(string $string, int $prefix = 128, bool $compressed = true) {
$string = hex2bin($string);
if ($prefix) {
$string = chr($prefix) . $string;
}
if ($compressed) {
$string .= chr(0x01);
}
$string = $string . substr(Hash::SHA256(Hash::SHA256($string)), 0, 4);
$base58 = Base58::encode(Crypto::bin2bc($string));
for ($i = 0; $i < strlen($string); $i++) {
if ($string[$i] != '\x00') {
break;
}
$base58 = '1' . $base58;
}
return $base58;
}
public static function decode(string $string, int $removeLeadingBytes = 1, int $removeTrailingBytes = 4, bool $removeCompression = true) {
$string = bin2hex(Crypto::bc2bin(Base58::decode($string)));
if ($removeLeadingBytes) {
$string = substr($string, $removeLeadingBytes * 2);
}
if ($removeTrailingBytes) {
$string = substr($string, 0, -($removeTrailingBytes * 2));
}
if ($removeCompression) {
$string = substr($string, 0, -2);
}
return $string;
}
}

88
library/crypto.php
Unescape View File

@ -0,0 +1,88 @@
<?php
class Crypto {
public static function bc2bin($num) {
return self::dec2base($num, 256);
}
public static function dec2base($dec, $base, $digits = false) {
if ($base < 2 || $base > 256) {
die('Invalid Base: ' . $base);
}
bcscale(0);
$value = '';
if (!$digits) {
$digits = self::digits($base);
}
while ($dec > $base - 1) {
$rest = bcmod($dec, $base);
$dec = bcdiv($dec, $base);
$value = $digits[$rest] . $value;
}
$value = $digits[intval($dec)] . $value;
return (string) $value;
}
public static function base2dec($value, $base, $digits = false) {
if ($base < 2 || $base > 256) {
die('Invalid Base: ' . $base);
}
bcscale(0);
if ($base < 37) {
$value = strtolower($value);
}
if (!$digits) {
$digits = self::digits($base);
}
$size = strlen($value);
$dec = '0';
for ($loop = 0; $loop < $size; $loop++) {
$element = strpos($digits, $value[$loop]);
$power = bcpow($base, $size - $loop - 1);
$dec = bcadd($dec, bcmul($element, $power));
}
return (string) $dec;
}
public static function digits($base) {
if ($base > 64) {
$digits = '';
for ($loop = 0; $loop < 256; $loop++) {
$digits .= chr($loop);
}
} else {
$digits = '0123456789abcdefghijklmnopqrstuvwxyz';
$digits .= 'ABCDEFGHIJKLMNOPQRSTUVWXYZ-_';
}
$digits = substr($digits, 0, $base);
return (string)$digits;
}
public static function bin2bc($num) {
return self::base2dec($num, 256);
}
}

19
library/hash.php
Unescape View File

@ -0,0 +1,19 @@
<?php
class Hash {
public static function SHA256(string $data, $raw = true): string {
return hash('sha256', $data, $raw);
}
public static function sha256d(string $data): string {
return hash('sha256', hash('sha256', $data, true), true);
}
public static function RIPEMD160(string $data, $raw = true): string {
return hash('ripemd160', $data, $raw);
}
}

15
library/helper.php
Unescape View File

@ -0,0 +1,15 @@
<?php
function decodeString($string) {
if (is_numeric($string) && $string < 0xFFFFFFFF) {
return mb_chr($string, 'ASCII');
} else {
return hex2bin($string);
}
}
function filterString($string) {
return strip_tags(html_entity_decode($string, ENT_QUOTES, 'UTF-8'));
}

187
library/mysql.php
Unescape View File

@ -0,0 +1,187 @@
<?php
class MySQL {
public function __construct() {
try {
$this->_db = new PDO('mysql:dbname=' . DB_NAME . ';host=' . DB_HOST . ';port=' . DB_PORT . ';charset=utf8', DB_USERNAME, DB_PASSWORD, [PDO::MYSQL_ATTR_INIT_COMMAND => 'SET NAMES utf8']);
$this->_db->setAttribute(PDO::ATTR_ERRMODE, PDO::ERRMODE_EXCEPTION);
$this->_db->setAttribute(PDO::ATTR_DEFAULT_FETCH_MODE, PDO::FETCH_ASSOC);
$this->_db->setAttribute(PDO::ATTR_TIMEOUT, 600);
} catch(PDOException $e) {
trigger_error($e->getMessage());
}
}
public function getLastBlock() {
try {
$query = $this->_db->query('SELECT MAX(`blockId`) AS `lastBlock` FROM `block`')->fetch();
return (int) $query['lastBlock'];
} catch(PDOException $e) {
trigger_error($e->getMessage());
return false;
}
}
public function getBlock($blockId) {
try {
$query = $this->_db->prepare('SELECT `blockId` FROM `block` WHERE `blockId` = ? LIMIT 1');
$query->execute([$blockId]);
return $query->rowCount() ? $query->fetch()['blockId'] : false;
} catch(PDOException $e) {
trigger_error($e->getMessage());
return false;
}
}
public function getNameSpace($hash) {
try {
$query = $this->_db->prepare('SELECT `nameSpaceId` FROM `namespace` WHERE `hash` = ? LIMIT 1');
$query->execute([$hash]);
return $query->rowCount() ? $query->fetch()['nameSpaceId'] : false;
} catch(PDOException $e) {
trigger_error($e->getMessage());
return false;
}
}
public function getData($txId) {
try {
$query = $this->_db->prepare('SELECT `dataId` FROM `data` WHERE `txId` = ? LIMIT 1');
$query->execute([$txId]);
return $query->rowCount() ? $query->fetch()['dataId'] : false;
} catch(PDOException $e) {
trigger_error($e->getMessage());
return false;
}
}
public function addBlock($blockId) {
try {
$query = $this->_db->prepare('INSERT INTO `block` SET `blockId` = ?,
`lostTransactions` = 0,
`timeIndexed` = UNIX_TIMESTAMP()');
$query->execute([$blockId]);
return $blockId;
} catch(PDOException $e) {
trigger_error($e->getMessage());
return false;
}
}
public function setLostTransactions($blockId, $lostTransactions) {
try {
$query = $this->_db->prepare('UPDATE `block` SET `lostTransactions` = ? WHERE `blockId` = ? LIMIT 1');
$query->execute([$lostTransactions, $blockId]);
return $blockId;
} catch(PDOException $e) {
trigger_error($e->getMessage());
return false;
}
}
public function addNameSpace($hash) {
try {
$query = $this->_db->prepare('INSERT INTO `namespace` SET `hash` = ?,
`timeIndexed` = UNIX_TIMESTAMP()');
$query->execute([$hash]);
return $this->_db->lastInsertId();
} catch(PDOException $e) {
trigger_error($e->getMessage());
return false;
}
}
public function addData($blockId, $nameSpaceId, $time, $size, $txid, $key, $value, $ns, $deleted = false) {
try {
$query = $this->_db->prepare('INSERT INTO `data` SET `blockId` = :blockId,
`nameSpaceId` = :nameSpaceId,
`time` = :time,
`size` = :size,
`txid` = :txid,
`key` = :key,
`value` = :value,
`deleted` = :deleted,
`ns` = :ns,
`timeIndexed` = UNIX_TIMESTAMP()');
$query->bindValue(':blockId', $blockId, PDO::PARAM_INT);
$query->bindValue(':nameSpaceId', $nameSpaceId, PDO::PARAM_INT);
$query->bindValue(':time', $time, PDO::PARAM_INT);
$query->bindValue(':size', $size, PDO::PARAM_INT);
$query->bindValue(':txid', $txid, PDO::PARAM_STR);
$query->bindValue(':key', $key, PDO::PARAM_STR);
$query->bindValue(':value', $value, PDO::PARAM_STR);
$query->bindValue(':deleted', (int) $deleted, PDO::PARAM_STR);
$query->bindValue(':ns', (int) $ns, PDO::PARAM_STR);
$query->execute();
return $this->_db->lastInsertId();
} catch(PDOException $e) {
trigger_error($e->getMessage());
return false;
}
}
public function setDataKeyDeleted($nameSpaceId, $key, $deleted) {
try {
$query = $this->_db->prepare('UPDATE `data` SET `deleted` = :deleted
WHERE `nameSpaceId` = :nameSpaceId AND `key` LIKE :key');
$query->bindValue(':nameSpaceId', $nameSpaceId, PDO::PARAM_INT);
$query->bindValue(':key', $key, PDO::PARAM_STR);
$query->bindValue(':deleted', (int) $deleted, PDO::PARAM_STR);
$query->execute();
return $query->rowCount();
} catch(PDOException $e) {
trigger_error($e->getMessage());
return false;
}
}
}
Write Preview
Loading…
Cancel
Save