kvazar
/
crawler-full-node
mirror of https://github.com/kvazar-network/crawler-full-node.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity
Browse Source

initial commit

pull/1/head
ghost 3 years ago
commit
feda96c33d
10 changed files with 566 additions and 0 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Unified View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 21
      LICENSE
  2. 1
      README.md
  3. 24
      config-default.php
  4. 98
      crawler.php
  5. 16
      library/base58.php
  6. 51
      library/base58check.php
  7. 88
      library/crypto.php
  8. 19
      library/hash.php
  9. 110
      library/keva.php
  10. 138
      library/mysql.php

21
LICENSE
Unescape View File

@ -0,0 +1,21 @@
MIT License
Copyright (c) 2021 kvazar-network
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.

1
README.md
Unescape View File

@ -0,0 +1 @@
# crawler-full-node

24
config-default.php
Unescape View File

@ -0,0 +1,24 @@
<?php
// Debug
ini_set('display_errors', '1');
ini_set('display_startup_errors', '1');
error_reporting(E_ALL);
// Application
define('KEVA_NS', ''); // KEVA_NS or empty to collect all the blockchain
define('STEP_BLOCK_LIMIT', 50); // Blocks per query
define('CRAWLER_DEBUG', true); // Debug output
// Kevacoin wallet
define('KEVA_HOST', '127.0.0.1');
define('KEVA_PORT', '9992');
define('KEVA_USERNAME', '');
define('KEVA_PASSWORD', '');
// Database
define('DB_HOST', '127.0.0.1');
define('DB_PORT', '3306');
define('DB_NAME', '');
define('DB_USERNAME', '');
define('DB_PASSWORD', '');

98
crawler.php
Unescape View File

@ -0,0 +1,98 @@
<?php
require_once('config.php');
require_once('library/mysql.php');
require_once('library/keva.php');
require_once('library/hash.php');
require_once('library/base58.php');
require_once('library/base58check.php');
require_once('library/crypto.php');
$db = new MySQL();
$node = new Keva();
$node->host = KEVA_HOST;
$node->username = KEVA_USERNAME;
$node->password = KEVA_PASSWORD;
$node->port = KEVA_PORT;
$blockLast = $db->getLastBlock();
$blockTotal = $node->getblockcount();
$response = [];
if ($blockTotal > $blockLast) {
for ($blockCurrent = $blockLast; $blockCurrent <= $blockLast + STEP_BLOCK_LIMIT; $blockCurrent++) {
if ($blockHash = $node->getblockhash($blockCurrent)) {
$blockData = $node->getblock($blockHash);
if (!$blockId = $db->getBlock($blockCurrent)) {
$blockId = $db->addBlock($blockCurrent);
}
foreach ($blockData['tx'] as $transaction) {
$transactionRaw = $node->getrawtransaction($transaction, 1);
foreach($transactionRaw['vout'] as $vout) {
$asmArray = explode(' ', $vout['scriptPubKey']['asm']);
if($asmArray[0] == 'OP_KEVA_NAMESPACE' || $asmArray[0] == 'OP_KEVA_PUT') { // OP_KEVA_DELETE
$hash = Base58Check::encode($asmArray[1], false , 0 , false);
$nameSpace = $node->keva_get($hash, '_KEVA_NS_');
$nameSpaceValue = strip_tags(html_entity_decode($nameSpace['value'], ENT_QUOTES, 'UTF-8'));
if ((empty(KEVA_NS) || (!empty(KEVA_NS) && KEVA_NS == $nameSpaceValue))) {
if (!$nameSpaceId = $db->getNameSpace($hash)) {
$nameSpaceId = $db->addNameSpace($hash, $nameSpaceValue);
}
if (!$db->getData($blockId, $nameSpaceId)) {
$db->addData($blockId,
$nameSpaceId,
$transactionRaw['time'],
$transactionRaw['size'],
$transactionRaw['txid'],
strip_tags(html_entity_decode(@hex2bin($asmArray[2]), ENT_QUOTES, 'UTF-8')),
strip_tags(html_entity_decode(@hex2bin($asmArray[3]), ENT_QUOTES, 'UTF-8')));
}
if (CRAWLER_DEBUG) {
$response[] = [
'blocktotal'=> $blockTotal,
'block' => $blockCurrent,
'blockhash' => $transactionRaw['blockhash'],
'txid' => $transactionRaw['txid'],
'version' => $transactionRaw['version'],
'size' => $transactionRaw['size'],
'time' => $transactionRaw['time'],
'blocktime' => $transactionRaw['blocktime'],
'namehash' => $hash,
'title' => $nameSpaceValue,
'key' => strip_tags(html_entity_decode(@hex2bin($asmArray[2]), ENT_QUOTES, 'UTF-8')),
'vale' => strip_tags(html_entity_decode(@hex2bin($asmArray[3]), ENT_QUOTES, 'UTF-8'))
];
}
}
}
}
}
} else {
// @TODO block not found
}
}
}
// Debug
if (CRAWLER_DEBUG) {
print_r($response);
}

16
library/base58.php
Unescape View File

@ -0,0 +1,16 @@
<?php
class Base58 {
const AVAILABLE_CHARS = '123456789ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz';
public static function encode($num, $length = 58): string {
return Crypto::dec2base($num, $length, self::AVAILABLE_CHARS);
}
public static function decode(string $addr, int $length = 58): string {
return Crypto::base2dec($addr, $length, self::AVAILABLE_CHARS);
}
}

51
library/base58check.php
Unescape View File

@ -0,0 +1,51 @@
<?php
class Base58Check {
public static function encode(string $string, int $prefix = 128, bool $compressed = true) {
$string = hex2bin($string);
if ($prefix) {
$string = chr($prefix) . $string;
}
if ($compressed) {
$string .= chr(0x01);
}
$string = $string . substr(Hash::SHA256(Hash::SHA256($string)), 0, 4);
$base58 = Base58::encode(Crypto::bin2bc($string));
for ($i = 0; $i < strlen($string); $i++) {
if ($string[$i] != '\x00') {
break;
}
$base58 = '1' . $base58;
}
return $base58;
}
public static function decode(string $string, int $removeLeadingBytes = 1, int $removeTrailingBytes = 4, bool $removeCompression = true) {
$string = bin2hex(Crypto::bc2bin(Base58::decode($string)));
if ($removeLeadingBytes) {
$string = substr($string, $removeLeadingBytes * 2);
}
if ($removeTrailingBytes) {
$string = substr($string, 0, -($removeTrailingBytes * 2));
}
if ($removeCompression) {
$string = substr($string, 0, -2);
}
return $string;
}
}

88
library/crypto.php
Unescape View File

@ -0,0 +1,88 @@
<?php
class Crypto {
public static function bc2bin($num) {
return self::dec2base($num, 256);
}
public static function dec2base($dec, $base, $digits = false) {
if ($base < 2 || $base > 256) {
die('Invalid Base: ' . $base);
}
bcscale(0);
$value = '';
if (!$digits) {
$digits = self::digits($base);
}
while ($dec > $base - 1) {
$rest = bcmod($dec, $base);
$dec = bcdiv($dec, $base);
$value = $digits[$rest] . $value;
}
$value = $digits[intval($dec)] . $value;
return (string) $value;
}
public static function base2dec($value, $base, $digits = false) {
if ($base < 2 || $base > 256) {
die('Invalid Base: ' . $base);
}
bcscale(0);
if ($base < 37) {
$value = strtolower($value);
}
if (!$digits) {
$digits = self::digits($base);
}
$size = strlen($value);
$dec = '0';
for ($loop = 0; $loop < $size; $loop++) {
$element = strpos($digits, $value[$loop]);
$power = bcpow($base, $size - $loop - 1);
$dec = bcadd($dec, bcmul($element, $power));
}
return (string) $dec;
}
public static function digits($base) {
if ($base > 64) {
$digits = '';
for ($loop = 0; $loop < 256; $loop++) {
$digits .= chr($loop);
}
} else {
$digits = '0123456789abcdefghijklmnopqrstuvwxyz';
$digits .= 'ABCDEFGHIJKLMNOPQRSTUVWXYZ-_';
}
$digits = substr($digits, 0, $base);
return (string)$digits;
}
public static function bin2bc($num) {
return self::base2dec($num, 256);
}
}

19
library/hash.php
Unescape View File

@ -0,0 +1,19 @@
<?php
class Hash {
public static function SHA256(string $data, $raw = true): string {
return hash('sha256', $data, $raw);
}
public static function sha256d(string $data): string {
return hash('sha256', hash('sha256', $data, true), true);
}
public static function RIPEMD160(string $data, $raw = true): string {
return hash('ripemd160', $data, $raw);
}
}

110
library/keva.php
Unescape View File

@ -0,0 +1,110 @@
<?php
class Keva {
public $username;
public $password;
public $host;
public $port;
public $url;
public $proto = 'http';
public $CACertificate = null;
public $status;
public $error;
public $rawResponse;
public $response;
private $id = 0;
public function setSSL($certificate = null) {
$this->proto = 'https';
$this->CACertificate = $certificate;
}
public function __call($method, $params) {
$this->status = null;
$this->error = null;
$this->rawResponse = null;
$this->response = null;
$params = array_values($params);
$this->id++;
$request = json_encode(array(
'method' => $method,
'params' => $params,
'id' => $this->id
));
$curl = curl_init("{$this->proto}://{$this->host}:{$this->port}/{$this->url}");
$options = array(
CURLOPT_HTTPAUTH => CURLAUTH_BASIC,
CURLOPT_USERPWD => $this->username . ':' . $this->password,
CURLOPT_RETURNTRANSFER => true,
CURLOPT_FOLLOWLOCATION => true,
CURLOPT_MAXREDIRS => 10,
CURLOPT_HTTPHEADER => array('Content-type: text/plain'),
CURLOPT_POST => true,
CURLOPT_POSTFIELDS => $request
);
if (ini_get('open_basedir')) {
unset($options[CURLOPT_FOLLOWLOCATION]);
}
if ($this->proto == 'https') {
if (!empty($this->CACertificate)) {
$options[CURLOPT_CAINFO] = $this->CACertificate;
$options[CURLOPT_CAPATH] = DIRNAME($this->CACertificate);
} else {
$options[CURLOPT_SSL_VERIFYPEER] = false;
}
}
curl_setopt_array($curl, $options);
$this->rawResponse = curl_exec($curl);
$this->response = json_decode($this->rawResponse, true);
$this->status = curl_getinfo($curl, CURLINFO_HTTP_CODE);
$curl_error = curl_error($curl);
curl_close($curl);
if (!empty($curl_error)) {
$this->error = $curl_error;
}
if ($this->response['error']) {
$this->error = $this->response['error']['message'];
} elseif ($this->status != 200) {
switch ($this->status) {
case 400:
$this->error = 'HTTP_BAD_REQUEST';
break;
case 401:
$this->error = 'HTTP_UNAUTHORIZED';
break;
case 403:
$this->error = 'HTTP_FORBIDDEN';
break;
case 404:
$this->error = 'HTTP_NOT_FOUND';
break;
}
}
if ($this->error) {
return false;
}
return $this->response['result'];
}
}

138
library/mysql.php
Unescape View File

@ -0,0 +1,138 @@
<?php
class MySQL {
public function __construct() {
try {
$this->_db = new PDO('mysql:dbname=' . DB_NAME . ';host=' . DB_HOST . ';port=' . DB_PORT . ';charset=utf8', DB_USERNAME, DB_PASSWORD, [PDO::MYSQL_ATTR_INIT_COMMAND => 'SET NAMES utf8']);
$this->_db->setAttribute(PDO::ATTR_ERRMODE, PDO::ERRMODE_EXCEPTION);
$this->_db->setAttribute(PDO::ATTR_DEFAULT_FETCH_MODE, PDO::FETCH_ASSOC);
$this->_db->setAttribute(PDO::ATTR_TIMEOUT, 600);
} catch(PDOException $e) {
trigger_error($e->getMessage());
}
}
public function getLastBlock() {
try {
$query = $this->_db->query('SELECT MAX(`blockId`) AS `lastBlock` FROM `block`')->fetch();
return (int) $query['lastBlock'] ? $query['lastBlock'] : 1;
} catch(PDOException $e) {
trigger_error($e->getMessage());
return false;
}
}
public function getBlock($blockId) {
try {
$query = $this->_db->prepare('SELECT `blockId` FROM `block` WHERE `blockId` = ? LIMIT 1');
$query->execute([$blockId]);
return $query->rowCount() ? $query->fetch()['blockId'] : false;
} catch(PDOException $e) {
trigger_error($e->getMessage());
return false;
}
}
public function getNameSpace($hash) {
try {
$query = $this->_db->prepare('SELECT `nameSpaceId` FROM `namespace` WHERE `hash` = ? LIMIT 1');
$query->execute([$hash]);
return $query->rowCount() ? $query->fetch()['nameSpaceId'] : false;
} catch(PDOException $e) {
trigger_error($e->getMessage());
return false;
}
}
public function getData($blockId, $nameSpaceId) {
try {
$query = $this->_db->prepare('SELECT `dataId` FROM `data` WHERE `blockId` = ? AND `nameSpaceId` = ? LIMIT 1');
$query->execute([$blockId, $nameSpaceId]);
return $query->rowCount() ? $query->fetch()['dataId'] : false;
} catch(PDOException $e) {
trigger_error($e->getMessage());
return false;
}
}
public function addBlock($blockId) {
try {
$query = $this->_db->prepare('INSERT INTO `block` SET `blockId` = ?,
`timeIndexed` = UNIX_TIMESTAMP()');
$query->execute([$blockId]);
return $blockId;
} catch(PDOException $e) {
trigger_error($e->getMessage());
return false;
}
}
public function addNameSpace($hash, $value) {
try {
$query = $this->_db->prepare('INSERT INTO `namespace` SET `hash` = ?,
`value` = ?,
`timeIndexed` = UNIX_TIMESTAMP()');
$query->execute([$hash, $value]);
return $this->_db->lastInsertId();
} catch(PDOException $e) {
trigger_error($e->getMessage());
return false;
}
}
public function addData($blockId, $nameSpaceId, $time, $size, $txid, $key, $value) {
try {
$query = $this->_db->prepare('INSERT INTO `data` SET `blockId` = ?,
`nameSpaceId` = ?,
`time` = ?,
`size` = ?,
`txid` = ?,
`key` = ?,
`value` = ?,
`timeIndexed` = UNIX_TIMESTAMP()');
$query->execute([$blockId, $nameSpaceId, $time, $size, $txid, $key, $value]);
return $this->_db->lastInsertId();
} catch(PDOException $e) {
trigger_error($e->getMessage());
return false;
}
}
}
Write Preview
Loading…
Cancel
Save