kvazar
/
crawler-full-node
mirror of https://github.com/kvazar-network/crawler-full-node.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity
Browse Source

apply latest updates from api-node

pull/1/head
ghost 3 years ago
parent
feda96c33d
commit
c29e0d6b58
6 changed files with 403 additions and 202 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Unified View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 14
      config-default.php
  2. 157
      crawler.php
  3. 15
      library/helper.php
  4. 110
      library/keva.php
  5. 156
      library/kevacoin.php
  6. 85
      library/mysql.php

14
config-default.php
Unescape View File

@ -6,19 +6,19 @@ ini_set('display_startup_errors', '1');
error_reporting(E_ALL); error_reporting(E_ALL);
// Application // Application
define('KEVA_NS', ''); // KEVA_NS or empty to collect all the blockchain
define('STEP_BLOCK_LIMIT', 50); // Blocks per query define('STEP_BLOCK_LIMIT', 50); // Blocks per query
define('CRAWLER_DEBUG', true); // Debug output define('CRAWLER_DEBUG', true); // Debug output
// Kevacoin wallet
define('KEVA_HOST', '127.0.0.1');
define('KEVA_PORT', '9992');
define('KEVA_USERNAME', '');
define('KEVA_PASSWORD', '');
// Database // Database
define('DB_HOST', '127.0.0.1'); define('DB_HOST', '127.0.0.1');
define('DB_PORT', '3306'); define('DB_PORT', '3306');
define('DB_NAME', ''); define('DB_NAME', '');
define('DB_USERNAME', ''); define('DB_USERNAME', '');
define('DB_PASSWORD', ''); define('DB_PASSWORD', '');
// Kevacoin wallet
define('KEVA_PROTOCOL', 'http');
define('KEVA_HOST', '127.0.0.1');
define('KEVA_PORT', '9992');
define('KEVA_USERNAME', '');
define('KEVA_PASSWORD', '');

157
crawler.php
Unescape View File

@ -1,67 +1,158 @@
<?php <?php
$semaphore = sem_get(1);
if (false !== sem_acquire($semaphore, 1)) {
require_once('config.php'); require_once('config.php');
require_once('library/mysql.php'); require_once('library/mysql.php');
require_once('library/keva.php'); require_once('library/kevacoin.php');
require_once('library/hash.php'); require_once('library/hash.php');
require_once('library/base58.php'); require_once('library/base58.php');
require_once('library/base58check.php'); require_once('library/base58check.php');
require_once('library/crypto.php'); require_once('library/crypto.php');
require_once('library/helper.php');
$db = new MySQL(); $db = new MySQL(DB_HOST, DB_PORT, DB_NAME, DB_USERNAME, DB_PASSWORD);
$node = new Keva(); $kevaCoin = new KevaCoin(KEVA_PROTOCOL, KEVA_HOST, KEVA_PORT, KEVA_USERNAME, KEVA_PASSWORD);
$node->host = KEVA_HOST;
$node->username = KEVA_USERNAME;
$node->password = KEVA_PASSWORD;
$node->port = KEVA_PORT;
$blockLast = $db->getLastBlock(); $blockLast = $db->getLastBlock();
$blockTotal = $node->getblockcount(); $blockTotal = $kevaCoin->getblockcount();
if (!$blockTotal) {
echo "API connection error.\n";
exit;
}
$response = []; $response = [];
if ($blockTotal > $blockLast) { if (CRAWLER_DEBUG) {
echo "scanning blockhain...\n";
}
for ($blockCurrent = ($blockLast + 1); $blockCurrent <= $blockLast + STEP_BLOCK_LIMIT; $blockCurrent++) {
if ($blockCurrent > $blockTotal) {
for ($blockCurrent = $blockLast; $blockCurrent <= $blockLast + STEP_BLOCK_LIMIT; $blockCurrent++) { if (CRAWLER_DEBUG) {
echo "database is up to date.\n";
}
if ($blockHash = $node->getblockhash($blockCurrent)) { break;
}
$blockData = $node->getblock($blockHash); if (CRAWLER_DEBUG) {
echo sprintf("reading block %s\n", $blockCurrent);
}
if (!$blockHash = $kevaCoin->getblockhash($blockCurrent)) {
if (CRAWLER_DEBUG) {
echo "could not read the block hash. waiting for reconnect.\n";
}
break;
}
if (!$blockData = $kevaCoin->getblock($blockHash)) {
if (CRAWLER_DEBUG) {
echo "could not read the block data. waiting for reconnect.\n";
}
break;
}
if (!$blockId = $db->getBlock($blockCurrent)) { if (!$blockId = $db->getBlock($blockCurrent)) {
$blockId = $db->addBlock($blockCurrent); $blockId = $db->addBlock($blockCurrent);
if (CRAWLER_DEBUG) {
echo sprintf("add block %s\n", $blockCurrent);
}
} }
$lostTransactions = 0;
foreach ($blockData['tx'] as $transaction) { foreach ($blockData['tx'] as $transaction) {
$transactionRaw = $node->getrawtransaction($transaction, 1); if (!$transactionRaw = $kevaCoin->getrawtransaction($transaction)) {
$lostTransactions++;
$db->setLostTransactions($blockId, $lostTransactions);
if (CRAWLER_DEBUG) {
echo sprintf("could not read the transaction %s. skipped.\n", $transaction);
}
break;
}
foreach($transactionRaw['vout'] as $vout) { foreach($transactionRaw['vout'] as $vout) {
$asmArray = explode(' ', $vout['scriptPubKey']['asm']); $asmArray = explode(' ', $vout['scriptPubKey']['asm']);
if($asmArray[0] == 'OP_KEVA_NAMESPACE' || $asmArray[0] == 'OP_KEVA_PUT') { // OP_KEVA_DELETE if (in_array($asmArray[0], ['OP_KEVA_NAMESPACE', 'OP_KEVA_PUT', 'OP_KEVA_DELETE'])) {
$hash = Base58Check::encode($asmArray[1], false , 0 , false); $hash = Base58Check::encode($asmArray[1], false , 0 , false);
$nameSpace = $node->keva_get($hash, '_KEVA_NS_');
$nameSpaceValue = strip_tags(html_entity_decode($nameSpace['value'], ENT_QUOTES, 'UTF-8')); switch ($asmArray[0]) {
case 'OP_KEVA_DELETE':
$key = filterString(decodeString($asmArray[2]));
$value = '';
break;
if ((empty(KEVA_NS) || (!empty(KEVA_NS) && KEVA_NS == $nameSpaceValue))) { case 'OP_KEVA_NAMESPACE':
$key = '_KEVA_NS_';
$value = filterString(decodeString($asmArray[2]));
break;
default:
$key = filterString(decodeString($asmArray[2]));
$value = filterString(decodeString($asmArray[3]));
}
if (!$nameSpaceId = $db->getNameSpace($hash)) { if (!$nameSpaceId = $db->getNameSpace($hash)) {
$nameSpaceId = $db->addNameSpace($hash, $nameSpaceValue); $nameSpaceId = $db->addNameSpace($hash);
if (CRAWLER_DEBUG) {
echo sprintf("add namespace %s\n", $hash);
}
} }
if (!$db->getData($blockId, $nameSpaceId)) { if (!$dataId = $db->getData($transactionRaw['txid'])) {
$db->addData($blockId, $dataId = $db->addData($blockId,
$nameSpaceId, $nameSpaceId,
$transactionRaw['time'], $transactionRaw['time'],
$transactionRaw['size'], $transactionRaw['size'],
$transactionRaw['txid'], $transactionRaw['txid'],
strip_tags(html_entity_decode(@hex2bin($asmArray[2]), ENT_QUOTES, 'UTF-8')), $key,
strip_tags(html_entity_decode(@hex2bin($asmArray[3]), ENT_QUOTES, 'UTF-8'))); $value,
($key == '_KEVA_NS_'),
empty($value));
if ($value) {
$db->setDataKeyDeleted($nameSpaceId, $key, false);
if (CRAWLER_DEBUG) {
echo sprintf("add new key/value %s\n", $transactionRaw['txid']);
}
} else {
$db->setDataKeyDeleted($nameSpaceId, $key, true);
if (CRAWLER_DEBUG) {
echo sprintf("delete key %s from namespace %s\n", $key, $hash);
}
}
} }
if (CRAWLER_DEBUG) { if (CRAWLER_DEBUG) {
@ -75,9 +166,8 @@ if ($blockTotal > $blockLast) {
'time' => $transactionRaw['time'], 'time' => $transactionRaw['time'],
'blocktime' => $transactionRaw['blocktime'], 'blocktime' => $transactionRaw['blocktime'],
'namehash' => $hash, 'namehash' => $hash,
'title' => $nameSpaceValue, 'key' => $key,
'key' => strip_tags(html_entity_decode(@hex2bin($asmArray[2]), ENT_QUOTES, 'UTF-8')), 'value' => $value
'vale' => strip_tags(html_entity_decode(@hex2bin($asmArray[3]), ENT_QUOTES, 'UTF-8'))
]; ];
} }
} }
@ -85,14 +175,15 @@ if ($blockTotal > $blockLast) {
} }
} }
} else {
// @TODO block not found
}
}
}
// Debug // Debug
if (CRAWLER_DEBUG) { if (CRAWLER_DEBUG) {
print_r($response); echo "scanning completed.\n";
# print_r($response);
}
sem_release($semaphore);
} else {
echo "database locked by the another process...\n";
} }

15
library/helper.php
Unescape View File

@ -0,0 +1,15 @@
<?php
function decodeString($string) {
if (is_numeric($string) && $string < 0xFFFFFFFF) {
return mb_chr($string, 'ASCII');
} else {
return hex2bin($string);
}
}
function filterString($string) {
return strip_tags(html_entity_decode($string, ENT_QUOTES, 'UTF-8'));
}

110
library/keva.php
Unescape View File

@ -1,110 +0,0 @@
<?php
class Keva {
public $username;
public $password;
public $host;
public $port;
public $url;
public $proto = 'http';
public $CACertificate = null;
public $status;
public $error;
public $rawResponse;
public $response;
private $id = 0;
public function setSSL($certificate = null) {
$this->proto = 'https';
$this->CACertificate = $certificate;
}
public function __call($method, $params) {
$this->status = null;
$this->error = null;
$this->rawResponse = null;
$this->response = null;
$params = array_values($params);
$this->id++;
$request = json_encode(array(
'method' => $method,
'params' => $params,
'id' => $this->id
));
$curl = curl_init("{$this->proto}://{$this->host}:{$this->port}/{$this->url}");
$options = array(
CURLOPT_HTTPAUTH => CURLAUTH_BASIC,
CURLOPT_USERPWD => $this->username . ':' . $this->password,
CURLOPT_RETURNTRANSFER => true,
CURLOPT_FOLLOWLOCATION => true,
CURLOPT_MAXREDIRS => 10,
CURLOPT_HTTPHEADER => array('Content-type: text/plain'),
CURLOPT_POST => true,
CURLOPT_POSTFIELDS => $request
);
if (ini_get('open_basedir')) {
unset($options[CURLOPT_FOLLOWLOCATION]);
}
if ($this->proto == 'https') {
if (!empty($this->CACertificate)) {
$options[CURLOPT_CAINFO] = $this->CACertificate;
$options[CURLOPT_CAPATH] = DIRNAME($this->CACertificate);
} else {
$options[CURLOPT_SSL_VERIFYPEER] = false;
}
}
curl_setopt_array($curl, $options);
$this->rawResponse = curl_exec($curl);
$this->response = json_decode($this->rawResponse, true);
$this->status = curl_getinfo($curl, CURLINFO_HTTP_CODE);
$curl_error = curl_error($curl);
curl_close($curl);
if (!empty($curl_error)) {
$this->error = $curl_error;
}
if ($this->response['error']) {
$this->error = $this->response['error']['message'];
} elseif ($this->status != 200) {
switch ($this->status) {
case 400:
$this->error = 'HTTP_BAD_REQUEST';
break;
case 401:
$this->error = 'HTTP_UNAUTHORIZED';
break;
case 403:
$this->error = 'HTTP_FORBIDDEN';
break;
case 404:
$this->error = 'HTTP_NOT_FOUND';
break;
}
}
if ($this->error) {
return false;
}
return $this->response['result'];
}
}

156
library/kevacoin.php
Unescape View File

@ -0,0 +1,156 @@
<?php
class KevaCoin {
private $_id = 0;
private $_curl;
private $_protocol;
private $_host;
private $_port;
public function __construct($protocol, $host, $port, $username, $password) {
$this->_protocol = $protocol;
$this->_host = $host;
$this->_port = $port;
$this->_curl = curl_init();
curl_setopt_array($this->_curl, [CURLOPT_RETURNTRANSFER => true,
CURLOPT_FOLLOWLOCATION => true,
CURLOPT_FRESH_CONNECT => true,
CURLOPT_HTTPAUTH => CURLAUTH_BASIC,
CURLOPT_USERPWD => $username . ':' . $password,
CURLOPT_RETURNTRANSFER => true,
CURLOPT_FOLLOWLOCATION => true,
//CURLOPT_VERBOSE => true,
CURLOPT_HTTPHEADER => [
'Content-Type: application/plain',
],
]);
}
public function __destruct() {
curl_close($this->_curl);
}
protected function prepare($url, $method, array $postfields = []) {
curl_setopt($this->_curl, CURLOPT_URL, $this->_protocol . '://' . $this->_host . ':' . $this->_port . $url);
curl_setopt($this->_curl, CURLOPT_CUSTOMREQUEST, $method);
if ($method == 'POST' && $postfields) {
curl_setopt($this->_curl, CURLOPT_POSTFIELDS, json_encode($postfields));
}
}
protected function execute($json = true) {
$response = curl_exec($this->_curl);
$errorNumber = curl_errno($this->_curl);
$errorText = curl_error($this->_curl);
if ($errorNumber > 0) {
//return false;
}
if ($response) {
if ($json) {
return json_decode($response, true);
} else {
return $response;
}
}
return false;
}
public function getblockcount() {
$this->_id++;
$this->prepare('', 'POST', [
'method' => 'getblockcount',
'params' => [],
'id' => $this->_id
]);
$response = $this->execute();
if (isset($response['result']) && is_int($response['result'])) {
return $response['result'];
} else {
return false;
}
}
public function getblockhash($block) {
$this->_id++;
$this->prepare('', 'POST', [
'method' => 'getblockhash',
'params' => [$block],
'id' => $this->_id
]);
$response = $this->execute();
if (isset($response['result']) && 64 == strlen($response['result'])) {
return $response['result'];
} else {
return false;
}
}
public function getblock($hash) {
$this->_id++;
$this->prepare('', 'POST', [
'method' => 'getblock',
'params' => [$hash],
'id' => $this->_id
]);
$response = $this->execute();
if (isset($response['result']) && is_array($response['result'])) {
return $response['result'];
} else {
return false;
}
}
public function getrawtransaction($txid, $decode = true) {
$this->_id++;
$this->prepare('', 'POST', [
'method' => 'getrawtransaction',
'params' => [$txid, $decode],
'id' => $this->_id
]);
$response = $this->execute();
if (isset($response['result']) && is_array($response['result'])) {
return $response['result'];
} else {
return false;
}
}
}

85
library/mysql.php
Unescape View File

@ -2,11 +2,11 @@
class MySQL { class MySQL {
public function __construct() { public function __construct($host, $port, $database, $username, $password) {
try { try {
$this->_db = new PDO('mysql:dbname=' . DB_NAME . ';host=' . DB_HOST . ';port=' . DB_PORT . ';charset=utf8', DB_USERNAME, DB_PASSWORD, [PDO::MYSQL_ATTR_INIT_COMMAND => 'SET NAMES utf8']); $this->_db = new PDO('mysql:dbname=' . $database . ';host=' . $host . ';port=' . $port . ';charset=utf8', $username, $password, [PDO::MYSQL_ATTR_INIT_COMMAND => 'SET NAMES utf8']);
$this->_db->setAttribute(PDO::ATTR_ERRMODE, PDO::ERRMODE_EXCEPTION); $this->_db->setAttribute(PDO::ATTR_ERRMODE, PDO::ERRMODE_EXCEPTION);
$this->_db->setAttribute(PDO::ATTR_DEFAULT_FETCH_MODE, PDO::FETCH_ASSOC); $this->_db->setAttribute(PDO::ATTR_DEFAULT_FETCH_MODE, PDO::FETCH_ASSOC);
$this->_db->setAttribute(PDO::ATTR_TIMEOUT, 600); $this->_db->setAttribute(PDO::ATTR_TIMEOUT, 600);
@ -22,7 +22,7 @@ class MySQL {
$query = $this->_db->query('SELECT MAX(`blockId`) AS `lastBlock` FROM `block`')->fetch(); $query = $this->_db->query('SELECT MAX(`blockId`) AS `lastBlock` FROM `block`')->fetch();
return (int) $query['lastBlock'] ? $query['lastBlock'] : 1; return (int) $query['lastBlock'];
} catch(PDOException $e) { } catch(PDOException $e) {
trigger_error($e->getMessage()); trigger_error($e->getMessage());
@ -62,13 +62,13 @@ class MySQL {
} }
} }
public function getData($blockId, $nameSpaceId) { public function getData($txId) {
try { try {
$query = $this->_db->prepare('SELECT `dataId` FROM `data` WHERE `blockId` = ? AND `nameSpaceId` = ? LIMIT 1'); $query = $this->_db->prepare('SELECT `dataId` FROM `data` WHERE `txId` = ? LIMIT 1');
$query->execute([$blockId, $nameSpaceId]); $query->execute([$txId]);
return $query->rowCount() ? $query->fetch()['dataId'] : false; return $query->rowCount() ? $query->fetch()['dataId'] : false;
@ -83,6 +83,7 @@ class MySQL {
try { try {
$query = $this->_db->prepare('INSERT INTO `block` SET `blockId` = ?, $query = $this->_db->prepare('INSERT INTO `block` SET `blockId` = ?,
`lostTransactions` = 0,
`timeIndexed` = UNIX_TIMESTAMP()'); `timeIndexed` = UNIX_TIMESTAMP()');
$query->execute([$blockId]); $query->execute([$blockId]);
@ -95,15 +96,30 @@ class MySQL {
} }
} }
public function addNameSpace($hash, $value) { public function setLostTransactions($blockId, $lostTransactions) {
try {
$query = $this->_db->prepare('UPDATE `block` SET `lostTransactions` = ? WHERE `blockId` = ? LIMIT 1');
$query->execute([$lostTransactions, $blockId]);
return $blockId;
} catch(PDOException $e) {
trigger_error($e->getMessage());
return false;
}
}
public function addNameSpace($hash) {
try { try {
$query = $this->_db->prepare('INSERT INTO `namespace` SET `hash` = ?, $query = $this->_db->prepare('INSERT INTO `namespace` SET `hash` = ?,
`value` = ?,
`timeIndexed` = UNIX_TIMESTAMP()'); `timeIndexed` = UNIX_TIMESTAMP()');
$query->execute([$hash, $value]); $query->execute([$hash]);
return $this->_db->lastInsertId(); return $this->_db->lastInsertId();
@ -113,20 +129,32 @@ class MySQL {
} }
} }
public function addData($blockId, $nameSpaceId, $time, $size, $txid, $key, $value) { public function addData($blockId, $nameSpaceId, $time, $size, $txid, $key, $value, $ns, $deleted = false) {
try { try {
$query = $this->_db->prepare('INSERT INTO `data` SET `blockId` = ?, $query = $this->_db->prepare('INSERT INTO `data` SET `blockId` = :blockId,
`nameSpaceId` = ?, `nameSpaceId` = :nameSpaceId,
`time` = ?, `time` = :time,
`size` = ?, `size` = :size,
`txid` = ?, `txid` = :txid,
`key` = ?, `key` = :key,
`value` = ?, `value` = :value,
`deleted` = :deleted,
`ns` = :ns,
`timeIndexed` = UNIX_TIMESTAMP()'); `timeIndexed` = UNIX_TIMESTAMP()');
$query->execute([$blockId, $nameSpaceId, $time, $size, $txid, $key, $value]); $query->bindValue(':blockId', $blockId, PDO::PARAM_INT);
$query->bindValue(':nameSpaceId', $nameSpaceId, PDO::PARAM_INT);
$query->bindValue(':time', $time, PDO::PARAM_INT);
$query->bindValue(':size', $size, PDO::PARAM_INT);
$query->bindValue(':txid', $txid, PDO::PARAM_STR);
$query->bindValue(':key', $key, PDO::PARAM_STR);
$query->bindValue(':value', $value, PDO::PARAM_STR);
$query->bindValue(':deleted', (int) $deleted, PDO::PARAM_STR);
$query->bindValue(':ns', (int) $ns, PDO::PARAM_STR);
$query->execute();
return $this->_db->lastInsertId(); return $this->_db->lastInsertId();
@ -135,4 +163,25 @@ class MySQL {
return false; return false;
} }
} }
public function setDataKeyDeleted($nameSpaceId, $key, $deleted) {
try {
$query = $this->_db->prepare('UPDATE `data` SET `deleted` = :deleted
WHERE `nameSpaceId` = :nameSpaceId AND `key` LIKE :key');
$query->bindValue(':nameSpaceId', $nameSpaceId, PDO::PARAM_INT);
$query->bindValue(':key', $key, PDO::PARAM_STR);
$query->bindValue(':deleted', (int) $deleted, PDO::PARAM_STR);
$query->execute();
return $query->rowCount();
} catch(PDOException $e) {
trigger_error($e->getMessage());
return false;
}
}
} }

Write Preview
Loading…
Cancel
Save