Browse Source

apply latest updates from api-node

pull/1/head
ghost 3 years ago
parent
commit
c29e0d6b58
  1. 14
      config-default.php
  2. 157
      crawler.php
  3. 15
      library/helper.php
  4. 110
      library/keva.php
  5. 156
      library/kevacoin.php
  6. 85
      library/mysql.php

14
config-default.php

@ -6,19 +6,19 @@ ini_set('display_startup_errors', '1'); @@ -6,19 +6,19 @@ ini_set('display_startup_errors', '1');
error_reporting(E_ALL);
// Application
define('KEVA_NS', ''); // KEVA_NS or empty to collect all the blockchain
define('STEP_BLOCK_LIMIT', 50); // Blocks per query
define('CRAWLER_DEBUG', true); // Debug output
// Kevacoin wallet
define('KEVA_HOST', '127.0.0.1');
define('KEVA_PORT', '9992');
define('KEVA_USERNAME', '');
define('KEVA_PASSWORD', '');
// Database
define('DB_HOST', '127.0.0.1');
define('DB_PORT', '3306');
define('DB_NAME', '');
define('DB_USERNAME', '');
define('DB_PASSWORD', '');
// Kevacoin wallet
define('KEVA_PROTOCOL', 'http');
define('KEVA_HOST', '127.0.0.1');
define('KEVA_PORT', '9992');
define('KEVA_USERNAME', '');
define('KEVA_PASSWORD', '');

157
crawler.php

@ -1,67 +1,158 @@ @@ -1,67 +1,158 @@
<?php
$semaphore = sem_get(1);
if (false !== sem_acquire($semaphore, 1)) {
require_once('config.php');
require_once('library/mysql.php');
require_once('library/keva.php');
require_once('library/kevacoin.php');
require_once('library/hash.php');
require_once('library/base58.php');
require_once('library/base58check.php');
require_once('library/crypto.php');
require_once('library/helper.php');
$db = new MySQL();
$node = new Keva();
$node->host = KEVA_HOST;
$node->username = KEVA_USERNAME;
$node->password = KEVA_PASSWORD;
$node->port = KEVA_PORT;
$db = new MySQL(DB_HOST, DB_PORT, DB_NAME, DB_USERNAME, DB_PASSWORD);
$kevaCoin = new KevaCoin(KEVA_PROTOCOL, KEVA_HOST, KEVA_PORT, KEVA_USERNAME, KEVA_PASSWORD);
$blockLast = $db->getLastBlock();
$blockTotal = $node->getblockcount();
$blockTotal = $kevaCoin->getblockcount();
if (!$blockTotal) {
echo "API connection error.\n";
exit;
}
$response = [];
if ($blockTotal > $blockLast) {
if (CRAWLER_DEBUG) {
echo "scanning blockhain...\n";
}
for ($blockCurrent = ($blockLast + 1); $blockCurrent <= $blockLast + STEP_BLOCK_LIMIT; $blockCurrent++) {
if ($blockCurrent > $blockTotal) {
for ($blockCurrent = $blockLast; $blockCurrent <= $blockLast + STEP_BLOCK_LIMIT; $blockCurrent++) {
if (CRAWLER_DEBUG) {
echo "database is up to date.\n";
}
if ($blockHash = $node->getblockhash($blockCurrent)) {
break;
}
$blockData = $node->getblock($blockHash);
if (CRAWLER_DEBUG) {
echo sprintf("reading block %s\n", $blockCurrent);
}
if (!$blockHash = $kevaCoin->getblockhash($blockCurrent)) {
if (CRAWLER_DEBUG) {
echo "could not read the block hash. waiting for reconnect.\n";
}
break;
}
if (!$blockData = $kevaCoin->getblock($blockHash)) {
if (CRAWLER_DEBUG) {
echo "could not read the block data. waiting for reconnect.\n";
}
break;
}
if (!$blockId = $db->getBlock($blockCurrent)) {
$blockId = $db->addBlock($blockCurrent);
if (CRAWLER_DEBUG) {
echo sprintf("add block %s\n", $blockCurrent);
}
}
$lostTransactions = 0;
foreach ($blockData['tx'] as $transaction) {
$transactionRaw = $node->getrawtransaction($transaction, 1);
if (!$transactionRaw = $kevaCoin->getrawtransaction($transaction)) {
$lostTransactions++;
$db->setLostTransactions($blockId, $lostTransactions);
if (CRAWLER_DEBUG) {
echo sprintf("could not read the transaction %s. skipped.\n", $transaction);
}
break;
}
foreach($transactionRaw['vout'] as $vout) {
$asmArray = explode(' ', $vout['scriptPubKey']['asm']);
if($asmArray[0] == 'OP_KEVA_NAMESPACE' || $asmArray[0] == 'OP_KEVA_PUT') { // OP_KEVA_DELETE
if (in_array($asmArray[0], ['OP_KEVA_NAMESPACE', 'OP_KEVA_PUT', 'OP_KEVA_DELETE'])) {
$hash = Base58Check::encode($asmArray[1], false , 0 , false);
$nameSpace = $node->keva_get($hash, '_KEVA_NS_');
$nameSpaceValue = strip_tags(html_entity_decode($nameSpace['value'], ENT_QUOTES, 'UTF-8'));
switch ($asmArray[0]) {
case 'OP_KEVA_DELETE':
$key = filterString(decodeString($asmArray[2]));
$value = '';
break;
if ((empty(KEVA_NS) || (!empty(KEVA_NS) && KEVA_NS == $nameSpaceValue))) {
case 'OP_KEVA_NAMESPACE':
$key = '_KEVA_NS_';
$value = filterString(decodeString($asmArray[2]));
break;
default:
$key = filterString(decodeString($asmArray[2]));
$value = filterString(decodeString($asmArray[3]));
}
if (!$nameSpaceId = $db->getNameSpace($hash)) {
$nameSpaceId = $db->addNameSpace($hash, $nameSpaceValue);
$nameSpaceId = $db->addNameSpace($hash);
if (CRAWLER_DEBUG) {
echo sprintf("add namespace %s\n", $hash);
}
}
if (!$db->getData($blockId, $nameSpaceId)) {
$db->addData($blockId,
if (!$dataId = $db->getData($transactionRaw['txid'])) {
$dataId = $db->addData($blockId,
$nameSpaceId,
$transactionRaw['time'],
$transactionRaw['size'],
$transactionRaw['txid'],
strip_tags(html_entity_decode(@hex2bin($asmArray[2]), ENT_QUOTES, 'UTF-8')),
strip_tags(html_entity_decode(@hex2bin($asmArray[3]), ENT_QUOTES, 'UTF-8')));
$key,
$value,
($key == '_KEVA_NS_'),
empty($value));
if ($value) {
$db->setDataKeyDeleted($nameSpaceId, $key, false);
if (CRAWLER_DEBUG) {
echo sprintf("add new key/value %s\n", $transactionRaw['txid']);
}
} else {
$db->setDataKeyDeleted($nameSpaceId, $key, true);
if (CRAWLER_DEBUG) {
echo sprintf("delete key %s from namespace %s\n", $key, $hash);
}
}
}
if (CRAWLER_DEBUG) {
@ -75,9 +166,8 @@ if ($blockTotal > $blockLast) { @@ -75,9 +166,8 @@ if ($blockTotal > $blockLast) {
'time' => $transactionRaw['time'],
'blocktime' => $transactionRaw['blocktime'],
'namehash' => $hash,
'title' => $nameSpaceValue,
'key' => strip_tags(html_entity_decode(@hex2bin($asmArray[2]), ENT_QUOTES, 'UTF-8')),
'vale' => strip_tags(html_entity_decode(@hex2bin($asmArray[3]), ENT_QUOTES, 'UTF-8'))
'key' => $key,
'value' => $value
];
}
}
@ -85,14 +175,15 @@ if ($blockTotal > $blockLast) { @@ -85,14 +175,15 @@ if ($blockTotal > $blockLast) {
}
}
} else {
// @TODO block not found
}
}
}
// Debug
if (CRAWLER_DEBUG) {
print_r($response);
echo "scanning completed.\n";
# print_r($response);
}
sem_release($semaphore);
} else {
echo "database locked by the another process...\n";
}

15
library/helper.php

@ -0,0 +1,15 @@ @@ -0,0 +1,15 @@
<?php
function decodeString($string) {
if (is_numeric($string) && $string < 0xFFFFFFFF) {
return mb_chr($string, 'ASCII');
} else {
return hex2bin($string);
}
}
function filterString($string) {
return strip_tags(html_entity_decode($string, ENT_QUOTES, 'UTF-8'));
}

110
library/keva.php

@ -1,110 +0,0 @@ @@ -1,110 +0,0 @@
<?php
class Keva {
public $username;
public $password;
public $host;
public $port;
public $url;
public $proto = 'http';
public $CACertificate = null;
public $status;
public $error;
public $rawResponse;
public $response;
private $id = 0;
public function setSSL($certificate = null) {
$this->proto = 'https';
$this->CACertificate = $certificate;
}
public function __call($method, $params) {
$this->status = null;
$this->error = null;
$this->rawResponse = null;
$this->response = null;
$params = array_values($params);
$this->id++;
$request = json_encode(array(
'method' => $method,
'params' => $params,
'id' => $this->id
));
$curl = curl_init("{$this->proto}://{$this->host}:{$this->port}/{$this->url}");
$options = array(
CURLOPT_HTTPAUTH => CURLAUTH_BASIC,
CURLOPT_USERPWD => $this->username . ':' . $this->password,
CURLOPT_RETURNTRANSFER => true,
CURLOPT_FOLLOWLOCATION => true,
CURLOPT_MAXREDIRS => 10,
CURLOPT_HTTPHEADER => array('Content-type: text/plain'),
CURLOPT_POST => true,
CURLOPT_POSTFIELDS => $request
);
if (ini_get('open_basedir')) {
unset($options[CURLOPT_FOLLOWLOCATION]);
}
if ($this->proto == 'https') {
if (!empty($this->CACertificate)) {
$options[CURLOPT_CAINFO] = $this->CACertificate;
$options[CURLOPT_CAPATH] = DIRNAME($this->CACertificate);
} else {
$options[CURLOPT_SSL_VERIFYPEER] = false;
}
}
curl_setopt_array($curl, $options);
$this->rawResponse = curl_exec($curl);
$this->response = json_decode($this->rawResponse, true);
$this->status = curl_getinfo($curl, CURLINFO_HTTP_CODE);
$curl_error = curl_error($curl);
curl_close($curl);
if (!empty($curl_error)) {
$this->error = $curl_error;
}
if ($this->response['error']) {
$this->error = $this->response['error']['message'];
} elseif ($this->status != 200) {
switch ($this->status) {
case 400:
$this->error = 'HTTP_BAD_REQUEST';
break;
case 401:
$this->error = 'HTTP_UNAUTHORIZED';
break;
case 403:
$this->error = 'HTTP_FORBIDDEN';
break;
case 404:
$this->error = 'HTTP_NOT_FOUND';
break;
}
}
if ($this->error) {
return false;
}
return $this->response['result'];
}
}

156
library/kevacoin.php

@ -0,0 +1,156 @@ @@ -0,0 +1,156 @@
<?php
class KevaCoin {
private $_id = 0;
private $_curl;
private $_protocol;
private $_host;
private $_port;
public function __construct($protocol, $host, $port, $username, $password) {
$this->_protocol = $protocol;
$this->_host = $host;
$this->_port = $port;
$this->_curl = curl_init();
curl_setopt_array($this->_curl, [CURLOPT_RETURNTRANSFER => true,
CURLOPT_FOLLOWLOCATION => true,
CURLOPT_FRESH_CONNECT => true,
CURLOPT_HTTPAUTH => CURLAUTH_BASIC,
CURLOPT_USERPWD => $username . ':' . $password,
CURLOPT_RETURNTRANSFER => true,
CURLOPT_FOLLOWLOCATION => true,
//CURLOPT_VERBOSE => true,
CURLOPT_HTTPHEADER => [
'Content-Type: application/plain',
],
]);
}
public function __destruct() {
curl_close($this->_curl);
}
protected function prepare($url, $method, array $postfields = []) {
curl_setopt($this->_curl, CURLOPT_URL, $this->_protocol . '://' . $this->_host . ':' . $this->_port . $url);
curl_setopt($this->_curl, CURLOPT_CUSTOMREQUEST, $method);
if ($method == 'POST' && $postfields) {
curl_setopt($this->_curl, CURLOPT_POSTFIELDS, json_encode($postfields));
}
}
protected function execute($json = true) {
$response = curl_exec($this->_curl);
$errorNumber = curl_errno($this->_curl);
$errorText = curl_error($this->_curl);
if ($errorNumber > 0) {
//return false;
}
if ($response) {
if ($json) {
return json_decode($response, true);
} else {
return $response;
}
}
return false;
}
public function getblockcount() {
$this->_id++;
$this->prepare('', 'POST', [
'method' => 'getblockcount',
'params' => [],
'id' => $this->_id
]);
$response = $this->execute();
if (isset($response['result']) && is_int($response['result'])) {
return $response['result'];
} else {
return false;
}
}
public function getblockhash($block) {
$this->_id++;
$this->prepare('', 'POST', [
'method' => 'getblockhash',
'params' => [$block],
'id' => $this->_id
]);
$response = $this->execute();
if (isset($response['result']) && 64 == strlen($response['result'])) {
return $response['result'];
} else {
return false;
}
}
public function getblock($hash) {
$this->_id++;
$this->prepare('', 'POST', [
'method' => 'getblock',
'params' => [$hash],
'id' => $this->_id
]);
$response = $this->execute();
if (isset($response['result']) && is_array($response['result'])) {
return $response['result'];
} else {
return false;
}
}
public function getrawtransaction($txid, $decode = true) {
$this->_id++;
$this->prepare('', 'POST', [
'method' => 'getrawtransaction',
'params' => [$txid, $decode],
'id' => $this->_id
]);
$response = $this->execute();
if (isset($response['result']) && is_array($response['result'])) {
return $response['result'];
} else {
return false;
}
}
}

85
library/mysql.php

@ -2,11 +2,11 @@ @@ -2,11 +2,11 @@
class MySQL {
public function __construct() {
public function __construct($host, $port, $database, $username, $password) {
try {
$this->_db = new PDO('mysql:dbname=' . DB_NAME . ';host=' . DB_HOST . ';port=' . DB_PORT . ';charset=utf8', DB_USERNAME, DB_PASSWORD, [PDO::MYSQL_ATTR_INIT_COMMAND => 'SET NAMES utf8']);
$this->_db = new PDO('mysql:dbname=' . $database . ';host=' . $host . ';port=' . $port . ';charset=utf8', $username, $password, [PDO::MYSQL_ATTR_INIT_COMMAND => 'SET NAMES utf8']);
$this->_db->setAttribute(PDO::ATTR_ERRMODE, PDO::ERRMODE_EXCEPTION);
$this->_db->setAttribute(PDO::ATTR_DEFAULT_FETCH_MODE, PDO::FETCH_ASSOC);
$this->_db->setAttribute(PDO::ATTR_TIMEOUT, 600);
@ -22,7 +22,7 @@ class MySQL { @@ -22,7 +22,7 @@ class MySQL {
$query = $this->_db->query('SELECT MAX(`blockId`) AS `lastBlock` FROM `block`')->fetch();
return (int) $query['lastBlock'] ? $query['lastBlock'] : 1;
return (int) $query['lastBlock'];
} catch(PDOException $e) {
trigger_error($e->getMessage());
@ -62,13 +62,13 @@ class MySQL { @@ -62,13 +62,13 @@ class MySQL {
}
}
public function getData($blockId, $nameSpaceId) {
public function getData($txId) {
try {
$query = $this->_db->prepare('SELECT `dataId` FROM `data` WHERE `blockId` = ? AND `nameSpaceId` = ? LIMIT 1');
$query = $this->_db->prepare('SELECT `dataId` FROM `data` WHERE `txId` = ? LIMIT 1');
$query->execute([$blockId, $nameSpaceId]);
$query->execute([$txId]);
return $query->rowCount() ? $query->fetch()['dataId'] : false;
@ -83,6 +83,7 @@ class MySQL { @@ -83,6 +83,7 @@ class MySQL {
try {
$query = $this->_db->prepare('INSERT INTO `block` SET `blockId` = ?,
`lostTransactions` = 0,
`timeIndexed` = UNIX_TIMESTAMP()');
$query->execute([$blockId]);
@ -95,15 +96,30 @@ class MySQL { @@ -95,15 +96,30 @@ class MySQL {
}
}
public function addNameSpace($hash, $value) {
public function setLostTransactions($blockId, $lostTransactions) {
try {
$query = $this->_db->prepare('UPDATE `block` SET `lostTransactions` = ? WHERE `blockId` = ? LIMIT 1');
$query->execute([$lostTransactions, $blockId]);
return $blockId;
} catch(PDOException $e) {
trigger_error($e->getMessage());
return false;
}
}
public function addNameSpace($hash) {
try {
$query = $this->_db->prepare('INSERT INTO `namespace` SET `hash` = ?,
`value` = ?,
`timeIndexed` = UNIX_TIMESTAMP()');
$query->execute([$hash, $value]);
$query->execute([$hash]);
return $this->_db->lastInsertId();
@ -113,20 +129,32 @@ class MySQL { @@ -113,20 +129,32 @@ class MySQL {
}
}
public function addData($blockId, $nameSpaceId, $time, $size, $txid, $key, $value) {
public function addData($blockId, $nameSpaceId, $time, $size, $txid, $key, $value, $ns, $deleted = false) {
try {
$query = $this->_db->prepare('INSERT INTO `data` SET `blockId` = ?,
`nameSpaceId` = ?,
`time` = ?,
`size` = ?,
`txid` = ?,
`key` = ?,
`value` = ?,
$query = $this->_db->prepare('INSERT INTO `data` SET `blockId` = :blockId,
`nameSpaceId` = :nameSpaceId,
`time` = :time,
`size` = :size,
`txid` = :txid,
`key` = :key,
`value` = :value,
`deleted` = :deleted,
`ns` = :ns,
`timeIndexed` = UNIX_TIMESTAMP()');
$query->execute([$blockId, $nameSpaceId, $time, $size, $txid, $key, $value]);
$query->bindValue(':blockId', $blockId, PDO::PARAM_INT);
$query->bindValue(':nameSpaceId', $nameSpaceId, PDO::PARAM_INT);
$query->bindValue(':time', $time, PDO::PARAM_INT);
$query->bindValue(':size', $size, PDO::PARAM_INT);
$query->bindValue(':txid', $txid, PDO::PARAM_STR);
$query->bindValue(':key', $key, PDO::PARAM_STR);
$query->bindValue(':value', $value, PDO::PARAM_STR);
$query->bindValue(':deleted', (int) $deleted, PDO::PARAM_STR);
$query->bindValue(':ns', (int) $ns, PDO::PARAM_STR);
$query->execute();
return $this->_db->lastInsertId();
@ -135,4 +163,25 @@ class MySQL { @@ -135,4 +163,25 @@ class MySQL {
return false;
}
}
public function setDataKeyDeleted($nameSpaceId, $key, $deleted) {
try {
$query = $this->_db->prepare('UPDATE `data` SET `deleted` = :deleted
WHERE `nameSpaceId` = :nameSpaceId AND `key` LIKE :key');
$query->bindValue(':nameSpaceId', $nameSpaceId, PDO::PARAM_INT);
$query->bindValue(':key', $key, PDO::PARAM_STR);
$query->bindValue(':deleted', (int) $deleted, PDO::PARAM_STR);
$query->execute();
return $query->rowCount();
} catch(PDOException $e) {
trigger_error($e->getMessage());
return false;
}
}
}

Loading…
Cancel
Save