diff --git a/config-default.php b/config-default.php index d8515c8..17d1785 100644 --- a/config-default.php +++ b/config-default.php @@ -6,19 +6,19 @@ ini_set('display_startup_errors', '1'); error_reporting(E_ALL); // Application -define('KEVA_NS', ''); // KEVA_NS or empty to collect all the blockchain define('STEP_BLOCK_LIMIT', 50); // Blocks per query define('CRAWLER_DEBUG', true); // Debug output -// Kevacoin wallet -define('KEVA_HOST', '127.0.0.1'); -define('KEVA_PORT', '9992'); -define('KEVA_USERNAME', ''); -define('KEVA_PASSWORD', ''); - // Database define('DB_HOST', '127.0.0.1'); define('DB_PORT', '3306'); define('DB_NAME', ''); define('DB_USERNAME', ''); define('DB_PASSWORD', ''); + +// Kevacoin wallet +define('KEVA_PROTOCOL', 'http'); +define('KEVA_HOST', '127.0.0.1'); +define('KEVA_PORT', '9992'); +define('KEVA_USERNAME', ''); +define('KEVA_PASSWORD', ''); diff --git a/crawler.php b/crawler.php index 620e6e2..4e068ca 100644 --- a/crawler.php +++ b/crawler.php @@ -1,98 +1,189 @@ host = KEVA_HOST; -$node->username = KEVA_USERNAME; -$node->password = KEVA_PASSWORD; -$node->port = KEVA_PORT; + require_once('config.php'); + require_once('library/mysql.php'); + require_once('library/kevacoin.php'); + require_once('library/hash.php'); + require_once('library/base58.php'); + require_once('library/base58check.php'); + require_once('library/crypto.php'); + require_once('library/helper.php'); -$blockLast = $db->getLastBlock(); -$blockTotal = $node->getblockcount(); + $db = new MySQL(DB_HOST, DB_PORT, DB_NAME, DB_USERNAME, DB_PASSWORD); + $kevaCoin = new KevaCoin(KEVA_PROTOCOL, KEVA_HOST, KEVA_PORT, KEVA_USERNAME, KEVA_PASSWORD); -$response = []; + $blockLast = $db->getLastBlock(); + $blockTotal = $kevaCoin->getblockcount(); -if ($blockTotal > $blockLast) { + if (!$blockTotal) { + echo "API connection error.\n"; + exit; + } + + $response = []; - for ($blockCurrent = $blockLast; $blockCurrent <= $blockLast + STEP_BLOCK_LIMIT; $blockCurrent++) { + if (CRAWLER_DEBUG) { + echo "scanning blockhain...\n"; + } - if ($blockHash = $node->getblockhash($blockCurrent)) { + for ($blockCurrent = ($blockLast + 1); $blockCurrent <= $blockLast + STEP_BLOCK_LIMIT; $blockCurrent++) { - $blockData = $node->getblock($blockHash); + if ($blockCurrent > $blockTotal) { - if (!$blockId = $db->getBlock($blockCurrent)) { - $blockId = $db->addBlock($blockCurrent); + if (CRAWLER_DEBUG) { + echo "database is up to date.\n"; } - foreach ($blockData['tx'] as $transaction) { + break; + } - $transactionRaw = $node->getrawtransaction($transaction, 1); + if (CRAWLER_DEBUG) { + echo sprintf("reading block %s\n", $blockCurrent); + } - foreach($transactionRaw['vout'] as $vout) { + if (!$blockHash = $kevaCoin->getblockhash($blockCurrent)) { - $asmArray = explode(' ', $vout['scriptPubKey']['asm']); + if (CRAWLER_DEBUG) { + echo "could not read the block hash. waiting for reconnect.\n"; + } - if($asmArray[0] == 'OP_KEVA_NAMESPACE' || $asmArray[0] == 'OP_KEVA_PUT') { // OP_KEVA_DELETE + break; + } - $hash = Base58Check::encode($asmArray[1], false , 0 , false); - $nameSpace = $node->keva_get($hash, '_KEVA_NS_'); + if (!$blockData = $kevaCoin->getblock($blockHash)) { - $nameSpaceValue = strip_tags(html_entity_decode($nameSpace['value'], ENT_QUOTES, 'UTF-8')); + if (CRAWLER_DEBUG) { + echo "could not read the block data. waiting for reconnect.\n"; + } - if ((empty(KEVA_NS) || (!empty(KEVA_NS) && KEVA_NS == $nameSpaceValue))) { + break; + } - if (!$nameSpaceId = $db->getNameSpace($hash)) { - $nameSpaceId = $db->addNameSpace($hash, $nameSpaceValue); - } + if (!$blockId = $db->getBlock($blockCurrent)) { + $blockId = $db->addBlock($blockCurrent); - if (!$db->getData($blockId, $nameSpaceId)) { - $db->addData($blockId, - $nameSpaceId, - $transactionRaw['time'], - $transactionRaw['size'], - $transactionRaw['txid'], - strip_tags(html_entity_decode(@hex2bin($asmArray[2]), ENT_QUOTES, 'UTF-8')), - strip_tags(html_entity_decode(@hex2bin($asmArray[3]), ENT_QUOTES, 'UTF-8'))); - } + if (CRAWLER_DEBUG) { + echo sprintf("add block %s\n", $blockCurrent); + } + } + + $lostTransactions = 0; + + foreach ($blockData['tx'] as $transaction) { + + if (!$transactionRaw = $kevaCoin->getrawtransaction($transaction)) { + + $lostTransactions++; + + $db->setLostTransactions($blockId, $lostTransactions); + + if (CRAWLER_DEBUG) { + echo sprintf("could not read the transaction %s. skipped.\n", $transaction); + } + + break; + } + + foreach($transactionRaw['vout'] as $vout) { + + $asmArray = explode(' ', $vout['scriptPubKey']['asm']); + + if (in_array($asmArray[0], ['OP_KEVA_NAMESPACE', 'OP_KEVA_PUT', 'OP_KEVA_DELETE'])) { + + $hash = Base58Check::encode($asmArray[1], false , 0 , false); + + switch ($asmArray[0]) { + + case 'OP_KEVA_DELETE': + + $key = filterString(decodeString($asmArray[2])); + $value = ''; + + break; + + case 'OP_KEVA_NAMESPACE': + + $key = '_KEVA_NS_'; + $value = filterString(decodeString($asmArray[2])); + + break; + + default: + + $key = filterString(decodeString($asmArray[2])); + $value = filterString(decodeString($asmArray[3])); + } + + if (!$nameSpaceId = $db->getNameSpace($hash)) { + $nameSpaceId = $db->addNameSpace($hash); if (CRAWLER_DEBUG) { - $response[] = [ - 'blocktotal'=> $blockTotal, - 'block' => $blockCurrent, - 'blockhash' => $transactionRaw['blockhash'], - 'txid' => $transactionRaw['txid'], - 'version' => $transactionRaw['version'], - 'size' => $transactionRaw['size'], - 'time' => $transactionRaw['time'], - 'blocktime' => $transactionRaw['blocktime'], - 'namehash' => $hash, - 'title' => $nameSpaceValue, - 'key' => strip_tags(html_entity_decode(@hex2bin($asmArray[2]), ENT_QUOTES, 'UTF-8')), - 'vale' => strip_tags(html_entity_decode(@hex2bin($asmArray[3]), ENT_QUOTES, 'UTF-8')) - ]; + echo sprintf("add namespace %s\n", $hash); } + } + + if (!$dataId = $db->getData($transactionRaw['txid'])) { + $dataId = $db->addData($blockId, + $nameSpaceId, + $transactionRaw['time'], + $transactionRaw['size'], + $transactionRaw['txid'], + $key, + $value, + ($key == '_KEVA_NS_'), + empty($value)); + + if ($value) { + + $db->setDataKeyDeleted($nameSpaceId, $key, false); + + if (CRAWLER_DEBUG) { + echo sprintf("add new key/value %s\n", $transactionRaw['txid']); + } + + } else { + + $db->setDataKeyDeleted($nameSpaceId, $key, true); + + if (CRAWLER_DEBUG) { + echo sprintf("delete key %s from namespace %s\n", $key, $hash); + } } } + + if (CRAWLER_DEBUG) { + $response[] = [ + 'blocktotal'=> $blockTotal, + 'block' => $blockCurrent, + 'blockhash' => $transactionRaw['blockhash'], + 'txid' => $transactionRaw['txid'], + 'version' => $transactionRaw['version'], + 'size' => $transactionRaw['size'], + 'time' => $transactionRaw['time'], + 'blocktime' => $transactionRaw['blocktime'], + 'namehash' => $hash, + 'key' => $key, + 'value' => $value + ]; + } } } + } + } - } else { - // @TODO block not found - } + // Debug + if (CRAWLER_DEBUG) { + echo "scanning completed.\n"; + # print_r($response); } -} -// Debug -if (CRAWLER_DEBUG) { - print_r($response); + sem_release($semaphore); + +} else { + echo "database locked by the another process...\n"; } diff --git a/library/helper.php b/library/helper.php new file mode 100644 index 0000000..6db5eda --- /dev/null +++ b/library/helper.php @@ -0,0 +1,15 @@ +proto = 'https'; - $this->CACertificate = $certificate; - } - - public function __call($method, $params) { - - $this->status = null; - $this->error = null; - $this->rawResponse = null; - $this->response = null; - - $params = array_values($params); - - $this->id++; - - $request = json_encode(array( - 'method' => $method, - 'params' => $params, - 'id' => $this->id - )); - - $curl = curl_init("{$this->proto}://{$this->host}:{$this->port}/{$this->url}"); - $options = array( - CURLOPT_HTTPAUTH => CURLAUTH_BASIC, - CURLOPT_USERPWD => $this->username . ':' . $this->password, - CURLOPT_RETURNTRANSFER => true, - CURLOPT_FOLLOWLOCATION => true, - CURLOPT_MAXREDIRS => 10, - CURLOPT_HTTPHEADER => array('Content-type: text/plain'), - CURLOPT_POST => true, - CURLOPT_POSTFIELDS => $request - ); - - if (ini_get('open_basedir')) { - unset($options[CURLOPT_FOLLOWLOCATION]); - } - - if ($this->proto == 'https') { - if (!empty($this->CACertificate)) { - $options[CURLOPT_CAINFO] = $this->CACertificate; - $options[CURLOPT_CAPATH] = DIRNAME($this->CACertificate); - } else { - $options[CURLOPT_SSL_VERIFYPEER] = false; - } - } - - curl_setopt_array($curl, $options); - - $this->rawResponse = curl_exec($curl); - $this->response = json_decode($this->rawResponse, true); - - $this->status = curl_getinfo($curl, CURLINFO_HTTP_CODE); - - $curl_error = curl_error($curl); - - curl_close($curl); - - if (!empty($curl_error)) { - $this->error = $curl_error; - } - - if ($this->response['error']) { - $this->error = $this->response['error']['message']; - } elseif ($this->status != 200) { - - switch ($this->status) { - case 400: - $this->error = 'HTTP_BAD_REQUEST'; - break; - case 401: - $this->error = 'HTTP_UNAUTHORIZED'; - break; - case 403: - $this->error = 'HTTP_FORBIDDEN'; - break; - case 404: - $this->error = 'HTTP_NOT_FOUND'; - break; - } - } - - if ($this->error) { - return false; - } - - return $this->response['result']; - } -} diff --git a/library/kevacoin.php b/library/kevacoin.php new file mode 100644 index 0000000..27e0627 --- /dev/null +++ b/library/kevacoin.php @@ -0,0 +1,156 @@ +_protocol = $protocol; + $this->_host = $host; + $this->_port = $port; + + $this->_curl = curl_init(); + + curl_setopt_array($this->_curl, [CURLOPT_RETURNTRANSFER => true, + CURLOPT_FOLLOWLOCATION => true, + CURLOPT_FRESH_CONNECT => true, + CURLOPT_HTTPAUTH => CURLAUTH_BASIC, + CURLOPT_USERPWD => $username . ':' . $password, + CURLOPT_RETURNTRANSFER => true, + CURLOPT_FOLLOWLOCATION => true, + //CURLOPT_VERBOSE => true, + CURLOPT_HTTPHEADER => [ + 'Content-Type: application/plain', + ], + ]); + } + + public function __destruct() { + curl_close($this->_curl); + } + + protected function prepare($url, $method, array $postfields = []) { + + curl_setopt($this->_curl, CURLOPT_URL, $this->_protocol . '://' . $this->_host . ':' . $this->_port . $url); + curl_setopt($this->_curl, CURLOPT_CUSTOMREQUEST, $method); + + if ($method == 'POST' && $postfields) { + curl_setopt($this->_curl, CURLOPT_POSTFIELDS, json_encode($postfields)); + } + } + + protected function execute($json = true) { + + $response = curl_exec($this->_curl); + $errorNumber = curl_errno($this->_curl); + $errorText = curl_error($this->_curl); + + if ($errorNumber > 0) { + //return false; + } + + if ($response) { + if ($json) { + return json_decode($response, true); + } else { + return $response; + } + } + + return false; + } + + public function getblockcount() { + + $this->_id++; + + $this->prepare('', 'POST', [ + 'method' => 'getblockcount', + 'params' => [], + 'id' => $this->_id + ]); + + $response = $this->execute(); + + if (isset($response['result']) && is_int($response['result'])) { + + return $response['result']; + + } else { + + return false; + } + } + + public function getblockhash($block) { + + $this->_id++; + + $this->prepare('', 'POST', [ + 'method' => 'getblockhash', + 'params' => [$block], + 'id' => $this->_id + ]); + + $response = $this->execute(); + + if (isset($response['result']) && 64 == strlen($response['result'])) { + + return $response['result']; + + } else { + + return false; + } + } + + public function getblock($hash) { + + $this->_id++; + + $this->prepare('', 'POST', [ + 'method' => 'getblock', + 'params' => [$hash], + 'id' => $this->_id + ]); + + $response = $this->execute(); + + if (isset($response['result']) && is_array($response['result'])) { + + return $response['result']; + + } else { + + return false; + } + } + + public function getrawtransaction($txid, $decode = true) { + + $this->_id++; + + $this->prepare('', 'POST', [ + 'method' => 'getrawtransaction', + 'params' => [$txid, $decode], + 'id' => $this->_id + ]); + + $response = $this->execute(); + + if (isset($response['result']) && is_array($response['result'])) { + + return $response['result']; + + } else { + + return false; + } + } +} diff --git a/library/mysql.php b/library/mysql.php index c332bfa..2ce0d97 100644 --- a/library/mysql.php +++ b/library/mysql.php @@ -1,12 +1,12 @@ _db = new PDO('mysql:dbname=' . DB_NAME . ';host=' . DB_HOST . ';port=' . DB_PORT . ';charset=utf8', DB_USERNAME, DB_PASSWORD, [PDO::MYSQL_ATTR_INIT_COMMAND => 'SET NAMES utf8']); + $this->_db = new PDO('mysql:dbname=' . $database . ';host=' . $host . ';port=' . $port . ';charset=utf8', $username, $password, [PDO::MYSQL_ATTR_INIT_COMMAND => 'SET NAMES utf8']); $this->_db->setAttribute(PDO::ATTR_ERRMODE, PDO::ERRMODE_EXCEPTION); $this->_db->setAttribute(PDO::ATTR_DEFAULT_FETCH_MODE, PDO::FETCH_ASSOC); $this->_db->setAttribute(PDO::ATTR_TIMEOUT, 600); @@ -22,7 +22,7 @@ class MySQL { $query = $this->_db->query('SELECT MAX(`blockId`) AS `lastBlock` FROM `block`')->fetch(); - return (int) $query['lastBlock'] ? $query['lastBlock'] : 1; + return (int) $query['lastBlock']; } catch(PDOException $e) { trigger_error($e->getMessage()); @@ -62,13 +62,13 @@ class MySQL { } } - public function getData($blockId, $nameSpaceId) { + public function getData($txId) { try { - $query = $this->_db->prepare('SELECT `dataId` FROM `data` WHERE `blockId` = ? AND `nameSpaceId` = ? LIMIT 1'); + $query = $this->_db->prepare('SELECT `dataId` FROM `data` WHERE `txId` = ? LIMIT 1'); - $query->execute([$blockId, $nameSpaceId]); + $query->execute([$txId]); return $query->rowCount() ? $query->fetch()['dataId'] : false; @@ -82,8 +82,9 @@ class MySQL { try { - $query = $this->_db->prepare('INSERT INTO `block` SET `blockId` = ?, - `timeIndexed` = UNIX_TIMESTAMP()'); + $query = $this->_db->prepare('INSERT INTO `block` SET `blockId` = ?, + `lostTransactions` = 0, + `timeIndexed` = UNIX_TIMESTAMP()'); $query->execute([$blockId]); @@ -95,15 +96,30 @@ class MySQL { } } - public function addNameSpace($hash, $value) { + public function setLostTransactions($blockId, $lostTransactions) { + + try { + + $query = $this->_db->prepare('UPDATE `block` SET `lostTransactions` = ? WHERE `blockId` = ? LIMIT 1'); + + $query->execute([$lostTransactions, $blockId]); + + return $blockId; + + } catch(PDOException $e) { + trigger_error($e->getMessage()); + return false; + } + } + + public function addNameSpace($hash) { try { $query = $this->_db->prepare('INSERT INTO `namespace` SET `hash` = ?, - `value` = ?, `timeIndexed` = UNIX_TIMESTAMP()'); - $query->execute([$hash, $value]); + $query->execute([$hash]); return $this->_db->lastInsertId(); @@ -113,20 +129,32 @@ class MySQL { } } - public function addData($blockId, $nameSpaceId, $time, $size, $txid, $key, $value) { + public function addData($blockId, $nameSpaceId, $time, $size, $txid, $key, $value, $ns, $deleted = false) { try { - $query = $this->_db->prepare('INSERT INTO `data` SET `blockId` = ?, - `nameSpaceId` = ?, - `time` = ?, - `size` = ?, - `txid` = ?, - `key` = ?, - `value` = ?, + $query = $this->_db->prepare('INSERT INTO `data` SET `blockId` = :blockId, + `nameSpaceId` = :nameSpaceId, + `time` = :time, + `size` = :size, + `txid` = :txid, + `key` = :key, + `value` = :value, + `deleted` = :deleted, + `ns` = :ns, `timeIndexed` = UNIX_TIMESTAMP()'); - $query->execute([$blockId, $nameSpaceId, $time, $size, $txid, $key, $value]); + $query->bindValue(':blockId', $blockId, PDO::PARAM_INT); + $query->bindValue(':nameSpaceId', $nameSpaceId, PDO::PARAM_INT); + $query->bindValue(':time', $time, PDO::PARAM_INT); + $query->bindValue(':size', $size, PDO::PARAM_INT); + $query->bindValue(':txid', $txid, PDO::PARAM_STR); + $query->bindValue(':key', $key, PDO::PARAM_STR); + $query->bindValue(':value', $value, PDO::PARAM_STR); + $query->bindValue(':deleted', (int) $deleted, PDO::PARAM_STR); + $query->bindValue(':ns', (int) $ns, PDO::PARAM_STR); + + $query->execute(); return $this->_db->lastInsertId(); @@ -135,4 +163,25 @@ class MySQL { return false; } } + + public function setDataKeyDeleted($nameSpaceId, $key, $deleted) { + + try { + + $query = $this->_db->prepare('UPDATE `data` SET `deleted` = :deleted + WHERE `nameSpaceId` = :nameSpaceId AND `key` LIKE :key'); + + $query->bindValue(':nameSpaceId', $nameSpaceId, PDO::PARAM_INT); + $query->bindValue(':key', $key, PDO::PARAM_STR); + $query->bindValue(':deleted', (int) $deleted, PDO::PARAM_STR); + + $query->execute(); + + return $query->rowCount(); + + } catch(PDOException $e) { + trigger_error($e->getMessage()); + return false; + } + } }