Browse Source

implement blockchain crawler

main
ghost 3 years ago
parent
commit
a432a78acf
  1. 84
      tool/crawler/blockchain.php
  2. 68
      tool/crawler/blockchain.py

84
tool/crawler/blockchain.php

@ -0,0 +1,84 @@ @@ -0,0 +1,84 @@
<?php
// Dependencies
require(dirname(dirname(__DIR__)) . '/src/config.php');
require(PROJECT_DIR . '/application/model/model.php');
require(PROJECT_DIR . '/application/model/user.php');
require(PROJECT_DIR . '/application/model/block.php');
require(PROJECT_DIR . '/system/curl.php');
require(PROJECT_DIR . '/system/twister.php');
// Init libraries
$_twister = new Twister(
new Curl(
TWISTER_PROTOCOL,
TWISTER_HOST,
TWISTER_PORT,
TWISTER_USER,
TWISTER_PASSWORD,
TWISTER_SSL
)
);
// Init models
$_modelUser = new ModelUser(
DB_DATABASE,
DB_HOST,
DB_PORT,
DB_USER,
DB_PASSWORD
);
$_modelBlock = new ModelBlock(
DB_DATABASE,
DB_HOST,
DB_PORT,
DB_USER,
DB_PASSWORD
);
print("import begin...\n");
$nextBlock = $_modelBlock->getNextBlock();
while (true) {
if (!$blockHash = $_twister->getBlockHash($nextBlock)) {
print("database up to date\n");
exit;
}
if (!$block = $_twister->getBlock($blockHash)) {
trigger_error(sprintf('could not receive block info on %s (%s)', $nextBlock, $blockHash));
exit;
}
// Add block
if ($blockId = $_modelBlock->addBlock($blockHash, time())) {
print(sprintf("add block %s\n", $blockId));
// Add users
foreach ($block['usernames'] as $userName) {
if (!$_modelUser->addUser($blockId, $userName)) {
trigger_error(sprintf('could not add user %s in block %s)', $userName, $blockId));
exit;
}
print(sprintf("add user %s\n", $userName));
}
// Update queue
$nextBlock++;
} else {
trigger_error(sprintf('could not add block %s (%s)', $nextBlock, $blockHash));
exit;
}
}

68
tool/crawler/blockchain.py

@ -1,68 +0,0 @@ @@ -1,68 +0,0 @@
#!/usr/bin/python
import sys, MySQLdb
db = MySQLdb.connect(host="localhost", # your host, usually localhost
user="root", # your username
passwd="password", # your password
db="twister-stat") # name of the data base
cursor = db.cursor()
blocksInStep = 1000000 # blocks processing by the one step
class MyDb:
nextBlock = 0
process = MyDb()
cursor.execute ("SELECT COUNT(*) + 1 AS nextBlock FROM block")
row = cursor.fetchone()
process.nextBlock = row[0]
try:
from bitcoinrpc.authproxy import AuthServiceProxy
except ImportError as exc:
sys.stderr.write("Error: install python-bitcoinrpc (https://github.com/jgarzik/python-bitcoinrpc)\n")
exit(-1)
serverUrl = "http://user:password@127.0.0.1:28332"
if len(sys.argv) > 1:
serverUrl = sys.argv[1]
twister = AuthServiceProxy(serverUrl)
print "blockchain reading..."
while True:
hash = twister.getblockhash(process.nextBlock)
block = twister.getblock(hash)
blocksInStep = blocksInStep - 1
if blocksInStep < 0:
break
print "add block", block["height"]
cursor.execute("INSERT INTO block SET hash = %s, time = %s", (block["hash"], block["time"]))
blockId = db.insert_id()
for userName in block["usernames"]:
print "add user", userName
cursor.execute("INSERT INTO user SET blockId = %s, username = %s", (blockId, userName))
if block.has_key("nextblockhash"):
process.nextBlock = process.nextBlock + 1
else:
print "database is up to date..."
break
db.commit()
db.close()
print "task completed."
Loading…
Cancel
Save