implement blockchain crawler

tool/crawler/blockchain.php

@@ -0,0 +1,84 @@
+<?php
+
+// Dependencies
+require(dirname(dirname(__DIR__)) . '/src/config.php');
+
+require(PROJECT_DIR . '/application/model/model.php');
+require(PROJECT_DIR . '/application/model/user.php');
+require(PROJECT_DIR . '/application/model/block.php');
+
+require(PROJECT_DIR . '/system/curl.php');
+require(PROJECT_DIR . '/system/twister.php');
+
+// Init libraries
+$_twister = new Twister(
+  new Curl(
+    TWISTER_PROTOCOL,
+    TWISTER_HOST,
+    TWISTER_PORT,
+    TWISTER_USER,
+    TWISTER_PASSWORD,
+    TWISTER_SSL
+  )
+);
+
+// Init models
+$_modelUser = new ModelUser(
+  DB_DATABASE,
+  DB_HOST,
+  DB_PORT,
+  DB_USER,
+  DB_PASSWORD
+);
+
+$_modelBlock = new ModelBlock(
+  DB_DATABASE,
+  DB_HOST,
+  DB_PORT,
+  DB_USER,
+  DB_PASSWORD
+);
+
+print("import begin...\n");
+
+$nextBlock = $_modelBlock->getNextBlock();
+
+while (true) {
+
+  if (!$blockHash = $_twister->getBlockHash($nextBlock)) {
+
+    print("database up to date\n");
+    exit;
+  }
+
+  if (!$block = $_twister->getBlock($blockHash)) {
+
+    trigger_error(sprintf('could not receive block info on %s (%s)', $nextBlock, $blockHash));
+    exit;
+  }
+
+  // Add block
+  if ($blockId = $_modelBlock->addBlock($blockHash, time())) {
+
+    print(sprintf("add block %s\n", $blockId));
+
+    // Add users
+    foreach ($block['usernames'] as $userName) {
+
+      if (!$_modelUser->addUser($blockId, $userName)) {
+        trigger_error(sprintf('could not add user %s in block %s)', $userName, $blockId));
+        exit;
+      }
+
+      print(sprintf("add user %s\n", $userName));
+    }
+
+    // Update queue
+    $nextBlock++;
+
+  } else {
+
+    trigger_error(sprintf('could not add block %s (%s)', $nextBlock, $blockHash));
+    exit;
+  }
+}

tool/crawler/blockchain.py

@@ -1,68 +0,0 @@
-#!/usr/bin/python
-
-import sys, MySQLdb
-
-db = MySQLdb.connect(host="localhost",  # your host, usually localhost
-                     user="root",       # your username
-                     passwd="password", # your password
-                     db="twister-stat") # name of the data base
-
-
-cursor = db.cursor()
-
-blocksInStep  = 1000000 # blocks processing by the one step
-
-class MyDb:
-    nextBlock = 0
-
-process = MyDb()
-
-cursor.execute ("SELECT COUNT(*) + 1 AS nextBlock FROM block")
-row = cursor.fetchone()
-
-process.nextBlock = row[0]
-
-try:
-    from bitcoinrpc.authproxy import AuthServiceProxy
-except ImportError as exc:
-    sys.stderr.write("Error: install python-bitcoinrpc (https://github.com/jgarzik/python-bitcoinrpc)\n")
-    exit(-1)
-
-serverUrl = "http://user:password@127.0.0.1:28332"
-if len(sys.argv) > 1:
-    serverUrl = sys.argv[1]
-
-twister = AuthServiceProxy(serverUrl)
-
-print "blockchain reading..."
-
-while True:
-
-    hash  = twister.getblockhash(process.nextBlock)
-    block = twister.getblock(hash)
-
-    blocksInStep = blocksInStep - 1
-
-    if blocksInStep < 0:
-        break
-
-    print "add block", block["height"]
-    cursor.execute("INSERT INTO block SET hash = %s, time = %s", (block["hash"], block["time"]))
-
-    blockId = db.insert_id()
-
-    for userName in block["usernames"]:
-
-        print "add user", userName
-        cursor.execute("INSERT INTO user SET blockId = %s, username = %s", (blockId, userName))
-
-    if block.has_key("nextblockhash"):
-        process.nextBlock = process.nextBlock + 1
-    else:
-        print "database is up to date..."
-        break
-
-db.commit()
-db.close()
-
-print "task completed."