From 59da018b1956169ff63a4c696f0f7a66de90fef6 Mon Sep 17 00:00:00 2001
From: ghost <noreply@localhost>
Date: Thu, 23 Dec 2021 02:24:12 +0200
Subject: [PATCH] add twister-stat crawler

---
 tool/crawler/blockchain.py | 68 ++++++++++++++++++++++++++++++++++++++
 1 file changed, 68 insertions(+)
 create mode 100644 tool/crawler/blockchain.py

diff --git a/tool/crawler/blockchain.py b/tool/crawler/blockchain.py
new file mode 100644
index 0000000..1e2b979
--- /dev/null
+++ b/tool/crawler/blockchain.py
@@ -0,0 +1,68 @@
+#!/usr/bin/python
+
+import sys, MySQLdb
+
+db = MySQLdb.connect(host="localhost",  # your host, usually localhost
+                     user="root",       # your username
+                     passwd="password", # your password
+                     db="twister-stat") # name of the data base
+
+
+cursor = db.cursor()
+
+blocksInStep  = 1000000 # blocks processing by the one step
+
+class MyDb:
+    nextBlock = 0
+
+process = MyDb()
+
+cursor.execute ("SELECT COUNT(*) + 1 AS nextBlock FROM block")
+row = cursor.fetchone()
+
+process.nextBlock = row[0]
+
+try:
+    from bitcoinrpc.authproxy import AuthServiceProxy
+except ImportError as exc:
+    sys.stderr.write("Error: install python-bitcoinrpc (https://github.com/jgarzik/python-bitcoinrpc)\n")
+    exit(-1)
+
+serverUrl = "http://user:password@127.0.0.1:28332"
+if len(sys.argv) > 1:
+    serverUrl = sys.argv[1]
+
+twister = AuthServiceProxy(serverUrl)
+
+print "blockchain reading..."
+
+while True:
+
+    hash  = twister.getblockhash(process.nextBlock)
+    block = twister.getblock(hash)
+
+    blocksInStep = blocksInStep - 1
+
+    if blocksInStep < 0:
+        break
+
+    print "add block", block["height"]
+    cursor.execute("INSERT INTO block SET hash = %s, time = %s", (block["hash"], block["time"]))
+
+    blockId = db.insert_id()
+
+    for userName in block["usernames"]:
+
+        print "add user", userName
+        cursor.execute("INSERT INTO user SET blockId = %s, username = %s", (blockId, userName))
+
+    if block.has_key("nextblockhash"):
+        process.nextBlock = process.nextBlock + 1
+    else:
+        print "database is up to date..."
+        break
+
+db.commit()
+db.close()
+
+print "task completed."
\ No newline at end of file