Browse Source

Make linearize scripts Python 3-compatible.

0.14
Doug 8 years ago committed by Douglas Roark
parent
commit
3c8f63ba7c
  1. 13
      contrib/linearize/README.md
  2. 28
      contrib/linearize/linearize-data.py
  3. 32
      contrib/linearize/linearize-hashes.py

13
contrib/linearize/README.md

@ -1,5 +1,6 @@
# Linearize # Linearize
Construct a linear, no-fork, best version of the Bitcoin blockchain. Construct a linear, no-fork, best version of the Bitcoin blockchain. The scripts
run using Python 3 but are compatible with Python 2.
## Step 1: Download hash list ## Step 1: Download hash list
@ -9,7 +10,8 @@ Required configuration file settings for linearize-hashes:
* RPC: `rpcuser`, `rpcpassword` * RPC: `rpcuser`, `rpcpassword`
Optional config file setting for linearize-hashes: Optional config file setting for linearize-hashes:
* RPC: `host`, `port` (Default: `127.0.0.1:8332`) * RPC: `host` (Default: `127.0.0.1`)
* RPC: `port` (Default: `8332`)
* Blockchain: `min_height`, `max_height` * Blockchain: `min_height`, `max_height`
* `rev_hash_bytes`: If true, the written block hash list will be * `rev_hash_bytes`: If true, the written block hash list will be
byte-reversed. (In other words, the hash returned by getblockhash will have its byte-reversed. (In other words, the hash returned by getblockhash will have its
@ -17,6 +19,9 @@ bytes reversed.) False by default. Intended for generation of
standalone hash lists but safe to use with linearize-data.py, which will output standalone hash lists but safe to use with linearize-data.py, which will output
the same data no matter which byte format is chosen. the same data no matter which byte format is chosen.
The `linearize-hashes` script requires a connection, local or remote, to a
JSON-RPC server. Running `bitcoind` or `bitcoin-qt -server` will be sufficient.
## Step 2: Copy local block data ## Step 2: Copy local block data
$ ./linearize-data.py linearize.cfg $ ./linearize-data.py linearize.cfg
@ -24,13 +29,13 @@ the same data no matter which byte format is chosen.
Required configuration file settings: Required configuration file settings:
* `output_file`: The file that will contain the final blockchain. * `output_file`: The file that will contain the final blockchain.
or or
* `output`: Output directory for linearized blocks/blkNNNNN.dat output. * `output`: Output directory for linearized `blocks/blkNNNNN.dat` output.
Optional config file setting for linearize-data: Optional config file setting for linearize-data:
* `file_timestamp`: Set each file's last-modified time to that of the most * `file_timestamp`: Set each file's last-modified time to that of the most
recent block in that file. recent block in that file.
* `genesis`: The hash of the genesis block in the blockchain. * `genesis`: The hash of the genesis block in the blockchain.
* `input: bitcoind blocks/ directory containing blkNNNNN.dat * `input`: bitcoind blocks/ directory containing blkNNNNN.dat
* `hashlist`: text file containing list of block hashes created by * `hashlist`: text file containing list of block hashes created by
linearize-hashes.py. linearize-hashes.py.
* `max_out_sz`: Maximum size for files created by the `output_file` option. * `max_out_sz`: Maximum size for files created by the `output_file` option.

28
contrib/linearize/linearize-data.py

@ -1,4 +1,4 @@
#!/usr/bin/env python #!/usr/bin/env python3
# #
# linearize-data.py: Construct a linear, no-fork version of the chain. # linearize-data.py: Construct a linear, no-fork version of the chain.
# #
@ -8,29 +8,33 @@
# #
from __future__ import print_function, division from __future__ import print_function, division
try: # Python 3
import http.client as httplib
except ImportError: # Python 2
import httplib
import json import json
import struct import struct
import re import re
import os import os
import os.path import os.path
import base64 import base64
import httplib
import sys import sys
import hashlib import hashlib
import datetime import datetime
import time import time
from collections import namedtuple from collections import namedtuple
from binascii import hexlify, unhexlify
settings = {} settings = {}
##### Switch endian-ness ##### ##### Switch endian-ness #####
def hex_switchEndian(s): def hex_switchEndian(s):
""" Switches the endianness of a hex string (in pairs of hex chars) """ """ Switches the endianness of a hex string (in pairs of hex chars) """
pairList = [s[i]+s[i+1] for i in range(0,len(s),2)] pairList = [s[i:i+2].encode() for i in range(0, len(s), 2)]
return ''.join(pairList[::-1]) return b''.join(pairList[::-1]).decode()
def uint32(x): def uint32(x):
return x & 0xffffffffL return x & 0xffffffff
def bytereverse(x): def bytereverse(x):
return uint32(( ((x) << 24) | (((x) << 8) & 0x00ff0000) | return uint32(( ((x) << 24) | (((x) << 8) & 0x00ff0000) |
@ -41,14 +45,14 @@ def bufreverse(in_buf):
for i in range(0, len(in_buf), 4): for i in range(0, len(in_buf), 4):
word = struct.unpack('@I', in_buf[i:i+4])[0] word = struct.unpack('@I', in_buf[i:i+4])[0]
out_words.append(struct.pack('@I', bytereverse(word))) out_words.append(struct.pack('@I', bytereverse(word)))
return ''.join(out_words) return b''.join(out_words)
def wordreverse(in_buf): def wordreverse(in_buf):
out_words = [] out_words = []
for i in range(0, len(in_buf), 4): for i in range(0, len(in_buf), 4):
out_words.append(in_buf[i:i+4]) out_words.append(in_buf[i:i+4])
out_words.reverse() out_words.reverse()
return ''.join(out_words) return b''.join(out_words)
def calc_hdr_hash(blk_hdr): def calc_hdr_hash(blk_hdr):
hash1 = hashlib.sha256() hash1 = hashlib.sha256()
@ -65,7 +69,7 @@ def calc_hash_str(blk_hdr):
hash = calc_hdr_hash(blk_hdr) hash = calc_hdr_hash(blk_hdr)
hash = bufreverse(hash) hash = bufreverse(hash)
hash = wordreverse(hash) hash = wordreverse(hash)
hash_str = hash.encode('hex') hash_str = hexlify(hash).decode('utf-8')
return hash_str return hash_str
def get_blk_dt(blk_hdr): def get_blk_dt(blk_hdr):
@ -217,7 +221,7 @@ class BlockDataCopier:
inMagic = inhdr[:4] inMagic = inhdr[:4]
if (inMagic != self.settings['netmagic']): if (inMagic != self.settings['netmagic']):
print("Invalid magic: " + inMagic.encode('hex')) print("Invalid magic: " + hexlify(inMagic).decode('utf-8'))
return return
inLenLE = inhdr[4:] inLenLE = inhdr[4:]
su = struct.unpack("<I", inLenLE) su = struct.unpack("<I", inLenLE)
@ -294,14 +298,14 @@ if __name__ == '__main__':
if 'split_timestamp' not in settings: if 'split_timestamp' not in settings:
settings['split_timestamp'] = 0 settings['split_timestamp'] = 0
if 'max_out_sz' not in settings: if 'max_out_sz' not in settings:
settings['max_out_sz'] = 1000L * 1000 * 1000 settings['max_out_sz'] = 1000 * 1000 * 1000
if 'out_of_order_cache_sz' not in settings: if 'out_of_order_cache_sz' not in settings:
settings['out_of_order_cache_sz'] = 100 * 1000 * 1000 settings['out_of_order_cache_sz'] = 100 * 1000 * 1000
settings['max_out_sz'] = long(settings['max_out_sz']) settings['max_out_sz'] = int(settings['max_out_sz'])
settings['split_timestamp'] = int(settings['split_timestamp']) settings['split_timestamp'] = int(settings['split_timestamp'])
settings['file_timestamp'] = int(settings['file_timestamp']) settings['file_timestamp'] = int(settings['file_timestamp'])
settings['netmagic'] = settings['netmagic'].decode('hex') settings['netmagic'] = unhexlify(settings['netmagic'].encode('utf-8'))
settings['out_of_order_cache_sz'] = int(settings['out_of_order_cache_sz']) settings['out_of_order_cache_sz'] = int(settings['out_of_order_cache_sz'])
if 'output_file' not in settings and 'output' not in settings: if 'output_file' not in settings and 'output' not in settings:

32
contrib/linearize/linearize-hashes.py

@ -1,4 +1,4 @@
#!/usr/bin/env python #!/usr/bin/env python3
# #
# linearize-hashes.py: List blocks in a linear, no-fork version of the chain. # linearize-hashes.py: List blocks in a linear, no-fork version of the chain.
# #
@ -8,11 +8,14 @@
# #
from __future__ import print_function from __future__ import print_function
try: # Python 3
import http.client as httplib
except ImportError: # Python 2
import httplib
import json import json
import struct import struct
import re import re
import base64 import base64
import httplib
import sys import sys
settings = {} settings = {}
@ -20,26 +23,32 @@ settings = {}
##### Switch endian-ness ##### ##### Switch endian-ness #####
def hex_switchEndian(s): def hex_switchEndian(s):
""" Switches the endianness of a hex string (in pairs of hex chars) """ """ Switches the endianness of a hex string (in pairs of hex chars) """
pairList = [s[i]+s[i+1] for i in range(0,len(s),2)] pairList = [s[i:i+2].encode() for i in range(0, len(s), 2)]
return ''.join(pairList[::-1]) return b''.join(pairList[::-1]).decode()
class BitcoinRPC: class BitcoinRPC:
def __init__(self, host, port, username, password): def __init__(self, host, port, username, password):
authpair = "%s:%s" % (username, password) authpair = "%s:%s" % (username, password)
self.authhdr = "Basic %s" % (base64.b64encode(authpair)) authpair = authpair.encode('utf-8')
self.conn = httplib.HTTPConnection(host, port, False, 30) self.authhdr = b"Basic " + base64.b64encode(authpair)
self.conn = httplib.HTTPConnection(host, port=port, timeout=30)
def execute(self, obj): def execute(self, obj):
self.conn.request('POST', '/', json.dumps(obj), try:
{ 'Authorization' : self.authhdr, self.conn.request('POST', '/', json.dumps(obj),
'Content-type' : 'application/json' }) { 'Authorization' : self.authhdr,
'Content-type' : 'application/json' })
except ConnectionRefusedError:
print('RPC connection refused. Check RPC settings and the server status.',
file=sys.stderr)
return None
resp = self.conn.getresponse() resp = self.conn.getresponse()
if resp is None: if resp is None:
print("JSON-RPC: no response", file=sys.stderr) print("JSON-RPC: no response", file=sys.stderr)
return None return None
body = resp.read() body = resp.read().decode('utf-8')
resp_obj = json.loads(body) resp_obj = json.loads(body)
return resp_obj return resp_obj
@ -70,6 +79,9 @@ def get_block_hashes(settings, max_blocks_per_call=10000):
batch.append(rpc.build_request(x, 'getblockhash', [height + x])) batch.append(rpc.build_request(x, 'getblockhash', [height + x]))
reply = rpc.execute(batch) reply = rpc.execute(batch)
if reply is None:
print('Cannot continue. Program will halt.')
return None
for x,resp_obj in enumerate(reply): for x,resp_obj in enumerate(reply):
if rpc.response_is_error(resp_obj): if rpc.response_is_error(resp_obj):

Loading…
Cancel
Save