Browse Source

Allow linearization scripts to support hash byte reversal

Currently, the linearization scripts require input hashes to be in one endian form. Add support for byte reversal.
0.14
Doug 8 years ago committed by Douglas Roark
parent
commit
d5aa19813c
  1. 40
      contrib/linearize/README.md
  2. 4
      contrib/linearize/example-linearize.cfg
  3. 19
      contrib/linearize/linearize-data.py
  4. 14
      contrib/linearize/linearize-hashes.py

40
contrib/linearize/README.md

@ -1,33 +1,43 @@
# Linearize # Linearize
Construct a linear, no-fork, best version of the blockchain. Construct a linear, no-fork, best version of the Bitcoin blockchain.
## Step 1: Download hash list ## Step 1: Download hash list
$ ./linearize-hashes.py linearize.cfg > hashlist.txt $ ./linearize-hashes.py linearize.cfg > hashlist.txt
Required configuration file settings for linearize-hashes: Required configuration file settings for linearize-hashes:
* RPC: rpcuser, rpcpassword * RPC: `rpcuser`, `rpcpassword`
Optional config file setting for linearize-hashes: Optional config file setting for linearize-hashes:
* RPC: host, port * RPC: `host`, `port` (Default: `127.0.0.1:8332`)
* Block chain: min_height, max_height * Blockchain: `min_height`, `max_height`
* `rev_hash_bytes`: If true, the written block hash list will be
byte-reversed. (In other words, the hash returned by getblockhash will have its
bytes reversed.) False by default. Intended for generation of
standalone hash lists but safe to use with linearize-data.py, which will output
the same data no matter which byte format is chosen.
## Step 2: Copy local block data ## Step 2: Copy local block data
$ ./linearize-data.py linearize.cfg $ ./linearize-data.py linearize.cfg
Required configuration file settings: Required configuration file settings:
* "input": bitcoind blocks/ directory containing blkNNNNN.dat * `output_file`: The file that will contain the final blockchain.
* "hashlist": text file containing list of block hashes, linearized-hashes.py
output.
* "output_file": bootstrap.dat
or or
* "output": output directory for linearized blocks/blkNNNNN.dat output * `output`: Output directory for linearized blocks/blkNNNNN.dat output.
Optional config file setting for linearize-data: Optional config file setting for linearize-data:
* "netmagic": network magic number * `file_timestamp`: Set each file's last-modified time to that of the most
* "max_out_sz": maximum output file size (default `1000*1000*1000`) recent block in that file.
* "split_timestamp": Split files when a new month is first seen, in addition to * `genesis`: The hash of the genesis block in the blockchain.
reaching a maximum file size. * `input: bitcoind blocks/ directory containing blkNNNNN.dat
* "file_timestamp": Set each file's last-modified time to that of the * `hashlist`: text file containing list of block hashes created by
most recent block in that file. linearize-hashes.py.
* `max_out_sz`: Maximum size for files created by the `output_file` option.
(Default: `1000*1000*1000 bytes`)
* `netmagic`: Network magic number.
* `rev_hash_bytes`: If true, the block hash list written by linearize-hashes.py
will be byte-reversed when read by linearize-data.py. See the linearize-hashes
entry for more information.
* `split_timestamp`: Split blockchain files when a new month is first seen, in
addition to reaching a maximum file size (`max_out_sz`).

4
contrib/linearize/example-linearize.cfg

@ -23,7 +23,9 @@ input=/home/example/.bitcoin/blocks
output_file=/home/example/Downloads/bootstrap.dat output_file=/home/example/Downloads/bootstrap.dat
hashlist=hashlist.txt hashlist=hashlist.txt
split_year=1
# Maxmimum size in bytes of out-of-order blocks cache in memory # Maxmimum size in bytes of out-of-order blocks cache in memory
out_of_order_cache_sz = 100000000 out_of_order_cache_sz = 100000000
# Do we want the reverse the hash bytes coming from getblockhash?
rev_hash_bytes = False

19
contrib/linearize/linearize-data.py

@ -23,6 +23,12 @@ from collections import namedtuple
settings = {} settings = {}
##### Switch endian-ness #####
def hex_switchEndian(s):
""" Switches the endianness of a hex string (in pairs of hex chars) """
pairList = [s[i]+s[i+1] for i in range(0,len(s),2)]
return ''.join(pairList[::-1])
def uint32(x): def uint32(x):
return x & 0xffffffffL return x & 0xffffffffL
@ -69,17 +75,21 @@ def get_blk_dt(blk_hdr):
dt_ym = datetime.datetime(dt.year, dt.month, 1) dt_ym = datetime.datetime(dt.year, dt.month, 1)
return (dt_ym, nTime) return (dt_ym, nTime)
# When getting the list of block hashes, undo any byte reversals.
def get_block_hashes(settings): def get_block_hashes(settings):
blkindex = [] blkindex = []
f = open(settings['hashlist'], "r") f = open(settings['hashlist'], "r")
for line in f: for line in f:
line = line.rstrip() line = line.rstrip()
if settings['rev_hash_bytes'] == 'true':
line = hex_switchEndian(line)
blkindex.append(line) blkindex.append(line)
print("Read " + str(len(blkindex)) + " hashes") print("Read " + str(len(blkindex)) + " hashes")
return blkindex return blkindex
# The block map shouldn't give or receive byte-reversed hashes.
def mkblockmap(blkindex): def mkblockmap(blkindex):
blkmap = {} blkmap = {}
for height,hash in enumerate(blkindex): for height,hash in enumerate(blkindex):
@ -265,6 +275,12 @@ if __name__ == '__main__':
settings[m.group(1)] = m.group(2) settings[m.group(1)] = m.group(2)
f.close() f.close()
# Force hash byte format setting to be lowercase to make comparisons easier.
# Also place upfront in case any settings need to know about it.
if 'rev_hash_bytes' not in settings:
settings['rev_hash_bytes'] = 'false'
settings['rev_hash_bytes'] = settings['rev_hash_bytes'].lower()
if 'netmagic' not in settings: if 'netmagic' not in settings:
settings['netmagic'] = 'f9beb4d9' settings['netmagic'] = 'f9beb4d9'
if 'genesis' not in settings: if 'genesis' not in settings:
@ -295,9 +311,8 @@ if __name__ == '__main__':
blkindex = get_block_hashes(settings) blkindex = get_block_hashes(settings)
blkmap = mkblockmap(blkindex) blkmap = mkblockmap(blkindex)
# Block hash map won't be byte-reversed. Neither should the genesis hash.
if not settings['genesis'] in blkmap: if not settings['genesis'] in blkmap:
print("Genesis block not found in hashlist") print("Genesis block not found in hashlist")
else: else:
BlockDataCopier(settings, blkindex, blkmap).run() BlockDataCopier(settings, blkindex, blkmap).run()

14
contrib/linearize/linearize-hashes.py

@ -17,6 +17,12 @@ import sys
settings = {} settings = {}
##### Switch endian-ness #####
def hex_switchEndian(s):
""" Switches the endianness of a hex string (in pairs of hex chars) """
pairList = [s[i]+s[i+1] for i in range(0,len(s),2)]
return ''.join(pairList[::-1])
class BitcoinRPC: class BitcoinRPC:
def __init__(self, host, port, username, password): def __init__(self, host, port, username, password):
authpair = "%s:%s" % (username, password) authpair = "%s:%s" % (username, password)
@ -70,6 +76,8 @@ def get_block_hashes(settings, max_blocks_per_call=10000):
print('JSON-RPC: error at height', height+x, ': ', resp_obj['error'], file=sys.stderr) print('JSON-RPC: error at height', height+x, ': ', resp_obj['error'], file=sys.stderr)
exit(1) exit(1)
assert(resp_obj['id'] == x) # assume replies are in-sequence assert(resp_obj['id'] == x) # assume replies are in-sequence
if settings['rev_hash_bytes'] == 'true':
resp_obj['result'] = hex_switchEndian(resp_obj['result'])
print(resp_obj['result']) print(resp_obj['result'])
height += num_blocks height += num_blocks
@ -101,6 +109,8 @@ if __name__ == '__main__':
settings['min_height'] = 0 settings['min_height'] = 0
if 'max_height' not in settings: if 'max_height' not in settings:
settings['max_height'] = 313000 settings['max_height'] = 313000
if 'rev_hash_bytes' not in settings:
settings['rev_hash_bytes'] = 'false'
if 'rpcuser' not in settings or 'rpcpassword' not in settings: if 'rpcuser' not in settings or 'rpcpassword' not in settings:
print("Missing username and/or password in cfg file", file=stderr) print("Missing username and/or password in cfg file", file=stderr)
sys.exit(1) sys.exit(1)
@ -109,5 +119,7 @@ if __name__ == '__main__':
settings['min_height'] = int(settings['min_height']) settings['min_height'] = int(settings['min_height'])
settings['max_height'] = int(settings['max_height']) settings['max_height'] = int(settings['max_height'])
get_block_hashes(settings) # Force hash byte format setting to be lowercase to make comparisons easier.
settings['rev_hash_bytes'] = settings['rev_hash_bytes'].lower()
get_block_hashes(settings)

Loading…
Cancel
Save