diff --git a/contrib/linearize/README.md b/contrib/linearize/README.md index 8d06d53b1..b5c6e7824 100644 --- a/contrib/linearize/README.md +++ b/contrib/linearize/README.md @@ -26,4 +26,7 @@ output. Optional config file setting for linearize-data: * "netmagic": network magic number +* "max_out_sz": maximum output file size (default 1000*1000*1000) +* "split_year": Split files when a new year is first seen, in addition to +reaching a maximum file size. diff --git a/contrib/linearize/example-linearize.cfg b/contrib/linearize/example-linearize.cfg index 9c3270d65..071345f23 100644 --- a/contrib/linearize/example-linearize.cfg +++ b/contrib/linearize/example-linearize.cfg @@ -13,4 +13,5 @@ netmagic=f9beb4d9 input=/home/example/.bitcoin/blocks output_file=/home/example/Downloads/bootstrap.dat hashlist=hashlist.txt +split_year=1 diff --git a/contrib/linearize/linearize-data.py b/contrib/linearize/linearize-data.py index 77bae6e3c..ea94f25fa 100755 --- a/contrib/linearize/linearize-data.py +++ b/contrib/linearize/linearize-data.py @@ -14,8 +14,7 @@ import base64 import httplib import sys import hashlib - -MAX_OUT_SZ = 128 * 1024 * 1024 +import datetime settings = {} @@ -41,9 +40,7 @@ def wordreverse(in_buf): out_words.reverse() return ''.join(out_words) -def calc_hdr_hash(rawblock): - blk_hdr = rawblock[:80] - +def calc_hdr_hash(blk_hdr): hash1 = hashlib.sha256() hash1.update(blk_hdr) hash1_o = hash1.digest() @@ -54,13 +51,18 @@ def calc_hdr_hash(rawblock): return hash2_o -def calc_hash_str(rawblock): - hash = calc_hdr_hash(rawblock) +def calc_hash_str(blk_hdr): + hash = calc_hdr_hash(blk_hdr) hash = bufreverse(hash) hash = wordreverse(hash) hash_str = hash.encode('hex') return hash_str +def get_blk_year(blk_hdr): + members = struct.unpack(" MAX_OUT_SZ): + if not fileOutput and ((outsz + inLen) > maxOutSz): outF.close() outF = None outFn = outFn + 1 outsz = 0 + + if splitYear: + blkYear = get_blk_year(blk_hdr) + if blkYear > lastYear: + print("New year " + str(blkYear) + " @ " + hash_str) + lastYear = blkYear + if outF: + outF.close() + outF = None + outFn = outFn + 1 + outsz = 0 + if not outF: if fileOutput: fname = settings['output_file'] @@ -164,7 +184,13 @@ if __name__ == '__main__': settings['input'] = 'input' if 'hashlist' not in settings: settings['hashlist'] = 'hashlist.txt' + if 'split_year' not in settings: + settings['split_year'] = 0 + if 'max_out_sz' not in settings: + settings['max_out_sz'] = 1000L * 1000 * 1000 + settings['max_out_sz'] = long(settings['max_out_sz']) + settings['split_year'] = int(settings['split_year']) settings['netmagic'] = settings['netmagic'].decode('hex') if 'output_file' not in settings and 'output' not in settings: