Merge pull request #4757

contrib/linearize/README.md

@@ -1,2 +1,32 @@
-### Linearize ###
+# Linearize
-Construct a linear, no-fork, best version of the blockchain.
+Construct a linear, no-fork, best version of the blockchain.
+
+## Step 1: Download hash list
+
+   $ ./linearize-hashes.py linearize.cfg > hashlist.txt
+
+Required configuration file settings for linearize-hashes:
+* RPC: rpcuser, rpcpassword
+
+Optional config file setting for linearize-hashes:
+* RPC: host, port
+* Block chain: min_height, max_height
+
+## Step 2: Copy local block data
+
+   $ ./linearize-data.py linearize.cfg
+
+Required configuration file settings:
+* "input": bitcoind blocks/ directory containing blkNNNNN.dat
+* "hashlist": text file containing list of block hashes, linearized-hashes.py
+output.
+* "output_file": bootstrap.dat
+      or
+* "output": output directory for linearized blocks/blkNNNNN.dat output
+
+Optional config file setting for linearize-data:
+* "netmagic": network magic number
+* "max_out_sz": maximum output file size (default 1000*1000*1000)
+* "split_year": Split files when a new year is first seen, in addition to
+reaching a maximum file size.
+

contrib/linearize/example-linearize.cfg

@@ -1,12 +1,17 @@
 
-# bitcoind RPC settings
+# bitcoind RPC settings (linearize-hashes)
 rpcuser=someuser
 rpcpassword=somepassword
 host=127.0.0.1
 port=8332
 
-# bootstrap.dat settings
+# bootstrap.dat hashlist settings (linearize-hashes)
+max_height=313000
+
+# bootstrap.dat input/output settings (linearize-data)
 netmagic=f9beb4d9
-max_height=279000
+input=/home/example/.bitcoin/blocks
-output=bootstrap.dat
+output_file=/home/example/Downloads/bootstrap.dat
+hashlist=hashlist.txt
+split_year=1
 

contrib/linearize/linearize-data.py

@@ -0,0 +1,208 @@
+#!/usr/bin/python
+#
+# linearize-data.py: Construct a linear, no-fork version of the chain.
+#
+# Copyright (c) 2013 The Bitcoin developers
+# Distributed under the MIT/X11 software license, see the accompanying
+# file COPYING or http://www.opensource.org/licenses/mit-license.php.
+#
+
+import json
+import struct
+import re
+import base64
+import httplib
+import sys
+import hashlib
+import datetime
+
+settings = {}
+
+
+def uint32(x):
+	return x & 0xffffffffL
+
+def bytereverse(x):
+	return uint32(( ((x) << 24) | (((x) << 8) & 0x00ff0000) |
+		       (((x) >> 8) & 0x0000ff00) | ((x) >> 24) ))
+
+def bufreverse(in_buf):
+	out_words = []
+	for i in range(0, len(in_buf), 4):
+		word = struct.unpack('@I', in_buf[i:i+4])[0]
+		out_words.append(struct.pack('@I', bytereverse(word)))
+	return ''.join(out_words)
+
+def wordreverse(in_buf):
+	out_words = []
+	for i in range(0, len(in_buf), 4):
+		out_words.append(in_buf[i:i+4])
+	out_words.reverse()
+	return ''.join(out_words)
+
+def calc_hdr_hash(blk_hdr):
+	hash1 = hashlib.sha256()
+	hash1.update(blk_hdr)
+	hash1_o = hash1.digest()
+
+	hash2 = hashlib.sha256()
+	hash2.update(hash1_o)
+	hash2_o = hash2.digest()
+
+	return hash2_o
+
+def calc_hash_str(blk_hdr):
+	hash = calc_hdr_hash(blk_hdr)
+	hash = bufreverse(hash)
+	hash = wordreverse(hash)
+	hash_str = hash.encode('hex')
+	return hash_str
+
+def get_blk_year(blk_hdr):
+	members = struct.unpack("<I", blk_hdr[68:68+4])
+	dt = datetime.datetime.fromtimestamp(members[0])
+	return dt.year
+
+def get_block_hashes(settings):
+	blkindex = []
+	f = open(settings['hashlist'], "r")
+	for line in f:
+		line = line.rstrip()
+		blkindex.append(line)
+
+	print("Read " + str(len(blkindex)) + " hashes")
+
+	return blkindex
+
+def mkblockset(blkindex):
+	blkmap = {}
+	for hash in blkindex:
+		blkmap[hash] = True
+	return blkmap
+
+def copydata(settings, blkindex, blkset):
+	inFn = 0
+	inF = None
+	outFn = 0
+	outsz = 0
+	outF = None
+	blkCount = 0
+
+	lastYear = 0
+	splitYear = False
+	fileOutput = True
+	maxOutSz = settings['max_out_sz']
+	if 'output' in settings:
+		fileOutput = False
+	if settings['split_year'] != 0:
+		splitYear = True
+
+	while True:
+		if not inF:
+			fname = "%s/blk%05d.dat" % (settings['input'], inFn)
+			print("Input file" + fname)
+			inF = open(fname, "rb")
+
+		inhdr = inF.read(8)
+		if (not inhdr or (inhdr[0] == "\0")):
+			inF.close()
+			inF = None
+			inFn = inFn + 1
+			continue
+
+		inMagic = inhdr[:4]
+		if (inMagic != settings['netmagic']):
+			print("Invalid magic:" + inMagic)
+			return
+		inLenLE = inhdr[4:]
+		su = struct.unpack("<I", inLenLE)
+		inLen = su[0]
+		rawblock = inF.read(inLen)
+		blk_hdr = rawblock[:80]
+
+		hash_str = calc_hash_str(blk_hdr)
+		if not hash_str in blkset:
+			print("Skipping unknown block " + hash_str)
+			continue
+
+		if not fileOutput and ((outsz + inLen) > maxOutSz):
+			outF.close()
+			outF = None
+			outFn = outFn + 1
+			outsz = 0
+
+		if splitYear:
+			blkYear = get_blk_year(blk_hdr)
+			if blkYear > lastYear:
+				print("New year " + str(blkYear) + " @ " + hash_str)
+				lastYear = blkYear
+				if outF:
+					outF.close()
+					outF = None
+					outFn = outFn + 1
+					outsz = 0
+
+		if not outF:
+			if fileOutput:
+				fname = settings['output_file']
+			else:
+				fname = "%s/blk%05d.dat" % (settings['output'], outFn)
+			print("Output file" + fname)
+			outF = open(fname, "wb")
+
+		outF.write(inhdr)
+		outF.write(rawblock)
+		outsz = outsz + inLen + 8
+
+		blkCount = blkCount + 1
+
+		if (blkCount % 1000) == 0:
+			print("Wrote " + str(blkCount) + " blocks")
+
+if __name__ == '__main__':
+	if len(sys.argv) != 2:
+		print "Usage: linearize-data.py CONFIG-FILE"
+		sys.exit(1)
+
+	f = open(sys.argv[1])
+	for line in f:
+		# skip comment lines
+		m = re.search('^\s*#', line)
+		if m:
+			continue
+
+		# parse key=value lines
+		m = re.search('^(\w+)\s*=\s*(\S.*)$', line)
+		if m is None:
+			continue
+		settings[m.group(1)] = m.group(2)
+	f.close()
+
+	if 'netmagic' not in settings:
+		settings['netmagic'] = 'f9beb4d9'
+	if 'input' not in settings:
+		settings['input'] = 'input'
+	if 'hashlist' not in settings:
+		settings['hashlist'] = 'hashlist.txt'
+	if 'split_year' not in settings:
+		settings['split_year'] = 0
+	if 'max_out_sz' not in settings:
+		settings['max_out_sz'] = 1000L * 1000 * 1000
+
+	settings['max_out_sz'] = long(settings['max_out_sz'])
+	settings['split_year'] = int(settings['split_year'])
+	settings['netmagic'] = settings['netmagic'].decode('hex')
+
+	if 'output_file' not in settings and 'output' not in settings:
+		print("Missing output file / directory")
+		sys.exit(1)
+
+	blkindex = get_block_hashes(settings)
+	blkset = mkblockset(blkindex)
+
+	if not "000000000019d6689c085ae165831e934ff763ae46a2a6c172b3f1b60a8ce26f" in blkset:
+		print("not found")
+	else:
+		copydata(settings, blkindex, blkset)
+
+

contrib/linearize/linearize-hashes.py

@@ -1,7 +1,6 @@
 #!/usr/bin/python
 #
-# linearize.py:  Construct a linear, no-fork, best version of the blockchain.
+# linearize-hashes.py:  List blocks in a linear, no-fork version of the chain.
-#
 #
 # Copyright (c) 2013 The Bitcoin developers
 # Distributed under the MIT/X11 software license, see the accompanying
@@ -15,9 +14,6 @@ import base64
 import httplib
 import sys
 
-ERR_SLEEP = 15
-MAX_NONCE = 1000000L
-
 settings = {}
 
 class BitcoinRPC:
@@ -62,34 +58,18 @@ class BitcoinRPC:
 	def getblockhash(self, index):
 		return self.rpc('getblockhash', [index])
 
-def getblock(rpc, settings, n):
+def get_block_hashes(settings):
-	hash = rpc.getblockhash(n)
-	hexdata = rpc.getblock(hash, False)
-	data = hexdata.decode('hex')
-
-	return data
-
-def get_blocks(settings):
 	rpc = BitcoinRPC(settings['host'], settings['port'],
 			 settings['rpcuser'], settings['rpcpassword'])
 
-	outf = open(settings['output'], 'ab')
-
 	for height in xrange(settings['min_height'], settings['max_height']+1):
-		data = getblock(rpc, settings, height)
+		hash = rpc.getblockhash(height)
-
-		outhdr = settings['netmagic']
-		outhdr += struct.pack("<i", len(data))
 
-		outf.write(outhdr)
+		print(hash)
-		outf.write(data)
-
-		if (height % 1000) == 0:
-			sys.stdout.write("Wrote block " + str(height) + "\n")
 
 if __name__ == '__main__':
 	if len(sys.argv) != 2:
-		print "Usage: linearize.py CONFIG-FILE"
+		print "Usage: linearize-hashes.py CONFIG-FILE"
 		sys.exit(1)
 
 	f = open(sys.argv[1])
@@ -106,10 +86,6 @@ if __name__ == '__main__':
 		settings[m.group(1)] = m.group(2)
 	f.close()
 
-	if 'netmagic' not in settings:
-		settings['netmagic'] = 'f9beb4d9'
-	if 'output' not in settings:
-		settings['output'] = 'bootstrap.dat'
 	if 'host' not in settings:
 		settings['host'] = '127.0.0.1'
 	if 'port' not in settings:
@@ -117,16 +93,14 @@ if __name__ == '__main__':
 	if 'min_height' not in settings:
 		settings['min_height'] = 0
 	if 'max_height' not in settings:
-		settings['max_height'] = 279000
+		settings['max_height'] = 313000
 	if 'rpcuser' not in settings or 'rpcpassword' not in settings:
 		print "Missing username and/or password in cfg file"
 		sys.exit(1)
 
-	settings['netmagic'] = settings['netmagic'].decode('hex')
 	settings['port'] = int(settings['port'])
 	settings['min_height'] = int(settings['min_height'])
 	settings['max_height'] = int(settings['max_height'])
 
-	get_blocks(settings)
+	get_block_hashes(settings)
-