Browse Source
Break into two steps: * Generate hash list * Build data file(s) from local bitcoind blocks/ directory. This supports building one large bootstrap.dat, or multiple smaller blocks/blkNNNNN.dat files.0.10
Jeff Garzik
10 years ago
4 changed files with 226 additions and 39 deletions
@ -1,2 +1,29 @@
@@ -1,2 +1,29 @@
|
||||
### Linearize ### |
||||
Construct a linear, no-fork, best version of the blockchain. |
||||
# Linearize |
||||
Construct a linear, no-fork, best version of the blockchain. |
||||
|
||||
## Step 1: Download hash list |
||||
|
||||
$ ./linearize-hashes.py linearize.cfg > hashlist.txt |
||||
|
||||
Required configuration file settings for linearize-hashes: |
||||
* RPC: rpcuser, rpcpassword |
||||
|
||||
Optional config file setting for linearize-hashes: |
||||
* RPC: host, port |
||||
* Block chain: min_height, max_height |
||||
|
||||
## Step 2: Copy local block data |
||||
|
||||
$ ./linearize-data.py linearize.cfg |
||||
|
||||
Required configuration file settings: |
||||
* "input": bitcoind blocks/ directory containing blkNNNNN.dat |
||||
* "hashlist": text file containing list of block hashes, linearized-hashes.py |
||||
output. |
||||
* "output_file": bootstrap.dat |
||||
or |
||||
* "output": output directory for linearized blocks/blkNNNNN.dat output |
||||
|
||||
Optional config file setting for linearize-data: |
||||
* "netmagic": network magic number |
||||
|
||||
|
@ -1,12 +1,16 @@
@@ -1,12 +1,16 @@
|
||||
|
||||
# bitcoind RPC settings |
||||
# bitcoind RPC settings (linearize-hashes) |
||||
rpcuser=someuser |
||||
rpcpassword=somepassword |
||||
host=127.0.0.1 |
||||
port=8332 |
||||
|
||||
# bootstrap.dat settings |
||||
# bootstrap.dat hashlist settings (linearize-hashes) |
||||
max_height=313000 |
||||
|
||||
# bootstrap.dat input/output settings (linearize-data) |
||||
netmagic=f9beb4d9 |
||||
max_height=279000 |
||||
output=bootstrap.dat |
||||
input=/home/example/.bitcoin/blocks |
||||
output_file=/home/example/Downloads/bootstrap.dat |
||||
hashlist=hashlist.txt |
||||
|
||||
|
@ -0,0 +1,182 @@
@@ -0,0 +1,182 @@
|
||||
#!/usr/bin/python |
||||
# |
||||
# linearize-data.py: Construct a linear, no-fork version of the chain. |
||||
# |
||||
# Copyright (c) 2013 The Bitcoin developers |
||||
# Distributed under the MIT/X11 software license, see the accompanying |
||||
# file COPYING or http://www.opensource.org/licenses/mit-license.php. |
||||
# |
||||
|
||||
import json |
||||
import struct |
||||
import re |
||||
import base64 |
||||
import httplib |
||||
import sys |
||||
import hashlib |
||||
|
||||
MAX_OUT_SZ = 128 * 1024 * 1024 |
||||
|
||||
settings = {} |
||||
|
||||
|
||||
def uint32(x): |
||||
return x & 0xffffffffL |
||||
|
||||
def bytereverse(x): |
||||
return uint32(( ((x) << 24) | (((x) << 8) & 0x00ff0000) | |
||||
(((x) >> 8) & 0x0000ff00) | ((x) >> 24) )) |
||||
|
||||
def bufreverse(in_buf): |
||||
out_words = [] |
||||
for i in range(0, len(in_buf), 4): |
||||
word = struct.unpack('@I', in_buf[i:i+4])[0] |
||||
out_words.append(struct.pack('@I', bytereverse(word))) |
||||
return ''.join(out_words) |
||||
|
||||
def wordreverse(in_buf): |
||||
out_words = [] |
||||
for i in range(0, len(in_buf), 4): |
||||
out_words.append(in_buf[i:i+4]) |
||||
out_words.reverse() |
||||
return ''.join(out_words) |
||||
|
||||
def calc_hdr_hash(rawblock): |
||||
blk_hdr = rawblock[:80] |
||||
|
||||
hash1 = hashlib.sha256() |
||||
hash1.update(blk_hdr) |
||||
hash1_o = hash1.digest() |
||||
|
||||
hash2 = hashlib.sha256() |
||||
hash2.update(hash1_o) |
||||
hash2_o = hash2.digest() |
||||
|
||||
return hash2_o |
||||
|
||||
def calc_hash_str(rawblock): |
||||
hash = calc_hdr_hash(rawblock) |
||||
hash = bufreverse(hash) |
||||
hash = wordreverse(hash) |
||||
hash_str = hash.encode('hex') |
||||
return hash_str |
||||
|
||||
def get_block_hashes(settings): |
||||
blkindex = [] |
||||
f = open(settings['hashlist'], "r") |
||||
for line in f: |
||||
line = line.rstrip() |
||||
blkindex.append(line) |
||||
|
||||
print("Read " + str(len(blkindex)) + " hashes") |
||||
|
||||
return blkindex |
||||
|
||||
def mkblockset(blkindex): |
||||
blkmap = {} |
||||
for hash in blkindex: |
||||
blkmap[hash] = True |
||||
return blkmap |
||||
|
||||
def copydata(settings, blkindex, blkset): |
||||
inFn = 0 |
||||
inF = None |
||||
outFn = 0 |
||||
outsz = 0 |
||||
outF = None |
||||
blkCount = 0 |
||||
|
||||
fileOutput = True |
||||
if 'output' in settings: |
||||
fileOutput = False |
||||
|
||||
while True: |
||||
if not inF: |
||||
fname = "%s/blk%05d.dat" % (settings['input'], inFn) |
||||
print("Input file" + fname) |
||||
inF = open(fname, "rb") |
||||
|
||||
inhdr = inF.read(8) |
||||
if (not inhdr or (inhdr[0] == "\0")): |
||||
inF.close() |
||||
inF = None |
||||
inFn = inFn + 1 |
||||
continue |
||||
|
||||
inMagic = inhdr[:4] |
||||
if (inMagic != settings['netmagic']): |
||||
print("Invalid magic:" + inMagic) |
||||
return |
||||
inLenLE = inhdr[4:] |
||||
su = struct.unpack("<I", inLenLE) |
||||
inLen = su[0] |
||||
rawblock = inF.read(inLen) |
||||
|
||||
hash_str = calc_hash_str(rawblock) |
||||
if not hash_str in blkset: |
||||
print("Skipping unknown block " + hash_str) |
||||
continue |
||||
|
||||
if not fileOutput and ((outsz + inLen) > MAX_OUT_SZ): |
||||
outF.close() |
||||
outF = None |
||||
outFn = outFn + 1 |
||||
outsz = 0 |
||||
if not outF: |
||||
if fileOutput: |
||||
fname = settings['output_file'] |
||||
else: |
||||
fname = "%s/blk%05d.dat" % (settings['output'], outFn) |
||||
print("Output file" + fname) |
||||
outF = open(fname, "wb") |
||||
|
||||
outF.write(inhdr) |
||||
outF.write(rawblock) |
||||
outsz = outsz + inLen + 8 |
||||
|
||||
blkCount = blkCount + 1 |
||||
|
||||
if (blkCount % 1000) == 0: |
||||
print("Wrote " + str(blkCount) + " blocks") |
||||
|
||||
if __name__ == '__main__': |
||||
if len(sys.argv) != 2: |
||||
print "Usage: linearize-data.py CONFIG-FILE" |
||||
sys.exit(1) |
||||
|
||||
f = open(sys.argv[1]) |
||||
for line in f: |
||||
# skip comment lines |
||||
m = re.search('^\s*#', line) |
||||
if m: |
||||
continue |
||||
|
||||
# parse key=value lines |
||||
m = re.search('^(\w+)\s*=\s*(\S.*)$', line) |
||||
if m is None: |
||||
continue |
||||
settings[m.group(1)] = m.group(2) |
||||
f.close() |
||||
|
||||
if 'netmagic' not in settings: |
||||
settings['netmagic'] = 'f9beb4d9' |
||||
if 'input' not in settings: |
||||
settings['input'] = 'input' |
||||
if 'hashlist' not in settings: |
||||
settings['hashlist'] = 'hashlist.txt' |
||||
|
||||
settings['netmagic'] = settings['netmagic'].decode('hex') |
||||
|
||||
if 'output_file' not in settings and 'output' not in settings: |
||||
print("Missing output file / directory") |
||||
sys.exit(1) |
||||
|
||||
blkindex = get_block_hashes(settings) |
||||
blkset = mkblockset(blkindex) |
||||
|
||||
if not "000000000019d6689c085ae165831e934ff763ae46a2a6c172b3f1b60a8ce26f" in blkset: |
||||
print("not found") |
||||
else: |
||||
copydata(settings, blkindex, blkset) |
||||
|
||||
|
Loading…
Reference in new issue