Browse Source

Merge #8815: Backports for 0.13.1

9dfa0c8 Implement NULLDUMMY softfork (Johnson Lau)
cef633c Fix broken sendcmpct test in p2p-compactblocks.py (Suhas Daftuar)
fb8706e Add p2p test for BIP 152 (compact blocks) (Suhas Daftuar)
f6be478 Add support for compactblocks to mininode (Suhas Daftuar)
4295a7a Tests: refactor compact size serialization in mininode (Suhas Daftuar)
ff893aa Implement SipHash in Python (Pieter Wuille)
198494c Allow changing BIP9 parameters on regtest (Suhas Daftuar)
23feab1 Remove maxuploadtargets recommended minimum (Jonas Schnelli)
a5ec248 Remove createwitnessaddress (Johnson Lau)
b394a96 Add basic test for IsStandard witness transaction blinding (instagibbs)
1672225 Do not store witness txn in rejection cache (Pieter Wuille)
0.13
Wladimir J. van der Laan 8 years ago
parent
commit
a916677ace
No known key found for this signature in database
GPG Key ID: 74810B012346C9A6
  1. 3
      doc/reduce-traffic.md
  2. 2
      qa/pull-tester/rpc-tests.py
  3. 148
      qa/rpc-tests/nulldummy.py
  4. 624
      qa/rpc-tests/p2p-compactblocks.py
  5. 18
      qa/rpc-tests/p2p-segwit.py
  6. 373
      qa/rpc-tests/test_framework/mininode.py
  7. 64
      qa/rpc-tests/test_framework/siphash.py
  8. 11
      src/chainparams.cpp
  9. 5
      src/chainparams.h
  10. 36
      src/init.cpp
  11. 21
      src/main.cpp
  12. 4
      src/net.cpp
  13. 38
      src/rpc/misc.cpp
  14. 4
      src/test/hash_tests.cpp

3
doc/reduce-traffic.md

@ -19,8 +19,7 @@ This is *not* a hard limit; only a threshold to minimize the outbound @@ -19,8 +19,7 @@ This is *not* a hard limit; only a threshold to minimize the outbound
traffic. When the limit is about to be reached, the uploaded data is cut by no
longer serving historic blocks (blocks older than one week).
Keep in mind that new nodes require other nodes that are willing to serve
historic blocks. **The recommended minimum is 144 blocks per day (max. 144MB
per day)**
historic blocks.
Whitelisted peers will never be disconnected, although their traffic counts for
calculating the target.

2
qa/pull-tester/rpc-tests.py

@ -142,6 +142,8 @@ testScripts = [ @@ -142,6 +142,8 @@ testScripts = [
'segwit.py',
'importprunedfunds.py',
'signmessages.py',
'p2p-compactblocks.py',
'nulldummy.py',
]
if ENABLE_ZMQ:
testScripts.append('zmq_test.py')

148
qa/rpc-tests/nulldummy.py

@ -0,0 +1,148 @@ @@ -0,0 +1,148 @@
#!/usr/bin/env python3
# Copyright (c) 2016 The Bitcoin Core developers
# Distributed under the MIT software license, see the accompanying
# file COPYING or http://www.opensource.org/licenses/mit-license.php.
from test_framework.test_framework import ComparisonTestFramework
from test_framework.util import *
from test_framework.mininode import CTransaction, NetworkThread
from test_framework.blocktools import create_coinbase, create_block, add_witness_commitment
from test_framework.comptool import TestManager
from test_framework.script import CScript
from io import BytesIO
import time
NULLDUMMY_ERROR = "64: non-mandatory-script-verify-flag (Dummy CHECKMULTISIG argument must be zero)"
def trueDummy(tx):
scriptSig = CScript(tx.vin[0].scriptSig)
newscript = []
for i in scriptSig:
if (len(newscript) == 0):
assert(len(i) == 0)
newscript.append(b'\x51')
else:
newscript.append(i)
tx.vin[0].scriptSig = CScript(newscript)
tx.rehash()
'''
This test is meant to exercise NULLDUMMY softfork.
Connect to a single node.
Generate 2 blocks (save the coinbases for later).
Generate 427 more blocks.
[Policy/Consensus] Check that NULLDUMMY compliant transactions are accepted in the 430th block.
[Policy] Check that non-NULLDUMMY transactions are rejected before activation.
[Consensus] Check that the new NULLDUMMY rules are not enforced on the 431st block.
[Policy/Consensus] Check that the new NULLDUMMY rules are enforced on the 432nd block.
'''
class NULLDUMMYTest(ComparisonTestFramework):
def __init__(self):
super().__init__()
self.num_nodes = 1
def setup_network(self):
# Must set the blockversion for this test
self.nodes = start_nodes(self.num_nodes, self.options.tmpdir,
extra_args=[['-debug', '-whitelist=127.0.0.1', '-walletprematurewitness']])
def run_test(self):
self.address = self.nodes[0].getnewaddress()
self.ms_address = self.nodes[0].addmultisigaddress(1,[self.address])
self.wit_address = self.nodes[0].addwitnessaddress(self.address)
self.wit_ms_address = self.nodes[0].addwitnessaddress(self.ms_address)
test = TestManager(self, self.options.tmpdir)
test.add_all_connections(self.nodes)
NetworkThread().start() # Start up network handling in another thread
self.coinbase_blocks = self.nodes[0].generate(2) # Block 2
coinbase_txid = []
for i in self.coinbase_blocks:
coinbase_txid.append(self.nodes[0].getblock(i)['tx'][0])
self.nodes[0].generate(427) # Block 429
self.lastblockhash = self.nodes[0].getbestblockhash()
self.tip = int("0x" + self.lastblockhash, 0)
self.lastblockheight = 429
self.lastblocktime = int(time.time()) + 429
print ("Test 1: NULLDUMMY compliant base transactions should be accepted to mempool and mined before activation [430]")
test1txs = [self.create_transaction(self.nodes[0], coinbase_txid[0], self.ms_address, 49)]
txid1 = self.tx_submit(self.nodes[0], test1txs[0])
test1txs.append(self.create_transaction(self.nodes[0], txid1, self.ms_address, 48))
txid2 = self.tx_submit(self.nodes[0], test1txs[1])
test1txs.append(self.create_transaction(self.nodes[0], coinbase_txid[1], self.wit_ms_address, 49))
txid3 = self.tx_submit(self.nodes[0], test1txs[2])
self.block_submit(self.nodes[0], test1txs, False, True)
print ("Test 2: Non-NULLDUMMY base multisig transaction should not be accepted to mempool before activation")
test2tx = self.create_transaction(self.nodes[0], txid2, self.ms_address, 48)
trueDummy(test2tx)
txid4 = self.tx_submit(self.nodes[0], test2tx, NULLDUMMY_ERROR)
print ("Test 3: Non-NULLDUMMY base transactions should be accepted in a block before activation [431]")
self.block_submit(self.nodes[0], [test2tx], False, True)
print ("Test 4: Non-NULLDUMMY base multisig transaction is invalid after activation")
test4tx = self.create_transaction(self.nodes[0], txid4, self.address, 47)
test6txs=[CTransaction(test4tx)]
trueDummy(test4tx)
self.tx_submit(self.nodes[0], test4tx, NULLDUMMY_ERROR)
self.block_submit(self.nodes[0], [test4tx])
print ("Test 5: Non-NULLDUMMY P2WSH multisig transaction invalid after activation")
test5tx = self.create_transaction(self.nodes[0], txid3, self.wit_address, 48)
test6txs.append(CTransaction(test5tx))
test5tx.wit.vtxinwit[0].scriptWitness.stack[0] = b'\x01'
self.tx_submit(self.nodes[0], test5tx, NULLDUMMY_ERROR)
self.block_submit(self.nodes[0], [test5tx], True)
print ("Test 6: NULLDUMMY compliant base/witness transactions should be accepted to mempool and in block after activation [432]")
for i in test6txs:
self.tx_submit(self.nodes[0], i)
self.block_submit(self.nodes[0], test6txs, True, True)
def create_transaction(self, node, txid, to_address, amount):
inputs = [{ "txid" : txid, "vout" : 0}]
outputs = { to_address : amount }
rawtx = node.createrawtransaction(inputs, outputs)
signresult = node.signrawtransaction(rawtx)
tx = CTransaction()
f = BytesIO(hex_str_to_bytes(signresult['hex']))
tx.deserialize(f)
return tx
def tx_submit(self, node, tx, msg = ""):
tx.rehash()
try:
node.sendrawtransaction(bytes_to_hex_str(tx.serialize_with_witness()), True)
except JSONRPCException as exp:
assert_equal(exp.error["message"], msg)
return tx.hash
def block_submit(self, node, txs, witness = False, accept = False):
block = create_block(self.tip, create_coinbase(self.lastblockheight + 1), self.lastblocktime + 1)
block.nVersion = 4
for tx in txs:
tx.rehash()
block.vtx.append(tx)
block.hashMerkleRoot = block.calc_merkle_root()
witness and add_witness_commitment(block)
block.rehash()
block.solve()
node.submitblock(bytes_to_hex_str(block.serialize(True)))
if (accept):
assert_equal(node.getbestblockhash(), block.hash)
self.tip = block.sha256
self.lastblockhash = block.hash
self.lastblocktime += 1
self.lastblockheight += 1
else:
assert_equal(node.getbestblockhash(), self.lastblockhash)
if __name__ == '__main__':
NULLDUMMYTest().main()

624
qa/rpc-tests/p2p-compactblocks.py

@ -0,0 +1,624 @@ @@ -0,0 +1,624 @@
#!/usr/bin/env python3
# Copyright (c) 2016 The Bitcoin Core developers
# Distributed under the MIT software license, see the accompanying
# file COPYING or http://www.opensource.org/licenses/mit-license.php.
from test_framework.mininode import *
from test_framework.test_framework import BitcoinTestFramework
from test_framework.util import *
from test_framework.blocktools import create_block, create_coinbase
from test_framework.siphash import siphash256
from test_framework.script import CScript, OP_TRUE
'''
CompactBlocksTest -- test compact blocks (BIP 152)
'''
# TestNode: A peer we use to send messages to bitcoind, and store responses.
class TestNode(SingleNodeConnCB):
def __init__(self):
SingleNodeConnCB.__init__(self)
self.last_sendcmpct = None
self.last_headers = None
self.last_inv = None
self.last_cmpctblock = None
self.block_announced = False
self.last_getdata = None
self.last_getblocktxn = None
self.last_block = None
self.last_blocktxn = None
def on_sendcmpct(self, conn, message):
self.last_sendcmpct = message
def on_block(self, conn, message):
self.last_block = message
def on_cmpctblock(self, conn, message):
self.last_cmpctblock = message
self.block_announced = True
def on_headers(self, conn, message):
self.last_headers = message
self.block_announced = True
def on_inv(self, conn, message):
self.last_inv = message
self.block_announced = True
def on_getdata(self, conn, message):
self.last_getdata = message
def on_getblocktxn(self, conn, message):
self.last_getblocktxn = message
def on_blocktxn(self, conn, message):
self.last_blocktxn = message
# Requires caller to hold mininode_lock
def received_block_announcement(self):
return self.block_announced
def clear_block_announcement(self):
with mininode_lock:
self.block_announced = False
self.last_inv = None
self.last_headers = None
self.last_cmpctblock = None
def get_headers(self, locator, hashstop):
msg = msg_getheaders()
msg.locator.vHave = locator
msg.hashstop = hashstop
self.connection.send_message(msg)
def send_header_for_blocks(self, new_blocks):
headers_message = msg_headers()
headers_message.headers = [CBlockHeader(b) for b in new_blocks]
self.send_message(headers_message)
def request_headers_and_sync(self, locator, hashstop=0):
self.clear_block_announcement()
self.get_headers(locator, hashstop)
assert(wait_until(self.received_block_announcement, timeout=30))
assert(self.received_block_announcement())
self.clear_block_announcement()
class CompactBlocksTest(BitcoinTestFramework):
def __init__(self):
super().__init__()
self.setup_clean_chain = True
self.num_nodes = 1
self.utxos = []
def setup_network(self):
self.nodes = []
# Turn off segwit in this test, as compact blocks don't currently work
# with segwit. (After BIP 152 is updated to support segwit, we can
# test behavior with and without segwit enabled by adding a second node
# to the test.)
self.nodes = start_nodes(self.num_nodes, self.options.tmpdir, [["-debug", "-logtimemicros=1", "-bip9params=segwit:0:0"]])
def build_block_on_tip(self):
height = self.nodes[0].getblockcount()
tip = self.nodes[0].getbestblockhash()
mtp = self.nodes[0].getblockheader(tip)['mediantime']
block = create_block(int(tip, 16), create_coinbase(height + 1), mtp + 1)
block.solve()
return block
# Create 10 more anyone-can-spend utxo's for testing.
def make_utxos(self):
block = self.build_block_on_tip()
self.test_node.send_and_ping(msg_block(block))
assert(int(self.nodes[0].getbestblockhash(), 16) == block.sha256)
self.nodes[0].generate(100)
total_value = block.vtx[0].vout[0].nValue
out_value = total_value // 10
tx = CTransaction()
tx.vin.append(CTxIn(COutPoint(block.vtx[0].sha256, 0), b''))
for i in range(10):
tx.vout.append(CTxOut(out_value, CScript([OP_TRUE])))
tx.rehash()
block2 = self.build_block_on_tip()
block2.vtx.append(tx)
block2.hashMerkleRoot = block2.calc_merkle_root()
block2.solve()
self.test_node.send_and_ping(msg_block(block2))
assert_equal(int(self.nodes[0].getbestblockhash(), 16), block2.sha256)
self.utxos.extend([[tx.sha256, i, out_value] for i in range(10)])
return
# Test "sendcmpct":
# - No compact block announcements or getdata(MSG_CMPCT_BLOCK) unless
# sendcmpct is sent.
# - If sendcmpct is sent with version > 1, the message is ignored.
# - If sendcmpct is sent with boolean 0, then block announcements are not
# made with compact blocks.
# - If sendcmpct is then sent with boolean 1, then new block announcements
# are made with compact blocks.
def test_sendcmpct(self):
print("Testing SENDCMPCT p2p message... ")
# Make sure we get a version 0 SENDCMPCT message from our peer
def received_sendcmpct():
return (self.test_node.last_sendcmpct is not None)
got_message = wait_until(received_sendcmpct, timeout=30)
assert(received_sendcmpct())
assert(got_message)
assert_equal(self.test_node.last_sendcmpct.version, 1)
tip = int(self.nodes[0].getbestblockhash(), 16)
def check_announcement_of_new_block(node, peer, predicate):
peer.clear_block_announcement()
node.generate(1)
got_message = wait_until(lambda: peer.block_announced, timeout=30)
assert(peer.block_announced)
assert(got_message)
with mininode_lock:
assert(predicate(peer))
# We shouldn't get any block announcements via cmpctblock yet.
check_announcement_of_new_block(self.nodes[0], self.test_node, lambda p: p.last_cmpctblock is None)
# Try one more time, this time after requesting headers.
self.test_node.request_headers_and_sync(locator=[tip])
check_announcement_of_new_block(self.nodes[0], self.test_node, lambda p: p.last_cmpctblock is None and p.last_inv is not None)
# Test a few ways of using sendcmpct that should NOT
# result in compact block announcements.
# Before each test, sync the headers chain.
self.test_node.request_headers_and_sync(locator=[tip])
# Now try a SENDCMPCT message with too-high version
sendcmpct = msg_sendcmpct()
sendcmpct.version = 2
self.test_node.send_and_ping(sendcmpct)
check_announcement_of_new_block(self.nodes[0], self.test_node, lambda p: p.last_cmpctblock is None)
# Headers sync before next test.
self.test_node.request_headers_and_sync(locator=[tip])
# Now try a SENDCMPCT message with valid version, but announce=False
self.test_node.send_and_ping(msg_sendcmpct())
check_announcement_of_new_block(self.nodes[0], self.test_node, lambda p: p.last_cmpctblock is None)
# Headers sync before next test.
self.test_node.request_headers_and_sync(locator=[tip])
# Finally, try a SENDCMPCT message with announce=True
sendcmpct.version = 1
sendcmpct.announce = True
self.test_node.send_and_ping(sendcmpct)
check_announcement_of_new_block(self.nodes[0], self.test_node, lambda p: p.last_cmpctblock is not None)
# Try one more time (no headers sync should be needed!)
check_announcement_of_new_block(self.nodes[0], self.test_node, lambda p: p.last_cmpctblock is not None)
# Try one more time, after turning on sendheaders
self.test_node.send_and_ping(msg_sendheaders())
check_announcement_of_new_block(self.nodes[0], self.test_node, lambda p: p.last_cmpctblock is not None)
# Now turn off announcements
sendcmpct.announce = False
self.test_node.send_and_ping(sendcmpct)
check_announcement_of_new_block(self.nodes[0], self.test_node, lambda p: p.last_cmpctblock is None and p.last_headers is not None)
# This test actually causes bitcoind to (reasonably!) disconnect us, so do this last.
def test_invalid_cmpctblock_message(self):
print("Testing invalid index in cmpctblock message...")
self.nodes[0].generate(101)
block = self.build_block_on_tip()
cmpct_block = P2PHeaderAndShortIDs()
cmpct_block.header = CBlockHeader(block)
cmpct_block.prefilled_txn_length = 1
# This index will be too high
prefilled_txn = PrefilledTransaction(1, block.vtx[0])
cmpct_block.prefilled_txn = [prefilled_txn]
self.test_node.send_and_ping(msg_cmpctblock(cmpct_block))
assert(int(self.nodes[0].getbestblockhash(), 16) == block.hashPrevBlock)
# Compare the generated shortids to what we expect based on BIP 152, given
# bitcoind's choice of nonce.
def test_compactblock_construction(self):
print("Testing compactblock headers and shortIDs are correct...")
# Generate a bunch of transactions.
self.nodes[0].generate(101)
num_transactions = 25
address = self.nodes[0].getnewaddress()
for i in range(num_transactions):
self.nodes[0].sendtoaddress(address, 0.1)
# Now mine a block, and look at the resulting compact block.
self.test_node.clear_block_announcement()
block_hash = int(self.nodes[0].generate(1)[0], 16)
# Store the raw block in our internal format.
block = FromHex(CBlock(), self.nodes[0].getblock("%02x" % block_hash, False))
[tx.calc_sha256() for tx in block.vtx]
block.rehash()
# Don't care which type of announcement came back for this test; just
# request the compact block if we didn't get one yet.
wait_until(self.test_node.received_block_announcement, timeout=30)
with mininode_lock:
if self.test_node.last_cmpctblock is None:
self.test_node.clear_block_announcement()
inv = CInv(4, block_hash) # 4 == "CompactBlock"
self.test_node.send_message(msg_getdata([inv]))
wait_until(self.test_node.received_block_announcement, timeout=30)
# Now we should have the compactblock
header_and_shortids = None
with mininode_lock:
assert(self.test_node.last_cmpctblock is not None)
# Convert the on-the-wire representation to absolute indexes
header_and_shortids = HeaderAndShortIDs(self.test_node.last_cmpctblock.header_and_shortids)
# Check that we got the right block!
header_and_shortids.header.calc_sha256()
assert_equal(header_and_shortids.header.sha256, block_hash)
# Make sure the prefilled_txn appears to have included the coinbase
assert(len(header_and_shortids.prefilled_txn) >= 1)
assert_equal(header_and_shortids.prefilled_txn[0].index, 0)
# Check that all prefilled_txn entries match what's in the block.
for entry in header_and_shortids.prefilled_txn:
entry.tx.calc_sha256()
assert_equal(entry.tx.sha256, block.vtx[entry.index].sha256)
# Check that the cmpctblock message announced all the transactions.
assert_equal(len(header_and_shortids.prefilled_txn) + len(header_and_shortids.shortids), len(block.vtx))
# And now check that all the shortids are as expected as well.
# Determine the siphash keys to use.
[k0, k1] = header_and_shortids.get_siphash_keys()
index = 0
while index < len(block.vtx):
if (len(header_and_shortids.prefilled_txn) > 0 and
header_and_shortids.prefilled_txn[0].index == index):
# Already checked prefilled transactions above
header_and_shortids.prefilled_txn.pop(0)
else:
shortid = calculate_shortid(k0, k1, block.vtx[index].sha256)
assert_equal(shortid, header_and_shortids.shortids[0])
header_and_shortids.shortids.pop(0)
index += 1
# Test that bitcoind requests compact blocks when we announce new blocks
# via header or inv, and that responding to getblocktxn causes the block
# to be successfully reconstructed.
def test_compactblock_requests(self):
print("Testing compactblock requests... ")
# Try announcing a block with an inv or header, expect a compactblock
# request
for announce in ["inv", "header"]:
block = self.build_block_on_tip()
with mininode_lock:
self.test_node.last_getdata = None
if announce == "inv":
self.test_node.send_message(msg_inv([CInv(2, block.sha256)]))
else:
self.test_node.send_header_for_blocks([block])
success = wait_until(lambda: self.test_node.last_getdata is not None, timeout=30)
assert(success)
assert_equal(len(self.test_node.last_getdata.inv), 1)
assert_equal(self.test_node.last_getdata.inv[0].type, 4)
assert_equal(self.test_node.last_getdata.inv[0].hash, block.sha256)
# Send back a compactblock message that omits the coinbase
comp_block = HeaderAndShortIDs()
comp_block.header = CBlockHeader(block)
comp_block.nonce = 0
comp_block.shortids = [1] # this is useless, and wrong
self.test_node.send_and_ping(msg_cmpctblock(comp_block.to_p2p()))
assert_equal(int(self.nodes[0].getbestblockhash(), 16), block.hashPrevBlock)
# Expect a getblocktxn message.
with mininode_lock:
assert(self.test_node.last_getblocktxn is not None)
absolute_indexes = self.test_node.last_getblocktxn.block_txn_request.to_absolute()
assert_equal(absolute_indexes, [0]) # should be a coinbase request
# Send the coinbase, and verify that the tip advances.
msg = msg_blocktxn()
msg.block_transactions.blockhash = block.sha256
msg.block_transactions.transactions = [block.vtx[0]]
self.test_node.send_and_ping(msg)
assert_equal(int(self.nodes[0].getbestblockhash(), 16), block.sha256)
# Create a chain of transactions from given utxo, and add to a new block.
def build_block_with_transactions(self, utxo, num_transactions):
block = self.build_block_on_tip()
for i in range(num_transactions):
tx = CTransaction()
tx.vin.append(CTxIn(COutPoint(utxo[0], utxo[1]), b''))
tx.vout.append(CTxOut(utxo[2] - 1000, CScript([OP_TRUE])))
tx.rehash()
utxo = [tx.sha256, 0, tx.vout[0].nValue]
block.vtx.append(tx)
block.hashMerkleRoot = block.calc_merkle_root()
block.solve()
return block
# Test that we only receive getblocktxn requests for transactions that the
# node needs, and that responding to them causes the block to be
# reconstructed.
def test_getblocktxn_requests(self):
print("Testing getblocktxn requests...")
# First try announcing compactblocks that won't reconstruct, and verify
# that we receive getblocktxn messages back.
utxo = self.utxos.pop(0)
block = self.build_block_with_transactions(utxo, 5)
self.utxos.append([block.vtx[-1].sha256, 0, block.vtx[-1].vout[0].nValue])
comp_block = HeaderAndShortIDs()
comp_block.initialize_from_block(block)
self.test_node.send_and_ping(msg_cmpctblock(comp_block.to_p2p()))
with mininode_lock:
assert(self.test_node.last_getblocktxn is not None)
absolute_indexes = self.test_node.last_getblocktxn.block_txn_request.to_absolute()
assert_equal(absolute_indexes, [1, 2, 3, 4, 5])
msg = msg_blocktxn()
msg.block_transactions = BlockTransactions(block.sha256, block.vtx[1:])
self.test_node.send_and_ping(msg)
assert_equal(int(self.nodes[0].getbestblockhash(), 16), block.sha256)
utxo = self.utxos.pop(0)
block = self.build_block_with_transactions(utxo, 5)
self.utxos.append([block.vtx[-1].sha256, 0, block.vtx[-1].vout[0].nValue])
# Now try interspersing the prefilled transactions
comp_block.initialize_from_block(block, prefill_list=[0, 1, 5])
self.test_node.send_and_ping(msg_cmpctblock(comp_block.to_p2p()))
with mininode_lock:
assert(self.test_node.last_getblocktxn is not None)
absolute_indexes = self.test_node.last_getblocktxn.block_txn_request.to_absolute()
assert_equal(absolute_indexes, [2, 3, 4])
msg.block_transactions = BlockTransactions(block.sha256, block.vtx[2:5])
self.test_node.send_and_ping(msg)
assert_equal(int(self.nodes[0].getbestblockhash(), 16), block.sha256)
# Now try giving one transaction ahead of time.
utxo = self.utxos.pop(0)
block = self.build_block_with_transactions(utxo, 5)
self.utxos.append([block.vtx[-1].sha256, 0, block.vtx[-1].vout[0].nValue])
self.test_node.send_and_ping(msg_tx(block.vtx[1]))
assert(block.vtx[1].hash in self.nodes[0].getrawmempool())
# Prefill 4 out of the 6 transactions, and verify that only the one
# that was not in the mempool is requested.
comp_block.initialize_from_block(block, prefill_list=[0, 2, 3, 4])
self.test_node.send_and_ping(msg_cmpctblock(comp_block.to_p2p()))
with mininode_lock:
assert(self.test_node.last_getblocktxn is not None)
absolute_indexes = self.test_node.last_getblocktxn.block_txn_request.to_absolute()
assert_equal(absolute_indexes, [5])
msg.block_transactions = BlockTransactions(block.sha256, [block.vtx[5]])
self.test_node.send_and_ping(msg)
assert_equal(int(self.nodes[0].getbestblockhash(), 16), block.sha256)
# Now provide all transactions to the node before the block is
# announced and verify reconstruction happens immediately.
utxo = self.utxos.pop(0)
block = self.build_block_with_transactions(utxo, 10)
self.utxos.append([block.vtx[-1].sha256, 0, block.vtx[-1].vout[0].nValue])
for tx in block.vtx[1:]:
self.test_node.send_message(msg_tx(tx))
self.test_node.sync_with_ping()
# Make sure all transactions were accepted.
mempool = self.nodes[0].getrawmempool()
for tx in block.vtx[1:]:
assert(tx.hash in mempool)
# Clear out last request.
with mininode_lock:
self.test_node.last_getblocktxn = None
# Send compact block
comp_block.initialize_from_block(block, prefill_list=[0])
self.test_node.send_and_ping(msg_cmpctblock(comp_block.to_p2p()))
with mininode_lock:
# Shouldn't have gotten a request for any transaction
assert(self.test_node.last_getblocktxn is None)
# Tip should have updated
assert_equal(int(self.nodes[0].getbestblockhash(), 16), block.sha256)
# Incorrectly responding to a getblocktxn shouldn't cause the block to be
# permanently failed.
def test_incorrect_blocktxn_response(self):
print("Testing handling of incorrect blocktxn responses...")
if (len(self.utxos) == 0):
self.make_utxos()
utxo = self.utxos.pop(0)
block = self.build_block_with_transactions(utxo, 10)
self.utxos.append([block.vtx[-1].sha256, 0, block.vtx[-1].vout[0].nValue])
# Relay the first 5 transactions from the block in advance
for tx in block.vtx[1:6]:
self.test_node.send_message(msg_tx(tx))
self.test_node.sync_with_ping()
# Make sure all transactions were accepted.
mempool = self.nodes[0].getrawmempool()
for tx in block.vtx[1:6]:
assert(tx.hash in mempool)
# Send compact block
comp_block = HeaderAndShortIDs()
comp_block.initialize_from_block(block, prefill_list=[0])
self.test_node.send_and_ping(msg_cmpctblock(comp_block.to_p2p()))
absolute_indexes = []
with mininode_lock:
assert(self.test_node.last_getblocktxn is not None)
absolute_indexes = self.test_node.last_getblocktxn.block_txn_request.to_absolute()
assert_equal(absolute_indexes, [6, 7, 8, 9, 10])
# Now give an incorrect response.
# Note that it's possible for bitcoind to be smart enough to know we're
# lying, since it could check to see if the shortid matches what we're
# sending, and eg disconnect us for misbehavior. If that behavior
# change were made, we could just modify this test by having a
# different peer provide the block further down, so that we're still
# verifying that the block isn't marked bad permanently. This is good
# enough for now.
msg = msg_blocktxn()
msg.block_transactions = BlockTransactions(block.sha256, [block.vtx[5]] + block.vtx[7:])
self.test_node.send_and_ping(msg)
# Tip should not have updated
assert_equal(int(self.nodes[0].getbestblockhash(), 16), block.hashPrevBlock)
# We should receive a getdata request
success = wait_until(lambda: self.test_node.last_getdata is not None, timeout=10)
assert(success)
assert_equal(len(self.test_node.last_getdata.inv), 1)
assert_equal(self.test_node.last_getdata.inv[0].type, 2)
assert_equal(self.test_node.last_getdata.inv[0].hash, block.sha256)
# Deliver the block
self.test_node.send_and_ping(msg_block(block))
assert_equal(int(self.nodes[0].getbestblockhash(), 16), block.sha256)
def test_getblocktxn_handler(self):
print("Testing getblocktxn handler...")
# bitcoind won't respond for blocks whose height is more than 15 blocks
# deep.
MAX_GETBLOCKTXN_DEPTH = 15
chain_height = self.nodes[0].getblockcount()
current_height = chain_height
while (current_height >= chain_height - MAX_GETBLOCKTXN_DEPTH):
block_hash = self.nodes[0].getblockhash(current_height)
block = FromHex(CBlock(), self.nodes[0].getblock(block_hash, False))
msg = msg_getblocktxn()
msg.block_txn_request = BlockTransactionsRequest(int(block_hash, 16), [])
num_to_request = random.randint(1, len(block.vtx))
msg.block_txn_request.from_absolute(sorted(random.sample(range(len(block.vtx)), num_to_request)))
self.test_node.send_message(msg)
success = wait_until(lambda: self.test_node.last_blocktxn is not None, timeout=10)
assert(success)
[tx.calc_sha256() for tx in block.vtx]
with mininode_lock:
assert_equal(self.test_node.last_blocktxn.block_transactions.blockhash, int(block_hash, 16))
all_indices = msg.block_txn_request.to_absolute()
for index in all_indices:
tx = self.test_node.last_blocktxn.block_transactions.transactions.pop(0)
tx.calc_sha256()
assert_equal(tx.sha256, block.vtx[index].sha256)
self.test_node.last_blocktxn = None
current_height -= 1
# Next request should be ignored, as we're past the allowed depth.
block_hash = self.nodes[0].getblockhash(current_height)
msg.block_txn_request = BlockTransactionsRequest(int(block_hash, 16), [0])
self.test_node.send_and_ping(msg)
with mininode_lock:
assert_equal(self.test_node.last_blocktxn, None)
def test_compactblocks_not_at_tip(self):
print("Testing compactblock requests/announcements not at chain tip...")
# Test that requesting old compactblocks doesn't work.
MAX_CMPCTBLOCK_DEPTH = 11
new_blocks = []
for i in range(MAX_CMPCTBLOCK_DEPTH):
self.test_node.clear_block_announcement()
new_blocks.append(self.nodes[0].generate(1)[0])
wait_until(self.test_node.received_block_announcement, timeout=30)
self.test_node.clear_block_announcement()
self.test_node.send_message(msg_getdata([CInv(4, int(new_blocks[0], 16))]))
success = wait_until(lambda: self.test_node.last_cmpctblock is not None, timeout=30)
assert(success)
self.test_node.clear_block_announcement()
self.nodes[0].generate(1)
wait_until(self.test_node.received_block_announcement, timeout=30)
self.test_node.clear_block_announcement()
self.test_node.send_message(msg_getdata([CInv(4, int(new_blocks[0], 16))]))
success = wait_until(lambda: self.test_node.last_block is not None, timeout=30)
assert(success)
with mininode_lock:
self.test_node.last_block.block.calc_sha256()
assert_equal(self.test_node.last_block.block.sha256, int(new_blocks[0], 16))
# Generate an old compactblock, and verify that it's not accepted.
cur_height = self.nodes[0].getblockcount()
hashPrevBlock = int(self.nodes[0].getblockhash(cur_height-5), 16)
block = self.build_block_on_tip()
block.hashPrevBlock = hashPrevBlock
block.solve()
comp_block = HeaderAndShortIDs()
comp_block.initialize_from_block(block)
self.test_node.send_and_ping(msg_cmpctblock(comp_block.to_p2p()))
tips = self.nodes[0].getchaintips()
found = False
for x in tips:
if x["hash"] == block.hash:
assert_equal(x["status"], "headers-only")
found = True
break
assert(found)
# Requesting this block via getblocktxn should silently fail
# (to avoid fingerprinting attacks).
msg = msg_getblocktxn()
msg.block_txn_request = BlockTransactionsRequest(block.sha256, [0])
with mininode_lock:
self.test_node.last_blocktxn = None
self.test_node.send_and_ping(msg)
with mininode_lock:
assert(self.test_node.last_blocktxn is None)
def run_test(self):
# Setup the p2p connections and start up the network thread.
self.test_node = TestNode()
connections = []
connections.append(NodeConn('127.0.0.1', p2p_port(0), self.nodes[0], self.test_node))
self.test_node.add_connection(connections[0])
NetworkThread().start() # Start up network handling in another thread
# Test logic begins here
self.test_node.wait_for_verack()
# We will need UTXOs to construct transactions in later tests.
self.make_utxos()
self.test_sendcmpct()
self.test_compactblock_construction()
self.test_compactblock_requests()
self.test_getblocktxn_requests()
self.test_getblocktxn_handler()
self.test_compactblocks_not_at_tip()
self.test_incorrect_blocktxn_response()
self.test_invalid_cmpctblock_message()
if __name__ == '__main__':
CompactBlocksTest().main()

18
qa/rpc-tests/p2p-segwit.py

@ -965,8 +965,24 @@ class SegWitTest(BitcoinTestFramework): @@ -965,8 +965,24 @@ class SegWitTest(BitcoinTestFramework):
tx3 = CTransaction()
tx3.vin.append(CTxIn(COutPoint(tx2.sha256, 0), b""))
tx3.vout.append(CTxOut(tx2.vout[0].nValue-1000, CScript([OP_TRUE])))
tx3.wit.vtxinwit.append(CTxInWitness())
# Add too-large for IsStandard witness and check that it does not enter reject filter
p2sh_program = CScript([OP_TRUE])
p2sh_pubkey = hash160(p2sh_program)
witness_program2 = CScript([b'a'*400000])
tx3.vout.append(CTxOut(tx2.vout[0].nValue-1000, CScript([OP_HASH160, p2sh_pubkey, OP_EQUAL])))
tx3.wit.vtxinwit[0].scriptWitness.stack = [witness_program2]
tx3.rehash()
# Node will not be blinded to the transaction
self.std_node.announce_tx_and_wait_for_getdata(tx3)
self.std_node.test_transaction_acceptance(tx3, True, False, b'tx-size')
self.std_node.announce_tx_and_wait_for_getdata(tx3)
self.std_node.test_transaction_acceptance(tx3, True, False, b'tx-size')
# Remove witness stuffing, instead add extra witness push on stack
tx3.vout[0] = CTxOut(tx2.vout[0].nValue-1000, CScript([OP_TRUE]))
tx3.wit.vtxinwit[0].scriptWitness.stack = [CScript([CScriptNum(1)]), witness_program ]
tx3.rehash()

373
qa/rpc-tests/test_framework/mininode.py

@ -36,9 +36,10 @@ from threading import RLock @@ -36,9 +36,10 @@ from threading import RLock
from threading import Thread
import logging
import copy
from test_framework.siphash import siphash256
BIP0031_VERSION = 60000
MY_VERSION = 60001 # past bip-31 for ping/pong
MY_VERSION = 70014 # past bip-31 for ping/pong
MY_SUBVERSION = b"/python-mininode-tester:0.0.3/"
MAX_INV_SZ = 50000
@ -52,7 +53,7 @@ NODE_BLOOM = (1 << 2) @@ -52,7 +53,7 @@ NODE_BLOOM = (1 << 2)
NODE_WITNESS = (1 << 3)
# Keep our own socket map for asyncore, so that we can track disconnects
# ourselves (to workaround an issue with closing an asyncore socket when
# ourselves (to workaround an issue with closing an asyncore socket when
# using select)
mininode_socket_map = dict()
@ -74,8 +75,19 @@ def ripemd160(s): @@ -74,8 +75,19 @@ def ripemd160(s):
def hash256(s):
return sha256(sha256(s))
def ser_compact_size(l):
r = b""
if l < 253:
r = struct.pack("B", l)
elif l < 0x10000:
r = struct.pack("<BH", 253, l)
elif l < 0x100000000:
r = struct.pack("<BI", 254, l)
else:
r = struct.pack("<BQ", 255, l)
return r
def deser_string(f):
def deser_compact_size(f):
nit = struct.unpack("<B", f.read(1))[0]
if nit == 253:
nit = struct.unpack("<H", f.read(2))[0]
@ -83,16 +95,14 @@ def deser_string(f): @@ -83,16 +95,14 @@ def deser_string(f):
nit = struct.unpack("<I", f.read(4))[0]
elif nit == 255:
nit = struct.unpack("<Q", f.read(8))[0]
return nit
def deser_string(f):
nit = deser_compact_size(f)
return f.read(nit)
def ser_string(s):
if len(s) < 253:
return struct.pack("B", len(s)) + s
elif len(s) < 0x10000:
return struct.pack("<BH", 253, len(s)) + s
elif len(s) < 0x100000000:
return struct.pack("<BI", 254, len(s)) + s
return struct.pack("<BQ", 255, len(s)) + s
return ser_compact_size(len(s)) + s
def deser_uint256(f):
r = 0
@ -125,13 +135,7 @@ def uint256_from_compact(c): @@ -125,13 +135,7 @@ def uint256_from_compact(c):
def deser_vector(f, c):
nit = struct.unpack("<B", f.read(1))[0]
if nit == 253:
nit = struct.unpack("<H", f.read(2))[0]
elif nit == 254:
nit = struct.unpack("<I", f.read(4))[0]
elif nit == 255:
nit = struct.unpack("<Q", f.read(8))[0]
nit = deser_compact_size(f)
r = []
for i in range(nit):
t = c()
@ -144,15 +148,7 @@ def deser_vector(f, c): @@ -144,15 +148,7 @@ def deser_vector(f, c):
# entries in the vector (we use this for serializing the vector of transactions
# for a witness block).
def ser_vector(l, ser_function_name=None):
r = b""
if len(l) < 253:
r = struct.pack("B", len(l))
elif len(l) < 0x10000:
r = struct.pack("<BH", 253, len(l))
elif len(l) < 0x100000000:
r = struct.pack("<BI", 254, len(l))
else:
r = struct.pack("<BQ", 255, len(l))
r = ser_compact_size(len(l))
for i in l:
if ser_function_name:
r += getattr(i, ser_function_name)()
@ -162,13 +158,7 @@ def ser_vector(l, ser_function_name=None): @@ -162,13 +158,7 @@ def ser_vector(l, ser_function_name=None):
def deser_uint256_vector(f):
nit = struct.unpack("<B", f.read(1))[0]
if nit == 253:
nit = struct.unpack("<H", f.read(2))[0]
elif nit == 254:
nit = struct.unpack("<I", f.read(4))[0]
elif nit == 255:
nit = struct.unpack("<Q", f.read(8))[0]
nit = deser_compact_size(f)
r = []
for i in range(nit):
t = deser_uint256(f)
@ -177,28 +167,14 @@ def deser_uint256_vector(f): @@ -177,28 +167,14 @@ def deser_uint256_vector(f):
def ser_uint256_vector(l):
r = b""
if len(l) < 253:
r = struct.pack("B", len(l))
elif len(l) < 0x10000:
r = struct.pack("<BH", 253, len(l))
elif len(l) < 0x100000000:
r = struct.pack("<BI", 254, len(l))
else:
r = struct.pack("<BQ", 255, len(l))
r = ser_compact_size(len(l))
for i in l:
r += ser_uint256(i)
return r
def deser_string_vector(f):
nit = struct.unpack("<B", f.read(1))[0]
if nit == 253:
nit = struct.unpack("<H", f.read(2))[0]
elif nit == 254:
nit = struct.unpack("<I", f.read(4))[0]
elif nit == 255:
nit = struct.unpack("<Q", f.read(8))[0]
nit = deser_compact_size(f)
r = []
for i in range(nit):
t = deser_string(f)
@ -207,28 +183,14 @@ def deser_string_vector(f): @@ -207,28 +183,14 @@ def deser_string_vector(f):
def ser_string_vector(l):
r = b""
if len(l) < 253:
r = struct.pack("B", len(l))
elif len(l) < 0x10000:
r = struct.pack("<BH", 253, len(l))
elif len(l) < 0x100000000:
r = struct.pack("<BI", 254, len(l))
else:
r = struct.pack("<BQ", 255, len(l))
r = ser_compact_size(len(l))
for sv in l:
r += ser_string(sv)
return r
def deser_int_vector(f):
nit = struct.unpack("<B", f.read(1))[0]
if nit == 253:
nit = struct.unpack("<H", f.read(2))[0]
elif nit == 254:
nit = struct.unpack("<I", f.read(4))[0]
elif nit == 255:
nit = struct.unpack("<Q", f.read(8))[0]
nit = deser_compact_size(f)
r = []
for i in range(nit):
t = struct.unpack("<i", f.read(4))[0]
@ -237,15 +199,7 @@ def deser_int_vector(f): @@ -237,15 +199,7 @@ def deser_int_vector(f):
def ser_int_vector(l):
r = b""
if len(l) < 253:
r = struct.pack("B", len(l))
elif len(l) < 0x10000:
r = struct.pack("<BH", 253, len(l))
elif len(l) < 0x100000000:
r = struct.pack("<BI", 254, len(l))
else:
r = struct.pack("<BQ", 255, len(l))
r = ser_compact_size(len(l))
for i in l:
r += struct.pack("<i", i)
return r
@ -294,7 +248,8 @@ class CInv(object): @@ -294,7 +248,8 @@ class CInv(object):
1: "TX",
2: "Block",
1|MSG_WITNESS_FLAG: "WitnessTx",
2|MSG_WITNESS_FLAG : "WitnessBlock"
2|MSG_WITNESS_FLAG : "WitnessBlock",
4: "CompactBlock"
}
def __init__(self, t=0, h=0):
@ -781,6 +736,187 @@ class CAlert(object): @@ -781,6 +736,187 @@ class CAlert(object):
% (len(self.vchMsg), len(self.vchSig))
class PrefilledTransaction(object):
def __init__(self, index=0, tx = None):
self.index = index
self.tx = tx
def deserialize(self, f):
self.index = deser_compact_size(f)
self.tx = CTransaction()
self.tx.deserialize(f)
def serialize(self, with_witness=False):
r = b""
r += ser_compact_size(self.index)
if with_witness:
r += self.tx.serialize_with_witness()
else:
r += self.tx.serialize_without_witness()
return r
def __repr__(self):
return "PrefilledTransaction(index=%d, tx=%s)" % (self.index, repr(self.tx))
# This is what we send on the wire, in a cmpctblock message.
class P2PHeaderAndShortIDs(object):
def __init__(self):
self.header = CBlockHeader()
self.nonce = 0
self.shortids_length = 0
self.shortids = []
self.prefilled_txn_length = 0
self.prefilled_txn = []
def deserialize(self, f):
self.header.deserialize(f)
self.nonce = struct.unpack("<Q", f.read(8))[0]
self.shortids_length = deser_compact_size(f)
for i in range(self.shortids_length):
# shortids are defined to be 6 bytes in the spec, so append
# two zero bytes and read it in as an 8-byte number
self.shortids.append(struct.unpack("<Q", f.read(6) + b'\x00\x00')[0])
self.prefilled_txn = deser_vector(f, PrefilledTransaction)
self.prefilled_txn_length = len(self.prefilled_txn)
def serialize(self, with_witness=False):
r = b""
r += self.header.serialize()
r += struct.pack("<Q", self.nonce)
r += ser_compact_size(self.shortids_length)
for x in self.shortids:
# We only want the first 6 bytes
r += struct.pack("<Q", x)[0:6]
r += ser_vector(self.prefilled_txn)
return r
def __repr__(self):
return "P2PHeaderAndShortIDs(header=%s, nonce=%d, shortids_length=%d, shortids=%s, prefilled_txn_length=%d, prefilledtxn=%s" % (repr(self.header), self.nonce, self.shortids_length, repr(self.shortids), self.prefilled_txn_length, repr(self.prefilled_txn))
# Calculate the BIP 152-compact blocks shortid for a given transaction hash
def calculate_shortid(k0, k1, tx_hash):
expected_shortid = siphash256(k0, k1, tx_hash)
expected_shortid &= 0x0000ffffffffffff
return expected_shortid
# This version gets rid of the array lengths, and reinterprets the differential
# encoding into indices that can be used for lookup.
class HeaderAndShortIDs(object):
def __init__(self, p2pheaders_and_shortids = None):
self.header = CBlockHeader()
self.nonce = 0
self.shortids = []
self.prefilled_txn = []
if p2pheaders_and_shortids != None:
self.header = p2pheaders_and_shortids.header
self.nonce = p2pheaders_and_shortids.nonce
self.shortids = p2pheaders_and_shortids.shortids
last_index = -1
for x in p2pheaders_and_shortids.prefilled_txn:
self.prefilled_txn.append(PrefilledTransaction(x.index + last_index + 1, x.tx))
last_index = self.prefilled_txn[-1].index
def to_p2p(self):
ret = P2PHeaderAndShortIDs()
ret.header = self.header
ret.nonce = self.nonce
ret.shortids_length = len(self.shortids)
ret.shortids = self.shortids
ret.prefilled_txn_length = len(self.prefilled_txn)
ret.prefilled_txn = []
last_index = -1
for x in self.prefilled_txn:
ret.prefilled_txn.append(PrefilledTransaction(x.index - last_index - 1, x.tx))
last_index = x.index
return ret
def get_siphash_keys(self):
header_nonce = self.header.serialize()
header_nonce += struct.pack("<Q", self.nonce)
hash_header_nonce_as_str = sha256(header_nonce)
key0 = struct.unpack("<Q", hash_header_nonce_as_str[0:8])[0]
key1 = struct.unpack("<Q", hash_header_nonce_as_str[8:16])[0]
return [ key0, key1 ]
def initialize_from_block(self, block, nonce=0, prefill_list = [0]):
self.header = CBlockHeader(block)
self.nonce = nonce
self.prefilled_txn = [ PrefilledTransaction(i, block.vtx[i]) for i in prefill_list ]
self.shortids = []
[k0, k1] = self.get_siphash_keys()
for i in range(len(block.vtx)):
if i not in prefill_list:
self.shortids.append(calculate_shortid(k0, k1, block.vtx[i].sha256))
def __repr__(self):
return "HeaderAndShortIDs(header=%s, nonce=%d, shortids=%s, prefilledtxn=%s" % (repr(self.header), self.nonce, repr(self.shortids), repr(self.prefilled_txn))
class BlockTransactionsRequest(object):
def __init__(self, blockhash=0, indexes = None):
self.blockhash = blockhash
self.indexes = indexes if indexes != None else []
def deserialize(self, f):
self.blockhash = deser_uint256(f)
indexes_length = deser_compact_size(f)
for i in range(indexes_length):
self.indexes.append(deser_compact_size(f))
def serialize(self):
r = b""
r += ser_uint256(self.blockhash)
r += ser_compact_size(len(self.indexes))
for x in self.indexes:
r += ser_compact_size(x)
return r
# helper to set the differentially encoded indexes from absolute ones
def from_absolute(self, absolute_indexes):
self.indexes = []
last_index = -1
for x in absolute_indexes:
self.indexes.append(x-last_index-1)
last_index = x
def to_absolute(self):
absolute_indexes = []
last_index = -1
for x in self.indexes:
absolute_indexes.append(x+last_index+1)
last_index = absolute_indexes[-1]
return absolute_indexes
def __repr__(self):
return "BlockTransactionsRequest(hash=%064x indexes=%s)" % (self.blockhash, repr(self.indexes))
class BlockTransactions(object):
def __init__(self, blockhash=0, transactions = None):
self.blockhash = blockhash
self.transactions = transactions if transactions != None else []
def deserialize(self, f):
self.blockhash = deser_uint256(f)
self.transactions = deser_vector(f, CTransaction)
def serialize(self, with_witness=False):
r = b""
r += ser_uint256(self.blockhash)
if with_witness:
r += ser_vector(self.transactions, "serialize_with_witness")
else:
r += ser_vector(self.transactions)
return r
def __repr__(self):
return "BlockTransactions(hash=%064x transactions=%s)" % (self.blockhash, repr(self.transactions))
# Objects that correspond to messages on the wire
class msg_version(object):
command = b"version"
@ -1215,6 +1351,79 @@ class msg_feefilter(object): @@ -1215,6 +1351,79 @@ class msg_feefilter(object):
def __repr__(self):
return "msg_feefilter(feerate=%08x)" % self.feerate
class msg_sendcmpct(object):
command = b"sendcmpct"
def __init__(self):
self.announce = False
self.version = 1
def deserialize(self, f):
self.announce = struct.unpack("<?", f.read(1))[0]
self.version = struct.unpack("<Q", f.read(8))[0]
def serialize(self):
r = b""
r += struct.pack("<?", self.announce)
r += struct.pack("<Q", self.version)
return r
def __repr__(self):
return "msg_sendcmpct(announce=%s, version=%lu)" % (self.announce, self.version)
class msg_cmpctblock(object):
command = b"cmpctblock"
def __init__(self, header_and_shortids = None):
self.header_and_shortids = header_and_shortids
def deserialize(self, f):
self.header_and_shortids = P2PHeaderAndShortIDs()
self.header_and_shortids.deserialize(f)
def serialize(self):
r = b""
r += self.header_and_shortids.serialize()
return r
def __repr__(self):
return "msg_cmpctblock(HeaderAndShortIDs=%s)" % repr(self.header_and_shortids)
class msg_getblocktxn(object):
command = b"getblocktxn"
def __init__(self):
self.block_txn_request = None
def deserialize(self, f):
self.block_txn_request = BlockTransactionsRequest()
self.block_txn_request.deserialize(f)
def serialize(self):
r = b""
r += self.block_txn_request.serialize()
return r
def __repr__(self):
return "msg_getblocktxn(block_txn_request=%s)" % (repr(self.block_txn_request))
class msg_blocktxn(object):
command = b"blocktxn"
def __init__(self):
self.block_transactions = BlockTransactions()
def deserialize(self, f):
self.block_transactions.deserialize(f)
def serialize(self):
r = b""
r += self.block_transactions.serialize()
return r
def __repr__(self):
return "msg_blocktxn(block_transactions=%s)" % (repr(self.block_transactions))
# This is what a callback should look like for NodeConn
# Reimplement the on_* functions to provide handling for events
class NodeConnCB(object):
@ -1295,6 +1504,10 @@ class NodeConnCB(object): @@ -1295,6 +1504,10 @@ class NodeConnCB(object):
def on_pong(self, conn, message): pass
def on_feefilter(self, conn, message): pass
def on_sendheaders(self, conn, message): pass
def on_sendcmpct(self, conn, message): pass
def on_cmpctblock(self, conn, message): pass
def on_getblocktxn(self, conn, message): pass
def on_blocktxn(self, conn, message): pass
# More useful callbacks and functions for NodeConnCB's which have a single NodeConn
class SingleNodeConnCB(NodeConnCB):
@ -1311,6 +1524,10 @@ class SingleNodeConnCB(NodeConnCB): @@ -1311,6 +1524,10 @@ class SingleNodeConnCB(NodeConnCB):
def send_message(self, message):
self.connection.send_message(message)
def send_and_ping(self, message):
self.send_message(message)
self.sync_with_ping()
def on_pong(self, conn, message):
self.last_pong = message
@ -1344,7 +1561,11 @@ class NodeConn(asyncore.dispatcher): @@ -1344,7 +1561,11 @@ class NodeConn(asyncore.dispatcher):
b"reject": msg_reject,
b"mempool": msg_mempool,
b"feefilter": msg_feefilter,
b"sendheaders": msg_sendheaders
b"sendheaders": msg_sendheaders,
b"sendcmpct": msg_sendcmpct,
b"cmpctblock": msg_cmpctblock,
b"getblocktxn": msg_getblocktxn,
b"blocktxn": msg_blocktxn
}
MAGIC_BYTES = {
"mainnet": b"\xf9\xbe\xb4\xd9", # mainnet

64
qa/rpc-tests/test_framework/siphash.py

@ -0,0 +1,64 @@ @@ -0,0 +1,64 @@
#!/usr/bin/env python3
# Copyright (c) 2016 The Bitcoin Core developers
# Distributed under the MIT software license, see the accompanying
# file COPYING or http://www.opensource.org/licenses/mit-license.php.
#
# siphash.py - Specialized SipHash-2-4 implementations
#
# This implements SipHash-2-4 for 256-bit integers.
def rotl64(n, b):
return n >> (64 - b) | (n & ((1 << (64 - b)) - 1)) << b
def siphash_round(v0, v1, v2, v3):
v0 = (v0 + v1) & ((1 << 64) - 1)
v1 = rotl64(v1, 13)
v1 ^= v0
v0 = rotl64(v0, 32)
v2 = (v2 + v3) & ((1 << 64) - 1)
v3 = rotl64(v3, 16)
v3 ^= v2
v0 = (v0 + v3) & ((1 << 64) - 1)
v3 = rotl64(v3, 21)
v3 ^= v0
v2 = (v2 + v1) & ((1 << 64) - 1)
v1 = rotl64(v1, 17)
v1 ^= v2
v2 = rotl64(v2, 32)
return (v0, v1, v2, v3)
def siphash256(k0, k1, h):
n0 = h & ((1 << 64) - 1)
n1 = (h >> 64) & ((1 << 64) - 1)
n2 = (h >> 128) & ((1 << 64) - 1)
n3 = (h >> 192) & ((1 << 64) - 1)
v0 = 0x736f6d6570736575 ^ k0
v1 = 0x646f72616e646f6d ^ k1
v2 = 0x6c7967656e657261 ^ k0
v3 = 0x7465646279746573 ^ k1 ^ n0
v0, v1, v2, v3 = siphash_round(v0, v1, v2, v3)
v0, v1, v2, v3 = siphash_round(v0, v1, v2, v3)
v0 ^= n0
v3 ^= n1
v0, v1, v2, v3 = siphash_round(v0, v1, v2, v3)
v0, v1, v2, v3 = siphash_round(v0, v1, v2, v3)
v0 ^= n1
v3 ^= n2
v0, v1, v2, v3 = siphash_round(v0, v1, v2, v3)
v0, v1, v2, v3 = siphash_round(v0, v1, v2, v3)
v0 ^= n2
v3 ^= n3
v0, v1, v2, v3 = siphash_round(v0, v1, v2, v3)
v0, v1, v2, v3 = siphash_round(v0, v1, v2, v3)
v0 ^= n3
v3 ^= 0x2000000000000000
v0, v1, v2, v3 = siphash_round(v0, v1, v2, v3)
v0, v1, v2, v3 = siphash_round(v0, v1, v2, v3)
v0 ^= 0x2000000000000000
v2 ^= 0xFF
v0, v1, v2, v3 = siphash_round(v0, v1, v2, v3)
v0, v1, v2, v3 = siphash_round(v0, v1, v2, v3)
v0, v1, v2, v3 = siphash_round(v0, v1, v2, v3)
v0, v1, v2, v3 = siphash_round(v0, v1, v2, v3)
return v0 ^ v1 ^ v2 ^ v3

11
src/chainparams.cpp

@ -303,6 +303,12 @@ public: @@ -303,6 +303,12 @@ public:
base58Prefixes[EXT_PUBLIC_KEY] = boost::assign::list_of(0x04)(0x35)(0x87)(0xCF).convert_to_container<std::vector<unsigned char> >();
base58Prefixes[EXT_SECRET_KEY] = boost::assign::list_of(0x04)(0x35)(0x83)(0x94).convert_to_container<std::vector<unsigned char> >();
}
void UpdateBIP9Parameters(Consensus::DeploymentPos d, int64_t nStartTime, int64_t nTimeout)
{
consensus.vDeployments[d].nStartTime = nStartTime;
consensus.vDeployments[d].nTimeout = nTimeout;
}
};
static CRegTestParams regTestParams;
@ -330,4 +336,9 @@ void SelectParams(const std::string& network) @@ -330,4 +336,9 @@ void SelectParams(const std::string& network)
SelectBaseParams(network);
pCurrentParams = &Params(network);
}
void UpdateRegtestBIP9Parameters(Consensus::DeploymentPos d, int64_t nStartTime, int64_t nTimeout)
{
regTestParams.UpdateBIP9Parameters(d, nStartTime, nTimeout);
}

5
src/chainparams.h

@ -112,4 +112,9 @@ CChainParams& Params(const std::string& chain); @@ -112,4 +112,9 @@ CChainParams& Params(const std::string& chain);
*/
void SelectParams(const std::string& chain);
/**
* Allows modifying the BIP9 regtest parameters.
*/
void UpdateRegtestBIP9Parameters(Consensus::DeploymentPos d, int64_t nStartTime, int64_t nTimeout);
#endif // BITCOIN_CHAINPARAMS_H

36
src/init.cpp

@ -410,6 +410,7 @@ std::string HelpMessage(HelpMessageMode mode) @@ -410,6 +410,7 @@ std::string HelpMessage(HelpMessageMode mode)
strUsage += HelpMessageOpt("-limitancestorsize=<n>", strprintf("Do not accept transactions whose size with all in-mempool ancestors exceeds <n> kilobytes (default: %u)", DEFAULT_ANCESTOR_SIZE_LIMIT));
strUsage += HelpMessageOpt("-limitdescendantcount=<n>", strprintf("Do not accept transactions if any ancestor would have <n> or more in-mempool descendants (default: %u)", DEFAULT_DESCENDANT_LIMIT));
strUsage += HelpMessageOpt("-limitdescendantsize=<n>", strprintf("Do not accept transactions if any ancestor would have more than <n> kilobytes of in-mempool descendants (default: %u).", DEFAULT_DESCENDANT_SIZE_LIMIT));
strUsage += HelpMessageOpt("-bip9params=deployment:start:end", "Use given start/end times for specified bip9 deployment (regtest-only)");
}
string debugCategories = "addrman, alert, bench, coindb, db, http, libevent, lock, mempool, mempoolrej, net, proxy, prune, rand, reindex, rpc, selectcoins, tor, zmq"; // Don't translate these and qt below
if (mode == HMM_BITCOIN_QT)
@ -990,6 +991,41 @@ bool AppInit2(boost::thread_group& threadGroup, CScheduler& scheduler) @@ -990,6 +991,41 @@ bool AppInit2(boost::thread_group& threadGroup, CScheduler& scheduler)
fEnableReplacement = (std::find(vstrReplacementModes.begin(), vstrReplacementModes.end(), "fee") != vstrReplacementModes.end());
}
if (!mapMultiArgs["-bip9params"].empty()) {
// Allow overriding bip9 parameters for testing
if (!Params().MineBlocksOnDemand()) {
return InitError("BIP9 parameters may only be overridden on regtest.");
}
const vector<string>& deployments = mapMultiArgs["-bip9params"];
for (auto i : deployments) {
std::vector<std::string> vDeploymentParams;
boost::split(vDeploymentParams, i, boost::is_any_of(":"));
if (vDeploymentParams.size() != 3) {
return InitError("BIP9 parameters malformed, expecting deployment:start:end");
}
int64_t nStartTime, nTimeout;
if (!ParseInt64(vDeploymentParams[1], &nStartTime)) {
return InitError(strprintf("Invalid nStartTime (%s)", vDeploymentParams[1]));
}
if (!ParseInt64(vDeploymentParams[2], &nTimeout)) {
return InitError(strprintf("Invalid nTimeout (%s)", vDeploymentParams[2]));
}
bool found = false;
for (int i=0; i<(int)Consensus::MAX_VERSION_BITS_DEPLOYMENTS; ++i)
{
if (vDeploymentParams[0].compare(VersionBitsDeploymentInfo[i].name) == 0) {
UpdateRegtestBIP9Parameters(Consensus::DeploymentPos(i), nStartTime, nTimeout);
found = true;
LogPrintf("Setting BIP9 activation parameters for %s to start=%ld, timeout=%ld\n", vDeploymentParams[0], nStartTime, nTimeout);
break;
}
}
if (!found) {
return InitError(strprintf("Invalid deployment (%s)", vDeploymentParams[0]));
}
}
}
// ********************************************************* Step 4: application initialization: dir lock, daemonize, pidfile, debug log
// Initialize elliptic curve code

21
src/main.cpp

@ -1506,9 +1506,9 @@ bool AcceptToMemoryPoolWorker(CTxMemPool& pool, CValidationState& state, const C @@ -1506,9 +1506,9 @@ bool AcceptToMemoryPoolWorker(CTxMemPool& pool, CValidationState& state, const C
// SCRIPT_VERIFY_CLEANSTACK requires SCRIPT_VERIFY_WITNESS, so we
// need to turn both off, and compare against just turning off CLEANSTACK
// to see if the failure is specifically due to witness validation.
if (CheckInputs(tx, state, view, true, scriptVerifyFlags & ~(SCRIPT_VERIFY_WITNESS | SCRIPT_VERIFY_CLEANSTACK), true, txdata) &&
if (tx.wit.IsNull() && CheckInputs(tx, state, view, true, scriptVerifyFlags & ~(SCRIPT_VERIFY_WITNESS | SCRIPT_VERIFY_CLEANSTACK), true, txdata) &&
!CheckInputs(tx, state, view, true, scriptVerifyFlags & ~SCRIPT_VERIFY_CLEANSTACK, true, txdata)) {
// Only the witness is wrong, so the transaction itself may be fine.
// Only the witness is missing, so the transaction itself may be fine.
state.SetCorruptionPossible();
}
return false;
@ -2399,6 +2399,7 @@ bool ConnectBlock(const CBlock& block, CValidationState& state, CBlockIndex* pin @@ -2399,6 +2399,7 @@ bool ConnectBlock(const CBlock& block, CValidationState& state, CBlockIndex* pin
// Start enforcing WITNESS rules using versionbits logic.
if (IsWitnessEnabled(pindex->pprev, chainparams.GetConsensus())) {
flags |= SCRIPT_VERIFY_WITNESS;
flags |= SCRIPT_VERIFY_NULLDUMMY;
}
int64_t nTime2 = GetTimeMicros(); nTimeForks += nTime2 - nTime1;
@ -5505,7 +5506,7 @@ bool static ProcessMessage(CNode* pfrom, string strCommand, CDataStream& vRecv, @@ -5505,7 +5506,7 @@ bool static ProcessMessage(CNode* pfrom, string strCommand, CDataStream& vRecv,
else if (!fMissingInputs2)
{
int nDos = 0;
if (stateDummy.IsInvalid(nDos) && nDos > 0 && (!state.CorruptionPossible() || State(fromPeer)->fHaveWitness))
if (stateDummy.IsInvalid(nDos) && nDos > 0)
{
// Punish peer that gave us an invalid orphan tx
Misbehaving(fromPeer, nDos);
@ -5516,7 +5517,10 @@ bool static ProcessMessage(CNode* pfrom, string strCommand, CDataStream& vRecv, @@ -5516,7 +5517,10 @@ bool static ProcessMessage(CNode* pfrom, string strCommand, CDataStream& vRecv,
// Probably non-standard or insufficient fee/priority
LogPrint("mempool", " removed orphan tx %s\n", orphanHash.ToString());
vEraseQueue.push_back(orphanHash);
if (!stateDummy.CorruptionPossible()) {
if (orphanTx.wit.IsNull() && !stateDummy.CorruptionPossible()) {
// Do not use rejection cache for witness transactions or
// witness-stripped transactions, as they can have been malleated.
// See https://github.com/bitcoin/bitcoin/issues/8279 for details.
assert(recentRejects);
recentRejects->insert(orphanHash);
}
@ -5554,7 +5558,10 @@ bool static ProcessMessage(CNode* pfrom, string strCommand, CDataStream& vRecv, @@ -5554,7 +5558,10 @@ bool static ProcessMessage(CNode* pfrom, string strCommand, CDataStream& vRecv,
LogPrint("mempool", "not keeping orphan with rejected parents %s\n",tx.GetHash().ToString());
}
} else {
if (!state.CorruptionPossible()) {
if (tx.wit.IsNull() && !state.CorruptionPossible()) {
// Do not use rejection cache for witness transactions or
// witness-stripped transactions, as they can have been malleated.
// See https://github.com/bitcoin/bitcoin/issues/8279 for details.
assert(recentRejects);
recentRejects->insert(tx.GetHash());
}
@ -5586,9 +5593,7 @@ bool static ProcessMessage(CNode* pfrom, string strCommand, CDataStream& vRecv, @@ -5586,9 +5593,7 @@ bool static ProcessMessage(CNode* pfrom, string strCommand, CDataStream& vRecv,
if (state.GetRejectCode() < REJECT_INTERNAL) // Never send AcceptToMemoryPool's internal codes over P2P
pfrom->PushMessage(NetMsgType::REJECT, strCommand, (unsigned char)state.GetRejectCode(),
state.GetRejectReason().substr(0, MAX_REJECT_MESSAGE_LENGTH), inv.hash);
if (nDoS > 0 && (!state.CorruptionPossible() || State(pfrom->id)->fHaveWitness)) {
// When a non-witness-supporting peer gives us a transaction that would
// be accepted if witness validation was off, we can't blame them for it.
if (nDoS > 0) {
Misbehaving(pfrom->GetId(), nDoS);
}
}

4
src/net.cpp

@ -2226,11 +2226,7 @@ void CNode::RecordBytesSent(uint64_t bytes) @@ -2226,11 +2226,7 @@ void CNode::RecordBytesSent(uint64_t bytes)
void CNode::SetMaxOutboundTarget(uint64_t limit)
{
LOCK(cs_totalBytesSent);
uint64_t recommendedMinimum = (nMaxOutboundTimeframe / 600) * MAX_BLOCK_SERIALIZED_SIZE;
nMaxOutboundLimit = limit;
if (limit > 0 && limit < recommendedMinimum)
LogPrintf("Max outbound target is very small (%s bytes) and will be overshot. Recommended minimum is %s bytes.\n", nMaxOutboundLimit, recommendedMinimum);
}
uint64_t CNode::GetMaxOutboundTarget()

38
src/rpc/misc.cpp

@ -320,43 +320,6 @@ UniValue createmultisig(const UniValue& params, bool fHelp) @@ -320,43 +320,6 @@ UniValue createmultisig(const UniValue& params, bool fHelp)
return result;
}
UniValue createwitnessaddress(const UniValue& params, bool fHelp)
{
if (fHelp || params.size() < 1 || params.size() > 1)
{
string msg = "createwitnessaddress \"script\"\n"
"\nCreates a witness address for a particular script.\n"
"It returns a json object with the address and witness script.\n"
"\nArguments:\n"
"1. \"script\" (string, required) A hex encoded script\n"
"\nResult:\n"
"{\n"
" \"address\":\"multisigaddress\", (string) The value of the new address (P2SH of witness script).\n"
" \"witnessScript\":\"script\" (string) The string value of the hex-encoded witness script.\n"
"}\n"
;
throw runtime_error(msg);
}
if (!IsHex(params[0].get_str())) {
throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Script must be hex-encoded");
}
std::vector<unsigned char> code = ParseHex(params[0].get_str());
CScript script(code.begin(), code.end());
CScript witscript = GetScriptForWitness(script);
CScriptID witscriptid(witscript);
CBitcoinAddress address(witscriptid);
UniValue result(UniValue::VOBJ);
result.push_back(Pair("address", address.ToString()));
result.push_back(Pair("witnessScript", HexStr(witscript.begin(), witscript.end())));
return result;
}
UniValue verifymessage(const UniValue& params, bool fHelp)
{
if (fHelp || params.size() != 3)
@ -490,7 +453,6 @@ static const CRPCCommand commands[] = @@ -490,7 +453,6 @@ static const CRPCCommand commands[] =
{ "control", "getinfo", &getinfo, true }, /* uses wallet if enabled */
{ "util", "validateaddress", &validateaddress, true }, /* uses wallet if enabled */
{ "util", "createmultisig", &createmultisig, true },
{ "util", "createwitnessaddress", &createwitnessaddress, true },
{ "util", "verifymessage", &verifymessage, true },
{ "util", "signmessagewithprivkey", &signmessagewithprivkey, true },

4
src/test/hash_tests.cpp

@ -122,6 +122,10 @@ BOOST_AUTO_TEST_CASE(siphash) @@ -122,6 +122,10 @@ BOOST_AUTO_TEST_CASE(siphash)
hasher3.Write(uint64_t(x)|(uint64_t(x+1)<<8)|(uint64_t(x+2)<<16)|(uint64_t(x+3)<<24)|
(uint64_t(x+4)<<32)|(uint64_t(x+5)<<40)|(uint64_t(x+6)<<48)|(uint64_t(x+7)<<56));
}
CHashWriter ss(SER_DISK, CLIENT_VERSION);
ss << CTransaction();
BOOST_CHECK_EQUAL(SipHashUint256(1, 2, ss.GetHash()), 0x79751e980c2a0a35ULL);
}
BOOST_AUTO_TEST_SUITE_END()

Loading…
Cancel
Save