Merge #8815: Backports for 0.13.1

9dfa0c8 Implement NULLDUMMY softfork (Johnson Lau)
cef633c Fix broken sendcmpct test in p2p-compactblocks.py (Suhas Daftuar)
fb8706e Add p2p test for BIP 152 (compact blocks) (Suhas Daftuar)
f6be478 Add support for compactblocks to mininode (Suhas Daftuar)
4295a7a Tests: refactor compact size serialization in mininode (Suhas Daftuar)
ff893aa Implement SipHash in Python (Pieter Wuille)
198494c Allow changing BIP9 parameters on regtest (Suhas Daftuar)
23feab1 Remove maxuploadtargets recommended minimum (Jonas Schnelli)
a5ec248 Remove createwitnessaddress (Johnson Lau)
b394a96 Add basic test for IsStandard witness transaction blinding (instagibbs)
1672225 Do not store witness txn in rejection cache (Pieter Wuille)

doc/reduce-traffic.md

@@ -19,8 +19,7 @@ This is *not* a hard limit; only a threshold to minimize the outbound
 traffic. When the limit is about to be reached, the uploaded data is cut by no
 longer serving historic blocks (blocks older than one week).
 Keep in mind that new nodes require other nodes that are willing to serve
-historic blocks. **The recommended minimum is 144 blocks per day (max. 144MB
-per day)**
+historic blocks.
 
 Whitelisted peers will never be disconnected, although their traffic counts for
 calculating the target.

qa/pull-tester/rpc-tests.py

@@ -142,6 +142,8 @@ testScripts = [
     'segwit.py',
     'importprunedfunds.py',
     'signmessages.py',
+    'p2p-compactblocks.py',
+    'nulldummy.py',
 ]
 if ENABLE_ZMQ:
     testScripts.append('zmq_test.py')

qa/rpc-tests/nulldummy.py

@@ -0,0 +1,148 @@
+#!/usr/bin/env python3
+# Copyright (c) 2016 The Bitcoin Core developers
+# Distributed under the MIT software license, see the accompanying
+# file COPYING or http://www.opensource.org/licenses/mit-license.php.
+
+from test_framework.test_framework import ComparisonTestFramework
+from test_framework.util import *
+from test_framework.mininode import CTransaction, NetworkThread
+from test_framework.blocktools import create_coinbase, create_block, add_witness_commitment
+from test_framework.comptool import TestManager
+from test_framework.script import CScript
+from io import BytesIO
+import time
+
+NULLDUMMY_ERROR = "64: non-mandatory-script-verify-flag (Dummy CHECKMULTISIG argument must be zero)"
+
+def trueDummy(tx):
+    scriptSig = CScript(tx.vin[0].scriptSig)
+    newscript = []
+    for i in scriptSig:
+        if (len(newscript) == 0):
+            assert(len(i) == 0)
+            newscript.append(b'\x51')
+        else:
+            newscript.append(i)
+    tx.vin[0].scriptSig = CScript(newscript)
+    tx.rehash()
+
+'''
+This test is meant to exercise NULLDUMMY softfork.
+Connect to a single node.
+Generate 2 blocks (save the coinbases for later).
+Generate 427 more blocks.
+[Policy/Consensus] Check that NULLDUMMY compliant transactions are accepted in the 430th block.
+[Policy] Check that non-NULLDUMMY transactions are rejected before activation.
+[Consensus] Check that the new NULLDUMMY rules are not enforced on the 431st block.
+[Policy/Consensus] Check that the new NULLDUMMY rules are enforced on the 432nd block.
+'''
+
+class NULLDUMMYTest(ComparisonTestFramework):
+
+    def __init__(self):
+        super().__init__()
+        self.num_nodes = 1
+
+    def setup_network(self):
+        # Must set the blockversion for this test
+        self.nodes = start_nodes(self.num_nodes, self.options.tmpdir,
+                                 extra_args=[['-debug', '-whitelist=127.0.0.1', '-walletprematurewitness']])
+
+    def run_test(self):
+        self.address = self.nodes[0].getnewaddress()
+        self.ms_address = self.nodes[0].addmultisigaddress(1,[self.address])
+        self.wit_address = self.nodes[0].addwitnessaddress(self.address)
+        self.wit_ms_address = self.nodes[0].addwitnessaddress(self.ms_address)
+
+        test = TestManager(self, self.options.tmpdir)
+        test.add_all_connections(self.nodes)
+        NetworkThread().start() # Start up network handling in another thread
+        self.coinbase_blocks = self.nodes[0].generate(2) # Block 2
+        coinbase_txid = []
+        for i in self.coinbase_blocks:
+            coinbase_txid.append(self.nodes[0].getblock(i)['tx'][0])
+        self.nodes[0].generate(427) # Block 429
+        self.lastblockhash = self.nodes[0].getbestblockhash()
+        self.tip = int("0x" + self.lastblockhash, 0)
+        self.lastblockheight = 429
+        self.lastblocktime = int(time.time()) + 429
+
+        print ("Test 1: NULLDUMMY compliant base transactions should be accepted to mempool and mined before activation [430]")
+        test1txs = [self.create_transaction(self.nodes[0], coinbase_txid[0], self.ms_address, 49)]
+        txid1 = self.tx_submit(self.nodes[0], test1txs[0])
+        test1txs.append(self.create_transaction(self.nodes[0], txid1, self.ms_address, 48))
+        txid2 = self.tx_submit(self.nodes[0], test1txs[1])
+        test1txs.append(self.create_transaction(self.nodes[0], coinbase_txid[1], self.wit_ms_address, 49))
+        txid3 = self.tx_submit(self.nodes[0], test1txs[2])
+        self.block_submit(self.nodes[0], test1txs, False, True)
+
+        print ("Test 2: Non-NULLDUMMY base multisig transaction should not be accepted to mempool before activation")
+        test2tx = self.create_transaction(self.nodes[0], txid2, self.ms_address, 48)
+        trueDummy(test2tx)
+        txid4 = self.tx_submit(self.nodes[0], test2tx, NULLDUMMY_ERROR)
+
+        print ("Test 3: Non-NULLDUMMY base transactions should be accepted in a block before activation [431]")
+        self.block_submit(self.nodes[0], [test2tx], False, True)
+
+        print ("Test 4: Non-NULLDUMMY base multisig transaction is invalid after activation")
+        test4tx = self.create_transaction(self.nodes[0], txid4, self.address, 47)
+        test6txs=[CTransaction(test4tx)]
+        trueDummy(test4tx)
+        self.tx_submit(self.nodes[0], test4tx, NULLDUMMY_ERROR)
+        self.block_submit(self.nodes[0], [test4tx])
+
+        print ("Test 5: Non-NULLDUMMY P2WSH multisig transaction invalid after activation")
+        test5tx = self.create_transaction(self.nodes[0], txid3, self.wit_address, 48)
+        test6txs.append(CTransaction(test5tx))
+        test5tx.wit.vtxinwit[0].scriptWitness.stack[0] = b'\x01'
+        self.tx_submit(self.nodes[0], test5tx, NULLDUMMY_ERROR)
+        self.block_submit(self.nodes[0], [test5tx], True)
+
+        print ("Test 6: NULLDUMMY compliant base/witness transactions should be accepted to mempool and in block after activation [432]")
+        for i in test6txs:
+            self.tx_submit(self.nodes[0], i)
+        self.block_submit(self.nodes[0], test6txs, True, True)
+
+
+    def create_transaction(self, node, txid, to_address, amount):
+        inputs = [{ "txid" : txid, "vout" : 0}]
+        outputs = { to_address : amount }
+        rawtx = node.createrawtransaction(inputs, outputs)
+        signresult = node.signrawtransaction(rawtx)
+        tx = CTransaction()
+        f = BytesIO(hex_str_to_bytes(signresult['hex']))
+        tx.deserialize(f)
+        return tx
+
+
+    def tx_submit(self, node, tx, msg = ""):
+        tx.rehash()
+        try:
+            node.sendrawtransaction(bytes_to_hex_str(tx.serialize_with_witness()), True)
+        except JSONRPCException as exp:
+            assert_equal(exp.error["message"], msg)
+        return tx.hash
+
+
+    def block_submit(self, node, txs, witness = False, accept = False):
+        block = create_block(self.tip, create_coinbase(self.lastblockheight + 1), self.lastblocktime + 1)
+        block.nVersion = 4
+        for tx in txs:
+            tx.rehash()
+            block.vtx.append(tx)
+        block.hashMerkleRoot = block.calc_merkle_root()
+        witness and add_witness_commitment(block)
+        block.rehash()
+        block.solve()
+        node.submitblock(bytes_to_hex_str(block.serialize(True)))
+        if (accept):
+            assert_equal(node.getbestblockhash(), block.hash)
+            self.tip = block.sha256
+            self.lastblockhash = block.hash
+            self.lastblocktime += 1
+            self.lastblockheight += 1
+        else:
+            assert_equal(node.getbestblockhash(), self.lastblockhash)
+
+if __name__ == '__main__':
+    NULLDUMMYTest().main()

qa/rpc-tests/p2p-compactblocks.py

@@ -0,0 +1,624 @@
+#!/usr/bin/env python3
+# Copyright (c) 2016 The Bitcoin Core developers
+# Distributed under the MIT software license, see the accompanying
+# file COPYING or http://www.opensource.org/licenses/mit-license.php.
+
+from test_framework.mininode import *
+from test_framework.test_framework import BitcoinTestFramework
+from test_framework.util import *
+from test_framework.blocktools import create_block, create_coinbase
+from test_framework.siphash import siphash256
+from test_framework.script import CScript, OP_TRUE
+
+'''
+CompactBlocksTest -- test compact blocks (BIP 152)
+'''
+
+
+# TestNode: A peer we use to send messages to bitcoind, and store responses.
+class TestNode(SingleNodeConnCB):
+    def __init__(self):
+        SingleNodeConnCB.__init__(self)
+        self.last_sendcmpct = None
+        self.last_headers = None
+        self.last_inv = None
+        self.last_cmpctblock = None
+        self.block_announced = False
+        self.last_getdata = None
+        self.last_getblocktxn = None
+        self.last_block = None
+        self.last_blocktxn = None
+
+    def on_sendcmpct(self, conn, message):
+        self.last_sendcmpct = message
+
+    def on_block(self, conn, message):
+        self.last_block = message
+
+    def on_cmpctblock(self, conn, message):
+        self.last_cmpctblock = message
+        self.block_announced = True
+
+    def on_headers(self, conn, message):
+        self.last_headers = message
+        self.block_announced = True
+
+    def on_inv(self, conn, message):
+        self.last_inv = message
+        self.block_announced = True
+
+    def on_getdata(self, conn, message):
+        self.last_getdata = message
+
+    def on_getblocktxn(self, conn, message):
+        self.last_getblocktxn = message
+
+    def on_blocktxn(self, conn, message):
+        self.last_blocktxn = message
+
+    # Requires caller to hold mininode_lock
+    def received_block_announcement(self):
+        return self.block_announced
+
+    def clear_block_announcement(self):
+        with mininode_lock:
+            self.block_announced = False
+            self.last_inv = None
+            self.last_headers = None
+            self.last_cmpctblock = None
+
+    def get_headers(self, locator, hashstop):
+        msg = msg_getheaders()
+        msg.locator.vHave = locator
+        msg.hashstop = hashstop
+        self.connection.send_message(msg)
+
+    def send_header_for_blocks(self, new_blocks):
+        headers_message = msg_headers()
+        headers_message.headers = [CBlockHeader(b) for b in new_blocks]
+        self.send_message(headers_message)
+
+    def request_headers_and_sync(self, locator, hashstop=0):
+        self.clear_block_announcement()
+        self.get_headers(locator, hashstop)
+        assert(wait_until(self.received_block_announcement, timeout=30))
+        assert(self.received_block_announcement())
+        self.clear_block_announcement()
+
+
+class CompactBlocksTest(BitcoinTestFramework):
+    def __init__(self):
+        super().__init__()
+        self.setup_clean_chain = True
+        self.num_nodes = 1
+        self.utxos = []
+
+    def setup_network(self):
+        self.nodes = []
+
+        # Turn off segwit in this test, as compact blocks don't currently work
+        # with segwit.  (After BIP 152 is updated to support segwit, we can
+        # test behavior with and without segwit enabled by adding a second node
+        # to the test.)
+        self.nodes = start_nodes(self.num_nodes, self.options.tmpdir, [["-debug", "-logtimemicros=1", "-bip9params=segwit:0:0"]])
+
+    def build_block_on_tip(self):
+        height = self.nodes[0].getblockcount()
+        tip = self.nodes[0].getbestblockhash()
+        mtp = self.nodes[0].getblockheader(tip)['mediantime']
+        block = create_block(int(tip, 16), create_coinbase(height + 1), mtp + 1)
+        block.solve()
+        return block
+
+    # Create 10 more anyone-can-spend utxo's for testing.
+    def make_utxos(self):
+        block = self.build_block_on_tip()
+        self.test_node.send_and_ping(msg_block(block))
+        assert(int(self.nodes[0].getbestblockhash(), 16) == block.sha256)
+        self.nodes[0].generate(100)
+
+        total_value = block.vtx[0].vout[0].nValue
+        out_value = total_value // 10
+        tx = CTransaction()
+        tx.vin.append(CTxIn(COutPoint(block.vtx[0].sha256, 0), b''))
+        for i in range(10):
+            tx.vout.append(CTxOut(out_value, CScript([OP_TRUE])))
+        tx.rehash()
+
+        block2 = self.build_block_on_tip()
+        block2.vtx.append(tx)
+        block2.hashMerkleRoot = block2.calc_merkle_root()
+        block2.solve()
+        self.test_node.send_and_ping(msg_block(block2))
+        assert_equal(int(self.nodes[0].getbestblockhash(), 16), block2.sha256)
+        self.utxos.extend([[tx.sha256, i, out_value] for i in range(10)])
+        return
+
+    # Test "sendcmpct":
+    # - No compact block announcements or getdata(MSG_CMPCT_BLOCK) unless
+    #   sendcmpct is sent.
+    # - If sendcmpct is sent with version > 1, the message is ignored.
+    # - If sendcmpct is sent with boolean 0, then block announcements are not
+    #   made with compact blocks.
+    # - If sendcmpct is then sent with boolean 1, then new block announcements
+    #   are made with compact blocks.
+    def test_sendcmpct(self):
+        print("Testing SENDCMPCT p2p message... ")
+
+        # Make sure we get a version 0 SENDCMPCT message from our peer
+        def received_sendcmpct():
+            return (self.test_node.last_sendcmpct is not None)
+        got_message = wait_until(received_sendcmpct, timeout=30)
+        assert(received_sendcmpct())
+        assert(got_message)
+        assert_equal(self.test_node.last_sendcmpct.version, 1)
+
+        tip = int(self.nodes[0].getbestblockhash(), 16)
+
+        def check_announcement_of_new_block(node, peer, predicate):
+            peer.clear_block_announcement()
+            node.generate(1)
+            got_message = wait_until(lambda: peer.block_announced, timeout=30)
+            assert(peer.block_announced)
+            assert(got_message)
+            with mininode_lock:
+                assert(predicate(peer))
+
+        # We shouldn't get any block announcements via cmpctblock yet.
+        check_announcement_of_new_block(self.nodes[0], self.test_node, lambda p: p.last_cmpctblock is None)
+
+        # Try one more time, this time after requesting headers.
+        self.test_node.request_headers_and_sync(locator=[tip])
+        check_announcement_of_new_block(self.nodes[0], self.test_node, lambda p: p.last_cmpctblock is None and p.last_inv is not None)
+
+        # Test a few ways of using sendcmpct that should NOT
+        # result in compact block announcements.
+        # Before each test, sync the headers chain.
+        self.test_node.request_headers_and_sync(locator=[tip])
+
+        # Now try a SENDCMPCT message with too-high version
+        sendcmpct = msg_sendcmpct()
+        sendcmpct.version = 2
+        self.test_node.send_and_ping(sendcmpct)
+        check_announcement_of_new_block(self.nodes[0], self.test_node, lambda p: p.last_cmpctblock is None)
+
+        # Headers sync before next test.
+        self.test_node.request_headers_and_sync(locator=[tip])
+
+        # Now try a SENDCMPCT message with valid version, but announce=False
+        self.test_node.send_and_ping(msg_sendcmpct())
+        check_announcement_of_new_block(self.nodes[0], self.test_node, lambda p: p.last_cmpctblock is None)
+
+        # Headers sync before next test.
+        self.test_node.request_headers_and_sync(locator=[tip])
+
+        # Finally, try a SENDCMPCT message with announce=True
+        sendcmpct.version = 1
+        sendcmpct.announce = True
+        self.test_node.send_and_ping(sendcmpct)
+        check_announcement_of_new_block(self.nodes[0], self.test_node, lambda p: p.last_cmpctblock is not None)
+
+        # Try one more time (no headers sync should be needed!)
+        check_announcement_of_new_block(self.nodes[0], self.test_node, lambda p: p.last_cmpctblock is not None)
+
+        # Try one more time, after turning on sendheaders
+        self.test_node.send_and_ping(msg_sendheaders())
+        check_announcement_of_new_block(self.nodes[0], self.test_node, lambda p: p.last_cmpctblock is not None)
+
+        # Now turn off announcements
+        sendcmpct.announce = False
+        self.test_node.send_and_ping(sendcmpct)
+        check_announcement_of_new_block(self.nodes[0], self.test_node, lambda p: p.last_cmpctblock is None and p.last_headers is not None)
+
+    # This test actually causes bitcoind to (reasonably!) disconnect us, so do this last.
+    def test_invalid_cmpctblock_message(self):
+        print("Testing invalid index in cmpctblock message...")
+        self.nodes[0].generate(101)
+        block = self.build_block_on_tip()
+
+        cmpct_block = P2PHeaderAndShortIDs()
+        cmpct_block.header = CBlockHeader(block)
+        cmpct_block.prefilled_txn_length = 1
+        # This index will be too high
+        prefilled_txn = PrefilledTransaction(1, block.vtx[0])
+        cmpct_block.prefilled_txn = [prefilled_txn]
+        self.test_node.send_and_ping(msg_cmpctblock(cmpct_block))
+        assert(int(self.nodes[0].getbestblockhash(), 16) == block.hashPrevBlock)
+
+    # Compare the generated shortids to what we expect based on BIP 152, given
+    # bitcoind's choice of nonce.
+    def test_compactblock_construction(self):
+        print("Testing compactblock headers and shortIDs are correct...")
+
+        # Generate a bunch of transactions.
+        self.nodes[0].generate(101)
+        num_transactions = 25
+        address = self.nodes[0].getnewaddress()
+        for i in range(num_transactions):
+            self.nodes[0].sendtoaddress(address, 0.1)
+
+        # Now mine a block, and look at the resulting compact block.
+        self.test_node.clear_block_announcement()
+        block_hash = int(self.nodes[0].generate(1)[0], 16)
+
+        # Store the raw block in our internal format.
+        block = FromHex(CBlock(), self.nodes[0].getblock("%02x" % block_hash, False))
+        [tx.calc_sha256() for tx in block.vtx]
+        block.rehash()
+
+        # Don't care which type of announcement came back for this test; just
+        # request the compact block if we didn't get one yet.
+        wait_until(self.test_node.received_block_announcement, timeout=30)
+
+        with mininode_lock:
+            if self.test_node.last_cmpctblock is None:
+                self.test_node.clear_block_announcement()
+                inv = CInv(4, block_hash)  # 4 == "CompactBlock"
+                self.test_node.send_message(msg_getdata([inv]))
+
+        wait_until(self.test_node.received_block_announcement, timeout=30)
+
+        # Now we should have the compactblock
+        header_and_shortids = None
+        with mininode_lock:
+            assert(self.test_node.last_cmpctblock is not None)
+            # Convert the on-the-wire representation to absolute indexes
+            header_and_shortids = HeaderAndShortIDs(self.test_node.last_cmpctblock.header_and_shortids)
+
+        # Check that we got the right block!
+        header_and_shortids.header.calc_sha256()
+        assert_equal(header_and_shortids.header.sha256, block_hash)
+
+        # Make sure the prefilled_txn appears to have included the coinbase
+        assert(len(header_and_shortids.prefilled_txn) >= 1)
+        assert_equal(header_and_shortids.prefilled_txn[0].index, 0)
+
+        # Check that all prefilled_txn entries match what's in the block.
+        for entry in header_and_shortids.prefilled_txn:
+            entry.tx.calc_sha256()
+            assert_equal(entry.tx.sha256, block.vtx[entry.index].sha256)
+
+        # Check that the cmpctblock message announced all the transactions.
+        assert_equal(len(header_and_shortids.prefilled_txn) + len(header_and_shortids.shortids), len(block.vtx))
+
+        # And now check that all the shortids are as expected as well.
+        # Determine the siphash keys to use.
+        [k0, k1] = header_and_shortids.get_siphash_keys()
+
+        index = 0
+        while index < len(block.vtx):
+            if (len(header_and_shortids.prefilled_txn) > 0 and
+                    header_and_shortids.prefilled_txn[0].index == index):
+                # Already checked prefilled transactions above
+                header_and_shortids.prefilled_txn.pop(0)
+            else:
+                shortid = calculate_shortid(k0, k1, block.vtx[index].sha256)
+                assert_equal(shortid, header_and_shortids.shortids[0])
+                header_and_shortids.shortids.pop(0)
+            index += 1
+
+    # Test that bitcoind requests compact blocks when we announce new blocks
+    # via header or inv, and that responding to getblocktxn causes the block
+    # to be successfully reconstructed.
+    def test_compactblock_requests(self):
+        print("Testing compactblock requests... ")
+
+        # Try announcing a block with an inv or header, expect a compactblock
+        # request
+        for announce in ["inv", "header"]:
+            block = self.build_block_on_tip()
+            with mininode_lock:
+                self.test_node.last_getdata = None
+
+            if announce == "inv":
+                self.test_node.send_message(msg_inv([CInv(2, block.sha256)]))
+            else:
+                self.test_node.send_header_for_blocks([block])
+            success = wait_until(lambda: self.test_node.last_getdata is not None, timeout=30)
+            assert(success)
+            assert_equal(len(self.test_node.last_getdata.inv), 1)
+            assert_equal(self.test_node.last_getdata.inv[0].type, 4)
+            assert_equal(self.test_node.last_getdata.inv[0].hash, block.sha256)
+
+            # Send back a compactblock message that omits the coinbase
+            comp_block = HeaderAndShortIDs()
+            comp_block.header = CBlockHeader(block)
+            comp_block.nonce = 0
+            comp_block.shortids = [1]  # this is useless, and wrong
+            self.test_node.send_and_ping(msg_cmpctblock(comp_block.to_p2p()))
+            assert_equal(int(self.nodes[0].getbestblockhash(), 16), block.hashPrevBlock)
+            # Expect a getblocktxn message.
+            with mininode_lock:
+                assert(self.test_node.last_getblocktxn is not None)
+                absolute_indexes = self.test_node.last_getblocktxn.block_txn_request.to_absolute()
+            assert_equal(absolute_indexes, [0])  # should be a coinbase request
+
+            # Send the coinbase, and verify that the tip advances.
+            msg = msg_blocktxn()
+            msg.block_transactions.blockhash = block.sha256
+            msg.block_transactions.transactions = [block.vtx[0]]
+            self.test_node.send_and_ping(msg)
+            assert_equal(int(self.nodes[0].getbestblockhash(), 16), block.sha256)
+
+    # Create a chain of transactions from given utxo, and add to a new block.
+    def build_block_with_transactions(self, utxo, num_transactions):
+        block = self.build_block_on_tip()
+
+        for i in range(num_transactions):
+            tx = CTransaction()
+            tx.vin.append(CTxIn(COutPoint(utxo[0], utxo[1]), b''))
+            tx.vout.append(CTxOut(utxo[2] - 1000, CScript([OP_TRUE])))
+            tx.rehash()
+            utxo = [tx.sha256, 0, tx.vout[0].nValue]
+            block.vtx.append(tx)
+
+        block.hashMerkleRoot = block.calc_merkle_root()
+        block.solve()
+        return block
+
+    # Test that we only receive getblocktxn requests for transactions that the
+    # node needs, and that responding to them causes the block to be
+    # reconstructed.
+    def test_getblocktxn_requests(self):
+        print("Testing getblocktxn requests...")
+
+        # First try announcing compactblocks that won't reconstruct, and verify
+        # that we receive getblocktxn messages back.
+        utxo = self.utxos.pop(0)
+
+        block = self.build_block_with_transactions(utxo, 5)
+        self.utxos.append([block.vtx[-1].sha256, 0, block.vtx[-1].vout[0].nValue])
+
+        comp_block = HeaderAndShortIDs()
+        comp_block.initialize_from_block(block)
+
+        self.test_node.send_and_ping(msg_cmpctblock(comp_block.to_p2p()))
+        with mininode_lock:
+            assert(self.test_node.last_getblocktxn is not None)
+            absolute_indexes = self.test_node.last_getblocktxn.block_txn_request.to_absolute()
+        assert_equal(absolute_indexes, [1, 2, 3, 4, 5])
+        msg = msg_blocktxn()
+        msg.block_transactions = BlockTransactions(block.sha256, block.vtx[1:])
+        self.test_node.send_and_ping(msg)
+        assert_equal(int(self.nodes[0].getbestblockhash(), 16), block.sha256)
+
+        utxo = self.utxos.pop(0)
+        block = self.build_block_with_transactions(utxo, 5)
+        self.utxos.append([block.vtx[-1].sha256, 0, block.vtx[-1].vout[0].nValue])
+
+        # Now try interspersing the prefilled transactions
+        comp_block.initialize_from_block(block, prefill_list=[0, 1, 5])
+        self.test_node.send_and_ping(msg_cmpctblock(comp_block.to_p2p()))
+        with mininode_lock:
+            assert(self.test_node.last_getblocktxn is not None)
+            absolute_indexes = self.test_node.last_getblocktxn.block_txn_request.to_absolute()
+        assert_equal(absolute_indexes, [2, 3, 4])
+        msg.block_transactions = BlockTransactions(block.sha256, block.vtx[2:5])
+        self.test_node.send_and_ping(msg)
+        assert_equal(int(self.nodes[0].getbestblockhash(), 16), block.sha256)
+
+        # Now try giving one transaction ahead of time.
+        utxo = self.utxos.pop(0)
+        block = self.build_block_with_transactions(utxo, 5)
+        self.utxos.append([block.vtx[-1].sha256, 0, block.vtx[-1].vout[0].nValue])
+        self.test_node.send_and_ping(msg_tx(block.vtx[1]))
+        assert(block.vtx[1].hash in self.nodes[0].getrawmempool())
+
+        # Prefill 4 out of the 6 transactions, and verify that only the one
+        # that was not in the mempool is requested.
+        comp_block.initialize_from_block(block, prefill_list=[0, 2, 3, 4])
+        self.test_node.send_and_ping(msg_cmpctblock(comp_block.to_p2p()))
+        with mininode_lock:
+            assert(self.test_node.last_getblocktxn is not None)
+            absolute_indexes = self.test_node.last_getblocktxn.block_txn_request.to_absolute()
+        assert_equal(absolute_indexes, [5])
+
+        msg.block_transactions = BlockTransactions(block.sha256, [block.vtx[5]])
+        self.test_node.send_and_ping(msg)
+        assert_equal(int(self.nodes[0].getbestblockhash(), 16), block.sha256)
+
+        # Now provide all transactions to the node before the block is
+        # announced and verify reconstruction happens immediately.
+        utxo = self.utxos.pop(0)
+        block = self.build_block_with_transactions(utxo, 10)
+        self.utxos.append([block.vtx[-1].sha256, 0, block.vtx[-1].vout[0].nValue])
+        for tx in block.vtx[1:]:
+            self.test_node.send_message(msg_tx(tx))
+        self.test_node.sync_with_ping()
+        # Make sure all transactions were accepted.
+        mempool = self.nodes[0].getrawmempool()
+        for tx in block.vtx[1:]:
+            assert(tx.hash in mempool)
+
+        # Clear out last request.
+        with mininode_lock:
+            self.test_node.last_getblocktxn = None
+
+        # Send compact block
+        comp_block.initialize_from_block(block, prefill_list=[0])
+        self.test_node.send_and_ping(msg_cmpctblock(comp_block.to_p2p()))
+        with mininode_lock:
+            # Shouldn't have gotten a request for any transaction
+            assert(self.test_node.last_getblocktxn is None)
+        # Tip should have updated
+        assert_equal(int(self.nodes[0].getbestblockhash(), 16), block.sha256)
+
+    # Incorrectly responding to a getblocktxn shouldn't cause the block to be
+    # permanently failed.
+    def test_incorrect_blocktxn_response(self):
+        print("Testing handling of incorrect blocktxn responses...")
+
+        if (len(self.utxos) == 0):
+            self.make_utxos()
+        utxo = self.utxos.pop(0)
+
+        block = self.build_block_with_transactions(utxo, 10)
+        self.utxos.append([block.vtx[-1].sha256, 0, block.vtx[-1].vout[0].nValue])
+        # Relay the first 5 transactions from the block in advance
+        for tx in block.vtx[1:6]:
+            self.test_node.send_message(msg_tx(tx))
+        self.test_node.sync_with_ping()
+        # Make sure all transactions were accepted.
+        mempool = self.nodes[0].getrawmempool()
+        for tx in block.vtx[1:6]:
+            assert(tx.hash in mempool)
+
+        # Send compact block
+        comp_block = HeaderAndShortIDs()
+        comp_block.initialize_from_block(block, prefill_list=[0])
+        self.test_node.send_and_ping(msg_cmpctblock(comp_block.to_p2p()))
+        absolute_indexes = []
+        with mininode_lock:
+            assert(self.test_node.last_getblocktxn is not None)
+            absolute_indexes = self.test_node.last_getblocktxn.block_txn_request.to_absolute()
+        assert_equal(absolute_indexes, [6, 7, 8, 9, 10])
+
+        # Now give an incorrect response.
+        # Note that it's possible for bitcoind to be smart enough to know we're
+        # lying, since it could check to see if the shortid matches what we're
+        # sending, and eg disconnect us for misbehavior.  If that behavior
+        # change were made, we could just modify this test by having a
+        # different peer provide the block further down, so that we're still
+        # verifying that the block isn't marked bad permanently. This is good
+        # enough for now.
+        msg = msg_blocktxn()
+        msg.block_transactions = BlockTransactions(block.sha256, [block.vtx[5]] + block.vtx[7:])
+        self.test_node.send_and_ping(msg)
+
+        # Tip should not have updated
+        assert_equal(int(self.nodes[0].getbestblockhash(), 16), block.hashPrevBlock)
+
+        # We should receive a getdata request
+        success = wait_until(lambda: self.test_node.last_getdata is not None, timeout=10)
+        assert(success)
+        assert_equal(len(self.test_node.last_getdata.inv), 1)
+        assert_equal(self.test_node.last_getdata.inv[0].type, 2)
+        assert_equal(self.test_node.last_getdata.inv[0].hash, block.sha256)
+
+        # Deliver the block
+        self.test_node.send_and_ping(msg_block(block))
+        assert_equal(int(self.nodes[0].getbestblockhash(), 16), block.sha256)
+
+    def test_getblocktxn_handler(self):
+        print("Testing getblocktxn handler...")
+
+        # bitcoind won't respond for blocks whose height is more than 15 blocks
+        # deep.
+        MAX_GETBLOCKTXN_DEPTH = 15
+        chain_height = self.nodes[0].getblockcount()
+        current_height = chain_height
+        while (current_height >= chain_height - MAX_GETBLOCKTXN_DEPTH):
+            block_hash = self.nodes[0].getblockhash(current_height)
+            block = FromHex(CBlock(), self.nodes[0].getblock(block_hash, False))
+
+            msg = msg_getblocktxn()
+            msg.block_txn_request = BlockTransactionsRequest(int(block_hash, 16), [])
+            num_to_request = random.randint(1, len(block.vtx))
+            msg.block_txn_request.from_absolute(sorted(random.sample(range(len(block.vtx)), num_to_request)))
+            self.test_node.send_message(msg)
+            success = wait_until(lambda: self.test_node.last_blocktxn is not None, timeout=10)
+            assert(success)
+
+            [tx.calc_sha256() for tx in block.vtx]
+            with mininode_lock:
+                assert_equal(self.test_node.last_blocktxn.block_transactions.blockhash, int(block_hash, 16))
+                all_indices = msg.block_txn_request.to_absolute()
+                for index in all_indices:
+                    tx = self.test_node.last_blocktxn.block_transactions.transactions.pop(0)
+                    tx.calc_sha256()
+                    assert_equal(tx.sha256, block.vtx[index].sha256)
+                self.test_node.last_blocktxn = None
+            current_height -= 1
+
+        # Next request should be ignored, as we're past the allowed depth.
+        block_hash = self.nodes[0].getblockhash(current_height)
+        msg.block_txn_request = BlockTransactionsRequest(int(block_hash, 16), [0])
+        self.test_node.send_and_ping(msg)
+        with mininode_lock:
+            assert_equal(self.test_node.last_blocktxn, None)
+
+    def test_compactblocks_not_at_tip(self):
+        print("Testing compactblock requests/announcements not at chain tip...")
+
+        # Test that requesting old compactblocks doesn't work.
+        MAX_CMPCTBLOCK_DEPTH = 11
+        new_blocks = []
+        for i in range(MAX_CMPCTBLOCK_DEPTH):
+            self.test_node.clear_block_announcement()
+            new_blocks.append(self.nodes[0].generate(1)[0])
+            wait_until(self.test_node.received_block_announcement, timeout=30)
+
+        self.test_node.clear_block_announcement()
+        self.test_node.send_message(msg_getdata([CInv(4, int(new_blocks[0], 16))]))
+        success = wait_until(lambda: self.test_node.last_cmpctblock is not None, timeout=30)
+        assert(success)
+
+        self.test_node.clear_block_announcement()
+        self.nodes[0].generate(1)
+        wait_until(self.test_node.received_block_announcement, timeout=30)
+        self.test_node.clear_block_announcement()
+        self.test_node.send_message(msg_getdata([CInv(4, int(new_blocks[0], 16))]))
+        success = wait_until(lambda: self.test_node.last_block is not None, timeout=30)
+        assert(success)
+        with mininode_lock:
+            self.test_node.last_block.block.calc_sha256()
+            assert_equal(self.test_node.last_block.block.sha256, int(new_blocks[0], 16))
+
+        # Generate an old compactblock, and verify that it's not accepted.
+        cur_height = self.nodes[0].getblockcount()
+        hashPrevBlock = int(self.nodes[0].getblockhash(cur_height-5), 16)
+        block = self.build_block_on_tip()
+        block.hashPrevBlock = hashPrevBlock
+        block.solve()
+
+        comp_block = HeaderAndShortIDs()
+        comp_block.initialize_from_block(block)
+        self.test_node.send_and_ping(msg_cmpctblock(comp_block.to_p2p()))
+
+        tips = self.nodes[0].getchaintips()
+        found = False
+        for x in tips:
+            if x["hash"] == block.hash:
+                assert_equal(x["status"], "headers-only")
+                found = True
+                break
+        assert(found)
+
+        # Requesting this block via getblocktxn should silently fail
+        # (to avoid fingerprinting attacks).
+        msg = msg_getblocktxn()
+        msg.block_txn_request = BlockTransactionsRequest(block.sha256, [0])
+        with mininode_lock:
+            self.test_node.last_blocktxn = None
+        self.test_node.send_and_ping(msg)
+        with mininode_lock:
+            assert(self.test_node.last_blocktxn is None)
+
+    def run_test(self):
+        # Setup the p2p connections and start up the network thread.
+        self.test_node = TestNode()
+
+        connections = []
+        connections.append(NodeConn('127.0.0.1', p2p_port(0), self.nodes[0], self.test_node))
+        self.test_node.add_connection(connections[0])
+
+        NetworkThread().start()  # Start up network handling in another thread
+
+        # Test logic begins here
+        self.test_node.wait_for_verack()
+
+        # We will need UTXOs to construct transactions in later tests.
+        self.make_utxos()
+
+        self.test_sendcmpct()
+        self.test_compactblock_construction()
+        self.test_compactblock_requests()
+        self.test_getblocktxn_requests()
+        self.test_getblocktxn_handler()
+        self.test_compactblocks_not_at_tip()
+        self.test_incorrect_blocktxn_response()
+        self.test_invalid_cmpctblock_message()
+
+
+if __name__ == '__main__':
+    CompactBlocksTest().main()

qa/rpc-tests/p2p-segwit.py

@@ -965,8 +965,24 @@ class SegWitTest(BitcoinTestFramework):
 
         tx3 = CTransaction()
         tx3.vin.append(CTxIn(COutPoint(tx2.sha256, 0), b""))
-        tx3.vout.append(CTxOut(tx2.vout[0].nValue-1000, CScript([OP_TRUE])))
         tx3.wit.vtxinwit.append(CTxInWitness())
+
+        # Add too-large for IsStandard witness and check that it does not enter reject filter
+        p2sh_program = CScript([OP_TRUE])
+        p2sh_pubkey = hash160(p2sh_program)
+        witness_program2 = CScript([b'a'*400000])
+        tx3.vout.append(CTxOut(tx2.vout[0].nValue-1000, CScript([OP_HASH160, p2sh_pubkey, OP_EQUAL])))
+        tx3.wit.vtxinwit[0].scriptWitness.stack = [witness_program2]
+        tx3.rehash()
+
+        # Node will not be blinded to the transaction
+        self.std_node.announce_tx_and_wait_for_getdata(tx3)
+        self.std_node.test_transaction_acceptance(tx3, True, False, b'tx-size')
+        self.std_node.announce_tx_and_wait_for_getdata(tx3)
+        self.std_node.test_transaction_acceptance(tx3, True, False, b'tx-size')
+
+        # Remove witness stuffing, instead add extra witness push on stack
+        tx3.vout[0] = CTxOut(tx2.vout[0].nValue-1000, CScript([OP_TRUE]))
         tx3.wit.vtxinwit[0].scriptWitness.stack = [CScript([CScriptNum(1)]), witness_program ]
         tx3.rehash()
 

qa/rpc-tests/test_framework/mininode.py

@@ -36,9 +36,10 @@ from threading import RLock
 from threading import Thread
 import logging
 import copy
+from test_framework.siphash import siphash256
 
 BIP0031_VERSION = 60000
-MY_VERSION = 60001  # past bip-31 for ping/pong
+MY_VERSION = 70014  # past bip-31 for ping/pong
 MY_SUBVERSION = b"/python-mininode-tester:0.0.3/"
 
 MAX_INV_SZ = 50000
@@ -52,7 +53,7 @@ NODE_BLOOM = (1 << 2)
 NODE_WITNESS = (1 << 3)
 
 # Keep our own socket map for asyncore, so that we can track disconnects
-# ourselves (to workaround an issue with closing an asyncore socket when 
+# ourselves (to workaround an issue with closing an asyncore socket when
 # using select)
 mininode_socket_map = dict()
 
@@ -74,8 +75,19 @@ def ripemd160(s):
 def hash256(s):
     return sha256(sha256(s))
 
+def ser_compact_size(l):
+    r = b""
+    if l < 253:
+        r = struct.pack("B", l)
+    elif l < 0x10000:
+        r = struct.pack("<BH", 253, l)
+    elif l < 0x100000000:
+        r = struct.pack("<BI", 254, l)
+    else:
+        r = struct.pack("<BQ", 255, l)
+    return r
 
-def deser_string(f):
+def deser_compact_size(f):
     nit = struct.unpack("<B", f.read(1))[0]
     if nit == 253:
         nit = struct.unpack("<H", f.read(2))[0]
@@ -83,16 +95,14 @@ def deser_string(f):
         nit = struct.unpack("<I", f.read(4))[0]
     elif nit == 255:
         nit = struct.unpack("<Q", f.read(8))[0]
+    return nit
+
+def deser_string(f):
+    nit = deser_compact_size(f)
     return f.read(nit)
 
 def ser_string(s):
-    if len(s) < 253:
-        return struct.pack("B", len(s)) + s
-    elif len(s) < 0x10000:
-        return struct.pack("<BH", 253, len(s)) + s
-    elif len(s) < 0x100000000:
-        return struct.pack("<BI", 254, len(s)) + s
-    return struct.pack("<BQ", 255, len(s)) + s
+    return ser_compact_size(len(s)) + s
 
 def deser_uint256(f):
     r = 0
@@ -125,13 +135,7 @@ def uint256_from_compact(c):
 
 
 def deser_vector(f, c):
-    nit = struct.unpack("<B", f.read(1))[0]
-    if nit == 253:
-        nit = struct.unpack("<H", f.read(2))[0]
-    elif nit == 254:
-        nit = struct.unpack("<I", f.read(4))[0]
-    elif nit == 255:
-        nit = struct.unpack("<Q", f.read(8))[0]
+    nit = deser_compact_size(f)
     r = []
     for i in range(nit):
         t = c()
@@ -144,15 +148,7 @@ def deser_vector(f, c):
 # entries in the vector (we use this for serializing the vector of transactions
 # for a witness block).
 def ser_vector(l, ser_function_name=None):
-    r = b""
-    if len(l) < 253:
-        r = struct.pack("B", len(l))
-    elif len(l) < 0x10000:
-        r = struct.pack("<BH", 253, len(l))
-    elif len(l) < 0x100000000:
-        r = struct.pack("<BI", 254, len(l))
-    else:
-        r = struct.pack("<BQ", 255, len(l))
+    r = ser_compact_size(len(l))
     for i in l:
         if ser_function_name:
             r += getattr(i, ser_function_name)()
@@ -162,13 +158,7 @@ def ser_vector(l, ser_function_name=None):
 
 
 def deser_uint256_vector(f):
-    nit = struct.unpack("<B", f.read(1))[0]
-    if nit == 253:
-        nit = struct.unpack("<H", f.read(2))[0]
-    elif nit == 254:
-        nit = struct.unpack("<I", f.read(4))[0]
-    elif nit == 255:
-        nit = struct.unpack("<Q", f.read(8))[0]
+    nit = deser_compact_size(f)
     r = []
     for i in range(nit):
         t = deser_uint256(f)
@@ -177,28 +167,14 @@ def deser_uint256_vector(f):
 
 
 def ser_uint256_vector(l):
-    r = b""
-    if len(l) < 253:
-        r = struct.pack("B", len(l))
-    elif len(l) < 0x10000:
-        r = struct.pack("<BH", 253, len(l))
-    elif len(l) < 0x100000000:
-        r = struct.pack("<BI", 254, len(l))
-    else:
-        r = struct.pack("<BQ", 255, len(l))
+    r = ser_compact_size(len(l))
     for i in l:
         r += ser_uint256(i)
     return r
 
 
 def deser_string_vector(f):
-    nit = struct.unpack("<B", f.read(1))[0]
-    if nit == 253:
-        nit = struct.unpack("<H", f.read(2))[0]
-    elif nit == 254:
-        nit = struct.unpack("<I", f.read(4))[0]
-    elif nit == 255:
-        nit = struct.unpack("<Q", f.read(8))[0]
+    nit = deser_compact_size(f)
     r = []
     for i in range(nit):
         t = deser_string(f)
@@ -207,28 +183,14 @@ def deser_string_vector(f):
 
 
 def ser_string_vector(l):
-    r = b""
-    if len(l) < 253:
-        r = struct.pack("B", len(l))
-    elif len(l) < 0x10000:
-        r = struct.pack("<BH", 253, len(l))
-    elif len(l) < 0x100000000:
-        r = struct.pack("<BI", 254, len(l))
-    else:
-        r = struct.pack("<BQ", 255, len(l))
+    r = ser_compact_size(len(l))
     for sv in l:
         r += ser_string(sv)
     return r
 
 
 def deser_int_vector(f):
-    nit = struct.unpack("<B", f.read(1))[0]
-    if nit == 253:
-        nit = struct.unpack("<H", f.read(2))[0]
-    elif nit == 254:
-        nit = struct.unpack("<I", f.read(4))[0]
-    elif nit == 255:
-        nit = struct.unpack("<Q", f.read(8))[0]
+    nit = deser_compact_size(f)
     r = []
     for i in range(nit):
         t = struct.unpack("<i", f.read(4))[0]
@@ -237,15 +199,7 @@ def deser_int_vector(f):
 
 
 def ser_int_vector(l):
-    r = b""
-    if len(l) < 253:
-        r = struct.pack("B", len(l))
-    elif len(l) < 0x10000:
-        r = struct.pack("<BH", 253, len(l))
-    elif len(l) < 0x100000000:
-        r = struct.pack("<BI", 254, len(l))
-    else:
-        r = struct.pack("<BQ", 255, len(l))
+    r = ser_compact_size(len(l))
     for i in l:
         r += struct.pack("<i", i)
     return r
@@ -294,7 +248,8 @@ class CInv(object):
         1: "TX",
         2: "Block",
         1|MSG_WITNESS_FLAG: "WitnessTx",
-        2|MSG_WITNESS_FLAG : "WitnessBlock"
+        2|MSG_WITNESS_FLAG : "WitnessBlock",
+        4: "CompactBlock"
     }
 
     def __init__(self, t=0, h=0):
@@ -781,6 +736,187 @@ class CAlert(object):
             % (len(self.vchMsg), len(self.vchSig))
 
 
+class PrefilledTransaction(object):
+    def __init__(self, index=0, tx = None):
+        self.index = index
+        self.tx = tx
+
+    def deserialize(self, f):
+        self.index = deser_compact_size(f)
+        self.tx = CTransaction()
+        self.tx.deserialize(f)
+
+    def serialize(self, with_witness=False):
+        r = b""
+        r += ser_compact_size(self.index)
+        if with_witness:
+            r += self.tx.serialize_with_witness()
+        else:
+            r += self.tx.serialize_without_witness()
+        return r
+
+    def __repr__(self):
+        return "PrefilledTransaction(index=%d, tx=%s)" % (self.index, repr(self.tx))
+
+# This is what we send on the wire, in a cmpctblock message.
+class P2PHeaderAndShortIDs(object):
+    def __init__(self):
+        self.header = CBlockHeader()
+        self.nonce = 0
+        self.shortids_length = 0
+        self.shortids = []
+        self.prefilled_txn_length = 0
+        self.prefilled_txn = []
+
+    def deserialize(self, f):
+        self.header.deserialize(f)
+        self.nonce = struct.unpack("<Q", f.read(8))[0]
+        self.shortids_length = deser_compact_size(f)
+        for i in range(self.shortids_length):
+            # shortids are defined to be 6 bytes in the spec, so append
+            # two zero bytes and read it in as an 8-byte number
+            self.shortids.append(struct.unpack("<Q", f.read(6) + b'\x00\x00')[0])
+        self.prefilled_txn = deser_vector(f, PrefilledTransaction)
+        self.prefilled_txn_length = len(self.prefilled_txn)
+
+    def serialize(self, with_witness=False):
+        r = b""
+        r += self.header.serialize()
+        r += struct.pack("<Q", self.nonce)
+        r += ser_compact_size(self.shortids_length)
+        for x in self.shortids:
+            # We only want the first 6 bytes
+            r += struct.pack("<Q", x)[0:6]
+        r += ser_vector(self.prefilled_txn)
+        return r
+
+    def __repr__(self):
+        return "P2PHeaderAndShortIDs(header=%s, nonce=%d, shortids_length=%d, shortids=%s, prefilled_txn_length=%d, prefilledtxn=%s" % (repr(self.header), self.nonce, self.shortids_length, repr(self.shortids), self.prefilled_txn_length, repr(self.prefilled_txn))
+
+
+# Calculate the BIP 152-compact blocks shortid for a given transaction hash
+def calculate_shortid(k0, k1, tx_hash):
+    expected_shortid = siphash256(k0, k1, tx_hash)
+    expected_shortid &= 0x0000ffffffffffff
+    return expected_shortid
+
+# This version gets rid of the array lengths, and reinterprets the differential
+# encoding into indices that can be used for lookup.
+class HeaderAndShortIDs(object):
+    def __init__(self, p2pheaders_and_shortids = None):
+        self.header = CBlockHeader()
+        self.nonce = 0
+        self.shortids = []
+        self.prefilled_txn = []
+
+        if p2pheaders_and_shortids != None:
+            self.header = p2pheaders_and_shortids.header
+            self.nonce = p2pheaders_and_shortids.nonce
+            self.shortids = p2pheaders_and_shortids.shortids
+            last_index = -1
+            for x in p2pheaders_and_shortids.prefilled_txn:
+                self.prefilled_txn.append(PrefilledTransaction(x.index + last_index + 1, x.tx))
+                last_index = self.prefilled_txn[-1].index
+
+    def to_p2p(self):
+        ret = P2PHeaderAndShortIDs()
+        ret.header = self.header
+        ret.nonce = self.nonce
+        ret.shortids_length = len(self.shortids)
+        ret.shortids = self.shortids
+        ret.prefilled_txn_length = len(self.prefilled_txn)
+        ret.prefilled_txn = []
+        last_index = -1
+        for x in self.prefilled_txn:
+            ret.prefilled_txn.append(PrefilledTransaction(x.index - last_index - 1, x.tx))
+            last_index = x.index
+        return ret
+
+    def get_siphash_keys(self):
+        header_nonce = self.header.serialize()
+        header_nonce += struct.pack("<Q", self.nonce)
+        hash_header_nonce_as_str = sha256(header_nonce)
+        key0 = struct.unpack("<Q", hash_header_nonce_as_str[0:8])[0]
+        key1 = struct.unpack("<Q", hash_header_nonce_as_str[8:16])[0]
+        return [ key0, key1 ]
+
+    def initialize_from_block(self, block, nonce=0, prefill_list = [0]):
+        self.header = CBlockHeader(block)
+        self.nonce = nonce
+        self.prefilled_txn = [ PrefilledTransaction(i, block.vtx[i]) for i in prefill_list ]
+        self.shortids = []
+        [k0, k1] = self.get_siphash_keys()
+        for i in range(len(block.vtx)):
+            if i not in prefill_list:
+                self.shortids.append(calculate_shortid(k0, k1, block.vtx[i].sha256))
+
+    def __repr__(self):
+        return "HeaderAndShortIDs(header=%s, nonce=%d, shortids=%s, prefilledtxn=%s" % (repr(self.header), self.nonce, repr(self.shortids), repr(self.prefilled_txn))
+
+
+class BlockTransactionsRequest(object):
+
+    def __init__(self, blockhash=0, indexes = None):
+        self.blockhash = blockhash
+        self.indexes = indexes if indexes != None else []
+
+    def deserialize(self, f):
+        self.blockhash = deser_uint256(f)
+        indexes_length = deser_compact_size(f)
+        for i in range(indexes_length):
+            self.indexes.append(deser_compact_size(f))
+
+    def serialize(self):
+        r = b""
+        r += ser_uint256(self.blockhash)
+        r += ser_compact_size(len(self.indexes))
+        for x in self.indexes:
+            r += ser_compact_size(x)
+        return r
+
+    # helper to set the differentially encoded indexes from absolute ones
+    def from_absolute(self, absolute_indexes):
+        self.indexes = []
+        last_index = -1
+        for x in absolute_indexes:
+            self.indexes.append(x-last_index-1)
+            last_index = x
+
+    def to_absolute(self):
+        absolute_indexes = []
+        last_index = -1
+        for x in self.indexes:
+            absolute_indexes.append(x+last_index+1)
+            last_index = absolute_indexes[-1]
+        return absolute_indexes
+
+    def __repr__(self):
+        return "BlockTransactionsRequest(hash=%064x indexes=%s)" % (self.blockhash, repr(self.indexes))
+
+
+class BlockTransactions(object):
+
+    def __init__(self, blockhash=0, transactions = None):
+        self.blockhash = blockhash
+        self.transactions = transactions if transactions != None else []
+
+    def deserialize(self, f):
+        self.blockhash = deser_uint256(f)
+        self.transactions = deser_vector(f, CTransaction)
+
+    def serialize(self, with_witness=False):
+        r = b""
+        r += ser_uint256(self.blockhash)
+        if with_witness:
+            r += ser_vector(self.transactions, "serialize_with_witness")
+        else:
+            r += ser_vector(self.transactions)
+        return r
+
+    def __repr__(self):
+        return "BlockTransactions(hash=%064x transactions=%s)" % (self.blockhash, repr(self.transactions))
+
+
 # Objects that correspond to messages on the wire
 class msg_version(object):
     command = b"version"
@@ -1215,6 +1351,79 @@ class msg_feefilter(object):
     def __repr__(self):
         return "msg_feefilter(feerate=%08x)" % self.feerate
 
+class msg_sendcmpct(object):
+    command = b"sendcmpct"
+
+    def __init__(self):
+        self.announce = False
+        self.version = 1
+
+    def deserialize(self, f):
+        self.announce = struct.unpack("<?", f.read(1))[0]
+        self.version = struct.unpack("<Q", f.read(8))[0]
+
+    def serialize(self):
+        r = b""
+        r += struct.pack("<?", self.announce)
+        r += struct.pack("<Q", self.version)
+        return r
+
+    def __repr__(self):
+        return "msg_sendcmpct(announce=%s, version=%lu)" % (self.announce, self.version)
+
+class msg_cmpctblock(object):
+    command = b"cmpctblock"
+
+    def __init__(self, header_and_shortids = None):
+        self.header_and_shortids = header_and_shortids
+
+    def deserialize(self, f):
+        self.header_and_shortids = P2PHeaderAndShortIDs()
+        self.header_and_shortids.deserialize(f)
+
+    def serialize(self):
+        r = b""
+        r += self.header_and_shortids.serialize()
+        return r
+
+    def __repr__(self):
+        return "msg_cmpctblock(HeaderAndShortIDs=%s)" % repr(self.header_and_shortids)
+
+class msg_getblocktxn(object):
+    command = b"getblocktxn"
+
+    def __init__(self):
+        self.block_txn_request = None
+
+    def deserialize(self, f):
+        self.block_txn_request = BlockTransactionsRequest()
+        self.block_txn_request.deserialize(f)
+
+    def serialize(self):
+        r = b""
+        r += self.block_txn_request.serialize()
+        return r
+
+    def __repr__(self):
+        return "msg_getblocktxn(block_txn_request=%s)" % (repr(self.block_txn_request))
+
+class msg_blocktxn(object):
+    command = b"blocktxn"
+
+    def __init__(self):
+        self.block_transactions = BlockTransactions()
+
+    def deserialize(self, f):
+        self.block_transactions.deserialize(f)
+
+    def serialize(self):
+        r = b""
+        r += self.block_transactions.serialize()
+        return r
+
+    def __repr__(self):
+        return "msg_blocktxn(block_transactions=%s)" % (repr(self.block_transactions))
+
 # This is what a callback should look like for NodeConn
 # Reimplement the on_* functions to provide handling for events
 class NodeConnCB(object):
@@ -1295,6 +1504,10 @@ class NodeConnCB(object):
     def on_pong(self, conn, message): pass
     def on_feefilter(self, conn, message): pass
     def on_sendheaders(self, conn, message): pass
+    def on_sendcmpct(self, conn, message): pass
+    def on_cmpctblock(self, conn, message): pass
+    def on_getblocktxn(self, conn, message): pass
+    def on_blocktxn(self, conn, message): pass
 
 # More useful callbacks and functions for NodeConnCB's which have a single NodeConn
 class SingleNodeConnCB(NodeConnCB):
@@ -1311,6 +1524,10 @@ class SingleNodeConnCB(NodeConnCB):
     def send_message(self, message):
         self.connection.send_message(message)
 
+    def send_and_ping(self, message):
+        self.send_message(message)
+        self.sync_with_ping()
+
     def on_pong(self, conn, message):
         self.last_pong = message
 
@@ -1344,7 +1561,11 @@ class NodeConn(asyncore.dispatcher):
         b"reject": msg_reject,
         b"mempool": msg_mempool,
         b"feefilter": msg_feefilter,
-        b"sendheaders": msg_sendheaders
+        b"sendheaders": msg_sendheaders,
+        b"sendcmpct": msg_sendcmpct,
+        b"cmpctblock": msg_cmpctblock,
+        b"getblocktxn": msg_getblocktxn,
+        b"blocktxn": msg_blocktxn
     }
     MAGIC_BYTES = {
         "mainnet": b"\xf9\xbe\xb4\xd9",   # mainnet

qa/rpc-tests/test_framework/siphash.py

@@ -0,0 +1,64 @@
+#!/usr/bin/env python3
+# Copyright (c) 2016 The Bitcoin Core developers
+# Distributed under the MIT software license, see the accompanying
+# file COPYING or http://www.opensource.org/licenses/mit-license.php.
+
+#
+# siphash.py - Specialized SipHash-2-4 implementations
+#
+# This implements SipHash-2-4 for 256-bit integers.
+
+def rotl64(n, b):
+    return n >> (64 - b) | (n & ((1 << (64 - b)) - 1)) << b
+
+def siphash_round(v0, v1, v2, v3):
+    v0 = (v0 + v1) & ((1 << 64) - 1)
+    v1 = rotl64(v1, 13)
+    v1 ^= v0
+    v0 = rotl64(v0, 32)
+    v2 = (v2 + v3) & ((1 << 64) - 1)
+    v3 = rotl64(v3, 16)
+    v3 ^= v2
+    v0 = (v0 + v3) & ((1 << 64) - 1)
+    v3 = rotl64(v3, 21)
+    v3 ^= v0
+    v2 = (v2 + v1) & ((1 << 64) - 1)
+    v1 = rotl64(v1, 17)
+    v1 ^= v2
+    v2 = rotl64(v2, 32)
+    return (v0, v1, v2, v3)
+
+def siphash256(k0, k1, h):
+    n0 = h & ((1 << 64) - 1)
+    n1 = (h >> 64) & ((1 << 64) - 1)
+    n2 = (h >> 128) & ((1 << 64) - 1)
+    n3 = (h >> 192) & ((1 << 64) - 1)
+    v0 = 0x736f6d6570736575 ^ k0
+    v1 = 0x646f72616e646f6d ^ k1
+    v2 = 0x6c7967656e657261 ^ k0
+    v3 = 0x7465646279746573 ^ k1 ^ n0
+    v0, v1, v2, v3 = siphash_round(v0, v1, v2, v3)
+    v0, v1, v2, v3 = siphash_round(v0, v1, v2, v3)
+    v0 ^= n0
+    v3 ^= n1
+    v0, v1, v2, v3 = siphash_round(v0, v1, v2, v3)
+    v0, v1, v2, v3 = siphash_round(v0, v1, v2, v3)
+    v0 ^= n1
+    v3 ^= n2
+    v0, v1, v2, v3 = siphash_round(v0, v1, v2, v3)
+    v0, v1, v2, v3 = siphash_round(v0, v1, v2, v3)
+    v0 ^= n2
+    v3 ^= n3
+    v0, v1, v2, v3 = siphash_round(v0, v1, v2, v3)
+    v0, v1, v2, v3 = siphash_round(v0, v1, v2, v3)
+    v0 ^= n3
+    v3 ^= 0x2000000000000000
+    v0, v1, v2, v3 = siphash_round(v0, v1, v2, v3)
+    v0, v1, v2, v3 = siphash_round(v0, v1, v2, v3)
+    v0 ^= 0x2000000000000000
+    v2 ^= 0xFF
+    v0, v1, v2, v3 = siphash_round(v0, v1, v2, v3)
+    v0, v1, v2, v3 = siphash_round(v0, v1, v2, v3)
+    v0, v1, v2, v3 = siphash_round(v0, v1, v2, v3)
+    v0, v1, v2, v3 = siphash_round(v0, v1, v2, v3)
+    return v0 ^ v1 ^ v2 ^ v3

src/chainparams.cpp

@@ -303,6 +303,12 @@ public:
         base58Prefixes[EXT_PUBLIC_KEY] = boost::assign::list_of(0x04)(0x35)(0x87)(0xCF).convert_to_container<std::vector<unsigned char> >();
         base58Prefixes[EXT_SECRET_KEY] = boost::assign::list_of(0x04)(0x35)(0x83)(0x94).convert_to_container<std::vector<unsigned char> >();
     }
+
+    void UpdateBIP9Parameters(Consensus::DeploymentPos d, int64_t nStartTime, int64_t nTimeout)
+    {
+        consensus.vDeployments[d].nStartTime = nStartTime;
+        consensus.vDeployments[d].nTimeout = nTimeout;
+    }
 };
 static CRegTestParams regTestParams;
 
@@ -330,4 +336,9 @@ void SelectParams(const std::string& network)
     SelectBaseParams(network);
     pCurrentParams = &Params(network);
 }
+
+void UpdateRegtestBIP9Parameters(Consensus::DeploymentPos d, int64_t nStartTime, int64_t nTimeout)
+{
+    regTestParams.UpdateBIP9Parameters(d, nStartTime, nTimeout);
+}
  

src/chainparams.h

@@ -112,4 +112,9 @@ CChainParams& Params(const std::string& chain);
  */
 void SelectParams(const std::string& chain);
 
+/**
+ * Allows modifying the BIP9 regtest parameters.
+ */
+void UpdateRegtestBIP9Parameters(Consensus::DeploymentPos d, int64_t nStartTime, int64_t nTimeout);
+
 #endif // BITCOIN_CHAINPARAMS_H

src/init.cpp

@@ -410,6 +410,7 @@ std::string HelpMessage(HelpMessageMode mode)
         strUsage += HelpMessageOpt("-limitancestorsize=<n>", strprintf("Do not accept transactions whose size with all in-mempool ancestors exceeds <n> kilobytes (default: %u)", DEFAULT_ANCESTOR_SIZE_LIMIT));
         strUsage += HelpMessageOpt("-limitdescendantcount=<n>", strprintf("Do not accept transactions if any ancestor would have <n> or more in-mempool descendants (default: %u)", DEFAULT_DESCENDANT_LIMIT));
         strUsage += HelpMessageOpt("-limitdescendantsize=<n>", strprintf("Do not accept transactions if any ancestor would have more than <n> kilobytes of in-mempool descendants (default: %u).", DEFAULT_DESCENDANT_SIZE_LIMIT));
+        strUsage += HelpMessageOpt("-bip9params=deployment:start:end", "Use given start/end times for specified bip9 deployment (regtest-only)");
     }
     string debugCategories = "addrman, alert, bench, coindb, db, http, libevent, lock, mempool, mempoolrej, net, proxy, prune, rand, reindex, rpc, selectcoins, tor, zmq"; // Don't translate these and qt below
     if (mode == HMM_BITCOIN_QT)
@@ -990,6 +991,41 @@ bool AppInit2(boost::thread_group& threadGroup, CScheduler& scheduler)
         fEnableReplacement = (std::find(vstrReplacementModes.begin(), vstrReplacementModes.end(), "fee") != vstrReplacementModes.end());
     }
 
+    if (!mapMultiArgs["-bip9params"].empty()) {
+        // Allow overriding bip9 parameters for testing
+        if (!Params().MineBlocksOnDemand()) {
+            return InitError("BIP9 parameters may only be overridden on regtest.");
+        }
+        const vector<string>& deployments = mapMultiArgs["-bip9params"];
+        for (auto i : deployments) {
+            std::vector<std::string> vDeploymentParams;
+            boost::split(vDeploymentParams, i, boost::is_any_of(":"));
+            if (vDeploymentParams.size() != 3) {
+                return InitError("BIP9 parameters malformed, expecting deployment:start:end");
+            }
+            int64_t nStartTime, nTimeout;
+            if (!ParseInt64(vDeploymentParams[1], &nStartTime)) {
+                return InitError(strprintf("Invalid nStartTime (%s)", vDeploymentParams[1]));
+            }
+            if (!ParseInt64(vDeploymentParams[2], &nTimeout)) {
+                return InitError(strprintf("Invalid nTimeout (%s)", vDeploymentParams[2]));
+            }
+            bool found = false;
+            for (int i=0; i<(int)Consensus::MAX_VERSION_BITS_DEPLOYMENTS; ++i)
+            {
+                if (vDeploymentParams[0].compare(VersionBitsDeploymentInfo[i].name) == 0) {
+                    UpdateRegtestBIP9Parameters(Consensus::DeploymentPos(i), nStartTime, nTimeout);
+                    found = true;
+                    LogPrintf("Setting BIP9 activation parameters for %s to start=%ld, timeout=%ld\n", vDeploymentParams[0], nStartTime, nTimeout);
+                    break;
+                }
+            }
+            if (!found) {
+                return InitError(strprintf("Invalid deployment (%s)", vDeploymentParams[0]));
+            }
+        }
+    }
+
     // ********************************************************* Step 4: application initialization: dir lock, daemonize, pidfile, debug log
 
     // Initialize elliptic curve code

src/main.cpp

@@ -1506,9 +1506,9 @@ bool AcceptToMemoryPoolWorker(CTxMemPool& pool, CValidationState& state, const C
             // SCRIPT_VERIFY_CLEANSTACK requires SCRIPT_VERIFY_WITNESS, so we
             // need to turn both off, and compare against just turning off CLEANSTACK
             // to see if the failure is specifically due to witness validation.
-            if (CheckInputs(tx, state, view, true, scriptVerifyFlags & ~(SCRIPT_VERIFY_WITNESS | SCRIPT_VERIFY_CLEANSTACK), true, txdata) &&
+            if (tx.wit.IsNull() && CheckInputs(tx, state, view, true, scriptVerifyFlags & ~(SCRIPT_VERIFY_WITNESS | SCRIPT_VERIFY_CLEANSTACK), true, txdata) &&
                 !CheckInputs(tx, state, view, true, scriptVerifyFlags & ~SCRIPT_VERIFY_CLEANSTACK, true, txdata)) {
-                // Only the witness is wrong, so the transaction itself may be fine.
+                // Only the witness is missing, so the transaction itself may be fine.
                 state.SetCorruptionPossible();
             }
             return false;
@@ -2399,6 +2399,7 @@ bool ConnectBlock(const CBlock& block, CValidationState& state, CBlockIndex* pin
     // Start enforcing WITNESS rules using versionbits logic.
     if (IsWitnessEnabled(pindex->pprev, chainparams.GetConsensus())) {
         flags |= SCRIPT_VERIFY_WITNESS;
+        flags |= SCRIPT_VERIFY_NULLDUMMY;
     }
 
     int64_t nTime2 = GetTimeMicros(); nTimeForks += nTime2 - nTime1;
@@ -5505,7 +5506,7 @@ bool static ProcessMessage(CNode* pfrom, string strCommand, CDataStream& vRecv,
                     else if (!fMissingInputs2)
                     {
                         int nDos = 0;
-                        if (stateDummy.IsInvalid(nDos) && nDos > 0 && (!state.CorruptionPossible() || State(fromPeer)->fHaveWitness))
+                        if (stateDummy.IsInvalid(nDos) && nDos > 0)
                         {
                             // Punish peer that gave us an invalid orphan tx
                             Misbehaving(fromPeer, nDos);
@@ -5516,7 +5517,10 @@ bool static ProcessMessage(CNode* pfrom, string strCommand, CDataStream& vRecv,
                         // Probably non-standard or insufficient fee/priority
                         LogPrint("mempool", "   removed orphan tx %s\n", orphanHash.ToString());
                         vEraseQueue.push_back(orphanHash);
-                        if (!stateDummy.CorruptionPossible()) {
+                        if (orphanTx.wit.IsNull() && !stateDummy.CorruptionPossible()) {
+                            // Do not use rejection cache for witness transactions or
+                            // witness-stripped transactions, as they can have been malleated.
+                            // See https://github.com/bitcoin/bitcoin/issues/8279 for details.
                             assert(recentRejects);
                             recentRejects->insert(orphanHash);
                         }
@@ -5554,7 +5558,10 @@ bool static ProcessMessage(CNode* pfrom, string strCommand, CDataStream& vRecv,
                 LogPrint("mempool", "not keeping orphan with rejected parents %s\n",tx.GetHash().ToString());
             }
         } else {
-            if (!state.CorruptionPossible()) {
+            if (tx.wit.IsNull() && !state.CorruptionPossible()) {
+                // Do not use rejection cache for witness transactions or
+                // witness-stripped transactions, as they can have been malleated.
+                // See https://github.com/bitcoin/bitcoin/issues/8279 for details.
                 assert(recentRejects);
                 recentRejects->insert(tx.GetHash());
             }
@@ -5586,9 +5593,7 @@ bool static ProcessMessage(CNode* pfrom, string strCommand, CDataStream& vRecv,
             if (state.GetRejectCode() < REJECT_INTERNAL) // Never send AcceptToMemoryPool's internal codes over P2P
                 pfrom->PushMessage(NetMsgType::REJECT, strCommand, (unsigned char)state.GetRejectCode(),
                                    state.GetRejectReason().substr(0, MAX_REJECT_MESSAGE_LENGTH), inv.hash);
-            if (nDoS > 0 && (!state.CorruptionPossible() || State(pfrom->id)->fHaveWitness)) {
-                // When a non-witness-supporting peer gives us a transaction that would
-                // be accepted if witness validation was off, we can't blame them for it.
+            if (nDoS > 0) {
                 Misbehaving(pfrom->GetId(), nDoS);
             }
         }

src/net.cpp

@@ -2226,11 +2226,7 @@ void CNode::RecordBytesSent(uint64_t bytes)
 void CNode::SetMaxOutboundTarget(uint64_t limit)
 {
     LOCK(cs_totalBytesSent);
-    uint64_t recommendedMinimum = (nMaxOutboundTimeframe / 600) * MAX_BLOCK_SERIALIZED_SIZE;
     nMaxOutboundLimit = limit;
-
-    if (limit > 0 && limit < recommendedMinimum)
-        LogPrintf("Max outbound target is very small (%s bytes) and will be overshot. Recommended minimum is %s bytes.\n", nMaxOutboundLimit, recommendedMinimum);
 }
 
 uint64_t CNode::GetMaxOutboundTarget()

src/rpc/misc.cpp

@@ -320,43 +320,6 @@ UniValue createmultisig(const UniValue& params, bool fHelp)
     return result;
 }
 
-UniValue createwitnessaddress(const UniValue& params, bool fHelp)
-{
-    if (fHelp || params.size() < 1 || params.size() > 1)
-    {
-        string msg = "createwitnessaddress \"script\"\n"
-            "\nCreates a witness address for a particular script.\n"
-            "It returns a json object with the address and witness script.\n"
-
-            "\nArguments:\n"
-            "1. \"script\"       (string, required) A hex encoded script\n"
-
-            "\nResult:\n"
-            "{\n"
-            "  \"address\":\"multisigaddress\",  (string) The value of the new address (P2SH of witness script).\n"
-            "  \"witnessScript\":\"script\"      (string) The string value of the hex-encoded witness script.\n"
-            "}\n"
-        ;
-        throw runtime_error(msg);
-    }
-
-    if (!IsHex(params[0].get_str())) {
-        throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Script must be hex-encoded");
-    }
-
-    std::vector<unsigned char> code = ParseHex(params[0].get_str());
-    CScript script(code.begin(), code.end());
-    CScript witscript = GetScriptForWitness(script);
-    CScriptID witscriptid(witscript);
-    CBitcoinAddress address(witscriptid);
-
-    UniValue result(UniValue::VOBJ);
-    result.push_back(Pair("address", address.ToString()));
-    result.push_back(Pair("witnessScript", HexStr(witscript.begin(), witscript.end())));
-
-    return result;
-}
-
 UniValue verifymessage(const UniValue& params, bool fHelp)
 {
     if (fHelp || params.size() != 3)
@@ -490,7 +453,6 @@ static const CRPCCommand commands[] =
     { "control",            "getinfo",                &getinfo,                true  }, /* uses wallet if enabled */
     { "util",               "validateaddress",        &validateaddress,        true  }, /* uses wallet if enabled */
     { "util",               "createmultisig",         &createmultisig,         true  },
-    { "util",               "createwitnessaddress",   &createwitnessaddress,   true  },
     { "util",               "verifymessage",          &verifymessage,          true  },
     { "util",               "signmessagewithprivkey", &signmessagewithprivkey, true  },
 

src/test/hash_tests.cpp

@@ -122,6 +122,10 @@ BOOST_AUTO_TEST_CASE(siphash)
         hasher3.Write(uint64_t(x)|(uint64_t(x+1)<<8)|(uint64_t(x+2)<<16)|(uint64_t(x+3)<<24)|
                      (uint64_t(x+4)<<32)|(uint64_t(x+5)<<40)|(uint64_t(x+6)<<48)|(uint64_t(x+7)<<56));
     }
+
+    CHashWriter ss(SER_DISK, CLIENT_VERSION);
+    ss << CTransaction();
+    BOOST_CHECK_EQUAL(SipHashUint256(1, 2, ss.GetHash()), 0x79751e980c2a0a35ULL);
 }
 
 BOOST_AUTO_TEST_SUITE_END()