From 1672225670cd34e21e397968433d47e6997f9a5f Mon Sep 17 00:00:00 2001 From: Pieter Wuille Date: Tue, 16 Aug 2016 17:38:57 +0200 Subject: [PATCH 01/11] Do not store witness txn in rejection cache Github-Pull: #8525 Rebased-From: 34521e4d7d176109dedf52ec8ef2b5052b9a30f3 --- src/main.cpp | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/src/main.cpp b/src/main.cpp index 82f9e147e..a664bf2de 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -1506,9 +1506,9 @@ bool AcceptToMemoryPoolWorker(CTxMemPool& pool, CValidationState& state, const C // SCRIPT_VERIFY_CLEANSTACK requires SCRIPT_VERIFY_WITNESS, so we // need to turn both off, and compare against just turning off CLEANSTACK // to see if the failure is specifically due to witness validation. - if (CheckInputs(tx, state, view, true, scriptVerifyFlags & ~(SCRIPT_VERIFY_WITNESS | SCRIPT_VERIFY_CLEANSTACK), true, txdata) && + if (tx.wit.IsNull() && CheckInputs(tx, state, view, true, scriptVerifyFlags & ~(SCRIPT_VERIFY_WITNESS | SCRIPT_VERIFY_CLEANSTACK), true, txdata) && !CheckInputs(tx, state, view, true, scriptVerifyFlags & ~SCRIPT_VERIFY_CLEANSTACK, true, txdata)) { - // Only the witness is wrong, so the transaction itself may be fine. + // Only the witness is missing, so the transaction itself may be fine. state.SetCorruptionPossible(); } return false; @@ -5505,7 +5505,7 @@ bool static ProcessMessage(CNode* pfrom, string strCommand, CDataStream& vRecv, else if (!fMissingInputs2) { int nDos = 0; - if (stateDummy.IsInvalid(nDos) && nDos > 0 && (!state.CorruptionPossible() || State(fromPeer)->fHaveWitness)) + if (stateDummy.IsInvalid(nDos) && nDos > 0) { // Punish peer that gave us an invalid orphan tx Misbehaving(fromPeer, nDos); @@ -5516,7 +5516,10 @@ bool static ProcessMessage(CNode* pfrom, string strCommand, CDataStream& vRecv, // Probably non-standard or insufficient fee/priority LogPrint("mempool", " removed orphan tx %s\n", orphanHash.ToString()); vEraseQueue.push_back(orphanHash); - if (!stateDummy.CorruptionPossible()) { + if (orphanTx.wit.IsNull() && !stateDummy.CorruptionPossible()) { + // Do not use rejection cache for witness transactions or + // witness-stripped transactions, as they can have been malleated. + // See https://github.com/bitcoin/bitcoin/issues/8279 for details. assert(recentRejects); recentRejects->insert(orphanHash); } @@ -5554,7 +5557,10 @@ bool static ProcessMessage(CNode* pfrom, string strCommand, CDataStream& vRecv, LogPrint("mempool", "not keeping orphan with rejected parents %s\n",tx.GetHash().ToString()); } } else { - if (!state.CorruptionPossible()) { + if (tx.wit.IsNull() && !state.CorruptionPossible()) { + // Do not use rejection cache for witness transactions or + // witness-stripped transactions, as they can have been malleated. + // See https://github.com/bitcoin/bitcoin/issues/8279 for details. assert(recentRejects); recentRejects->insert(tx.GetHash()); } @@ -5586,9 +5592,7 @@ bool static ProcessMessage(CNode* pfrom, string strCommand, CDataStream& vRecv, if (state.GetRejectCode() < REJECT_INTERNAL) // Never send AcceptToMemoryPool's internal codes over P2P pfrom->PushMessage(NetMsgType::REJECT, strCommand, (unsigned char)state.GetRejectCode(), state.GetRejectReason().substr(0, MAX_REJECT_MESSAGE_LENGTH), inv.hash); - if (nDoS > 0 && (!state.CorruptionPossible() || State(pfrom->id)->fHaveWitness)) { - // When a non-witness-supporting peer gives us a transaction that would - // be accepted if witness validation was off, we can't blame them for it. + if (nDoS > 0) { Misbehaving(pfrom->GetId(), nDoS); } } From b394a96396ce166bf2b519190fdc2a124f57eb5e Mon Sep 17 00:00:00 2001 From: instagibbs Date: Thu, 18 Aug 2016 12:04:33 -0400 Subject: [PATCH 02/11] Add basic test for IsStandard witness transaction blinding Github-Pull: #8525 Rebased-From: ca10a03addf70421893791c2c499e82fc494d60b --- qa/rpc-tests/p2p-segwit.py | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/qa/rpc-tests/p2p-segwit.py b/qa/rpc-tests/p2p-segwit.py index 68d8b9a00..8af46f031 100755 --- a/qa/rpc-tests/p2p-segwit.py +++ b/qa/rpc-tests/p2p-segwit.py @@ -965,8 +965,24 @@ class SegWitTest(BitcoinTestFramework): tx3 = CTransaction() tx3.vin.append(CTxIn(COutPoint(tx2.sha256, 0), b"")) - tx3.vout.append(CTxOut(tx2.vout[0].nValue-1000, CScript([OP_TRUE]))) tx3.wit.vtxinwit.append(CTxInWitness()) + + # Add too-large for IsStandard witness and check that it does not enter reject filter + p2sh_program = CScript([OP_TRUE]) + p2sh_pubkey = hash160(p2sh_program) + witness_program2 = CScript([b'a'*400000]) + tx3.vout.append(CTxOut(tx2.vout[0].nValue-1000, CScript([OP_HASH160, p2sh_pubkey, OP_EQUAL]))) + tx3.wit.vtxinwit[0].scriptWitness.stack = [witness_program2] + tx3.rehash() + + # Node will not be blinded to the transaction + self.std_node.announce_tx_and_wait_for_getdata(tx3) + self.std_node.test_transaction_acceptance(tx3, True, False, b'tx-size') + self.std_node.announce_tx_and_wait_for_getdata(tx3) + self.std_node.test_transaction_acceptance(tx3, True, False, b'tx-size') + + # Remove witness stuffing, instead add extra witness push on stack + tx3.vout[0] = CTxOut(tx2.vout[0].nValue-1000, CScript([OP_TRUE])) tx3.wit.vtxinwit[0].scriptWitness.stack = [CScript([CScriptNum(1)]), witness_program ] tx3.rehash() From a5ec2483235d5585e985715a5f66d41dbe1c424e Mon Sep 17 00:00:00 2001 From: Johnson Lau Date: Mon, 12 Sep 2016 02:03:55 +0800 Subject: [PATCH 03/11] Remove createwitnessaddress This RPC command is unsafe as it will return an address even if the script is invalid. Github-Pull: #8699 Rebased-From: 86c3f8db0bf64693313a81d5fe92ef603499030a --- src/rpc/misc.cpp | 38 -------------------------------------- 1 file changed, 38 deletions(-) diff --git a/src/rpc/misc.cpp b/src/rpc/misc.cpp index a8c5bcd17..06489566b 100644 --- a/src/rpc/misc.cpp +++ b/src/rpc/misc.cpp @@ -320,43 +320,6 @@ UniValue createmultisig(const UniValue& params, bool fHelp) return result; } -UniValue createwitnessaddress(const UniValue& params, bool fHelp) -{ - if (fHelp || params.size() < 1 || params.size() > 1) - { - string msg = "createwitnessaddress \"script\"\n" - "\nCreates a witness address for a particular script.\n" - "It returns a json object with the address and witness script.\n" - - "\nArguments:\n" - "1. \"script\" (string, required) A hex encoded script\n" - - "\nResult:\n" - "{\n" - " \"address\":\"multisigaddress\", (string) The value of the new address (P2SH of witness script).\n" - " \"witnessScript\":\"script\" (string) The string value of the hex-encoded witness script.\n" - "}\n" - ; - throw runtime_error(msg); - } - - if (!IsHex(params[0].get_str())) { - throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Script must be hex-encoded"); - } - - std::vector code = ParseHex(params[0].get_str()); - CScript script(code.begin(), code.end()); - CScript witscript = GetScriptForWitness(script); - CScriptID witscriptid(witscript); - CBitcoinAddress address(witscriptid); - - UniValue result(UniValue::VOBJ); - result.push_back(Pair("address", address.ToString())); - result.push_back(Pair("witnessScript", HexStr(witscript.begin(), witscript.end()))); - - return result; -} - UniValue verifymessage(const UniValue& params, bool fHelp) { if (fHelp || params.size() != 3) @@ -490,7 +453,6 @@ static const CRPCCommand commands[] = { "control", "getinfo", &getinfo, true }, /* uses wallet if enabled */ { "util", "validateaddress", &validateaddress, true }, /* uses wallet if enabled */ { "util", "createmultisig", &createmultisig, true }, - { "util", "createwitnessaddress", &createwitnessaddress, true }, { "util", "verifymessage", &verifymessage, true }, { "util", "signmessagewithprivkey", &signmessagewithprivkey, true }, From 23feab1f38cd63f55631a1b5125ef1bd5ce049df Mon Sep 17 00:00:00 2001 From: Jonas Schnelli Date: Tue, 13 Sep 2016 18:08:17 +0200 Subject: [PATCH 04/11] Remove maxuploadtargets recommended minimum Github-Pull: #8712 Rebased-From: 1b6bcdd3aa379a50c960e23d7c55db8294e76f7f --- doc/reduce-traffic.md | 3 +-- src/net.cpp | 4 ---- 2 files changed, 1 insertion(+), 6 deletions(-) diff --git a/doc/reduce-traffic.md b/doc/reduce-traffic.md index 2d86588eb..697099bea 100644 --- a/doc/reduce-traffic.md +++ b/doc/reduce-traffic.md @@ -19,8 +19,7 @@ This is *not* a hard limit; only a threshold to minimize the outbound traffic. When the limit is about to be reached, the uploaded data is cut by no longer serving historic blocks (blocks older than one week). Keep in mind that new nodes require other nodes that are willing to serve -historic blocks. **The recommended minimum is 144 blocks per day (max. 144MB -per day)** +historic blocks. Whitelisted peers will never be disconnected, although their traffic counts for calculating the target. diff --git a/src/net.cpp b/src/net.cpp index a0ab544f5..5e38ec077 100644 --- a/src/net.cpp +++ b/src/net.cpp @@ -2226,11 +2226,7 @@ void CNode::RecordBytesSent(uint64_t bytes) void CNode::SetMaxOutboundTarget(uint64_t limit) { LOCK(cs_totalBytesSent); - uint64_t recommendedMinimum = (nMaxOutboundTimeframe / 600) * MAX_BLOCK_SERIALIZED_SIZE; nMaxOutboundLimit = limit; - - if (limit > 0 && limit < recommendedMinimum) - LogPrintf("Max outbound target is very small (%s bytes) and will be overshot. Recommended minimum is %s bytes.\n", nMaxOutboundLimit, recommendedMinimum); } uint64_t CNode::GetMaxOutboundTarget() From 198494ce5323cafbf90a7b543d1b07355eb3db6c Mon Sep 17 00:00:00 2001 From: Suhas Daftuar Date: Mon, 25 Jul 2016 17:22:37 -0400 Subject: [PATCH 05/11] Allow changing BIP9 parameters on regtest Github-Pull: #8418 Rebased-From: 56c87e92110f05d7452f1e85bf755246ffc77206 --- src/chainparams.cpp | 11 +++++++++++ src/chainparams.h | 5 +++++ src/init.cpp | 36 ++++++++++++++++++++++++++++++++++++ 3 files changed, 52 insertions(+) diff --git a/src/chainparams.cpp b/src/chainparams.cpp index 0cb0fccd1..7b493ed8a 100644 --- a/src/chainparams.cpp +++ b/src/chainparams.cpp @@ -303,6 +303,12 @@ public: base58Prefixes[EXT_PUBLIC_KEY] = boost::assign::list_of(0x04)(0x35)(0x87)(0xCF).convert_to_container >(); base58Prefixes[EXT_SECRET_KEY] = boost::assign::list_of(0x04)(0x35)(0x83)(0x94).convert_to_container >(); } + + void UpdateBIP9Parameters(Consensus::DeploymentPos d, int64_t nStartTime, int64_t nTimeout) + { + consensus.vDeployments[d].nStartTime = nStartTime; + consensus.vDeployments[d].nTimeout = nTimeout; + } }; static CRegTestParams regTestParams; @@ -330,4 +336,9 @@ void SelectParams(const std::string& network) SelectBaseParams(network); pCurrentParams = &Params(network); } + +void UpdateRegtestBIP9Parameters(Consensus::DeploymentPos d, int64_t nStartTime, int64_t nTimeout) +{ + regTestParams.UpdateBIP9Parameters(d, nStartTime, nTimeout); +} diff --git a/src/chainparams.h b/src/chainparams.h index 638893e9a..0c3820b7c 100644 --- a/src/chainparams.h +++ b/src/chainparams.h @@ -112,4 +112,9 @@ CChainParams& Params(const std::string& chain); */ void SelectParams(const std::string& chain); +/** + * Allows modifying the BIP9 regtest parameters. + */ +void UpdateRegtestBIP9Parameters(Consensus::DeploymentPos d, int64_t nStartTime, int64_t nTimeout); + #endif // BITCOIN_CHAINPARAMS_H diff --git a/src/init.cpp b/src/init.cpp index 9b6eca2e1..f2b13b627 100644 --- a/src/init.cpp +++ b/src/init.cpp @@ -410,6 +410,7 @@ std::string HelpMessage(HelpMessageMode mode) strUsage += HelpMessageOpt("-limitancestorsize=", strprintf("Do not accept transactions whose size with all in-mempool ancestors exceeds kilobytes (default: %u)", DEFAULT_ANCESTOR_SIZE_LIMIT)); strUsage += HelpMessageOpt("-limitdescendantcount=", strprintf("Do not accept transactions if any ancestor would have or more in-mempool descendants (default: %u)", DEFAULT_DESCENDANT_LIMIT)); strUsage += HelpMessageOpt("-limitdescendantsize=", strprintf("Do not accept transactions if any ancestor would have more than kilobytes of in-mempool descendants (default: %u).", DEFAULT_DESCENDANT_SIZE_LIMIT)); + strUsage += HelpMessageOpt("-bip9params=deployment:start:end", "Use given start/end times for specified bip9 deployment (regtest-only)"); } string debugCategories = "addrman, alert, bench, coindb, db, http, libevent, lock, mempool, mempoolrej, net, proxy, prune, rand, reindex, rpc, selectcoins, tor, zmq"; // Don't translate these and qt below if (mode == HMM_BITCOIN_QT) @@ -990,6 +991,41 @@ bool AppInit2(boost::thread_group& threadGroup, CScheduler& scheduler) fEnableReplacement = (std::find(vstrReplacementModes.begin(), vstrReplacementModes.end(), "fee") != vstrReplacementModes.end()); } + if (!mapMultiArgs["-bip9params"].empty()) { + // Allow overriding bip9 parameters for testing + if (!Params().MineBlocksOnDemand()) { + return InitError("BIP9 parameters may only be overridden on regtest."); + } + const vector& deployments = mapMultiArgs["-bip9params"]; + for (auto i : deployments) { + std::vector vDeploymentParams; + boost::split(vDeploymentParams, i, boost::is_any_of(":")); + if (vDeploymentParams.size() != 3) { + return InitError("BIP9 parameters malformed, expecting deployment:start:end"); + } + int64_t nStartTime, nTimeout; + if (!ParseInt64(vDeploymentParams[1], &nStartTime)) { + return InitError(strprintf("Invalid nStartTime (%s)", vDeploymentParams[1])); + } + if (!ParseInt64(vDeploymentParams[2], &nTimeout)) { + return InitError(strprintf("Invalid nTimeout (%s)", vDeploymentParams[2])); + } + bool found = false; + for (int i=0; i<(int)Consensus::MAX_VERSION_BITS_DEPLOYMENTS; ++i) + { + if (vDeploymentParams[0].compare(VersionBitsDeploymentInfo[i].name) == 0) { + UpdateRegtestBIP9Parameters(Consensus::DeploymentPos(i), nStartTime, nTimeout); + found = true; + LogPrintf("Setting BIP9 activation parameters for %s to start=%ld, timeout=%ld\n", vDeploymentParams[0], nStartTime, nTimeout); + break; + } + } + if (!found) { + return InitError(strprintf("Invalid deployment (%s)", vDeploymentParams[0])); + } + } + } + // ********************************************************* Step 4: application initialization: dir lock, daemonize, pidfile, debug log // Initialize elliptic curve code From ff893aa5574f5c422b72dd10b00eac805fc737e5 Mon Sep 17 00:00:00 2001 From: Pieter Wuille Date: Mon, 27 Jun 2016 16:06:17 +0200 Subject: [PATCH 06/11] Implement SipHash in Python Github-Pull: #8418 Rebased-From: 9c8593d2b4e25ef628172ceadbedf0ef078d01ef --- qa/rpc-tests/test_framework/siphash.py | 64 ++++++++++++++++++++++++++ src/test/hash_tests.cpp | 4 ++ 2 files changed, 68 insertions(+) create mode 100644 qa/rpc-tests/test_framework/siphash.py diff --git a/qa/rpc-tests/test_framework/siphash.py b/qa/rpc-tests/test_framework/siphash.py new file mode 100644 index 000000000..9c0574bd9 --- /dev/null +++ b/qa/rpc-tests/test_framework/siphash.py @@ -0,0 +1,64 @@ +#!/usr/bin/env python3 +# Copyright (c) 2016 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. + +# +# siphash.py - Specialized SipHash-2-4 implementations +# +# This implements SipHash-2-4 for 256-bit integers. + +def rotl64(n, b): + return n >> (64 - b) | (n & ((1 << (64 - b)) - 1)) << b + +def siphash_round(v0, v1, v2, v3): + v0 = (v0 + v1) & ((1 << 64) - 1) + v1 = rotl64(v1, 13) + v1 ^= v0 + v0 = rotl64(v0, 32) + v2 = (v2 + v3) & ((1 << 64) - 1) + v3 = rotl64(v3, 16) + v3 ^= v2 + v0 = (v0 + v3) & ((1 << 64) - 1) + v3 = rotl64(v3, 21) + v3 ^= v0 + v2 = (v2 + v1) & ((1 << 64) - 1) + v1 = rotl64(v1, 17) + v1 ^= v2 + v2 = rotl64(v2, 32) + return (v0, v1, v2, v3) + +def siphash256(k0, k1, h): + n0 = h & ((1 << 64) - 1) + n1 = (h >> 64) & ((1 << 64) - 1) + n2 = (h >> 128) & ((1 << 64) - 1) + n3 = (h >> 192) & ((1 << 64) - 1) + v0 = 0x736f6d6570736575 ^ k0 + v1 = 0x646f72616e646f6d ^ k1 + v2 = 0x6c7967656e657261 ^ k0 + v3 = 0x7465646279746573 ^ k1 ^ n0 + v0, v1, v2, v3 = siphash_round(v0, v1, v2, v3) + v0, v1, v2, v3 = siphash_round(v0, v1, v2, v3) + v0 ^= n0 + v3 ^= n1 + v0, v1, v2, v3 = siphash_round(v0, v1, v2, v3) + v0, v1, v2, v3 = siphash_round(v0, v1, v2, v3) + v0 ^= n1 + v3 ^= n2 + v0, v1, v2, v3 = siphash_round(v0, v1, v2, v3) + v0, v1, v2, v3 = siphash_round(v0, v1, v2, v3) + v0 ^= n2 + v3 ^= n3 + v0, v1, v2, v3 = siphash_round(v0, v1, v2, v3) + v0, v1, v2, v3 = siphash_round(v0, v1, v2, v3) + v0 ^= n3 + v3 ^= 0x2000000000000000 + v0, v1, v2, v3 = siphash_round(v0, v1, v2, v3) + v0, v1, v2, v3 = siphash_round(v0, v1, v2, v3) + v0 ^= 0x2000000000000000 + v2 ^= 0xFF + v0, v1, v2, v3 = siphash_round(v0, v1, v2, v3) + v0, v1, v2, v3 = siphash_round(v0, v1, v2, v3) + v0, v1, v2, v3 = siphash_round(v0, v1, v2, v3) + v0, v1, v2, v3 = siphash_round(v0, v1, v2, v3) + return v0 ^ v1 ^ v2 ^ v3 diff --git a/src/test/hash_tests.cpp b/src/test/hash_tests.cpp index 82d61209b..fa9624f13 100644 --- a/src/test/hash_tests.cpp +++ b/src/test/hash_tests.cpp @@ -122,6 +122,10 @@ BOOST_AUTO_TEST_CASE(siphash) hasher3.Write(uint64_t(x)|(uint64_t(x+1)<<8)|(uint64_t(x+2)<<16)|(uint64_t(x+3)<<24)| (uint64_t(x+4)<<32)|(uint64_t(x+5)<<40)|(uint64_t(x+6)<<48)|(uint64_t(x+7)<<56)); } + + CHashWriter ss(SER_DISK, CLIENT_VERSION); + ss << CTransaction(); + BOOST_CHECK_EQUAL(SipHashUint256(1, 2, ss.GetHash()), 0x79751e980c2a0a35ULL); } BOOST_AUTO_TEST_SUITE_END() From 4295a7aea55be34c95a0624e95ea0b9587e644b6 Mon Sep 17 00:00:00 2001 From: Suhas Daftuar Date: Fri, 17 Jun 2016 21:17:25 -0400 Subject: [PATCH 07/11] Tests: refactor compact size serialization in mininode Github-Pull: #8418 Rebased-From: a8689fdf8e10300b73750161a73a23467ecd1efe --- qa/rpc-tests/test_framework/mininode.py | 97 +++++++------------------ 1 file changed, 25 insertions(+), 72 deletions(-) diff --git a/qa/rpc-tests/test_framework/mininode.py b/qa/rpc-tests/test_framework/mininode.py index cdd5292cd..67aaab698 100755 --- a/qa/rpc-tests/test_framework/mininode.py +++ b/qa/rpc-tests/test_framework/mininode.py @@ -74,8 +74,19 @@ def ripemd160(s): def hash256(s): return sha256(sha256(s)) +def ser_compact_size(l): + r = b"" + if l < 253: + r = struct.pack("B", l) + elif l < 0x10000: + r = struct.pack(" Date: Tue, 12 Jul 2016 16:04:38 -0400 Subject: [PATCH 08/11] Add support for compactblocks to mininode Github-Pull: #8418 Rebased-From: 9a22a6c0891256f02f4906c1c13fb22a9722ec7c --- qa/rpc-tests/test_framework/mininode.py | 276 +++++++++++++++++++++++- 1 file changed, 272 insertions(+), 4 deletions(-) diff --git a/qa/rpc-tests/test_framework/mininode.py b/qa/rpc-tests/test_framework/mininode.py index 67aaab698..caffab353 100755 --- a/qa/rpc-tests/test_framework/mininode.py +++ b/qa/rpc-tests/test_framework/mininode.py @@ -36,9 +36,10 @@ from threading import RLock from threading import Thread import logging import copy +from test_framework.siphash import siphash256 BIP0031_VERSION = 60000 -MY_VERSION = 60001 # past bip-31 for ping/pong +MY_VERSION = 70014 # past bip-31 for ping/pong MY_SUBVERSION = b"/python-mininode-tester:0.0.3/" MAX_INV_SZ = 50000 @@ -52,7 +53,7 @@ NODE_BLOOM = (1 << 2) NODE_WITNESS = (1 << 3) # Keep our own socket map for asyncore, so that we can track disconnects -# ourselves (to workaround an issue with closing an asyncore socket when +# ourselves (to workaround an issue with closing an asyncore socket when # using select) mininode_socket_map = dict() @@ -247,7 +248,8 @@ class CInv(object): 1: "TX", 2: "Block", 1|MSG_WITNESS_FLAG: "WitnessTx", - 2|MSG_WITNESS_FLAG : "WitnessBlock" + 2|MSG_WITNESS_FLAG : "WitnessBlock", + 4: "CompactBlock" } def __init__(self, t=0, h=0): @@ -734,6 +736,187 @@ class CAlert(object): % (len(self.vchMsg), len(self.vchSig)) +class PrefilledTransaction(object): + def __init__(self, index=0, tx = None): + self.index = index + self.tx = tx + + def deserialize(self, f): + self.index = deser_compact_size(f) + self.tx = CTransaction() + self.tx.deserialize(f) + + def serialize(self, with_witness=False): + r = b"" + r += ser_compact_size(self.index) + if with_witness: + r += self.tx.serialize_with_witness() + else: + r += self.tx.serialize_without_witness() + return r + + def __repr__(self): + return "PrefilledTransaction(index=%d, tx=%s)" % (self.index, repr(self.tx)) + +# This is what we send on the wire, in a cmpctblock message. +class P2PHeaderAndShortIDs(object): + def __init__(self): + self.header = CBlockHeader() + self.nonce = 0 + self.shortids_length = 0 + self.shortids = [] + self.prefilled_txn_length = 0 + self.prefilled_txn = [] + + def deserialize(self, f): + self.header.deserialize(f) + self.nonce = struct.unpack(" Date: Tue, 12 Jul 2016 16:05:02 -0400 Subject: [PATCH 09/11] Add p2p test for BIP 152 (compact blocks) Github-Pull: #8418 Rebased-From: 45c7ddd109465e03551f5b39c2e650c243b4a078 --- qa/pull-tester/rpc-tests.py | 1 + qa/rpc-tests/p2p-compactblocks.py | 608 ++++++++++++++++++++++++++++++ 2 files changed, 609 insertions(+) create mode 100755 qa/rpc-tests/p2p-compactblocks.py diff --git a/qa/pull-tester/rpc-tests.py b/qa/pull-tester/rpc-tests.py index 634d67589..1e57a3fc2 100755 --- a/qa/pull-tester/rpc-tests.py +++ b/qa/pull-tester/rpc-tests.py @@ -142,6 +142,7 @@ testScripts = [ 'segwit.py', 'importprunedfunds.py', 'signmessages.py', + 'p2p-compactblocks.py', ] if ENABLE_ZMQ: testScripts.append('zmq_test.py') diff --git a/qa/rpc-tests/p2p-compactblocks.py b/qa/rpc-tests/p2p-compactblocks.py new file mode 100755 index 000000000..7fe7ecc16 --- /dev/null +++ b/qa/rpc-tests/p2p-compactblocks.py @@ -0,0 +1,608 @@ +#!/usr/bin/env python3 +# Copyright (c) 2016 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. + +from test_framework.mininode import * +from test_framework.test_framework import BitcoinTestFramework +from test_framework.util import * +from test_framework.blocktools import create_block, create_coinbase +from test_framework.siphash import siphash256 +from test_framework.script import CScript, OP_TRUE + +''' +CompactBlocksTest -- test compact blocks (BIP 152) +''' + + +# TestNode: A peer we use to send messages to bitcoind, and store responses. +class TestNode(SingleNodeConnCB): + def __init__(self): + SingleNodeConnCB.__init__(self) + self.last_sendcmpct = None + self.last_headers = None + self.last_inv = None + self.last_cmpctblock = None + self.block_announced = False + self.last_getdata = None + self.last_getblocktxn = None + self.last_block = None + self.last_blocktxn = None + + def on_sendcmpct(self, conn, message): + self.last_sendcmpct = message + + def on_block(self, conn, message): + self.last_block = message + + def on_cmpctblock(self, conn, message): + self.last_cmpctblock = message + self.block_announced = True + + def on_headers(self, conn, message): + self.last_headers = message + self.block_announced = True + + def on_inv(self, conn, message): + self.last_inv = message + self.block_announced = True + + def on_getdata(self, conn, message): + self.last_getdata = message + + def on_getblocktxn(self, conn, message): + self.last_getblocktxn = message + + def on_blocktxn(self, conn, message): + self.last_blocktxn = message + + # Requires caller to hold mininode_lock + def received_block_announcement(self): + return self.block_announced + + def clear_block_announcement(self): + with mininode_lock: + self.block_announced = False + self.last_inv = None + self.last_headers = None + self.last_cmpctblock = None + + def get_headers(self, locator, hashstop): + msg = msg_getheaders() + msg.locator.vHave = locator + msg.hashstop = hashstop + self.connection.send_message(msg) + + def send_header_for_blocks(self, new_blocks): + headers_message = msg_headers() + headers_message.headers = [CBlockHeader(b) for b in new_blocks] + self.send_message(headers_message) + + +class CompactBlocksTest(BitcoinTestFramework): + def __init__(self): + super().__init__() + self.setup_clean_chain = True + self.num_nodes = 1 + self.utxos = [] + + def setup_network(self): + self.nodes = [] + + # Turn off segwit in this test, as compact blocks don't currently work + # with segwit. (After BIP 152 is updated to support segwit, we can + # test behavior with and without segwit enabled by adding a second node + # to the test.) + self.nodes = start_nodes(self.num_nodes, self.options.tmpdir, [["-debug", "-logtimemicros=1", "-bip9params=segwit:0:0"]]) + + def build_block_on_tip(self): + height = self.nodes[0].getblockcount() + tip = self.nodes[0].getbestblockhash() + mtp = self.nodes[0].getblockheader(tip)['mediantime'] + block = create_block(int(tip, 16), create_coinbase(height + 1), mtp + 1) + block.solve() + return block + + # Create 10 more anyone-can-spend utxo's for testing. + def make_utxos(self): + block = self.build_block_on_tip() + self.test_node.send_and_ping(msg_block(block)) + assert(int(self.nodes[0].getbestblockhash(), 16) == block.sha256) + self.nodes[0].generate(100) + + total_value = block.vtx[0].vout[0].nValue + out_value = total_value // 10 + tx = CTransaction() + tx.vin.append(CTxIn(COutPoint(block.vtx[0].sha256, 0), b'')) + for i in range(10): + tx.vout.append(CTxOut(out_value, CScript([OP_TRUE]))) + tx.rehash() + + block2 = self.build_block_on_tip() + block2.vtx.append(tx) + block2.hashMerkleRoot = block2.calc_merkle_root() + block2.solve() + self.test_node.send_and_ping(msg_block(block2)) + assert_equal(int(self.nodes[0].getbestblockhash(), 16), block2.sha256) + self.utxos.extend([[tx.sha256, i, out_value] for i in range(10)]) + return + + # Test "sendcmpct": + # - No compact block announcements or getdata(MSG_CMPCT_BLOCK) unless + # sendcmpct is sent. + # - If sendcmpct is sent with version > 0, the message is ignored. + # - If sendcmpct is sent with boolean 0, then block announcements are not + # made with compact blocks. + # - If sendcmpct is then sent with boolean 1, then new block announcements + # are made with compact blocks. + def test_sendcmpct(self): + print("Testing SENDCMPCT p2p message... ") + + # Make sure we get a version 0 SENDCMPCT message from our peer + def received_sendcmpct(): + return (self.test_node.last_sendcmpct is not None) + got_message = wait_until(received_sendcmpct, timeout=30) + assert(got_message) + assert_equal(self.test_node.last_sendcmpct.version, 1) + + tip = int(self.nodes[0].getbestblockhash(), 16) + + def check_announcement_of_new_block(node, peer, predicate): + self.test_node.clear_block_announcement() + node.generate(1) + got_message = wait_until(peer.received_block_announcement, timeout=30) + assert(got_message) + with mininode_lock: + assert(predicate) + + # We shouldn't get any block announcements via cmpctblock yet. + check_announcement_of_new_block(self.nodes[0], self.test_node, lambda: self.test_node.last_cmpctblock is None) + + # Try one more time, this time after requesting headers. + self.test_node.clear_block_announcement() + self.test_node.get_headers(locator=[tip], hashstop=0) + wait_until(self.test_node.received_block_announcement, timeout=30) + self.test_node.clear_block_announcement() + + check_announcement_of_new_block(self.nodes[0], self.test_node, lambda: self.test_node.last_cmpctblock is None and self.test_node.last_inv is not None) + + # Now try a SENDCMPCT message with too-high version + sendcmpct = msg_sendcmpct() + sendcmpct.version = 2 + self.test_node.send_message(sendcmpct) + + check_announcement_of_new_block(self.nodes[0], self.test_node, lambda: self.test_node.last_cmpctblock is None) + + # Now try a SENDCMPCT message with valid version, but announce=False + self.test_node.send_message(msg_sendcmpct()) + check_announcement_of_new_block(self.nodes[0], self.test_node, lambda: self.test_node.last_cmpctblock is None) + + # Finally, try a SENDCMPCT message with announce=True + sendcmpct.version = 1 + sendcmpct.announce = True + self.test_node.send_message(sendcmpct) + check_announcement_of_new_block(self.nodes[0], self.test_node, lambda: self.test_node.last_cmpctblock is not None) + + # Try one more time + check_announcement_of_new_block(self.nodes[0], self.test_node, lambda: self.test_node.last_cmpctblock is not None) + + # Try one more time, after turning on sendheaders + self.test_node.send_message(msg_sendheaders()) + check_announcement_of_new_block(self.nodes[0], self.test_node, lambda: self.test_node.last_cmpctblock is not None) + + # Now turn off announcements + sendcmpct.announce = False + check_announcement_of_new_block(self.nodes[0], self.test_node, lambda: self.test_node.last_cmpctblock is None and self.test_node.last_headers is not None) + + # This test actually causes bitcoind to (reasonably!) disconnect us, so do this last. + def test_invalid_cmpctblock_message(self): + print("Testing invalid index in cmpctblock message...") + self.nodes[0].generate(101) + block = self.build_block_on_tip() + + cmpct_block = P2PHeaderAndShortIDs() + cmpct_block.header = CBlockHeader(block) + cmpct_block.prefilled_txn_length = 1 + # This index will be too high + prefilled_txn = PrefilledTransaction(1, block.vtx[0]) + cmpct_block.prefilled_txn = [prefilled_txn] + self.test_node.send_and_ping(msg_cmpctblock(cmpct_block)) + assert(int(self.nodes[0].getbestblockhash(), 16) == block.hashPrevBlock) + + # Compare the generated shortids to what we expect based on BIP 152, given + # bitcoind's choice of nonce. + def test_compactblock_construction(self): + print("Testing compactblock headers and shortIDs are correct...") + + # Generate a bunch of transactions. + self.nodes[0].generate(101) + num_transactions = 25 + address = self.nodes[0].getnewaddress() + for i in range(num_transactions): + self.nodes[0].sendtoaddress(address, 0.1) + + # Now mine a block, and look at the resulting compact block. + self.test_node.clear_block_announcement() + block_hash = int(self.nodes[0].generate(1)[0], 16) + + # Store the raw block in our internal format. + block = FromHex(CBlock(), self.nodes[0].getblock("%02x" % block_hash, False)) + [tx.calc_sha256() for tx in block.vtx] + block.rehash() + + # Don't care which type of announcement came back for this test; just + # request the compact block if we didn't get one yet. + wait_until(self.test_node.received_block_announcement, timeout=30) + + with mininode_lock: + if self.test_node.last_cmpctblock is None: + self.test_node.clear_block_announcement() + inv = CInv(4, block_hash) # 4 == "CompactBlock" + self.test_node.send_message(msg_getdata([inv])) + + wait_until(self.test_node.received_block_announcement, timeout=30) + + # Now we should have the compactblock + header_and_shortids = None + with mininode_lock: + assert(self.test_node.last_cmpctblock is not None) + # Convert the on-the-wire representation to absolute indexes + header_and_shortids = HeaderAndShortIDs(self.test_node.last_cmpctblock.header_and_shortids) + + # Check that we got the right block! + header_and_shortids.header.calc_sha256() + assert_equal(header_and_shortids.header.sha256, block_hash) + + # Make sure the prefilled_txn appears to have included the coinbase + assert(len(header_and_shortids.prefilled_txn) >= 1) + assert_equal(header_and_shortids.prefilled_txn[0].index, 0) + + # Check that all prefilled_txn entries match what's in the block. + for entry in header_and_shortids.prefilled_txn: + entry.tx.calc_sha256() + assert_equal(entry.tx.sha256, block.vtx[entry.index].sha256) + + # Check that the cmpctblock message announced all the transactions. + assert_equal(len(header_and_shortids.prefilled_txn) + len(header_and_shortids.shortids), len(block.vtx)) + + # And now check that all the shortids are as expected as well. + # Determine the siphash keys to use. + [k0, k1] = header_and_shortids.get_siphash_keys() + + index = 0 + while index < len(block.vtx): + if (len(header_and_shortids.prefilled_txn) > 0 and + header_and_shortids.prefilled_txn[0].index == index): + # Already checked prefilled transactions above + header_and_shortids.prefilled_txn.pop(0) + else: + shortid = calculate_shortid(k0, k1, block.vtx[index].sha256) + assert_equal(shortid, header_and_shortids.shortids[0]) + header_and_shortids.shortids.pop(0) + index += 1 + + # Test that bitcoind requests compact blocks when we announce new blocks + # via header or inv, and that responding to getblocktxn causes the block + # to be successfully reconstructed. + def test_compactblock_requests(self): + print("Testing compactblock requests... ") + + # Try announcing a block with an inv or header, expect a compactblock + # request + for announce in ["inv", "header"]: + block = self.build_block_on_tip() + with mininode_lock: + self.test_node.last_getdata = None + + if announce == "inv": + self.test_node.send_message(msg_inv([CInv(2, block.sha256)])) + else: + self.test_node.send_header_for_blocks([block]) + success = wait_until(lambda: self.test_node.last_getdata is not None, timeout=30) + assert(success) + assert_equal(len(self.test_node.last_getdata.inv), 1) + assert_equal(self.test_node.last_getdata.inv[0].type, 4) + assert_equal(self.test_node.last_getdata.inv[0].hash, block.sha256) + + # Send back a compactblock message that omits the coinbase + comp_block = HeaderAndShortIDs() + comp_block.header = CBlockHeader(block) + comp_block.nonce = 0 + comp_block.shortids = [1] # this is useless, and wrong + self.test_node.send_and_ping(msg_cmpctblock(comp_block.to_p2p())) + assert_equal(int(self.nodes[0].getbestblockhash(), 16), block.hashPrevBlock) + # Expect a getblocktxn message. + with mininode_lock: + assert(self.test_node.last_getblocktxn is not None) + absolute_indexes = self.test_node.last_getblocktxn.block_txn_request.to_absolute() + assert_equal(absolute_indexes, [0]) # should be a coinbase request + + # Send the coinbase, and verify that the tip advances. + msg = msg_blocktxn() + msg.block_transactions.blockhash = block.sha256 + msg.block_transactions.transactions = [block.vtx[0]] + self.test_node.send_and_ping(msg) + assert_equal(int(self.nodes[0].getbestblockhash(), 16), block.sha256) + + # Create a chain of transactions from given utxo, and add to a new block. + def build_block_with_transactions(self, utxo, num_transactions): + block = self.build_block_on_tip() + + for i in range(num_transactions): + tx = CTransaction() + tx.vin.append(CTxIn(COutPoint(utxo[0], utxo[1]), b'')) + tx.vout.append(CTxOut(utxo[2] - 1000, CScript([OP_TRUE]))) + tx.rehash() + utxo = [tx.sha256, 0, tx.vout[0].nValue] + block.vtx.append(tx) + + block.hashMerkleRoot = block.calc_merkle_root() + block.solve() + return block + + # Test that we only receive getblocktxn requests for transactions that the + # node needs, and that responding to them causes the block to be + # reconstructed. + def test_getblocktxn_requests(self): + print("Testing getblocktxn requests...") + + # First try announcing compactblocks that won't reconstruct, and verify + # that we receive getblocktxn messages back. + utxo = self.utxos.pop(0) + + block = self.build_block_with_transactions(utxo, 5) + self.utxos.append([block.vtx[-1].sha256, 0, block.vtx[-1].vout[0].nValue]) + + comp_block = HeaderAndShortIDs() + comp_block.initialize_from_block(block) + + self.test_node.send_and_ping(msg_cmpctblock(comp_block.to_p2p())) + with mininode_lock: + assert(self.test_node.last_getblocktxn is not None) + absolute_indexes = self.test_node.last_getblocktxn.block_txn_request.to_absolute() + assert_equal(absolute_indexes, [1, 2, 3, 4, 5]) + msg = msg_blocktxn() + msg.block_transactions = BlockTransactions(block.sha256, block.vtx[1:]) + self.test_node.send_and_ping(msg) + assert_equal(int(self.nodes[0].getbestblockhash(), 16), block.sha256) + + utxo = self.utxos.pop(0) + block = self.build_block_with_transactions(utxo, 5) + self.utxos.append([block.vtx[-1].sha256, 0, block.vtx[-1].vout[0].nValue]) + + # Now try interspersing the prefilled transactions + comp_block.initialize_from_block(block, prefill_list=[0, 1, 5]) + self.test_node.send_and_ping(msg_cmpctblock(comp_block.to_p2p())) + with mininode_lock: + assert(self.test_node.last_getblocktxn is not None) + absolute_indexes = self.test_node.last_getblocktxn.block_txn_request.to_absolute() + assert_equal(absolute_indexes, [2, 3, 4]) + msg.block_transactions = BlockTransactions(block.sha256, block.vtx[2:5]) + self.test_node.send_and_ping(msg) + assert_equal(int(self.nodes[0].getbestblockhash(), 16), block.sha256) + + # Now try giving one transaction ahead of time. + utxo = self.utxos.pop(0) + block = self.build_block_with_transactions(utxo, 5) + self.utxos.append([block.vtx[-1].sha256, 0, block.vtx[-1].vout[0].nValue]) + self.test_node.send_and_ping(msg_tx(block.vtx[1])) + assert(block.vtx[1].hash in self.nodes[0].getrawmempool()) + + # Prefill 4 out of the 6 transactions, and verify that only the one + # that was not in the mempool is requested. + comp_block.initialize_from_block(block, prefill_list=[0, 2, 3, 4]) + self.test_node.send_and_ping(msg_cmpctblock(comp_block.to_p2p())) + with mininode_lock: + assert(self.test_node.last_getblocktxn is not None) + absolute_indexes = self.test_node.last_getblocktxn.block_txn_request.to_absolute() + assert_equal(absolute_indexes, [5]) + + msg.block_transactions = BlockTransactions(block.sha256, [block.vtx[5]]) + self.test_node.send_and_ping(msg) + assert_equal(int(self.nodes[0].getbestblockhash(), 16), block.sha256) + + # Now provide all transactions to the node before the block is + # announced and verify reconstruction happens immediately. + utxo = self.utxos.pop(0) + block = self.build_block_with_transactions(utxo, 10) + self.utxos.append([block.vtx[-1].sha256, 0, block.vtx[-1].vout[0].nValue]) + for tx in block.vtx[1:]: + self.test_node.send_message(msg_tx(tx)) + self.test_node.sync_with_ping() + # Make sure all transactions were accepted. + mempool = self.nodes[0].getrawmempool() + for tx in block.vtx[1:]: + assert(tx.hash in mempool) + + # Clear out last request. + with mininode_lock: + self.test_node.last_getblocktxn = None + + # Send compact block + comp_block.initialize_from_block(block, prefill_list=[0]) + self.test_node.send_and_ping(msg_cmpctblock(comp_block.to_p2p())) + with mininode_lock: + # Shouldn't have gotten a request for any transaction + assert(self.test_node.last_getblocktxn is None) + # Tip should have updated + assert_equal(int(self.nodes[0].getbestblockhash(), 16), block.sha256) + + # Incorrectly responding to a getblocktxn shouldn't cause the block to be + # permanently failed. + def test_incorrect_blocktxn_response(self): + print("Testing handling of incorrect blocktxn responses...") + + if (len(self.utxos) == 0): + self.make_utxos() + utxo = self.utxos.pop(0) + + block = self.build_block_with_transactions(utxo, 10) + self.utxos.append([block.vtx[-1].sha256, 0, block.vtx[-1].vout[0].nValue]) + # Relay the first 5 transactions from the block in advance + for tx in block.vtx[1:6]: + self.test_node.send_message(msg_tx(tx)) + self.test_node.sync_with_ping() + # Make sure all transactions were accepted. + mempool = self.nodes[0].getrawmempool() + for tx in block.vtx[1:6]: + assert(tx.hash in mempool) + + # Send compact block + comp_block = HeaderAndShortIDs() + comp_block.initialize_from_block(block, prefill_list=[0]) + self.test_node.send_and_ping(msg_cmpctblock(comp_block.to_p2p())) + absolute_indexes = [] + with mininode_lock: + assert(self.test_node.last_getblocktxn is not None) + absolute_indexes = self.test_node.last_getblocktxn.block_txn_request.to_absolute() + assert_equal(absolute_indexes, [6, 7, 8, 9, 10]) + + # Now give an incorrect response. + # Note that it's possible for bitcoind to be smart enough to know we're + # lying, since it could check to see if the shortid matches what we're + # sending, and eg disconnect us for misbehavior. If that behavior + # change were made, we could just modify this test by having a + # different peer provide the block further down, so that we're still + # verifying that the block isn't marked bad permanently. This is good + # enough for now. + msg = msg_blocktxn() + msg.block_transactions = BlockTransactions(block.sha256, [block.vtx[5]] + block.vtx[7:]) + self.test_node.send_and_ping(msg) + + # Tip should not have updated + assert_equal(int(self.nodes[0].getbestblockhash(), 16), block.hashPrevBlock) + + # We should receive a getdata request + success = wait_until(lambda: self.test_node.last_getdata is not None, timeout=10) + assert(success) + assert_equal(len(self.test_node.last_getdata.inv), 1) + assert_equal(self.test_node.last_getdata.inv[0].type, 2) + assert_equal(self.test_node.last_getdata.inv[0].hash, block.sha256) + + # Deliver the block + self.test_node.send_and_ping(msg_block(block)) + assert_equal(int(self.nodes[0].getbestblockhash(), 16), block.sha256) + + def test_getblocktxn_handler(self): + print("Testing getblocktxn handler...") + + # bitcoind won't respond for blocks whose height is more than 15 blocks + # deep. + MAX_GETBLOCKTXN_DEPTH = 15 + chain_height = self.nodes[0].getblockcount() + current_height = chain_height + while (current_height >= chain_height - MAX_GETBLOCKTXN_DEPTH): + block_hash = self.nodes[0].getblockhash(current_height) + block = FromHex(CBlock(), self.nodes[0].getblock(block_hash, False)) + + msg = msg_getblocktxn() + msg.block_txn_request = BlockTransactionsRequest(int(block_hash, 16), []) + num_to_request = random.randint(1, len(block.vtx)) + msg.block_txn_request.from_absolute(sorted(random.sample(range(len(block.vtx)), num_to_request))) + self.test_node.send_message(msg) + success = wait_until(lambda: self.test_node.last_blocktxn is not None, timeout=10) + assert(success) + + [tx.calc_sha256() for tx in block.vtx] + with mininode_lock: + assert_equal(self.test_node.last_blocktxn.block_transactions.blockhash, int(block_hash, 16)) + all_indices = msg.block_txn_request.to_absolute() + for index in all_indices: + tx = self.test_node.last_blocktxn.block_transactions.transactions.pop(0) + tx.calc_sha256() + assert_equal(tx.sha256, block.vtx[index].sha256) + self.test_node.last_blocktxn = None + current_height -= 1 + + # Next request should be ignored, as we're past the allowed depth. + block_hash = self.nodes[0].getblockhash(current_height) + msg.block_txn_request = BlockTransactionsRequest(int(block_hash, 16), [0]) + self.test_node.send_and_ping(msg) + with mininode_lock: + assert_equal(self.test_node.last_blocktxn, None) + + def test_compactblocks_not_at_tip(self): + print("Testing compactblock requests/announcements not at chain tip...") + + # Test that requesting old compactblocks doesn't work. + MAX_CMPCTBLOCK_DEPTH = 11 + new_blocks = [] + for i in range(MAX_CMPCTBLOCK_DEPTH): + self.test_node.clear_block_announcement() + new_blocks.append(self.nodes[0].generate(1)[0]) + wait_until(self.test_node.received_block_announcement, timeout=30) + + self.test_node.clear_block_announcement() + self.test_node.send_message(msg_getdata([CInv(4, int(new_blocks[0], 16))])) + success = wait_until(lambda: self.test_node.last_cmpctblock is not None, timeout=30) + assert(success) + + self.test_node.clear_block_announcement() + self.nodes[0].generate(1) + wait_until(self.test_node.received_block_announcement, timeout=30) + self.test_node.clear_block_announcement() + self.test_node.send_message(msg_getdata([CInv(4, int(new_blocks[0], 16))])) + success = wait_until(lambda: self.test_node.last_block is not None, timeout=30) + assert(success) + with mininode_lock: + self.test_node.last_block.block.calc_sha256() + assert_equal(self.test_node.last_block.block.sha256, int(new_blocks[0], 16)) + + # Generate an old compactblock, and verify that it's not accepted. + cur_height = self.nodes[0].getblockcount() + hashPrevBlock = int(self.nodes[0].getblockhash(cur_height-5), 16) + block = self.build_block_on_tip() + block.hashPrevBlock = hashPrevBlock + block.solve() + + comp_block = HeaderAndShortIDs() + comp_block.initialize_from_block(block) + self.test_node.send_and_ping(msg_cmpctblock(comp_block.to_p2p())) + + tips = self.nodes[0].getchaintips() + found = False + for x in tips: + if x["hash"] == block.hash: + assert_equal(x["status"], "headers-only") + found = True + break + assert(found) + + # Requesting this block via getblocktxn should silently fail + # (to avoid fingerprinting attacks). + msg = msg_getblocktxn() + msg.block_txn_request = BlockTransactionsRequest(block.sha256, [0]) + with mininode_lock: + self.test_node.last_blocktxn = None + self.test_node.send_and_ping(msg) + with mininode_lock: + assert(self.test_node.last_blocktxn is None) + + def run_test(self): + # Setup the p2p connections and start up the network thread. + self.test_node = TestNode() + + connections = [] + connections.append(NodeConn('127.0.0.1', p2p_port(0), self.nodes[0], self.test_node)) + self.test_node.add_connection(connections[0]) + + NetworkThread().start() # Start up network handling in another thread + + # Test logic begins here + self.test_node.wait_for_verack() + + # We will need UTXOs to construct transactions in later tests. + self.make_utxos() + + self.test_sendcmpct() + self.test_compactblock_construction() + self.test_compactblock_requests() + self.test_getblocktxn_requests() + self.test_getblocktxn_handler() + self.test_compactblocks_not_at_tip() + self.test_incorrect_blocktxn_response() + self.test_invalid_cmpctblock_message() + + +if __name__ == '__main__': + CompactBlocksTest().main() From cef633ce63937eea37048fe07da2488fefb4ec7c Mon Sep 17 00:00:00 2001 From: Suhas Daftuar Date: Wed, 14 Sep 2016 21:00:53 -0400 Subject: [PATCH 10/11] Fix broken sendcmpct test in p2p-compactblocks.py Python lambda use was incorrect. sendcmpct messages need to be synchronized with RPC calls to generate(). Headers need to be synced (eg with getheaders) for cmpctblock announcements to start. Last test omitted sending a sendcmpct message. Github-Pull: #8739 Rebased-From: 157254a4bfdfc4ca3ad5bf2d84e82f290bd0c7f2) --- qa/rpc-tests/p2p-compactblocks.py | 58 ++++++++++++++++++++----------- 1 file changed, 37 insertions(+), 21 deletions(-) diff --git a/qa/rpc-tests/p2p-compactblocks.py b/qa/rpc-tests/p2p-compactblocks.py index 7fe7ecc16..bf4fb43ad 100755 --- a/qa/rpc-tests/p2p-compactblocks.py +++ b/qa/rpc-tests/p2p-compactblocks.py @@ -78,6 +78,13 @@ class TestNode(SingleNodeConnCB): headers_message.headers = [CBlockHeader(b) for b in new_blocks] self.send_message(headers_message) + def request_headers_and_sync(self, locator, hashstop=0): + self.clear_block_announcement() + self.get_headers(locator, hashstop) + assert(wait_until(self.received_block_announcement, timeout=30)) + assert(self.received_block_announcement()) + self.clear_block_announcement() + class CompactBlocksTest(BitcoinTestFramework): def __init__(self): @@ -130,7 +137,7 @@ class CompactBlocksTest(BitcoinTestFramework): # Test "sendcmpct": # - No compact block announcements or getdata(MSG_CMPCT_BLOCK) unless # sendcmpct is sent. - # - If sendcmpct is sent with version > 0, the message is ignored. + # - If sendcmpct is sent with version > 1, the message is ignored. # - If sendcmpct is sent with boolean 0, then block announcements are not # made with compact blocks. # - If sendcmpct is then sent with boolean 1, then new block announcements @@ -142,57 +149,66 @@ class CompactBlocksTest(BitcoinTestFramework): def received_sendcmpct(): return (self.test_node.last_sendcmpct is not None) got_message = wait_until(received_sendcmpct, timeout=30) + assert(received_sendcmpct()) assert(got_message) assert_equal(self.test_node.last_sendcmpct.version, 1) tip = int(self.nodes[0].getbestblockhash(), 16) def check_announcement_of_new_block(node, peer, predicate): - self.test_node.clear_block_announcement() + peer.clear_block_announcement() node.generate(1) - got_message = wait_until(peer.received_block_announcement, timeout=30) + got_message = wait_until(lambda: peer.block_announced, timeout=30) + assert(peer.block_announced) assert(got_message) with mininode_lock: - assert(predicate) + assert(predicate(peer)) # We shouldn't get any block announcements via cmpctblock yet. - check_announcement_of_new_block(self.nodes[0], self.test_node, lambda: self.test_node.last_cmpctblock is None) + check_announcement_of_new_block(self.nodes[0], self.test_node, lambda p: p.last_cmpctblock is None) # Try one more time, this time after requesting headers. - self.test_node.clear_block_announcement() - self.test_node.get_headers(locator=[tip], hashstop=0) - wait_until(self.test_node.received_block_announcement, timeout=30) - self.test_node.clear_block_announcement() + self.test_node.request_headers_and_sync(locator=[tip]) + check_announcement_of_new_block(self.nodes[0], self.test_node, lambda p: p.last_cmpctblock is None and p.last_inv is not None) - check_announcement_of_new_block(self.nodes[0], self.test_node, lambda: self.test_node.last_cmpctblock is None and self.test_node.last_inv is not None) + # Test a few ways of using sendcmpct that should NOT + # result in compact block announcements. + # Before each test, sync the headers chain. + self.test_node.request_headers_and_sync(locator=[tip]) # Now try a SENDCMPCT message with too-high version sendcmpct = msg_sendcmpct() sendcmpct.version = 2 - self.test_node.send_message(sendcmpct) + self.test_node.send_and_ping(sendcmpct) + check_announcement_of_new_block(self.nodes[0], self.test_node, lambda p: p.last_cmpctblock is None) - check_announcement_of_new_block(self.nodes[0], self.test_node, lambda: self.test_node.last_cmpctblock is None) + # Headers sync before next test. + self.test_node.request_headers_and_sync(locator=[tip]) # Now try a SENDCMPCT message with valid version, but announce=False - self.test_node.send_message(msg_sendcmpct()) - check_announcement_of_new_block(self.nodes[0], self.test_node, lambda: self.test_node.last_cmpctblock is None) + self.test_node.send_and_ping(msg_sendcmpct()) + check_announcement_of_new_block(self.nodes[0], self.test_node, lambda p: p.last_cmpctblock is None) + + # Headers sync before next test. + self.test_node.request_headers_and_sync(locator=[tip]) # Finally, try a SENDCMPCT message with announce=True sendcmpct.version = 1 sendcmpct.announce = True - self.test_node.send_message(sendcmpct) - check_announcement_of_new_block(self.nodes[0], self.test_node, lambda: self.test_node.last_cmpctblock is not None) + self.test_node.send_and_ping(sendcmpct) + check_announcement_of_new_block(self.nodes[0], self.test_node, lambda p: p.last_cmpctblock is not None) - # Try one more time - check_announcement_of_new_block(self.nodes[0], self.test_node, lambda: self.test_node.last_cmpctblock is not None) + # Try one more time (no headers sync should be needed!) + check_announcement_of_new_block(self.nodes[0], self.test_node, lambda p: p.last_cmpctblock is not None) # Try one more time, after turning on sendheaders - self.test_node.send_message(msg_sendheaders()) - check_announcement_of_new_block(self.nodes[0], self.test_node, lambda: self.test_node.last_cmpctblock is not None) + self.test_node.send_and_ping(msg_sendheaders()) + check_announcement_of_new_block(self.nodes[0], self.test_node, lambda p: p.last_cmpctblock is not None) # Now turn off announcements sendcmpct.announce = False - check_announcement_of_new_block(self.nodes[0], self.test_node, lambda: self.test_node.last_cmpctblock is None and self.test_node.last_headers is not None) + self.test_node.send_and_ping(sendcmpct) + check_announcement_of_new_block(self.nodes[0], self.test_node, lambda p: p.last_cmpctblock is None and p.last_headers is not None) # This test actually causes bitcoind to (reasonably!) disconnect us, so do this last. def test_invalid_cmpctblock_message(self): From 9dfa0c8d90177a12c3a3b1ac59c32727077d6803 Mon Sep 17 00:00:00 2001 From: Johnson Lau Date: Wed, 31 Aug 2016 19:38:23 +0800 Subject: [PATCH 11/11] Implement NULLDUMMY softfork Github-Pull: #8636 Rebased-From: 482f852da65457eb2fbea6b259e7568133fb81c4 --- qa/pull-tester/rpc-tests.py | 1 + qa/rpc-tests/nulldummy.py | 148 ++++++++++++++++++++++++++++++++++++ src/main.cpp | 1 + 3 files changed, 150 insertions(+) create mode 100755 qa/rpc-tests/nulldummy.py diff --git a/qa/pull-tester/rpc-tests.py b/qa/pull-tester/rpc-tests.py index 1e57a3fc2..dbd6de559 100755 --- a/qa/pull-tester/rpc-tests.py +++ b/qa/pull-tester/rpc-tests.py @@ -143,6 +143,7 @@ testScripts = [ 'importprunedfunds.py', 'signmessages.py', 'p2p-compactblocks.py', + 'nulldummy.py', ] if ENABLE_ZMQ: testScripts.append('zmq_test.py') diff --git a/qa/rpc-tests/nulldummy.py b/qa/rpc-tests/nulldummy.py new file mode 100755 index 000000000..eaed7a8c7 --- /dev/null +++ b/qa/rpc-tests/nulldummy.py @@ -0,0 +1,148 @@ +#!/usr/bin/env python3 +# Copyright (c) 2016 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. + +from test_framework.test_framework import ComparisonTestFramework +from test_framework.util import * +from test_framework.mininode import CTransaction, NetworkThread +from test_framework.blocktools import create_coinbase, create_block, add_witness_commitment +from test_framework.comptool import TestManager +from test_framework.script import CScript +from io import BytesIO +import time + +NULLDUMMY_ERROR = "64: non-mandatory-script-verify-flag (Dummy CHECKMULTISIG argument must be zero)" + +def trueDummy(tx): + scriptSig = CScript(tx.vin[0].scriptSig) + newscript = [] + for i in scriptSig: + if (len(newscript) == 0): + assert(len(i) == 0) + newscript.append(b'\x51') + else: + newscript.append(i) + tx.vin[0].scriptSig = CScript(newscript) + tx.rehash() + +''' +This test is meant to exercise NULLDUMMY softfork. +Connect to a single node. +Generate 2 blocks (save the coinbases for later). +Generate 427 more blocks. +[Policy/Consensus] Check that NULLDUMMY compliant transactions are accepted in the 430th block. +[Policy] Check that non-NULLDUMMY transactions are rejected before activation. +[Consensus] Check that the new NULLDUMMY rules are not enforced on the 431st block. +[Policy/Consensus] Check that the new NULLDUMMY rules are enforced on the 432nd block. +''' + +class NULLDUMMYTest(ComparisonTestFramework): + + def __init__(self): + super().__init__() + self.num_nodes = 1 + + def setup_network(self): + # Must set the blockversion for this test + self.nodes = start_nodes(self.num_nodes, self.options.tmpdir, + extra_args=[['-debug', '-whitelist=127.0.0.1', '-walletprematurewitness']]) + + def run_test(self): + self.address = self.nodes[0].getnewaddress() + self.ms_address = self.nodes[0].addmultisigaddress(1,[self.address]) + self.wit_address = self.nodes[0].addwitnessaddress(self.address) + self.wit_ms_address = self.nodes[0].addwitnessaddress(self.ms_address) + + test = TestManager(self, self.options.tmpdir) + test.add_all_connections(self.nodes) + NetworkThread().start() # Start up network handling in another thread + self.coinbase_blocks = self.nodes[0].generate(2) # Block 2 + coinbase_txid = [] + for i in self.coinbase_blocks: + coinbase_txid.append(self.nodes[0].getblock(i)['tx'][0]) + self.nodes[0].generate(427) # Block 429 + self.lastblockhash = self.nodes[0].getbestblockhash() + self.tip = int("0x" + self.lastblockhash, 0) + self.lastblockheight = 429 + self.lastblocktime = int(time.time()) + 429 + + print ("Test 1: NULLDUMMY compliant base transactions should be accepted to mempool and mined before activation [430]") + test1txs = [self.create_transaction(self.nodes[0], coinbase_txid[0], self.ms_address, 49)] + txid1 = self.tx_submit(self.nodes[0], test1txs[0]) + test1txs.append(self.create_transaction(self.nodes[0], txid1, self.ms_address, 48)) + txid2 = self.tx_submit(self.nodes[0], test1txs[1]) + test1txs.append(self.create_transaction(self.nodes[0], coinbase_txid[1], self.wit_ms_address, 49)) + txid3 = self.tx_submit(self.nodes[0], test1txs[2]) + self.block_submit(self.nodes[0], test1txs, False, True) + + print ("Test 2: Non-NULLDUMMY base multisig transaction should not be accepted to mempool before activation") + test2tx = self.create_transaction(self.nodes[0], txid2, self.ms_address, 48) + trueDummy(test2tx) + txid4 = self.tx_submit(self.nodes[0], test2tx, NULLDUMMY_ERROR) + + print ("Test 3: Non-NULLDUMMY base transactions should be accepted in a block before activation [431]") + self.block_submit(self.nodes[0], [test2tx], False, True) + + print ("Test 4: Non-NULLDUMMY base multisig transaction is invalid after activation") + test4tx = self.create_transaction(self.nodes[0], txid4, self.address, 47) + test6txs=[CTransaction(test4tx)] + trueDummy(test4tx) + self.tx_submit(self.nodes[0], test4tx, NULLDUMMY_ERROR) + self.block_submit(self.nodes[0], [test4tx]) + + print ("Test 5: Non-NULLDUMMY P2WSH multisig transaction invalid after activation") + test5tx = self.create_transaction(self.nodes[0], txid3, self.wit_address, 48) + test6txs.append(CTransaction(test5tx)) + test5tx.wit.vtxinwit[0].scriptWitness.stack[0] = b'\x01' + self.tx_submit(self.nodes[0], test5tx, NULLDUMMY_ERROR) + self.block_submit(self.nodes[0], [test5tx], True) + + print ("Test 6: NULLDUMMY compliant base/witness transactions should be accepted to mempool and in block after activation [432]") + for i in test6txs: + self.tx_submit(self.nodes[0], i) + self.block_submit(self.nodes[0], test6txs, True, True) + + + def create_transaction(self, node, txid, to_address, amount): + inputs = [{ "txid" : txid, "vout" : 0}] + outputs = { to_address : amount } + rawtx = node.createrawtransaction(inputs, outputs) + signresult = node.signrawtransaction(rawtx) + tx = CTransaction() + f = BytesIO(hex_str_to_bytes(signresult['hex'])) + tx.deserialize(f) + return tx + + + def tx_submit(self, node, tx, msg = ""): + tx.rehash() + try: + node.sendrawtransaction(bytes_to_hex_str(tx.serialize_with_witness()), True) + except JSONRPCException as exp: + assert_equal(exp.error["message"], msg) + return tx.hash + + + def block_submit(self, node, txs, witness = False, accept = False): + block = create_block(self.tip, create_coinbase(self.lastblockheight + 1), self.lastblocktime + 1) + block.nVersion = 4 + for tx in txs: + tx.rehash() + block.vtx.append(tx) + block.hashMerkleRoot = block.calc_merkle_root() + witness and add_witness_commitment(block) + block.rehash() + block.solve() + node.submitblock(bytes_to_hex_str(block.serialize(True))) + if (accept): + assert_equal(node.getbestblockhash(), block.hash) + self.tip = block.sha256 + self.lastblockhash = block.hash + self.lastblocktime += 1 + self.lastblockheight += 1 + else: + assert_equal(node.getbestblockhash(), self.lastblockhash) + +if __name__ == '__main__': + NULLDUMMYTest().main() \ No newline at end of file diff --git a/src/main.cpp b/src/main.cpp index a664bf2de..c9869d04f 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -2399,6 +2399,7 @@ bool ConnectBlock(const CBlock& block, CValidationState& state, CBlockIndex* pin // Start enforcing WITNESS rules using versionbits logic. if (IsWitnessEnabled(pindex->pprev, chainparams.GetConsensus())) { flags |= SCRIPT_VERIFY_WITNESS; + flags |= SCRIPT_VERIFY_NULLDUMMY; } int64_t nTime2 = GetTimeMicros(); nTimeForks += nTime2 - nTime1;