Python-based regression tests

skeleton.py : a do-nothing test skeleton
listtransactions.py : start of regression test for listtransactions call

qa/rpc-tests/.gitignore

@@ -0,0 +1,2 @@
+*.pyc
+cache

qa/rpc-tests/README.md

@@ -1,26 +1,36 @@
 Regression tests of RPC interface
 =================================
 
-Bash scripts that use the RPC interface and command-line bitcoin-cli to test
+python-bitcoinrpc: git subtree of https://github.com/jgarzik/python-bitcoinrpc
-full functionality in -regtest mode.
+Changes to python-bitcoinrpc should be made upstream, and then
+pulled here using git subtree
 
-wallet.sh : Exercise wallet send/receive code.
+skeleton.py : Copy this to create new regression tests.
 
-txnmall.sh : Test proper accounting of malleable transactions
+listtransactions.py : Tests for the listtransactions RPC call
+
+util.py : generally useful functions
 
+Bash-based tests, to be ported to Python:
+-----------------------------------------
+wallet.sh : Exercise wallet send/receive code.
+walletbackup.sh : Exercise wallet backup / dump / import
+txnmall.sh : Test proper accounting of malleable transactions
 conflictedbalance.sh : More testing of malleable transaction handling
 
-util.sh : useful re-usable bash functions
+Notes
+=====
 
+A 200-block -regtest blockchain and wallets for four nodes
+is created the first time a regression test is run and
+is stored in the cache/ directory. Each node has 25 mature
+blocks (25*50=1250 BTC) in their wallet.
 
-Tips for creating new tests
+After the first run, the cache/ blockchain and wallets are
-===========================
+copied into a temporary directory and used as the initial
+test state.
 
-To cleanup after a failed or interrupted test:
+If you get into a bad state, you should be able
+to recover with:
+  rm -rf cache
   killall bitcoind
-  rm -rf test.*
-
-The most difficult part of writing reproducible tests is
-keeping multiple nodes in sync. See WaitBlocks,
-WaitPeers, and WaitMemPools for how other tests
-deal with this.

qa/rpc-tests/listtransactions.py

@@ -0,0 +1,151 @@
+#!/usr/bin/env python
+
+# Exercise the listtransactions API
+
+# Add python-bitcoinrpc to module search path:
+import os
+import sys
+sys.path.append(os.path.join(os.path.dirname(os.path.abspath(__file__)), "python-bitcoinrpc"))
+
+import json
+import shutil
+import subprocess
+import tempfile
+import traceback
+
+from bitcoinrpc.authproxy import AuthServiceProxy, JSONRPCException
+from util import *
+
+
+def check_array_result(object_array, to_match, expected):
+    """
+    Pass in array of JSON objects, a dictionary with key/value pairs
+    to match against, and another dictionary with expected key/value
+    pairs.
+    """
+    num_matched = 0
+    for item in object_array:
+        all_match = True
+        for key,value in to_match.items():
+            if item[key] != value:
+                all_match = False
+        if not all_match:
+            continue
+        for key,value in expected.items():
+            if item[key] != value:
+                raise AssertionError("%s : expected %s=%s"%(str(item), str(key), str(value)))
+            num_matched = num_matched+1
+    if num_matched == 0:
+        raise AssertionError("No objects matched %s"%(str(to_match)))
+
+def run_test(nodes):
+    # Simple send, 0 to 1:
+    txid = nodes[0].sendtoaddress(nodes[1].getnewaddress(), 0.1)
+    sync_mempools(nodes)
+    check_array_result(nodes[0].listtransactions(),
+                       {"txid":txid},
+                       {"category":"send","account":"","amount":Decimal("-0.1"),"confirmations":0})
+    check_array_result(nodes[1].listtransactions(),
+                       {"txid":txid},
+                       {"category":"receive","account":"","amount":Decimal("0.1"),"confirmations":0})
+    # mine a block, confirmations should change:
+    nodes[0].setgenerate(True, 1)
+    sync_blocks(nodes)
+    check_array_result(nodes[0].listtransactions(),
+                       {"txid":txid},
+                       {"category":"send","account":"","amount":Decimal("-0.1"),"confirmations":1})
+    check_array_result(nodes[1].listtransactions(),
+                       {"txid":txid},
+                       {"category":"receive","account":"","amount":Decimal("0.1"),"confirmations":1})
+
+    # send-to-self:
+    txid = nodes[0].sendtoaddress(nodes[0].getnewaddress(), 0.2)
+    check_array_result(nodes[0].listtransactions(),
+                       {"txid":txid, "category":"send"},
+                       {"amount":Decimal("-0.2")})
+    check_array_result(nodes[0].listtransactions(),
+                       {"txid":txid, "category":"receive"},
+                       {"amount":Decimal("0.2")})
+
+    # sendmany from node1: twice to self, twice to node2:
+    send_to = { nodes[0].getnewaddress() : 0.11, nodes[1].getnewaddress() : 0.22,
+                nodes[0].getaccountaddress("from1") : 0.33, nodes[1].getaccountaddress("toself") : 0.44 }
+    txid = nodes[1].sendmany("", send_to)
+    sync_mempools(nodes)
+    check_array_result(nodes[1].listtransactions(),
+                       {"category":"send","amount":Decimal("-0.11")},
+                       {"txid":txid} )
+    check_array_result(nodes[0].listtransactions(),
+                       {"category":"receive","amount":Decimal("0.11")},
+                       {"txid":txid} )
+    check_array_result(nodes[1].listtransactions(),
+                       {"category":"send","amount":Decimal("-0.22")},
+                       {"txid":txid} )
+    check_array_result(nodes[1].listtransactions(),
+                       {"category":"receive","amount":Decimal("0.22")},
+                       {"txid":txid} )
+    check_array_result(nodes[1].listtransactions(),
+                       {"category":"send","amount":Decimal("-0.33")},
+                       {"txid":txid} )
+    check_array_result(nodes[0].listtransactions(),
+                       {"category":"receive","amount":Decimal("0.33")},
+                       {"txid":txid, "account" : "from1"} )
+    check_array_result(nodes[1].listtransactions(),
+                       {"category":"send","amount":Decimal("-0.44")},
+                       {"txid":txid, "account" : ""} )
+    check_array_result(nodes[1].listtransactions(),
+                       {"category":"receive","amount":Decimal("0.44")},
+                       {"txid":txid, "account" : "toself"} )
+    
+
+def main():
+    import optparse
+
+    parser = optparse.OptionParser(usage="%prog [options]")
+    parser.add_option("--nocleanup", dest="nocleanup", default=False, action="store_true",
+                      help="Leave bitcoinds and test.* datadir on exit or error")
+    parser.add_option("--srcdir", dest="srcdir", default="../../src",
+                      help="Source directory containing bitcoind/bitcoin-cli (default: %default%)")
+    parser.add_option("--tmpdir", dest="tmpdir", default=tempfile.mkdtemp(prefix="test"),
+                      help="Root directory for datadirs")
+    (options, args) = parser.parse_args()
+
+    os.environ['PATH'] = options.srcdir+":"+os.environ['PATH']
+
+    check_json_precision()
+
+    success = False
+    try:
+        print("Initializing test directory "+options.tmpdir)
+        if not os.path.isdir(options.tmpdir):
+            os.makedirs(options.tmpdir)
+        initialize_chain(options.tmpdir)
+
+        nodes = start_nodes(2, options.tmpdir)
+        connect_nodes(nodes[1], 0)
+        sync_blocks(nodes)
+        run_test(nodes)
+
+        success = True
+
+    except AssertionError as e:
+        print("Assertion failed: "+e.message)
+    except Exception as e:
+        print("Unexpected exception caught during testing: "+str(e))
+        stack = traceback.extract_tb(sys.exc_info()[2])
+        print(stack[-1])
+
+    if not options.nocleanup:
+        print("Cleaning up")
+        stop_nodes()
+        shutil.rmtree(options.tmpdir)
+
+    if success:
+        print("Tests successful")
+        sys.exit(0)
+    else:
+        print("Failed")
+        sys.exit(1)
+
+if __name__ == '__main__':
+    main()

qa/rpc-tests/skeleton.py

@@ -0,0 +1,79 @@
+#!/usr/bin/env python
+
+# Skeleton for python-based regression tests using
+# JSON-RPC
+
+
+# Add python-bitcoinrpc to module search path:
+import os
+import sys
+sys.path.append(os.path.join(os.path.dirname(os.path.abspath(__file__)), "python-bitcoinrpc"))
+
+import json
+import shutil
+import subprocess
+import tempfile
+import traceback
+
+from bitcoinrpc.authproxy import AuthServiceProxy, JSONRPCException
+from util import *
+
+
+def run_test(nodes):
+    # Replace this as appropriate
+    for node in nodes:
+        assert_equal(node.getblockcount(), 200)
+        assert_equal(node.getbalance(), 25*50)
+
+def main():
+    import optparse
+
+    parser = optparse.OptionParser(usage="%prog [options]")
+    parser.add_option("--nocleanup", dest="nocleanup", default=False, action="store_true",
+                      help="Leave bitcoinds and test.* datadir on exit or error")
+    parser.add_option("--srcdir", dest="srcdir", default="../../src",
+                      help="Source directory containing bitcoind/bitcoin-cli (default: %default%)")
+    parser.add_option("--tmpdir", dest="tmpdir", default=tempfile.mkdtemp(prefix="test"),
+                      help="Root directory for datadirs")
+    (options, args) = parser.parse_args()
+
+    os.environ['PATH'] = options.srcdir+":"+os.environ['PATH']
+
+    check_json_precision()
+
+    success = False
+    try:
+        print("Initializing test directory "+options.tmpdir)
+        if not os.path.isdir(options.tmpdir):
+            os.makedirs(options.tmpdir)
+        initialize_chain(options.tmpdir)
+
+        nodes = start_nodes(2, options.tmpdir)
+        connect_nodes(nodes[1], 0)
+        sync_blocks(nodes)
+
+        run_test(nodes)
+
+        success = True
+
+    except AssertionError as e:
+        print("Assertion failed: "+e.message)
+    except Exception as e:
+        print("Unexpected exception caught during testing: "+str(e))
+        stack = traceback.extract_tb(sys.exc_info()[2])
+        print(stack[-1])
+
+    if not options.nocleanup:
+        print("Cleaning up")
+        stop_nodes()
+        shutil.rmtree(options.tmpdir)
+
+    if success:
+        print("Tests successful")
+        sys.exit(0)
+    else:
+        print("Failed")
+        sys.exit(1)
+
+if __name__ == '__main__':
+    main()

qa/rpc-tests/util.py

@@ -0,0 +1,136 @@
+#
+# Helpful routines for regression testing
+#
+
+# Add python-bitcoinrpc to module search path:
+import os
+import sys
+sys.path.append(os.path.join(os.path.dirname(os.path.abspath(__file__)), "python-bitcoinrpc"))
+
+from decimal import Decimal
+import json
+import shutil
+import subprocess
+import time
+
+from bitcoinrpc.authproxy import AuthServiceProxy, JSONRPCException
+from util import *
+
+START_P2P_PORT=11000
+START_RPC_PORT=11100
+
+def check_json_precision():
+    """Make sure json library being used does not lose precision converting BTC values"""
+    n = Decimal("20000000.00000003")
+    satoshis = int(json.loads(json.dumps(float(n)))*1.0e8)
+    if satoshis != 2000000000000003:
+        raise RuntimeError("JSON encode/decode loses precision")
+
+def sync_blocks(rpc_connections):
+    """
+    Wait until everybody has the same block count
+    """
+    while True:
+        counts = [ x.getblockcount() for x in rpc_connections ]
+        if counts == [ counts[0] ]*len(counts):
+            break
+        time.sleep(1)
+
+def sync_mempools(rpc_connections):
+    """
+    Wait until everybody has the same transactions in their memory
+    pools
+    """
+    while True:
+        pool = set(rpc_connections[0].getrawmempool())
+        num_match = 1
+        for i in range(1, len(rpc_connections)):
+            if set(rpc_connections[i].getrawmempool()) == pool:
+                num_match = num_match+1
+        if num_match == len(rpc_connections):
+            break
+        time.sleep(1)
+        
+
+def initialize_chain(test_dir):
+    """
+    Create (or copy from cache) a 200-block-long chain and
+    4 wallets.
+    bitcoind and bitcoin-cli must be in search path.
+    """
+
+    if not os.path.isdir(os.path.join("cache", "node0")):
+        # Create cache directories, run bitcoinds:
+        bitcoinds = []
+        for i in range(4):
+            datadir = os.path.join("cache", "node"+str(i))
+            os.makedirs(datadir)
+            with open(os.path.join(datadir, "bitcoin.conf"), 'w') as f:
+                f.write("regtest=1\n");
+                f.write("rpcuser=rt\n");
+                f.write("rpcpassword=rt\n");
+                f.write("port="+str(START_P2P_PORT+i)+"\n");
+                f.write("rpcport="+str(START_RPC_PORT+i)+"\n");
+            args = [ "bitcoind", "-keypool=1", "-datadir="+datadir ]
+            if i > 0:
+                args.append("-connect=127.0.0.1:"+str(START_P2P_PORT))
+            bitcoinds.append(subprocess.Popen(args))
+            subprocess.check_output([ "bitcoin-cli", "-datadir="+datadir,
+                                      "-rpcwait", "getblockcount"])
+
+        rpcs = []
+        for i in range(4):
+            try:
+                url = "http://rt:rt@127.0.0.1:%d"%(START_RPC_PORT+i,)
+                rpcs.append(AuthServiceProxy(url))
+            except:
+                sys.stderr.write("Error connecting to "+url+"\n")
+                sys.exit(1)
+
+        import pdb; pdb.set_trace()
+
+        # Create a 200-block-long chain; each of the 4 nodes
+        # gets 25 mature blocks and 25 immature.
+        for i in range(4):
+            rpcs[i].setgenerate(True, 25)
+            sync_blocks(rpcs)
+        for i in range(4):
+            rpcs[i].setgenerate(True, 25)
+            sync_blocks(rpcs)
+        # Shut them down
+        for i in range(4):
+            rpcs[i].stop()
+
+    for i in range(4):
+        from_dir = os.path.join("cache", "node"+str(i))
+        to_dir = os.path.join(test_dir,  "node"+str(i))
+        shutil.copytree(from_dir, to_dir)
+
+bitcoind_processes = []
+
+def start_nodes(num_nodes, dir):
+    # Start bitcoinds, and wait for RPC interface to be up and running:
+    for i in range(num_nodes):
+        datadir = os.path.join(dir, "node"+str(i))
+        args = [ "bitcoind", "-datadir="+datadir ]
+        bitcoind_processes.append(subprocess.Popen(args))
+        subprocess.check_output([ "bitcoin-cli", "-datadir="+datadir,
+                                  "-rpcwait", "getblockcount"])
+    # Create&return JSON-RPC connections
+    rpc_connections = []
+    for i in range(num_nodes):
+        url = "http://rt:rt@127.0.0.1:%d"%(START_RPC_PORT+i,)
+        rpc_connections.append(AuthServiceProxy(url))
+    return rpc_connections
+
+def stop_nodes():
+    for process in bitcoind_processes:
+        process.kill()
+
+def connect_nodes(from_connection, node_num):
+    ip_port = "127.0.0.1:"+str(START_P2P_PORT+node_num)
+    from_connection.addnode(ip_port, "onetry")
+
+def assert_equal(thing1, thing2):
+    if thing1 != thing2:
+        raise AssertionError("%s != %s"%(str(thing1),str(thing2)))