Merge #8482: [qa] Use single cache dir for chains

fad8cf6 [qa] Use single cache dir for chains (MarcoFalke)
fa2d68f [qa] Adjust timeouts for micro-optimization of run time (MarcoFalke)
fae596f [qa] Sort scripts by time for pull_tester and don't overwrite setup_chain (MarcoFalke)

.gitignore

@@ -103,10 +103,9 @@ linux-build
 win32-build
 qa/pull-tester/run-bitcoind-for-test.sh
 qa/pull-tester/tests_config.py
-qa/pull-tester/cache/*
 qa/pull-tester/test.*/*
 qa/tmp
-cache/
+qa/cache/*
 share/BitcoindComparisonTool.jar
 
 !src/leveldb*/Makefile

qa/pull-tester/rpc-tests.py

@@ -112,6 +112,8 @@ testScripts = [
     'receivedby.py',
     'mempool_resurrect_test.py',
     'txn_doublespend.py --mineblock',
+    'p2p-segwit.py',
+    'segwit.py',
     'txn_clone.py',
     'getchaintips.py',
     'rawtransactions.py',
@@ -133,13 +135,12 @@ testScripts = [
     'disablewallet.py',
     'sendheaders.py',
     'keypool.py',
+    'p2p-mempool.py',
     'prioritise_transaction.py',
     'invalidblockrequest.py',
     'invalidtxrequest.py',
     'abandonconflict.py',
     'p2p-versionbits-warning.py',
-    'p2p-segwit.py',
-    'segwit.py',
     'importprunedfunds.py',
     'signmessages.py',
     'p2p-compactblocks.py',
@@ -194,6 +195,7 @@ def runtests():
         coverage = RPCCoverage()
         print("Initializing coverage directory at %s\n" % coverage.dir)
     flags = ["--srcdir=%s/src" % BUILDDIR] + passon_args
+    flags.append("--cachedir=%s/qa/cache" % BUILDDIR)
     if coverage:
         flags.append(coverage.flag)
 

qa/rpc-tests/p2p-mempool.py

@@ -72,8 +72,11 @@ class TestNode(NodeConnCB):
         self.send_message(msg_mempool())
 
 class P2PMempoolTests(BitcoinTestFramework):
-    def setup_chain(self):
+
-        initialize_chain_clean(self.options.tmpdir, 2)
+    def __init__(self):
+        super().__init__()
+        self.setup_clean_chain = True
+        self.num_nodes = 2
 
     def setup_network(self):
         # Start a node with maxuploadtarget of 200 MB (/24h)

qa/rpc-tests/p2p-segwit.py

@@ -168,8 +168,11 @@ class UTXO(object):
 
 
 class SegWitTest(BitcoinTestFramework):
-    def setup_chain(self):
+
-        initialize_chain_clean(self.options.tmpdir, 3)
+    def __init__(self):
+        super().__init__()
+        self.setup_clean_chain = True
+        self.num_nodes = 3
 
     def add_options(self, parser):
         parser.add_option("--oldbinary", dest="oldbinary",

qa/rpc-tests/segwit.py

@@ -76,9 +76,10 @@ def find_unspent(node, min_value):
 
 class SegWitTest(BitcoinTestFramework):
 
-    def setup_chain(self):
+    def __init__(self):
-        print("Initializing test directory "+self.options.tmpdir)
+        super().__init__()
-        initialize_chain_clean(self.options.tmpdir, 3)
+        self.setup_clean_chain = True
+        self.num_nodes = 3
 
     def setup_network(self):
         self.nodes = []

qa/rpc-tests/test_framework/test_framework.py

@@ -48,7 +48,7 @@ class BitcoinTestFramework(object):
         if self.setup_clean_chain:
             initialize_chain_clean(self.options.tmpdir, self.num_nodes)
         else:
-            initialize_chain(self.options.tmpdir, self.num_nodes)
+            initialize_chain(self.options.tmpdir, self.num_nodes, self.options.cachedir)
 
     def stop_node(self, num_node):
         stop_node(self.nodes[num_node], num_node)
@@ -112,6 +112,8 @@ class BitcoinTestFramework(object):
                           help="Don't stop bitcoinds after the test execution")
         parser.add_option("--srcdir", dest="srcdir", default=os.path.normpath(os.path.dirname(os.path.realpath(__file__))+"/../../../src"),
                           help="Source directory containing bitcoind/bitcoin-cli (default: %default)")
+        parser.add_option("--cachedir", dest="cachedir", default=os.path.normpath(os.path.dirname(os.path.realpath(__file__))+"/../../cache"),
+                          help="Directory for caching pregenerated datadirs")
         parser.add_option("--tmpdir", dest="tmpdir", default=tempfile.mkdtemp(prefix="test"),
                           help="Root directory for datadirs")
         parser.add_option("--tracerpc", dest="trace_rpc", default=False, action="store_true",

qa/rpc-tests/test_framework/util.py

@@ -121,7 +121,7 @@ def hex_str_to_bytes(hex_str):
 def str_to_b64str(string):
     return b64encode(string.encode('utf-8')).decode('ascii')
 
-def sync_blocks(rpc_connections, wait=1, timeout=60):
+def sync_blocks(rpc_connections, wait=0.125, timeout=60):
     """
     Wait until everybody has the same tip
     """
@@ -133,7 +133,7 @@ def sync_blocks(rpc_connections, wait=1, timeout=60):
         timeout -= wait
     raise AssertionError("Block sync failed")
 
-def sync_mempools(rpc_connections, wait=1, timeout=60):
+def sync_mempools(rpc_connections, wait=0.5, timeout=60):
     """
     Wait until everybody has the same transactions in their memory
     pools
@@ -201,7 +201,7 @@ def wait_for_bitcoind_start(process, url, i):
                 raise # unkown JSON RPC exception
         time.sleep(0.25)
 
-def initialize_chain(test_dir, num_nodes):
+def initialize_chain(test_dir, num_nodes, cachedir):
     """
     Create a cache of a 200-block-long chain (with wallet) for MAX_NODES
     Afterward, create num_nodes copies from the cache
@@ -210,7 +210,7 @@ def initialize_chain(test_dir, num_nodes):
     assert num_nodes <= MAX_NODES
     create_cache = False
     for i in range(MAX_NODES):
-        if not os.path.isdir(os.path.join('cache', 'node'+str(i))):
+        if not os.path.isdir(os.path.join(cachedir, 'node'+str(i))):
             create_cache = True
             break
 
@@ -218,12 +218,12 @@ def initialize_chain(test_dir, num_nodes):
 
         #find and delete old cache directories if any exist
         for i in range(MAX_NODES):
-            if os.path.isdir(os.path.join("cache","node"+str(i))):
+            if os.path.isdir(os.path.join(cachedir,"node"+str(i))):
-                shutil.rmtree(os.path.join("cache","node"+str(i)))
+                shutil.rmtree(os.path.join(cachedir,"node"+str(i)))
 
         # Create cache directories, run bitcoinds:
         for i in range(MAX_NODES):
-            datadir=initialize_datadir("cache", i)
+            datadir=initialize_datadir(cachedir, i)
             args = [ os.getenv("BITCOIND", "bitcoind"), "-server", "-keypool=1", "-datadir="+datadir, "-discover=0" ]
             if i > 0:
                 args.append("-connect=127.0.0.1:"+str(p2p_port(0)))
@@ -265,13 +265,13 @@ def initialize_chain(test_dir, num_nodes):
         wait_bitcoinds()
         disable_mocktime()
         for i in range(MAX_NODES):
-            os.remove(log_filename("cache", i, "debug.log"))
+            os.remove(log_filename(cachedir, i, "debug.log"))
-            os.remove(log_filename("cache", i, "db.log"))
+            os.remove(log_filename(cachedir, i, "db.log"))
-            os.remove(log_filename("cache", i, "peers.dat"))
+            os.remove(log_filename(cachedir, i, "peers.dat"))
-            os.remove(log_filename("cache", i, "fee_estimates.dat"))
+            os.remove(log_filename(cachedir, i, "fee_estimates.dat"))
 
     for i in range(num_nodes):
-        from_dir = os.path.join("cache", "node"+str(i))
+        from_dir = os.path.join(cachedir, "node"+str(i))
         to_dir = os.path.join(test_dir,  "node"+str(i))
         shutil.copytree(from_dir, to_dir)
         initialize_datadir(test_dir, i) # Overwrite port/rpcport in bitcoin.conf