Merge #10423: [tests] skipped tests should clean up after themselves

930deb9 [tests] skipped tests should clean up after themselves (John Newbery)

Tree-SHA512: ed486c1cf144c223efa738a66e8bb407089a23002871ef5b15c43d4c08641038a20253fc286853b457d66f37a52c81c16f7988aef90635ac262df6ce7bab9b18

test/functional/rpcbind_test.py

@@ -7,7 +7,7 @@
 import socket
 import sys
 
-from test_framework.test_framework import BitcoinTestFramework
+from test_framework.test_framework import BitcoinTestFramework, SkipTest
 from test_framework.util import *
 from test_framework.netutil import *
 
@@ -56,8 +56,7 @@ class RPCBindTest(BitcoinTestFramework):
     def run_test(self):
         # due to OS-specific network stats queries, this test works only on Linux
         if not sys.platform.startswith('linux'):
-            self.log.warning("This test can only be run on linux. Skipping test.")
-            sys.exit(self.TEST_EXIT_SKIPPED)
+            raise SkipTest("This test can only be run on linux.")
         # find the first non-loopback interface for testing
         non_loopback_ip = None
         for name,ip in all_interfaces():
@@ -65,15 +64,13 @@ class RPCBindTest(BitcoinTestFramework):
                 non_loopback_ip = ip
                 break
         if non_loopback_ip is None:
-            self.log.warning("This test requires at least one non-loopback IPv4 interface. Skipping test.")
-            sys.exit(self.TEST_EXIT_SKIPPED)
+            raise SkipTest("This test requires at least one non-loopback IPv4 interface.")
         try:
             s = socket.socket(socket.AF_INET6, socket.SOCK_DGRAM)
             s.connect(("::1",1))
             s.close
         except OSError:
-            self.log.warning("This test requires IPv6 support. Skipping test.")
-            sys.exit(self.TEST_EXIT_SKIPPED)
+            raise SkipTest("This test requires IPv6 support.")
 
         self.log.info("Using interface %s for testing" % non_loopback_ip)
 

test/functional/test_framework/test_framework.py

@@ -5,6 +5,7 @@
 """Base class for RPC testing."""
 
 from collections import deque
+from enum import Enum
 import logging
 import optparse
 import os
@@ -41,6 +42,15 @@ from .util import (
 )
 from .authproxy import JSONRPCException
 
+class TestStatus(Enum):
+    PASSED = 1
+    FAILED = 2
+    SKIPPED = 3
+
+TEST_EXIT_PASSED = 0
+TEST_EXIT_FAILED = 1
+TEST_EXIT_SKIPPED = 77
+
 class BitcoinTestFramework(object):
     """Base class for a bitcoin test script.
 
@@ -57,11 +67,6 @@ class BitcoinTestFramework(object):
     This class also contains various public and private helper methods."""
 
     # Methods to override in subclass test scripts.
-
-    TEST_EXIT_PASSED = 0
-    TEST_EXIT_FAILED = 1
-    TEST_EXIT_SKIPPED = 77
-
     def __init__(self):
         self.num_nodes = 4
         self.setup_clean_chain = False
@@ -139,15 +144,18 @@ class BitcoinTestFramework(object):
             self.options.tmpdir = tempfile.mkdtemp(prefix="test")
         self._start_logging()
 
-        success = False
+        success = TestStatus.FAILED
 
         try:
             self.setup_chain()
             self.setup_network()
             self.run_test()
-            success = True
+            success = TestStatus.PASSED
         except JSONRPCException as e:
             self.log.exception("JSONRPC error")
+        except SkipTest as e:
+            self.log.warning("Test Skipped: %s" % e.message)
+            success = TestStatus.SKIPPED
         except AssertionError as e:
             self.log.exception("Assertion failed")
         except KeyError as e:
@@ -159,11 +167,12 @@ class BitcoinTestFramework(object):
 
         if not self.options.noshutdown:
             self.log.info("Stopping nodes")
-            self.stop_nodes()
+            if self.nodes:
+                self.stop_nodes()
         else:
             self.log.info("Note: bitcoinds were not stopped and may still be running")
 
-        if not self.options.nocleanup and not self.options.noshutdown and success:
+        if not self.options.nocleanup and not self.options.noshutdown and success != TestStatus.FAILED:
             self.log.info("Cleaning up")
             shutil.rmtree(self.options.tmpdir)
         else:
@@ -183,13 +192,17 @@ class BitcoinTestFramework(object):
                     except OSError:
                         print("Opening file %s failed." % fn)
                         traceback.print_exc()
-        if success:
+
+        if success == TestStatus.PASSED:
             self.log.info("Tests successful")
-            sys.exit(self.TEST_EXIT_PASSED)
+            sys.exit(TEST_EXIT_PASSED)
+        elif success == TestStatus.SKIPPED:
+            self.log.info("Test skipped")
+            sys.exit(TEST_EXIT_SKIPPED)
         else:
             self.log.error("Test failed. Test logging available at %s/test_framework.log", self.options.tmpdir)
             logging.shutdown()
-            sys.exit(self.TEST_EXIT_FAILED)
+            sys.exit(TEST_EXIT_FAILED)
 
     # Public helper methods. These can be accessed by the subclass test scripts.
 
@@ -346,6 +359,11 @@ class BitcoinTestFramework(object):
 # 2 binaries: 1 test binary, 1 ref binary
 # n>2 binaries: 1 test binary, n-1 ref binaries
 
+class SkipTest(Exception):
+    """This exception is raised to skip a test"""
+    def __init__(self, message):
+        self.message = message
+
 class ComparisonTestFramework(BitcoinTestFramework):
 
     def __init__(self):

test/functional/zmq_test.py

@@ -6,9 +6,8 @@
 import configparser
 import os
 import struct
-import sys
 
-from test_framework.test_framework import BitcoinTestFramework
+from test_framework.test_framework import BitcoinTestFramework, SkipTest
 from test_framework.util import *
 
 class ZMQTest (BitcoinTestFramework):
@@ -24,8 +23,7 @@ class ZMQTest (BitcoinTestFramework):
         try:
             import zmq
         except ImportError:
-            self.log.warning("python3-zmq module not available. Skipping zmq tests!")
-            sys.exit(self.TEST_EXIT_SKIPPED)
+            raise SkipTest("python3-zmq module not available.")
 
         # Check that bitcoin has been built with ZMQ enabled
         config = configparser.ConfigParser()
@@ -34,8 +32,7 @@ class ZMQTest (BitcoinTestFramework):
         config.read_file(open(self.options.configfile))
 
         if not config["components"].getboolean("ENABLE_ZMQ"):
-            self.log.warning("bitcoind has not been built with zmq enabled. Skipping zmq tests!")
-            sys.exit(self.TEST_EXIT_SKIPPED)
+            raise SkipTest("bitcoind has not been built with zmq enabled.")
 
         self.zmqContext = zmq.Context()
         self.zmqSubSocket = self.zmqContext.socket(zmq.SUB)