From 232b6665bc3e5b134821dc7584968fb439fd5f44 Mon Sep 17 00:00:00 2001 From: John Newbery Date: Wed, 22 Mar 2017 10:26:02 -0400 Subject: [PATCH 1/2] Allow test cases to be skipped Currently, functional test cases can either pass or fail. There are occasions when it is helpful to skip tests, for example if the system they are running on does not meet the requirements for the test. The rest of the test suite can run without being marked as a failure. This commit adds framework for tests to skip if their requirements aren't met. --- .../test_framework/test_framework.py | 9 ++++--- test/functional/test_runner.py | 26 ++++++++++++------- 2 files changed, 23 insertions(+), 12 deletions(-) diff --git a/test/functional/test_framework/test_framework.py b/test/functional/test_framework/test_framework.py index fd2e80354..198c0cb5a 100755 --- a/test/functional/test_framework/test_framework.py +++ b/test/functional/test_framework/test_framework.py @@ -28,9 +28,12 @@ from .util import ( ) from .authproxy import JSONRPCException - class BitcoinTestFramework(object): + TEST_EXIT_PASSED = 0 + TEST_EXIT_FAILED = 1 + TEST_EXIT_SKIPPED = 77 + def __init__(self): self.num_nodes = 4 self.setup_clean_chain = False @@ -183,11 +186,11 @@ class BitcoinTestFramework(object): print("".join(deque(open(f), MAX_LINES_TO_PRINT))) if success: self.log.info("Tests successful") - sys.exit(0) + sys.exit(self.TEST_EXIT_PASSED) else: self.log.error("Test failed. Test logging available at %s/test_framework.log", self.options.tmpdir) logging.shutdown() - sys.exit(1) + sys.exit(self.TEST_EXIT_FAILED) def _start_logging(self): # Add logger and logging handlers diff --git a/test/functional/test_runner.py b/test/functional/test_runner.py index 12eb92028..41885e524 100755 --- a/test/functional/test_runner.py +++ b/test/functional/test_runner.py @@ -24,6 +24,9 @@ import subprocess import tempfile import re +TEST_EXIT_PASSED = 0 +TEST_EXIT_SKIPPED = 77 + BASE_SCRIPTS= [ # Scripts that are run by the travis build process. # Longest test should go first, to favor running tests in parallel @@ -245,20 +248,20 @@ def run_tests(test_list, src_dir, build_dir, exeext, jobs=1, enable_coverage=Fal job_queue = TestHandler(jobs, tests_dir, test_list, flags) max_len_name = len(max(test_list, key=len)) - results = BOLD[1] + "%s | %s | %s\n\n" % ("TEST".ljust(max_len_name), "PASSED", "DURATION") + BOLD[0] + results = BOLD[1] + "%s | %s | %s\n\n" % ("TEST".ljust(max_len_name), "PASSED ", "DURATION") + BOLD[0] for _ in range(len(test_list)): - (name, stdout, stderr, passed, duration) = job_queue.get_next() - all_passed = all_passed and passed + (name, stdout, stderr, status, duration) = job_queue.get_next() + all_passed = all_passed and status != "Failed" time_sum += duration print('\n' + BOLD[1] + name + BOLD[0] + ":") - print('' if passed else stdout + '\n', end='') + print('' if status == "Passed" else stdout + '\n', end='') print('' if stderr == '' else 'stderr:\n' + stderr + '\n', end='') - print("Pass: %s%s%s, Duration: %s s\n" % (BOLD[1], passed, BOLD[0], duration)) + print("Status: %s%s%s, Duration: %s s\n" % (BOLD[1], status, BOLD[0], duration)) - results += "%s | %s | %s s\n" % (name.ljust(max_len_name), str(passed).ljust(6), duration) + results += "%s | %s | %s s\n" % (name.ljust(max_len_name), status.ljust(7), duration) - results += BOLD[1] + "\n%s | %s | %s s (accumulated)" % ("ALL".ljust(max_len_name), str(all_passed).ljust(6), time_sum) + BOLD[0] + results += BOLD[1] + "\n%s | %s | %s s (accumulated)" % ("ALL".ljust(max_len_name), str(all_passed).ljust(7), time_sum) + BOLD[0] print(results) print("\nRuntime: %s s" % (int(time.time() - time0))) @@ -315,10 +318,15 @@ class TestHandler: log_out.seek(0), log_err.seek(0) [stdout, stderr] = [l.read().decode('utf-8') for l in (log_out, log_err)] log_out.close(), log_err.close() - passed = stderr == "" and proc.returncode == 0 + if proc.returncode == TEST_EXIT_PASSED and stderr == "": + status = "Passed" + elif proc.returncode == TEST_EXIT_SKIPPED: + status = "Skipped" + else: + status = "Failed" self.num_running -= 1 self.jobs.remove(j) - return name, stdout, stderr, passed, int(time.time() - time0) + return name, stdout, stderr, status, int(time.time() - time0) print('.', end='', flush=True) From 0c1ade6a4b3921da6f8f2b17c8fc23e15edef426 Mon Sep 17 00:00:00 2001 From: John Newbery Date: Wed, 22 Mar 2017 10:29:39 -0400 Subject: [PATCH 2/2] Skip rpcbind_test if OS/network requirements are not met. --- test/functional/rpcbind_test.py | 18 ++++++++++++++++-- test/functional/test_runner.py | 2 +- 2 files changed, 17 insertions(+), 3 deletions(-) diff --git a/test/functional/rpcbind_test.py b/test/functional/rpcbind_test.py index 8720a345c..efc36481d 100755 --- a/test/functional/rpcbind_test.py +++ b/test/functional/rpcbind_test.py @@ -4,6 +4,9 @@ # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test running bitcoind with the -rpcbind and -rpcallowip options.""" +import socket +import sys + from test_framework.test_framework import BitcoinTestFramework from test_framework.util import * from test_framework.netutil import * @@ -52,7 +55,9 @@ class RPCBindTest(BitcoinTestFramework): def run_test(self): # due to OS-specific network stats queries, this test works only on Linux - assert(sys.platform.startswith('linux')) + if not sys.platform.startswith('linux'): + self.log.warning("This test can only be run on linux. Skipping test.") + sys.exit(self.TEST_EXIT_SKIPPED) # find the first non-loopback interface for testing non_loopback_ip = None for name,ip in all_interfaces(): @@ -60,7 +65,16 @@ class RPCBindTest(BitcoinTestFramework): non_loopback_ip = ip break if non_loopback_ip is None: - assert(not 'This test requires at least one non-loopback IPv4 interface') + self.log.warning("This test requires at least one non-loopback IPv4 interface. Skipping test.") + sys.exit(self.TEST_EXIT_SKIPPED) + try: + s = socket.socket(socket.AF_INET6, socket.SOCK_DGRAM) + s.connect(("::1",1)) + s.close + except OSError: + self.log.warning("This test requires IPv6 support. Skipping test.") + sys.exit(self.TEST_EXIT_SKIPPED) + self.log.info("Using interface %s for testing" % non_loopback_ip) defaultport = rpc_port(0) diff --git a/test/functional/test_runner.py b/test/functional/test_runner.py index 41885e524..5bd71fe32 100755 --- a/test/functional/test_runner.py +++ b/test/functional/test_runner.py @@ -248,7 +248,7 @@ def run_tests(test_list, src_dir, build_dir, exeext, jobs=1, enable_coverage=Fal job_queue = TestHandler(jobs, tests_dir, test_list, flags) max_len_name = len(max(test_list, key=len)) - results = BOLD[1] + "%s | %s | %s\n\n" % ("TEST".ljust(max_len_name), "PASSED ", "DURATION") + BOLD[0] + results = BOLD[1] + "%s | %s | %s\n\n" % ("TEST".ljust(max_len_name), "STATUS ", "DURATION") + BOLD[0] for _ in range(len(test_list)): (name, stdout, stderr, status, duration) = job_queue.get_next() all_passed = all_passed and status != "Failed"