Browse Source

Allow test cases to be skipped

Currently, functional test cases can either pass or fail. There are
occasions when it is helpful to skip tests, for example if the
system they are running on does not meet the requirements for the test.
The rest of the test suite can run without being marked as a failure.

This commit adds framework for tests to skip if their requirements
aren't met.
0.15
John Newbery 8 years ago
parent
commit
232b6665bc
  1. 9
      test/functional/test_framework/test_framework.py
  2. 24
      test/functional/test_runner.py

9
test/functional/test_framework/test_framework.py

@ -28,9 +28,12 @@ from .util import ( @@ -28,9 +28,12 @@ from .util import (
)
from .authproxy import JSONRPCException
class BitcoinTestFramework(object):
TEST_EXIT_PASSED = 0
TEST_EXIT_FAILED = 1
TEST_EXIT_SKIPPED = 77
def __init__(self):
self.num_nodes = 4
self.setup_clean_chain = False
@ -183,11 +186,11 @@ class BitcoinTestFramework(object): @@ -183,11 +186,11 @@ class BitcoinTestFramework(object):
print("".join(deque(open(f), MAX_LINES_TO_PRINT)))
if success:
self.log.info("Tests successful")
sys.exit(0)
sys.exit(self.TEST_EXIT_PASSED)
else:
self.log.error("Test failed. Test logging available at %s/test_framework.log", self.options.tmpdir)
logging.shutdown()
sys.exit(1)
sys.exit(self.TEST_EXIT_FAILED)
def _start_logging(self):
# Add logger and logging handlers

24
test/functional/test_runner.py

@ -24,6 +24,9 @@ import subprocess @@ -24,6 +24,9 @@ import subprocess
import tempfile
import re
TEST_EXIT_PASSED = 0
TEST_EXIT_SKIPPED = 77
BASE_SCRIPTS= [
# Scripts that are run by the travis build process.
# Longest test should go first, to favor running tests in parallel
@ -247,18 +250,18 @@ def run_tests(test_list, src_dir, build_dir, exeext, jobs=1, enable_coverage=Fal @@ -247,18 +250,18 @@ def run_tests(test_list, src_dir, build_dir, exeext, jobs=1, enable_coverage=Fal
max_len_name = len(max(test_list, key=len))
results = BOLD[1] + "%s | %s | %s\n\n" % ("TEST".ljust(max_len_name), "PASSED ", "DURATION") + BOLD[0]
for _ in range(len(test_list)):
(name, stdout, stderr, passed, duration) = job_queue.get_next()
all_passed = all_passed and passed
(name, stdout, stderr, status, duration) = job_queue.get_next()
all_passed = all_passed and status != "Failed"
time_sum += duration
print('\n' + BOLD[1] + name + BOLD[0] + ":")
print('' if passed else stdout + '\n', end='')
print('' if status == "Passed" else stdout + '\n', end='')
print('' if stderr == '' else 'stderr:\n' + stderr + '\n', end='')
print("Pass: %s%s%s, Duration: %s s\n" % (BOLD[1], passed, BOLD[0], duration))
print("Status: %s%s%s, Duration: %s s\n" % (BOLD[1], status, BOLD[0], duration))
results += "%s | %s | %s s\n" % (name.ljust(max_len_name), str(passed).ljust(6), duration)
results += "%s | %s | %s s\n" % (name.ljust(max_len_name), status.ljust(7), duration)
results += BOLD[1] + "\n%s | %s | %s s (accumulated)" % ("ALL".ljust(max_len_name), str(all_passed).ljust(6), time_sum) + BOLD[0]
results += BOLD[1] + "\n%s | %s | %s s (accumulated)" % ("ALL".ljust(max_len_name), str(all_passed).ljust(7), time_sum) + BOLD[0]
print(results)
print("\nRuntime: %s s" % (int(time.time() - time0)))
@ -315,10 +318,15 @@ class TestHandler: @@ -315,10 +318,15 @@ class TestHandler:
log_out.seek(0), log_err.seek(0)
[stdout, stderr] = [l.read().decode('utf-8') for l in (log_out, log_err)]
log_out.close(), log_err.close()
passed = stderr == "" and proc.returncode == 0
if proc.returncode == TEST_EXIT_PASSED and stderr == "":
status = "Passed"
elif proc.returncode == TEST_EXIT_SKIPPED:
status = "Skipped"
else:
status = "Failed"
self.num_running -= 1
self.jobs.remove(j)
return name, stdout, stderr, passed, int(time.time() - time0)
return name, stdout, stderr, status, int(time.time() - time0)
print('.', end='', flush=True)

Loading…
Cancel
Save