From 63062bda1ac0b57cb92e663596650a6e42508f15 Mon Sep 17 00:00:00 2001 From: John Newbery Date: Wed, 5 Apr 2017 15:19:26 -0400 Subject: [PATCH 1/3] [tests] color test results and sort alphabetically --- test/functional/test_runner.py | 86 ++++++++++++++++++++++++---------- 1 file changed, 61 insertions(+), 25 deletions(-) diff --git a/test/functional/test_runner.py b/test/functional/test_runner.py index b1a5d2de9..7cf022182 100755 --- a/test/functional/test_runner.py +++ b/test/functional/test_runner.py @@ -25,6 +25,12 @@ import tempfile import re import logging +BOLD = ("", "") +if os.name == 'posix': + # primitive formatting on supported + # terminal via ANSI escape sequences: + BOLD = ('\033[0m', '\033[1m') + TEST_EXIT_PASSED = 0 TEST_EXIT_SKIPPED = 77 @@ -231,11 +237,6 @@ def main(): run_tests(test_list, config["environment"]["SRCDIR"], config["environment"]["BUILDDIR"], config["environment"]["EXEEXT"], args.jobs, args.coverage, passon_args) def run_tests(test_list, src_dir, build_dir, exeext, jobs=1, enable_coverage=False, args=[]): - BOLD = ("","") - if os.name == 'posix': - # primitive formatting on supported - # terminal via ANSI escape sequences: - BOLD = ('\033[0m', '\033[1m') #Set env vars if "BITCOIND" not in os.environ: @@ -258,33 +259,26 @@ def run_tests(test_list, src_dir, build_dir, exeext, jobs=1, enable_coverage=Fal subprocess.check_output([tests_dir + 'create_cache.py'] + flags) #Run Tests - all_passed = True - time_sum = 0 - time0 = time.time() - job_queue = TestHandler(jobs, tests_dir, test_list, flags) + time0 = time.time() + test_results = [] max_len_name = len(max(test_list, key=len)) - results = "\n" + BOLD[1] + "%s | %s | %s\n\n" % ("TEST".ljust(max_len_name), "STATUS ", "DURATION") + BOLD[0] + for _ in range(len(test_list)): - (name, stdout, stderr, status, duration) = job_queue.get_next() - all_passed = all_passed and status != "Failed" - time_sum += duration - - if status == "Passed": - logging.debug("\n%s%s%s passed, Duration: %s s" % (BOLD[1], name, BOLD[0], duration)) - elif status == "Skipped": - logging.debug("\n%s%s%s skipped" % (BOLD[1], name, BOLD[0])) + test_result, stdout, stderr = job_queue.get_next() + test_results.append(test_result) + + if test_result.status == "Passed": + logging.debug("\n%s%s%s passed, Duration: %s s" % (BOLD[1], test_result.name, BOLD[0], test_result.time)) + elif test_result.status == "Skipped": + logging.debug("\n%s%s%s skipped" % (BOLD[1], test_result.name, BOLD[0])) else: - print("\n%s%s%s failed, Duration: %s s\n" % (BOLD[1], name, BOLD[0], duration)) + print("\n%s%s%s failed, Duration: %s s\n" % (BOLD[1], test_result.name, BOLD[0], test_result.time)) print(BOLD[1] + 'stdout:\n' + BOLD[0] + stdout + '\n') print(BOLD[1] + 'stderr:\n' + BOLD[0] + stderr + '\n') - results += "%s | %s | %s s\n" % (name.ljust(max_len_name), status.ljust(7), duration) - - results += BOLD[1] + "\n%s | %s | %s s (accumulated)" % ("ALL".ljust(max_len_name), str(all_passed).ljust(7), time_sum) + BOLD[0] - print(results) - print("\nRuntime: %s s" % (int(time.time() - time0))) + print_results(test_results, max_len_name, (int(time.time() - time0))) if coverage: coverage.report_rpc_coverage() @@ -292,8 +286,27 @@ def run_tests(test_list, src_dir, build_dir, exeext, jobs=1, enable_coverage=Fal logging.debug("Cleaning up coverage data") coverage.cleanup() + all_passed = all(map(lambda test_result: test_result.status == "Passed", test_results)) + sys.exit(not all_passed) +def print_results(test_results, max_len_name, runtime): + results = "\n" + BOLD[1] + "%s | %s | %s\n\n" % ("TEST".ljust(max_len_name), "STATUS ", "DURATION") + BOLD[0] + + test_results.sort(key=lambda result: result.name.lower()) + all_passed = True + time_sum = 0 + + for test_result in test_results: + all_passed = all_passed and test_result.status != "Failed" + time_sum += test_result.time + test_result.padding = max_len_name + results += str(test_result) + + results += BOLD[1] + "\n%s | %s | %s s (accumulated) \n" % ("ALL".ljust(max_len_name), str(all_passed).ljust(7), time_sum) + BOLD[0] + results += "Runtime: %s s\n" % (runtime) + print(results) + class TestHandler: """ Trigger the testscrips passed in via the list. @@ -348,9 +361,32 @@ class TestHandler: status = "Failed" self.num_running -= 1 self.jobs.remove(j) - return name, stdout, stderr, status, int(time.time() - time0) + + return TestResult(name, status, int(time.time() - time0)), stdout, stderr print('.', end='', flush=True) +class TestResult(): + def __init__(self, name, status, time): + self.name = name + self.status = status + self.time = time + self.padding = 0 + + def __repr__(self): + COLOR = ("", "") + if os.name == 'posix': + # primitive formatting on supported + # terminal via ANSI escape sequences: + if self.status == "Passed": + COLOR = ('\033[0m', '\033[0;34m') + elif self.status == "Failed": + COLOR = ('\033[0m', '\033[0;31m') + elif self.status == "Skipped": + COLOR = ('\033[0m', '\033[1;30m') + + return COLOR[1] + "%s | %s | %s s\n" % (self.name.ljust(self.padding), self.status.ljust(7), self.time) + COLOR[0] + + def check_script_list(src_dir): """Check scripts directory. From bb92d839d5aecd41954091435a35e86b35a0a7ca Mon Sep 17 00:00:00 2001 From: John Newbery Date: Thu, 6 Apr 2017 13:31:47 -0400 Subject: [PATCH 2/3] [tests] Add unicode symbols for tests passing/failing/skipping --- test/functional/test_runner.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/test/functional/test_runner.py b/test/functional/test_runner.py index 7cf022182..d40ca9bbc 100755 --- a/test/functional/test_runner.py +++ b/test/functional/test_runner.py @@ -291,7 +291,7 @@ def run_tests(test_list, src_dir, build_dir, exeext, jobs=1, enable_coverage=Fal sys.exit(not all_passed) def print_results(test_results, max_len_name, runtime): - results = "\n" + BOLD[1] + "%s | %s | %s\n\n" % ("TEST".ljust(max_len_name), "STATUS ", "DURATION") + BOLD[0] + results = "\n" + BOLD[1] + "%s | %s | %s\n\n" % ("TEST".ljust(max_len_name), "STATUS ", "DURATION") + BOLD[0] test_results.sort(key=lambda result: result.name.lower()) all_passed = True @@ -384,7 +384,15 @@ class TestResult(): elif self.status == "Skipped": COLOR = ('\033[0m', '\033[1;30m') - return COLOR[1] + "%s | %s | %s s\n" % (self.name.ljust(self.padding), self.status.ljust(7), self.time) + COLOR[0] + SYMBOL = " " + if self.status == "Passed": + SYMBOL = "✓ " + elif self.status == "Failed": + SYMBOL = "✖ " + elif self.status == "Skipped": + SYMBOL = "○ " + + return COLOR[1] + "%s | %s%s | %s s\n" % (self.name.ljust(self.padding), SYMBOL, self.status.ljust(7), self.time) + COLOR[0] def check_script_list(src_dir): From d80baaa514260334077799abbce4d7f16c2a538d Mon Sep 17 00:00:00 2001 From: John Newbery Date: Fri, 7 Apr 2017 09:15:29 -0400 Subject: [PATCH 3/3] fixup - align summary row correctly and make colors/glyphs globals --- test/functional/test_runner.py | 35 +++++++++++++++++----------------- 1 file changed, 17 insertions(+), 18 deletions(-) diff --git a/test/functional/test_runner.py b/test/functional/test_runner.py index d40ca9bbc..c0bbc623a 100755 --- a/test/functional/test_runner.py +++ b/test/functional/test_runner.py @@ -25,11 +25,18 @@ import tempfile import re import logging -BOLD = ("", "") +# Formatting. Default colors to empty strings. +BOLD, BLUE, RED, GREY = ("", ""), ("", ""), ("", ""), ("", "") +TICK = "✓ " +CROSS = "✖ " +CIRCLE = "○ " if os.name == 'posix': # primitive formatting on supported # terminal via ANSI escape sequences: BOLD = ('\033[0m', '\033[1m') + BLUE = ('\033[0m', '\033[0;34m') + RED = ('\033[0m', '\033[0;31m') + GREY = ('\033[0m', '\033[1;30m') TEST_EXIT_PASSED = 0 TEST_EXIT_SKIPPED = 77 @@ -303,7 +310,8 @@ def print_results(test_results, max_len_name, runtime): test_result.padding = max_len_name results += str(test_result) - results += BOLD[1] + "\n%s | %s | %s s (accumulated) \n" % ("ALL".ljust(max_len_name), str(all_passed).ljust(7), time_sum) + BOLD[0] + status = TICK + "Passed" if all_passed else CROSS + "Failed" + results += BOLD[1] + "\n%s | %s | %s s (accumulated) \n" % ("ALL".ljust(max_len_name), status.ljust(9), time_sum) + BOLD[0] results += "Runtime: %s s\n" % (runtime) print(results) @@ -373,26 +381,17 @@ class TestResult(): self.padding = 0 def __repr__(self): - COLOR = ("", "") - if os.name == 'posix': - # primitive formatting on supported - # terminal via ANSI escape sequences: - if self.status == "Passed": - COLOR = ('\033[0m', '\033[0;34m') - elif self.status == "Failed": - COLOR = ('\033[0m', '\033[0;31m') - elif self.status == "Skipped": - COLOR = ('\033[0m', '\033[1;30m') - - SYMBOL = " " if self.status == "Passed": - SYMBOL = "✓ " + color = BLUE + glyph = TICK elif self.status == "Failed": - SYMBOL = "✖ " + color = RED + glyph = CROSS elif self.status == "Skipped": - SYMBOL = "○ " + color = GREY + glyph = CIRCLE - return COLOR[1] + "%s | %s%s | %s s\n" % (self.name.ljust(self.padding), SYMBOL, self.status.ljust(7), self.time) + COLOR[0] + return color[1] + "%s | %s%s | %s s\n" % (self.name.ljust(self.padding), glyph, self.status.ljust(7), self.time) + color[0] def check_script_list(src_dir):