Merge #10096: Check that all test scripts in test/functional are being run

29d6634 Check that all test scripts in test/functional are being run (John Newbery)

Tree-SHA512: 9231d3a119632be031c51c4f7e95a8adae58489a8ec36fc967d499c0708bae2941a3bf28f11dcd4efd59141eb54c3c920f2629f5cd8a0139d30397a19591666d

test/functional/test_runner.py

@@ -128,6 +128,13 @@ EXTENDED_SCRIPTS = [
 
 ALL_SCRIPTS = BASE_SCRIPTS + ZMQ_SCRIPTS + EXTENDED_SCRIPTS
 
+NON_SCRIPTS = [
+    # These are python files that live in the functional tests directory, but are not test scripts.
+    "combine_logs.py",
+    "create_cache.py",
+    "test_runner.py",
+]
+
 def main():
     # Parse arguments and pass through unrecognised args
     parser = argparse.ArgumentParser(add_help=False,
@@ -218,6 +225,8 @@ def main():
         subprocess.check_call([(config["environment"]["SRCDIR"] + '/test/functional/' + test_list[0].split()[0])] + ['-h'])
         sys.exit(0)
 
+    check_script_list(config["environment"]["SRCDIR"])
+
     run_tests(test_list, config["environment"]["SRCDIR"], config["environment"]["BUILDDIR"], config["environment"]["EXEEXT"], args.jobs, args.coverage, passon_args)
 
 def run_tests(test_list, src_dir, build_dir, exeext, jobs=1, enable_coverage=False, args=[]):
@@ -341,6 +350,18 @@ class TestHandler:
                     return name, stdout, stderr, status, int(time.time() - time0)
             print('.', end='', flush=True)
 
+def check_script_list(src_dir):
+    """Check scripts directory.
+
+    Check that there are no scripts in the functional tests directory which are
+    not being run by pull-tester.py."""
+    script_dir = src_dir + '/test/functional/'
+    python_files = set([t for t in os.listdir(script_dir) if t[-3:] == ".py"])
+    missed_tests = list(python_files - set(map(lambda x: x.split()[0], ALL_SCRIPTS + NON_SCRIPTS)))
+    if len(missed_tests) != 0:
+        print("The following scripts are not being run:" + str(missed_tests))
+        print("Check the test lists in test_runner.py")
+        sys.exit(1)
 
 class RPCCoverage(object):
     """