From 1581ecbc33edd9d6257e50b11d8854fbccaf8ad8 Mon Sep 17 00:00:00 2001 From: John Newbery Date: Mon, 30 Jan 2017 14:57:27 -0800 Subject: [PATCH 1/5] Use configparser in rpc-tests.py Remove the use of wildcard imports in rpc-tests.py and replace with configparser. --- Makefile.am | 3 --- configure.ac | 2 +- qa/pull-tester/rpc-tests.py | 33 ++++++++++++++---------------- qa/pull-tester/tests_config.ini.in | 25 ++++++++++++++++++++++ qa/pull-tester/tests_config.py.in | 14 ------------- 5 files changed, 41 insertions(+), 36 deletions(-) create mode 100644 qa/pull-tester/tests_config.ini.in delete mode 100644 qa/pull-tester/tests_config.py.in diff --git a/Makefile.am b/Makefile.am index 6a8c1b761..1ee2dfb73 100644 --- a/Makefile.am +++ b/Makefile.am @@ -227,9 +227,6 @@ EXTRA_DIST = $(top_srcdir)/share/genbuild.sh qa/pull-tester/rpc-tests.py qa/rpc- CLEANFILES = $(OSX_DMG) $(BITCOIN_WIN_INSTALLER) -# This file is problematic for out-of-tree builds if it exists. -DISTCLEANFILES = qa/pull-tester/tests_config.pyc - .INTERMEDIATE: $(COVERAGE_INFO) DISTCHECK_CONFIGURE_FLAGS = --enable-man diff --git a/configure.ac b/configure.ac index 346695043..1452ff74c 100644 --- a/configure.ac +++ b/configure.ac @@ -1087,7 +1087,7 @@ AC_SUBST(ZMQ_LIBS) AC_SUBST(PROTOBUF_LIBS) AC_SUBST(QR_LIBS) AC_CONFIG_FILES([Makefile src/Makefile doc/man/Makefile share/setup.nsi share/qt/Info.plist src/test/buildenv.py]) -AC_CONFIG_FILES([qa/pull-tester/tests_config.py],[chmod +x qa/pull-tester/tests_config.py]) +AC_CONFIG_FILES([qa/pull-tester/tests_config.ini],[chmod +x qa/pull-tester/tests_config.ini]) AC_CONFIG_FILES([contrib/devtools/split-debug.sh],[chmod +x contrib/devtools/split-debug.sh]) AC_CONFIG_LINKS([qa/pull-tester/rpc-tests.py:qa/pull-tester/rpc-tests.py]) diff --git a/qa/pull-tester/rpc-tests.py b/qa/pull-tester/rpc-tests.py index 26bc6a73d..521574740 100755 --- a/qa/pull-tester/rpc-tests.py +++ b/qa/pull-tester/rpc-tests.py @@ -21,6 +21,7 @@ For a description of arguments recognized by test scripts, see """ +import configparser import os import time import shutil @@ -29,26 +30,22 @@ import subprocess import tempfile import re -sys.path.append("qa/pull-tester/") -from tests_config import * - BOLD = ("","") if os.name == 'posix': # primitive formatting on supported # terminal via ANSI escape sequences: BOLD = ('\033[0m', '\033[1m') -RPC_TESTS_DIR = SRCDIR + '/qa/rpc-tests/' +# Read config generated by configure. +config = configparser.ConfigParser() +config.read_file(open(os.path.dirname(__file__) + "/tests_config.ini")) + +ENABLE_WALLET = config["components"]["ENABLE_WALLET"] == "True" +ENABLE_UTILS = config["components"]["ENABLE_UTILS"] == "True" +ENABLE_BITCOIND = config["components"]["ENABLE_BITCOIND"] == "True" +ENABLE_ZMQ = config["components"]["ENABLE_ZMQ"] == "True" -#If imported values are not defined then set to zero (or disabled) -if 'ENABLE_WALLET' not in vars(): - ENABLE_WALLET=0 -if 'ENABLE_BITCOIND' not in vars(): - ENABLE_BITCOIND=0 -if 'ENABLE_UTILS' not in vars(): - ENABLE_UTILS=0 -if 'ENABLE_ZMQ' not in vars(): - ENABLE_ZMQ=0 +RPC_TESTS_DIR = config["environment"]["SRCDIR"] + '/qa/rpc-tests/' ENABLE_COVERAGE=0 @@ -76,15 +73,15 @@ for arg in sys.argv[1:]: #Set env vars if "BITCOIND" not in os.environ: - os.environ["BITCOIND"] = BUILDDIR + '/src/bitcoind' + EXEEXT + os.environ["BITCOIND"] = config["environment"]["BUILDDIR"] + '/src/bitcoind' + config["environment"]["EXEEXT"] -if EXEEXT == ".exe" and "-win" not in opts: +if config["environment"]["EXEEXT"] == ".exe" and "-win" not in opts: # https://github.com/bitcoin/bitcoin/commit/d52802551752140cf41f0d9a225a43e84404d3e9 # https://github.com/bitcoin/bitcoin/pull/5677#issuecomment-136646964 print("Win tests currently disabled by default. Use -win option to enable") sys.exit(0) -if not (ENABLE_WALLET == 1 and ENABLE_UTILS == 1 and ENABLE_BITCOIND == 1): +if not (ENABLE_WALLET and ENABLE_UTILS and ENABLE_BITCOIND): print("No rpc tests to run. Wallet, utils, and bitcoind must all be enabled") sys.exit(0) @@ -209,8 +206,8 @@ def runtests(): if ENABLE_COVERAGE: coverage = RPCCoverage() print("Initializing coverage directory at %s\n" % coverage.dir) - flags = ["--srcdir=%s/src" % BUILDDIR] + passon_args - flags.append("--cachedir=%s/qa/cache" % BUILDDIR) + flags = ["--srcdir=%s/src" % config["environment"]["BUILDDIR"]] + passon_args + flags.append("--cachedir=%s/qa/cache" % config["environment"]["BUILDDIR"]) if coverage: flags.append(coverage.flag) diff --git a/qa/pull-tester/tests_config.ini.in b/qa/pull-tester/tests_config.ini.in new file mode 100644 index 000000000..8317caaeb --- /dev/null +++ b/qa/pull-tester/tests_config.ini.in @@ -0,0 +1,25 @@ +# Copyright (c) 2013-2016 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. + +# These environment variables are set by the build process and read by +# rpc-tests.py + +[DEFAULT] +# Provides default values for whether different components are enabled +ENABLE_WALLET=False +ENABLE_UTILS=False +ENABLE_BITCOIND=False +ENABLE_ZMQ=False + +[environment] +SRCDIR=@abs_top_srcdir@ +BUILDDIR=@abs_top_builddir@ +EXEEXT=@EXEEXT@ + +[components] +# Which components are enabled. These are commented out by `configure` if they were disabled when running config. +@ENABLE_WALLET_TRUE@ENABLE_WALLET=True +@BUILD_BITCOIN_UTILS_TRUE@ENABLE_UTILS=True +@BUILD_BITCOIND_TRUE@ENABLE_BITCOIND=True +@ENABLE_ZMQ_TRUE@ENABLE_ZMQ=True diff --git a/qa/pull-tester/tests_config.py.in b/qa/pull-tester/tests_config.py.in deleted file mode 100644 index a0d0a3d98..000000000 --- a/qa/pull-tester/tests_config.py.in +++ /dev/null @@ -1,14 +0,0 @@ -#!/usr/bin/env python3 -# Copyright (c) 2013-2016 The Bitcoin Core developers -# Distributed under the MIT software license, see the accompanying -# file COPYING or http://www.opensource.org/licenses/mit-license.php. - -SRCDIR="@abs_top_srcdir@" -BUILDDIR="@abs_top_builddir@" -EXEEXT="@EXEEXT@" - -# These will turn into comments if they were disabled when configuring. -@ENABLE_WALLET_TRUE@ENABLE_WALLET=1 -@BUILD_BITCOIN_UTILS_TRUE@ENABLE_UTILS=1 -@BUILD_BITCOIND_TRUE@ENABLE_BITCOIND=1 -@ENABLE_ZMQ_TRUE@ENABLE_ZMQ=1 From 91bfffff5d1de034661e7fd6b319447b47ae6e33 Mon Sep 17 00:00:00 2001 From: John Newbery Date: Tue, 31 Jan 2017 10:15:40 -0800 Subject: [PATCH 2/5] Use argparse in rpc_tests.py This commit replaces the roll-your-own argument parsing in rpc_tests.py with Python's standard library argparse. --- qa/pull-tester/rpc-tests.py | 83 ++++++++++++++++++++----------------- 1 file changed, 44 insertions(+), 39 deletions(-) diff --git a/qa/pull-tester/rpc-tests.py b/qa/pull-tester/rpc-tests.py index 521574740..862859919 100755 --- a/qa/pull-tester/rpc-tests.py +++ b/qa/pull-tester/rpc-tests.py @@ -21,6 +21,7 @@ For a description of arguments recognized by test scripts, see """ +import argparse import configparser import os import time @@ -30,6 +31,19 @@ import subprocess import tempfile import re +# Parse arguments and pass through unrecognised args +parser = argparse.ArgumentParser(add_help=False) +parser.add_argument('--coverage', action='store_true') +parser.add_argument('-extended', action='store_true') +parser.add_argument('--help', '-h', '-?', action='store_true') +parser.add_argument('--parallel', type=int, default=4) +parser.add_argument('-win', action='store_true') +(args, unknown_args) = parser.parse_known_args() + +#Create a set to store arguments and create the passon string +tests = set(arg for arg in unknown_args if arg[:2] != "--") +passon_args = [arg for arg in unknown_args if arg[:2] == "--"] + BOLD = ("","") if os.name == 'posix': # primitive formatting on supported @@ -47,35 +61,14 @@ ENABLE_ZMQ = config["components"]["ENABLE_ZMQ"] == "True" RPC_TESTS_DIR = config["environment"]["SRCDIR"] + '/qa/rpc-tests/' -ENABLE_COVERAGE=0 - -#Create a set to store arguments and create the passon string -opts = set() -passon_args = [] -PASSON_REGEX = re.compile("^--") -PARALLEL_REGEX = re.compile('^-parallel=') - -print_help = False -run_parallel = 4 - -for arg in sys.argv[1:]: - if arg == "--help" or arg == "-h" or arg == "-?": - print_help = True - break - if arg == '--coverage': - ENABLE_COVERAGE = 1 - elif PASSON_REGEX.match(arg): - passon_args.append(arg) - elif PARALLEL_REGEX.match(arg): - run_parallel = int(arg.split(sep='=', maxsplit=1)[1]) - else: - opts.add(arg) +print_help = args.help +run_parallel = args.parallel #Set env vars if "BITCOIND" not in os.environ: os.environ["BITCOIND"] = config["environment"]["BUILDDIR"] + '/src/bitcoind' + config["environment"]["EXEEXT"] -if config["environment"]["EXEEXT"] == ".exe" and "-win" not in opts: +if config["environment"]["EXEEXT"] == ".exe" and not args.win: # https://github.com/bitcoin/bitcoin/commit/d52802551752140cf41f0d9a225a43e84404d3e9 # https://github.com/bitcoin/bitcoin/pull/5677#issuecomment-136646964 print("Win tests currently disabled by default. Use -win option to enable") @@ -95,7 +88,7 @@ if ENABLE_ZMQ: # ENABLE_ZMQ=0 raise -testScripts = [ +BASE_SCRIPTS= [ # longest test should go first, to favor running tests in parallel 'wallet-hd.py', 'walletbackup.py', @@ -152,10 +145,9 @@ testScripts = [ 'rpcnamedargs.py', 'listsinceblock.py', ] -if ENABLE_ZMQ: - testScripts.append('zmq_test.py') +ZMQ_SCRIPTS = ["zmq_test.py"] -testScriptsExt = [ +EXTENDED_SCRIPTS = [ 'pruning.py', # vv Tests less than 20m vv 'smartfees.py', @@ -184,26 +176,39 @@ testScriptsExt = [ 'replace-by-fee.py', ] +ALL_SCRIPTS = BASE_SCRIPTS + ZMQ_SCRIPTS + EXTENDED_SCRIPTS def runtests(): - test_list = [] - if '-extended' in opts: - test_list = testScripts + testScriptsExt - elif len(opts) == 0 or (len(opts) == 1 and "-win" in opts): - test_list = testScripts + # Build list of tests + if len(tests) != 0: + # Individual tests have been specified. Run specified tests that exist + # in the ALL_SCRIPTS list. Accept the name with or without .py extension. + test_list = [t for t in ALL_SCRIPTS if + (t in tests or re.sub(".py$", "", t) in tests)] + if len(test_list) == 0: + print("No valid test scripts specified. Check that your test is in one " + "of the test lists in rpc-tests.py or run rpc-tests.py with no arguments to run all tests") + sys.exit(0) else: - for t in testScripts + testScriptsExt: - if t in opts or re.sub(".py$", "", t) in opts: - test_list.append(t) - - if print_help: + # No individual tests have been specified. Run base tests, and + # optionally ZMQ tests and extended tests. + test_list = BASE_SCRIPTS + if ENABLE_ZMQ: + test_list += ZMQ_SCRIPTS + if args.extended: + test_list += EXTENDED_SCRIPTS + # TODO: BASE_SCRIPTS and EXTENDED_SCRIPTS are sorted by runtime + # (for parallel running efficiency). This combined list will is no + # longer sorted. + + if args.help: # Only print help of the first script and exit subprocess.check_call((RPC_TESTS_DIR + test_list[0]).split() + ['-h']) sys.exit(0) coverage = None - if ENABLE_COVERAGE: + if args.coverage: coverage = RPCCoverage() print("Initializing coverage directory at %s\n" % coverage.dir) flags = ["--srcdir=%s/src" % config["environment"]["BUILDDIR"]] + passon_args From afd38e7cc875e79f282ebf1c63fa10bf2d2af8d2 Mon Sep 17 00:00:00 2001 From: John Newbery Date: Tue, 31 Jan 2017 11:32:49 -0800 Subject: [PATCH 3/5] Improve rpc-tests.py arguments A few miscellaneous improvements to rpc-tests.py command line arguments: - make all arguments start with double dash for consistency - improve help text and output - add nozmq argument to explicitly exclude the ZMQ tests - change 'parallel' to 'jobs' --- qa/pull-tester/rpc-tests.py | 48 ++++++++++++++++++------------------- 1 file changed, 23 insertions(+), 25 deletions(-) diff --git a/qa/pull-tester/rpc-tests.py b/qa/pull-tester/rpc-tests.py index 862859919..637999ae4 100755 --- a/qa/pull-tester/rpc-tests.py +++ b/qa/pull-tester/rpc-tests.py @@ -2,19 +2,11 @@ # Copyright (c) 2014-2016 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. - """ -Run Regression Test Suite +rpc-tests.py - run regression test suite This module calls down into individual test cases via subprocess. It will -forward all unrecognized arguments onto the individual test scripts, other -than: - - - `-extended`: run the "extended" test suite in addition to the basic one. - - `-win`: signal that this is running in a Windows environment, and we - should run the tests. - - `--coverage`: this generates a basic coverage report for the RPC - interface. +forward all unrecognized arguments onto the individual test scripts. For a description of arguments recognized by test scripts, see `qa/pull-tester/test_framework/test_framework.py:BitcoinTestFramework.main`. @@ -32,12 +24,18 @@ import tempfile import re # Parse arguments and pass through unrecognised args -parser = argparse.ArgumentParser(add_help=False) -parser.add_argument('--coverage', action='store_true') -parser.add_argument('-extended', action='store_true') -parser.add_argument('--help', '-h', '-?', action='store_true') -parser.add_argument('--parallel', type=int, default=4) -parser.add_argument('-win', action='store_true') +parser = argparse.ArgumentParser(add_help=False, + usage='%(prog)s [rpc-test.py options] [script options] [scripts]', + description=__doc__, + epilog=''' +Help text and arguments for individual test script:''', + formatter_class=argparse.RawTextHelpFormatter) +parser.add_argument('--coverage', action='store_true', help='generate a basic coverage report for the RPC interface') +parser.add_argument('--extended', action='store_true', help='run the extended test suite in addition to the basic tests') +parser.add_argument('--help', '-h', '-?', action='store_true', help='print help text and exit') +parser.add_argument('--jobs', '-j', type=int, default=4, help='how many test scripts to run in parallel. Default=4.') +parser.add_argument('--nozmq', action='store_true', help='do not run the zmq tests') +parser.add_argument('--win', action='store_true', help='signal that this is running in a Windows environment and that we should run the tests') (args, unknown_args) = parser.parse_known_args() #Create a set to store arguments and create the passon string @@ -57,12 +55,12 @@ config.read_file(open(os.path.dirname(__file__) + "/tests_config.ini")) ENABLE_WALLET = config["components"]["ENABLE_WALLET"] == "True" ENABLE_UTILS = config["components"]["ENABLE_UTILS"] == "True" ENABLE_BITCOIND = config["components"]["ENABLE_BITCOIND"] == "True" -ENABLE_ZMQ = config["components"]["ENABLE_ZMQ"] == "True" +ENABLE_ZMQ = config["components"]["ENABLE_ZMQ"] == "True" and not args.nozmq RPC_TESTS_DIR = config["environment"]["SRCDIR"] + '/qa/rpc-tests/' print_help = args.help -run_parallel = args.parallel +jobs = args.jobs #Set env vars if "BITCOIND" not in os.environ: @@ -71,7 +69,7 @@ if "BITCOIND" not in os.environ: if config["environment"]["EXEEXT"] == ".exe" and not args.win: # https://github.com/bitcoin/bitcoin/commit/d52802551752140cf41f0d9a225a43e84404d3e9 # https://github.com/bitcoin/bitcoin/pull/5677#issuecomment-136646964 - print("Win tests currently disabled by default. Use -win option to enable") + print("Win tests currently disabled by default. Use --win option to enable") sys.exit(0) if not (ENABLE_WALLET and ENABLE_UTILS and ENABLE_BITCOIND): @@ -83,9 +81,8 @@ if ENABLE_ZMQ: try: import zmq except ImportError: - print("ERROR: \"import zmq\" failed. Set ENABLE_ZMQ=0 or " - "to run zmq tests, see dependency info in /qa/README.md.") - # ENABLE_ZMQ=0 + print("ERROR: \"import zmq\" failed. Use -nozmq to run without the ZMQ tests." + "To run zmq tests, see dependency info in /qa/README.md.") raise BASE_SCRIPTS= [ @@ -202,7 +199,8 @@ def runtests(): # longer sorted. if args.help: - # Only print help of the first script and exit + # Print help for rpc-tests.py, then print help of the first script and exit. + parser.print_help() subprocess.check_call((RPC_TESTS_DIR + test_list[0]).split() + ['-h']) sys.exit(0) @@ -216,7 +214,7 @@ def runtests(): if coverage: flags.append(coverage.flag) - if len(test_list) > 1 and run_parallel > 1: + if len(test_list) > 1 and jobs > 1: # Populate cache subprocess.check_output([RPC_TESTS_DIR + 'create_cache.py'] + flags) @@ -224,7 +222,7 @@ def runtests(): max_len_name = len(max(test_list, key=len)) time_sum = 0 time0 = time.time() - job_queue = RPCTestHandler(run_parallel, test_list, flags) + job_queue = RPCTestHandler(jobs, test_list, flags) results = BOLD[1] + "%s | %s | %s\n\n" % ("TEST".ljust(max_len_name), "PASSED", "DURATION") + BOLD[0] all_passed = True for _ in range(len(test_list)): From 3de3ccdfaaa5edcf4e0924860d960e7e0db5a9b9 Mon Sep 17 00:00:00 2001 From: John Newbery Date: Mon, 6 Feb 2017 09:07:14 -0500 Subject: [PATCH 4/5] Refactor rpc-tests.py - add main() - remove global variables --- qa/pull-tester/rpc-tests.py | 185 +++++++++++++++++++----------------- 1 file changed, 100 insertions(+), 85 deletions(-) diff --git a/qa/pull-tester/rpc-tests.py b/qa/pull-tester/rpc-tests.py index 637999ae4..1b4c90564 100755 --- a/qa/pull-tester/rpc-tests.py +++ b/qa/pull-tester/rpc-tests.py @@ -23,69 +23,8 @@ import subprocess import tempfile import re -# Parse arguments and pass through unrecognised args -parser = argparse.ArgumentParser(add_help=False, - usage='%(prog)s [rpc-test.py options] [script options] [scripts]', - description=__doc__, - epilog=''' -Help text and arguments for individual test script:''', - formatter_class=argparse.RawTextHelpFormatter) -parser.add_argument('--coverage', action='store_true', help='generate a basic coverage report for the RPC interface') -parser.add_argument('--extended', action='store_true', help='run the extended test suite in addition to the basic tests') -parser.add_argument('--help', '-h', '-?', action='store_true', help='print help text and exit') -parser.add_argument('--jobs', '-j', type=int, default=4, help='how many test scripts to run in parallel. Default=4.') -parser.add_argument('--nozmq', action='store_true', help='do not run the zmq tests') -parser.add_argument('--win', action='store_true', help='signal that this is running in a Windows environment and that we should run the tests') -(args, unknown_args) = parser.parse_known_args() - -#Create a set to store arguments and create the passon string -tests = set(arg for arg in unknown_args if arg[:2] != "--") -passon_args = [arg for arg in unknown_args if arg[:2] == "--"] - -BOLD = ("","") -if os.name == 'posix': - # primitive formatting on supported - # terminal via ANSI escape sequences: - BOLD = ('\033[0m', '\033[1m') - -# Read config generated by configure. -config = configparser.ConfigParser() -config.read_file(open(os.path.dirname(__file__) + "/tests_config.ini")) - -ENABLE_WALLET = config["components"]["ENABLE_WALLET"] == "True" -ENABLE_UTILS = config["components"]["ENABLE_UTILS"] == "True" -ENABLE_BITCOIND = config["components"]["ENABLE_BITCOIND"] == "True" -ENABLE_ZMQ = config["components"]["ENABLE_ZMQ"] == "True" and not args.nozmq - -RPC_TESTS_DIR = config["environment"]["SRCDIR"] + '/qa/rpc-tests/' - -print_help = args.help -jobs = args.jobs - -#Set env vars -if "BITCOIND" not in os.environ: - os.environ["BITCOIND"] = config["environment"]["BUILDDIR"] + '/src/bitcoind' + config["environment"]["EXEEXT"] - -if config["environment"]["EXEEXT"] == ".exe" and not args.win: - # https://github.com/bitcoin/bitcoin/commit/d52802551752140cf41f0d9a225a43e84404d3e9 - # https://github.com/bitcoin/bitcoin/pull/5677#issuecomment-136646964 - print("Win tests currently disabled by default. Use --win option to enable") - sys.exit(0) - -if not (ENABLE_WALLET and ENABLE_UTILS and ENABLE_BITCOIND): - print("No rpc tests to run. Wallet, utils, and bitcoind must all be enabled") - sys.exit(0) - -# python3-zmq may not be installed. Handle this gracefully and with some helpful info -if ENABLE_ZMQ: - try: - import zmq - except ImportError: - print("ERROR: \"import zmq\" failed. Use -nozmq to run without the ZMQ tests." - "To run zmq tests, see dependency info in /qa/README.md.") - raise - BASE_SCRIPTS= [ + # Scripts that are run by the travis build process # longest test should go first, to favor running tests in parallel 'wallet-hd.py', 'walletbackup.py', @@ -142,9 +81,15 @@ BASE_SCRIPTS= [ 'rpcnamedargs.py', 'listsinceblock.py', ] -ZMQ_SCRIPTS = ["zmq_test.py"] + +ZMQ_SCRIPTS = [ + # ZMQ test can only be run if bitcoin was built with zmq-enabled. + # call rpc_tests.py with -nozmq to explicitly exclude these tests. + "zmq_test.py"] EXTENDED_SCRIPTS = [ + # These tests are not run by the travis build process. + # Longest test should go first, to favor running tests in parallel 'pruning.py', # vv Tests less than 20m vv 'smartfees.py', @@ -175,7 +120,55 @@ EXTENDED_SCRIPTS = [ ALL_SCRIPTS = BASE_SCRIPTS + ZMQ_SCRIPTS + EXTENDED_SCRIPTS -def runtests(): +def main(): + # Parse arguments and pass through unrecognised args + parser = argparse.ArgumentParser(add_help=False, + usage='%(prog)s [rpc-test.py options] [script options] [scripts]', + description=__doc__, + epilog=''' + Help text and arguments for individual test script:''', + formatter_class=argparse.RawTextHelpFormatter) + parser.add_argument('--coverage', action='store_true', help='generate a basic coverage report for the RPC interface') + parser.add_argument('--extended', action='store_true', help='run the extended test suite in addition to the basic tests') + parser.add_argument('--help', '-h', '-?', action='store_true', help='print help text and exit') + parser.add_argument('--nozmq', action='store_true', help='do not run the zmq tests') + parser.add_argument('--jobs', '-j', type=int, default=4, help='how many test scripts to run in parallel. Default=4.') + parser.add_argument('--win', action='store_true', help='signal that this is running in a Windows environment and that we should run the tests') + (args, unknown_args) = parser.parse_known_args() + + # Create a set to store arguments and create the passon string + tests = set(arg for arg in unknown_args if arg[:2] != "--") + passon_args = [arg for arg in unknown_args if arg[:2] == "--"] + + # Read config generated by configure. + config = configparser.ConfigParser() + config.read_file(open(os.path.dirname(__file__) + "/tests_config.ini")) + + enable_wallet = config["components"]["ENABLE_WALLET"] == "True" + enable_utils = config["components"]["ENABLE_UTILS"] == "True" + enable_bitcoind = config["components"]["ENABLE_BITCOIND"] == "True" + enable_zmq = config["components"]["ENABLE_ZMQ"] == "True" and not args.nozmq + + if config["environment"]["EXEEXT"] == ".exe" and not args.win: + # https://github.com/bitcoin/bitcoin/commit/d52802551752140cf41f0d9a225a43e84404d3e9 + # https://github.com/bitcoin/bitcoin/pull/5677#issuecomment-136646964 + print("Win tests currently disabled by default. Use --win option to enable") + sys.exit(0) + + if not (enable_wallet and enable_utils and enable_bitcoind): + print("No rpc tests to run. Wallet, utils, and bitcoind must all be enabled") + print("Rerun `configure` with -enable-wallet, -with-utils and -with-daemon and rerun make") + sys.exit(0) + + # python3-zmq may not be installed. Handle this gracefully and with some helpful info + if enable_zmq: + try: + import zmq + except ImportError: + print("ERROR: \"import zmq\" failed. Use -nozmq to run without the ZMQ tests." + "To run zmq tests, see dependency info in /qa/README.md.") + raise + # Build list of tests if len(tests) != 0: # Individual tests have been specified. Run specified tests that exist @@ -185,12 +178,15 @@ def runtests(): if len(test_list) == 0: print("No valid test scripts specified. Check that your test is in one " "of the test lists in rpc-tests.py or run rpc-tests.py with no arguments to run all tests") + print("Scripts not found:") + print(tests) sys.exit(0) + else: # No individual tests have been specified. Run base tests, and # optionally ZMQ tests and extended tests. test_list = BASE_SCRIPTS - if ENABLE_ZMQ: + if enable_zmq: test_list += ZMQ_SCRIPTS if args.extended: test_list += EXTENDED_SCRIPTS @@ -201,30 +197,47 @@ def runtests(): if args.help: # Print help for rpc-tests.py, then print help of the first script and exit. parser.print_help() - subprocess.check_call((RPC_TESTS_DIR + test_list[0]).split() + ['-h']) + subprocess.check_call((config["environment"]["SRCDIR"] + '/qa/rpc-tests/' + test_list[0]).split() + ['-h']) sys.exit(0) - coverage = None + runtests(test_list, config["environment"]["SRCDIR"], config["environment"]["BUILDDIR"], config["environment"]["EXEEXT"], args.jobs, args.coverage, passon_args) + +def runtests(test_list, src_dir, build_dir, exeext, jobs=1, enable_coverage=False, args=[]): + BOLD = ("","") + if os.name == 'posix': + # primitive formatting on supported + # terminal via ANSI escape sequences: + BOLD = ('\033[0m', '\033[1m') + + #Set env vars + if "BITCOIND" not in os.environ: + os.environ["BITCOIND"] = build_dir + '/src/bitcoind' + exeext + + tests_dir = src_dir + '/qa/rpc-tests/' - if args.coverage: + flags = ["--srcdir=" + src_dir] + args + flags.append("--cachedir=%s/qa/cache" % build_dir) + + if enable_coverage: coverage = RPCCoverage() - print("Initializing coverage directory at %s\n" % coverage.dir) - flags = ["--srcdir=%s/src" % config["environment"]["BUILDDIR"]] + passon_args - flags.append("--cachedir=%s/qa/cache" % config["environment"]["BUILDDIR"]) - if coverage: flags.append(coverage.flag) + print("Initializing coverage directory at %s\n" % coverage.dir) + else: + coverage = None if len(test_list) > 1 and jobs > 1: # Populate cache - subprocess.check_output([RPC_TESTS_DIR + 'create_cache.py'] + flags) + subprocess.check_output([tests_dir + 'create_cache.py'] + flags) #Run Tests - max_len_name = len(max(test_list, key=len)) + all_passed = True time_sum = 0 time0 = time.time() - job_queue = RPCTestHandler(jobs, test_list, flags) + + job_queue = RPCTestHandler(jobs, tests_dir, test_list, flags) + + max_len_name = len(max(test_list, key=len)) results = BOLD[1] + "%s | %s | %s\n\n" % ("TEST".ljust(max_len_name), "PASSED", "DURATION") + BOLD[0] - all_passed = True for _ in range(len(test_list)): (name, stdout, stderr, passed, duration) = job_queue.get_next() all_passed = all_passed and passed @@ -233,8 +246,10 @@ def runtests(): print('\n' + BOLD[1] + name + BOLD[0] + ":") print('' if passed else stdout + '\n', end='') print('' if stderr == '' else 'stderr:\n' + stderr + '\n', end='') - results += "%s | %s | %s s\n" % (name.ljust(max_len_name), str(passed).ljust(6), duration) print("Pass: %s%s%s, Duration: %s s\n" % (BOLD[1], passed, BOLD[0], duration)) + + results += "%s | %s | %s s\n" % (name.ljust(max_len_name), str(passed).ljust(6), duration) + results += BOLD[1] + "\n%s | %s | %s s (accumulated)" % ("ALL".ljust(max_len_name), str(all_passed).ljust(6), time_sum) + BOLD[0] print(results) print("\nRuntime: %s s" % (int(time.time() - time0))) @@ -247,15 +262,15 @@ def runtests(): sys.exit(not all_passed) - class RPCTestHandler: """ Trigger the testscrips passed in via the list. """ - def __init__(self, num_tests_parallel, test_list=None, flags=None): + def __init__(self, num_tests_parallel, tests_dir, test_list=None, flags=None): assert(num_tests_parallel >= 1) self.num_jobs = num_tests_parallel + self.tests_dir = tests_dir self.test_list = test_list self.flags = flags self.num_running = 0 @@ -275,7 +290,7 @@ class RPCTestHandler: log_stderr = tempfile.SpooledTemporaryFile(max_size=2**16) self.jobs.append((t, time.time(), - subprocess.Popen((RPC_TESTS_DIR + t).split() + self.flags + port_seed, + subprocess.Popen((self.tests_dir + t).split() + self.flags + port_seed, universal_newlines=True, stdout=log_stdout, stderr=log_stderr), @@ -340,10 +355,10 @@ class RPCCoverage(object): """ # This is shared from `qa/rpc-tests/test-framework/coverage.py` - REFERENCE_FILENAME = 'rpc_interface.txt' - COVERAGE_FILE_PREFIX = 'coverage.' + reference_filename = 'rpc_interface.txt' + coverage_file_prefix = 'coverage.' - coverage_ref_filename = os.path.join(self.dir, REFERENCE_FILENAME) + coverage_ref_filename = os.path.join(self.dir, reference_filename) coverage_filenames = set() all_cmds = set() covered_cmds = set() @@ -356,7 +371,7 @@ class RPCCoverage(object): for root, dirs, files in os.walk(self.dir): for filename in files: - if filename.startswith(COVERAGE_FILE_PREFIX): + if filename.startswith(coverage_file_prefix): coverage_filenames.add(os.path.join(root, filename)) for filename in coverage_filenames: @@ -367,4 +382,4 @@ class RPCCoverage(object): if __name__ == '__main__': - runtests() + main() From a6a3e58c483700bd5f2bc7d117a621e0c4af60e3 Mon Sep 17 00:00:00 2001 From: John Newbery Date: Fri, 17 Feb 2017 14:22:56 -0500 Subject: [PATCH 5/5] Various review markups for rpc-tests.py improvements --- qa/pull-tester/rpc-tests.py | 32 ++++++++++++++++-------------- qa/pull-tester/tests_config.ini.in | 15 ++++---------- 2 files changed, 21 insertions(+), 26 deletions(-) diff --git a/qa/pull-tester/rpc-tests.py b/qa/pull-tester/rpc-tests.py index 1b4c90564..18e08f06c 100755 --- a/qa/pull-tester/rpc-tests.py +++ b/qa/pull-tester/rpc-tests.py @@ -8,6 +8,8 @@ rpc-tests.py - run regression test suite This module calls down into individual test cases via subprocess. It will forward all unrecognized arguments onto the individual test scripts. +RPC tests are disabled on Windows by default. Use --force to run them anyway. + For a description of arguments recognized by test scripts, see `qa/pull-tester/test_framework/test_framework.py:BitcoinTestFramework.main`. @@ -24,8 +26,8 @@ import tempfile import re BASE_SCRIPTS= [ - # Scripts that are run by the travis build process - # longest test should go first, to favor running tests in parallel + # Scripts that are run by the travis build process. + # Longest test should go first, to favor running tests in parallel 'wallet-hd.py', 'walletbackup.py', # vv Tests less than 5m vv @@ -130,11 +132,11 @@ def main(): formatter_class=argparse.RawTextHelpFormatter) parser.add_argument('--coverage', action='store_true', help='generate a basic coverage report for the RPC interface') parser.add_argument('--extended', action='store_true', help='run the extended test suite in addition to the basic tests') + parser.add_argument('--force', '-f', action='store_true', help='run tests even on platforms where they are disabled by default (e.g. windows).') parser.add_argument('--help', '-h', '-?', action='store_true', help='print help text and exit') - parser.add_argument('--nozmq', action='store_true', help='do not run the zmq tests') parser.add_argument('--jobs', '-j', type=int, default=4, help='how many test scripts to run in parallel. Default=4.') - parser.add_argument('--win', action='store_true', help='signal that this is running in a Windows environment and that we should run the tests') - (args, unknown_args) = parser.parse_known_args() + parser.add_argument('--nozmq', action='store_true', help='do not run the zmq tests') + args, unknown_args = parser.parse_known_args() # Create a set to store arguments and create the passon string tests = set(arg for arg in unknown_args if arg[:2] != "--") @@ -144,15 +146,15 @@ def main(): config = configparser.ConfigParser() config.read_file(open(os.path.dirname(__file__) + "/tests_config.ini")) - enable_wallet = config["components"]["ENABLE_WALLET"] == "True" - enable_utils = config["components"]["ENABLE_UTILS"] == "True" - enable_bitcoind = config["components"]["ENABLE_BITCOIND"] == "True" - enable_zmq = config["components"]["ENABLE_ZMQ"] == "True" and not args.nozmq + enable_wallet = config["components"].getboolean("ENABLE_WALLET") + enable_utils = config["components"].getboolean("ENABLE_UTILS") + enable_bitcoind = config["components"].getboolean("ENABLE_BITCOIND") + enable_zmq = config["components"].getboolean("ENABLE_ZMQ") and not args.nozmq - if config["environment"]["EXEEXT"] == ".exe" and not args.win: + if config["environment"]["EXEEXT"] == ".exe" and not args.force: # https://github.com/bitcoin/bitcoin/commit/d52802551752140cf41f0d9a225a43e84404d3e9 # https://github.com/bitcoin/bitcoin/pull/5677#issuecomment-136646964 - print("Win tests currently disabled by default. Use --win option to enable") + print("Tests currently disabled on Windows by default. Use --force option to enable") sys.exit(0) if not (enable_wallet and enable_utils and enable_bitcoind): @@ -170,12 +172,12 @@ def main(): raise # Build list of tests - if len(tests) != 0: + if tests: # Individual tests have been specified. Run specified tests that exist # in the ALL_SCRIPTS list. Accept the name with or without .py extension. test_list = [t for t in ALL_SCRIPTS if (t in tests or re.sub(".py$", "", t) in tests)] - if len(test_list) == 0: + if not test_list: print("No valid test scripts specified. Check that your test is in one " "of the test lists in rpc-tests.py or run rpc-tests.py with no arguments to run all tests") print("Scripts not found:") @@ -200,9 +202,9 @@ def main(): subprocess.check_call((config["environment"]["SRCDIR"] + '/qa/rpc-tests/' + test_list[0]).split() + ['-h']) sys.exit(0) - runtests(test_list, config["environment"]["SRCDIR"], config["environment"]["BUILDDIR"], config["environment"]["EXEEXT"], args.jobs, args.coverage, passon_args) + run_tests(test_list, config["environment"]["SRCDIR"], config["environment"]["BUILDDIR"], config["environment"]["EXEEXT"], args.jobs, args.coverage, passon_args) -def runtests(test_list, src_dir, build_dir, exeext, jobs=1, enable_coverage=False, args=[]): +def run_tests(test_list, src_dir, build_dir, exeext, jobs=1, enable_coverage=False, args=[]): BOLD = ("","") if os.name == 'posix': # primitive formatting on supported diff --git a/qa/pull-tester/tests_config.ini.in b/qa/pull-tester/tests_config.ini.in index 8317caaeb..e3e457d0b 100644 --- a/qa/pull-tester/tests_config.ini.in +++ b/qa/pull-tester/tests_config.ini.in @@ -5,13 +5,6 @@ # These environment variables are set by the build process and read by # rpc-tests.py -[DEFAULT] -# Provides default values for whether different components are enabled -ENABLE_WALLET=False -ENABLE_UTILS=False -ENABLE_BITCOIND=False -ENABLE_ZMQ=False - [environment] SRCDIR=@abs_top_srcdir@ BUILDDIR=@abs_top_builddir@ @@ -19,7 +12,7 @@ EXEEXT=@EXEEXT@ [components] # Which components are enabled. These are commented out by `configure` if they were disabled when running config. -@ENABLE_WALLET_TRUE@ENABLE_WALLET=True -@BUILD_BITCOIN_UTILS_TRUE@ENABLE_UTILS=True -@BUILD_BITCOIND_TRUE@ENABLE_BITCOIND=True -@ENABLE_ZMQ_TRUE@ENABLE_ZMQ=True +@ENABLE_WALLET_TRUE@ENABLE_WALLET=true +@BUILD_BITCOIN_UTILS_TRUE@ENABLE_UTILS=true +@BUILD_BITCOIND_TRUE@ENABLE_BITCOIND=true +@ENABLE_ZMQ_TRUE@ENABLE_ZMQ=true