|
|
@ -1,4 +1,5 @@ |
|
|
|
# Copyright 2014 BitPay, Inc. |
|
|
|
# Copyright 2014 BitPay, Inc. |
|
|
|
|
|
|
|
# Copyright 2016 The Bitcoin Core developers |
|
|
|
# Distributed under the MIT software license, see the accompanying |
|
|
|
# Distributed under the MIT software license, see the accompanying |
|
|
|
# file COPYING or http://www.opensource.org/licenses/mit-license.php. |
|
|
|
# file COPYING or http://www.opensource.org/licenses/mit-license.php. |
|
|
|
from __future__ import division,print_function,unicode_literals |
|
|
|
from __future__ import division,print_function,unicode_literals |
|
|
@ -11,94 +12,110 @@ import difflib |
|
|
|
import logging |
|
|
|
import logging |
|
|
|
|
|
|
|
|
|
|
|
def parse_output(a, fmt): |
|
|
|
def parse_output(a, fmt): |
|
|
|
if fmt == 'json': # json: compare parsed data |
|
|
|
"""Parse the output according to specified format. |
|
|
|
return json.loads(a) |
|
|
|
|
|
|
|
elif fmt == 'hex': # hex: parse and compare binary data |
|
|
|
Raise an error if the output can't be parsed.""" |
|
|
|
return binascii.a2b_hex(a.strip()) |
|
|
|
if fmt == 'json': # json: compare parsed data |
|
|
|
else: |
|
|
|
return json.loads(a) |
|
|
|
raise NotImplementedError("Don't know how to compare %s" % fmt) |
|
|
|
elif fmt == 'hex': # hex: parse and compare binary data |
|
|
|
|
|
|
|
return binascii.a2b_hex(a.strip()) |
|
|
|
|
|
|
|
else: |
|
|
|
|
|
|
|
raise NotImplementedError("Don't know how to compare %s" % fmt) |
|
|
|
|
|
|
|
|
|
|
|
def bctest(testDir, testObj, exeext): |
|
|
|
def bctest(testDir, testObj, exeext): |
|
|
|
|
|
|
|
"""Runs a single test, comparing output and RC to expected output and RC. |
|
|
|
|
|
|
|
|
|
|
|
execprog = testObj['exec'] + exeext |
|
|
|
Raises an error if input can't be read, executable fails, or output/RC |
|
|
|
execargs = testObj['args'] |
|
|
|
are not as expected. Error is caught by bctester() and reported. |
|
|
|
execrun = [execprog] + execargs |
|
|
|
""" |
|
|
|
stdinCfg = None |
|
|
|
# Get the exec names and arguments |
|
|
|
inputData = None |
|
|
|
execprog = testObj['exec'] + exeext |
|
|
|
if "input" in testObj: |
|
|
|
execargs = testObj['args'] |
|
|
|
filename = testDir + "/" + testObj['input'] |
|
|
|
execrun = [execprog] + execargs |
|
|
|
inputData = open(filename).read() |
|
|
|
|
|
|
|
stdinCfg = subprocess.PIPE |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
outputFn = None |
|
|
|
# Read the input data (if there is any) |
|
|
|
outputData = None |
|
|
|
stdinCfg = None |
|
|
|
if "output_cmp" in testObj: |
|
|
|
inputData = None |
|
|
|
outputFn = testObj['output_cmp'] |
|
|
|
if "input" in testObj: |
|
|
|
outputType = os.path.splitext(outputFn)[1][1:] # output type from file extension (determines how to compare) |
|
|
|
filename = testDir + "/" + testObj['input'] |
|
|
|
try: |
|
|
|
inputData = open(filename).read() |
|
|
|
outputData = open(testDir + "/" + outputFn).read() |
|
|
|
stdinCfg = subprocess.PIPE |
|
|
|
except: |
|
|
|
|
|
|
|
logging.error("Output file " + outputFn + " can not be opened") |
|
|
|
|
|
|
|
raise |
|
|
|
|
|
|
|
if not outputData: |
|
|
|
|
|
|
|
logging.error("Output data missing for " + outputFn) |
|
|
|
|
|
|
|
raise Exception |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
proc = subprocess.Popen(execrun, stdin=stdinCfg, stdout=subprocess.PIPE, stderr=subprocess.PIPE,universal_newlines=True) |
|
|
|
# Read the expected output data (if there is any) |
|
|
|
try: |
|
|
|
outputFn = None |
|
|
|
outs = proc.communicate(input=inputData) |
|
|
|
outputData = None |
|
|
|
except OSError: |
|
|
|
if "output_cmp" in testObj: |
|
|
|
logging.error("OSError, Failed to execute " + execprog) |
|
|
|
outputFn = testObj['output_cmp'] |
|
|
|
raise |
|
|
|
outputType = os.path.splitext(outputFn)[1][1:] # output type from file extension (determines how to compare) |
|
|
|
|
|
|
|
try: |
|
|
|
|
|
|
|
outputData = open(testDir + "/" + outputFn).read() |
|
|
|
|
|
|
|
except: |
|
|
|
|
|
|
|
logging.error("Output file " + outputFn + " can not be opened") |
|
|
|
|
|
|
|
raise |
|
|
|
|
|
|
|
if not outputData: |
|
|
|
|
|
|
|
logging.error("Output data missing for " + outputFn) |
|
|
|
|
|
|
|
raise Exception |
|
|
|
|
|
|
|
|
|
|
|
if outputData: |
|
|
|
# Run the test |
|
|
|
try: |
|
|
|
proc = subprocess.Popen(execrun, stdin=stdinCfg, stdout=subprocess.PIPE, stderr=subprocess.PIPE,universal_newlines=True) |
|
|
|
a_parsed = parse_output(outs[0], outputType) |
|
|
|
try: |
|
|
|
except Exception as e: |
|
|
|
outs = proc.communicate(input=inputData) |
|
|
|
logging.error('Error parsing command output as %s: %s' % (outputType,e)) |
|
|
|
except OSError: |
|
|
|
raise |
|
|
|
logging.error("OSError, Failed to execute " + execprog) |
|
|
|
try: |
|
|
|
raise |
|
|
|
b_parsed = parse_output(outputData, outputType) |
|
|
|
|
|
|
|
except Exception as e: |
|
|
|
|
|
|
|
logging.error('Error parsing expected output %s as %s: %s' % (outputFn,outputType,e)) |
|
|
|
|
|
|
|
raise |
|
|
|
|
|
|
|
if a_parsed != b_parsed: |
|
|
|
|
|
|
|
logging.error("Output data mismatch for " + outputFn + " (format " + outputType + ")") |
|
|
|
|
|
|
|
raise Exception |
|
|
|
|
|
|
|
if outs[0] != outputData: |
|
|
|
|
|
|
|
error_message = "Output formatting mismatch for " + outputFn + ":\n" |
|
|
|
|
|
|
|
error_message += "".join(difflib.context_diff(outputData.splitlines(True), |
|
|
|
|
|
|
|
outs[0].splitlines(True), |
|
|
|
|
|
|
|
fromfile=outputFn, |
|
|
|
|
|
|
|
tofile="returned")) |
|
|
|
|
|
|
|
logging.error(error_message) |
|
|
|
|
|
|
|
raise Exception |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
wantRC = 0 |
|
|
|
if outputData: |
|
|
|
if "return_code" in testObj: |
|
|
|
# Parse command output and expected output |
|
|
|
wantRC = testObj['return_code'] |
|
|
|
try: |
|
|
|
if proc.returncode != wantRC: |
|
|
|
a_parsed = parse_output(outs[0], outputType) |
|
|
|
logging.error("Return code mismatch for " + outputFn) |
|
|
|
except Exception as e: |
|
|
|
raise Exception |
|
|
|
logging.error('Error parsing command output as %s: %s' % (outputType,e)) |
|
|
|
|
|
|
|
raise |
|
|
|
|
|
|
|
try: |
|
|
|
|
|
|
|
b_parsed = parse_output(outputData, outputType) |
|
|
|
|
|
|
|
except Exception as e: |
|
|
|
|
|
|
|
logging.error('Error parsing expected output %s as %s: %s' % (outputFn,outputType,e)) |
|
|
|
|
|
|
|
raise |
|
|
|
|
|
|
|
# Compare data |
|
|
|
|
|
|
|
if a_parsed != b_parsed: |
|
|
|
|
|
|
|
logging.error("Output data mismatch for " + outputFn + " (format " + outputType + ")") |
|
|
|
|
|
|
|
raise Exception |
|
|
|
|
|
|
|
# Compare formatting |
|
|
|
|
|
|
|
if outs[0] != outputData: |
|
|
|
|
|
|
|
error_message = "Output formatting mismatch for " + outputFn + ":\n" |
|
|
|
|
|
|
|
error_message += "".join(difflib.context_diff(outputData.splitlines(True), |
|
|
|
|
|
|
|
outs[0].splitlines(True), |
|
|
|
|
|
|
|
fromfile=outputFn, |
|
|
|
|
|
|
|
tofile="returned")) |
|
|
|
|
|
|
|
logging.error(error_message) |
|
|
|
|
|
|
|
raise Exception |
|
|
|
|
|
|
|
|
|
|
|
def bctester(testDir, input_basename, buildenv): |
|
|
|
# Compare the return code to the expected return code |
|
|
|
input_filename = testDir + "/" + input_basename |
|
|
|
wantRC = 0 |
|
|
|
raw_data = open(input_filename).read() |
|
|
|
if "return_code" in testObj: |
|
|
|
input_data = json.loads(raw_data) |
|
|
|
wantRC = testObj['return_code'] |
|
|
|
|
|
|
|
if proc.returncode != wantRC: |
|
|
|
|
|
|
|
logging.error("Return code mismatch for " + outputFn) |
|
|
|
|
|
|
|
raise Exception |
|
|
|
|
|
|
|
|
|
|
|
failed_testcases = [] |
|
|
|
def bctester(testDir, input_basename, buildenv): |
|
|
|
|
|
|
|
""" Loads and parses the input file, runs all tests and reports results""" |
|
|
|
|
|
|
|
input_filename = testDir + "/" + input_basename |
|
|
|
|
|
|
|
raw_data = open(input_filename).read() |
|
|
|
|
|
|
|
input_data = json.loads(raw_data) |
|
|
|
|
|
|
|
|
|
|
|
for testObj in input_data: |
|
|
|
failed_testcases = [] |
|
|
|
try: |
|
|
|
|
|
|
|
bctest(testDir, testObj, buildenv.exeext) |
|
|
|
|
|
|
|
logging.info("PASSED: " + testObj["description"]) |
|
|
|
|
|
|
|
except: |
|
|
|
|
|
|
|
logging.info("FAILED: " + testObj["description"]) |
|
|
|
|
|
|
|
failed_testcases.append(testObj["description"]) |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
if failed_testcases: |
|
|
|
for testObj in input_data: |
|
|
|
logging.error("FAILED TESTCASES: [" + ", ".join(failed_testcases) + "]") |
|
|
|
try: |
|
|
|
sys.exit(1) |
|
|
|
bctest(testDir, testObj, buildenv.exeext) |
|
|
|
else: |
|
|
|
logging.info("PASSED: " + testObj["description"]) |
|
|
|
sys.exit(0) |
|
|
|
except: |
|
|
|
|
|
|
|
logging.info("FAILED: " + testObj["description"]) |
|
|
|
|
|
|
|
failed_testcases.append(testObj["description"]) |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
if failed_testcases: |
|
|
|
|
|
|
|
logging.error("FAILED TESTCASES: [" + ", ".join(failed_testcases) + "]") |
|
|
|
|
|
|
|
sys.exit(1) |
|
|
|
|
|
|
|
else: |
|
|
|
|
|
|
|
sys.exit(0) |
|
|
|