From c8914b9dbbf6106dac3c62769f7ce3bacd8fbf9b Mon Sep 17 00:00:00 2001 From: Andrew Chow Date: Fri, 2 Jun 2017 14:13:08 -0700 Subject: [PATCH 1/3] Have `make cov` optionally include branch coverage statistics Added an option to configure to allow for branch coverage statistics gathering. Disabled logprint macro when coverage testing is on so that unnecessary branches are not analyzed. --- Makefile.am | 32 ++++++++++++++++---------------- configure.ac | 12 ++++++++++++ src/util.h | 12 ++++++++++++ 3 files changed, 40 insertions(+), 16 deletions(-) diff --git a/Makefile.am b/Makefile.am index 40114a551..27d9f5af8 100644 --- a/Makefile.am +++ b/Makefile.am @@ -171,47 +171,47 @@ baseline.info: $(LCOV) -c -i -d $(abs_builddir)/src -o $@ baseline_filtered.info: baseline.info - $(LCOV) -r $< "/usr/include/*" -o $@ + $(LCOV) -r $< "/usr/include/*" $(LCOV_OPTS) -o $@ leveldb_baseline.info: baseline_filtered.info $(LCOV) -c -i -d $(abs_builddir)/src/leveldb -b $(abs_builddir)/src/leveldb -o $@ leveldb_baseline_filtered.info: leveldb_baseline.info - $(LCOV) -r $< "/usr/include/*" -o $@ + $(LCOV) -r $< "/usr/include/*" $(LCOV_OPTS) -o $@ baseline_filtered_combined.info: leveldb_baseline_filtered.info baseline_filtered.info - $(LCOV) -a leveldb_baseline_filtered.info -a baseline_filtered.info -o $@ + $(LCOV) -a $(LCOV_OPTS) leveldb_baseline_filtered.info -a baseline_filtered.info -o $@ test_bitcoin.info: baseline_filtered_combined.info $(MAKE) -C src/ check - $(LCOV) -c -d $(abs_builddir)/src -t test_bitcoin -o $@ - $(LCOV) -z -d $(abs_builddir)/src - $(LCOV) -z -d $(abs_builddir)/src/leveldb + $(LCOV) -c $(LCOV_OPTS) -d $(abs_builddir)/src -t test_bitcoin -o $@ + $(LCOV) -z $(LCOV_OPTS) -d $(abs_builddir)/src + $(LCOV) -z $(LCOV_OPTS) -d $(abs_builddir)/src/leveldb test_bitcoin_filtered.info: test_bitcoin.info - $(LCOV) -r $< "/usr/include/*" -o $@ + $(LCOV) -r $< "/usr/include/*" $(LCOV_OPTS) -o $@ functional_test.info: test_bitcoin_filtered.info - -@TIMEOUT=15 python test/functional/test_runner.py $(EXTENDED_FUNCTIONAL_TESTS) - $(LCOV) -c -d $(abs_builddir)/src --t functional-tests -o $@ - $(LCOV) -z -d $(abs_builddir)/src - $(LCOV) -z -d $(abs_builddir)/src/leveldb + -@TIMEOUT=15 test/functional/test_runner.py $(EXTENDED_FUNCTIONAL_TESTS) + $(LCOV) -c $(LCOV_OPTS) -d $(abs_builddir)/src --t functional-tests -o $@ + $(LCOV) -z $(LCOV_OPTS) -d $(abs_builddir)/src + $(LCOV) -z $(LCOV_OPTS) -d $(abs_builddir)/src/leveldb functional_test_filtered.info: functional_test.info - $(LCOV) -r $< "/usr/include/*" -o $@ + $(LCOV) -r $< "/usr/include/*" $(LCOV_OPTS) -o $@ test_bitcoin_coverage.info: baseline_filtered_combined.info test_bitcoin_filtered.info - $(LCOV) -a baseline_filtered.info -a leveldb_baseline_filtered.info -a test_bitcoin_filtered.info -o $@ + $(LCOV) -a $(LCOV_OPTS) baseline_filtered.info -a leveldb_baseline_filtered.info -a test_bitcoin_filtered.info -o $@ total_coverage.info: baseline_filtered_combined.info test_bitcoin_filtered.info functional_test_filtered.info - $(LCOV) -a baseline_filtered.info -a leveldb_baseline_filtered.info -a test_bitcoin_filtered.info -a functional_test_filtered.info -o $@ | $(GREP) "\%" | $(AWK) '{ print substr($$3,2,50) "/" $$5 }' > coverage_percent.txt + $(LCOV) -a $(LCOV_OPTS) baseline_filtered.info -a leveldb_baseline_filtered.info -a test_bitcoin_filtered.info -a functional_test_filtered.info -o $@ | $(GREP) "\%" | $(AWK) '{ print substr($$3,2,50) "/" $$5 }' > coverage_percent.txt test_bitcoin.coverage/.dirstamp: test_bitcoin_coverage.info - $(GENHTML) -s $< -o $(@D) + $(GENHTML) -s $(LCOV_OPTS) $< -o $(@D) @touch $@ total.coverage/.dirstamp: total_coverage.info - $(GENHTML) -s $< -o $(@D) + $(GENHTML) -s $(LCOV_OPTS) $< -o $(@D) @touch $@ cov: test_bitcoin.coverage/.dirstamp total.coverage/.dirstamp diff --git a/configure.ac b/configure.ac index 160be397b..0f6149ea6 100644 --- a/configure.ac +++ b/configure.ac @@ -158,6 +158,12 @@ AC_ARG_ENABLE([lcov], [enable lcov testing (default is no)])], [use_lcov=yes], [use_lcov=no]) + +AC_ARG_ENABLE([lcov-branch-coverage], + [AS_HELP_STRING([--enable-lcov-branch-coverage], + [enable lcov testing branch coverage (default is no)])], + [use_lcov_branch=yes], + [use_lcov_branch=no]) AC_ARG_ENABLE([glibc-back-compat], [AS_HELP_STRING([--enable-glibc-back-compat], @@ -436,6 +442,12 @@ if test x$use_lcov = xyes; then [AC_MSG_ERROR("lcov testing requested but --coverage linker flag does not work")]) AX_CHECK_COMPILE_FLAG([--coverage],[CXXFLAGS="$CXXFLAGS --coverage"], [AC_MSG_ERROR("lcov testing requested but --coverage flag does not work")]) + AC_DEFINE(USE_COVERAGE, 1, [Define this symbol if coverage is enabled]) + CXXFLAGS="$CXXFLAGS -Og" +fi + +if test x$use_lcov_branch != xno; then + AC_SUBST(LCOV_OPTS, "$LCOV_OPTS --rc lcov_branch_coverage=1") fi dnl Check for endianness diff --git a/src/util.h b/src/util.h index 4386ddd55..a1c59bbd1 100644 --- a/src/util.h +++ b/src/util.h @@ -123,6 +123,17 @@ int LogPrintStr(const std::string &str); /** Get format string from VA_ARGS for error reporting */ template std::string FormatStringFromLogArgs(const char *fmt, const Args&... args) { return fmt; } +static inline void MarkUsed() {} +template static inline void MarkUsed(const T& t, const Args&... args) +{ + (void)t; + MarkUsed(args...); +} + +#ifdef USE_COVERAGE +#define LogPrintf(...) do { MarkUsed(__VA_ARGS__); } while(0) +#define LogPrint(category, ...) do { MarkUsed(__VA_ARGS__); } while(0) +#else #define LogPrintf(...) do { \ std::string _log_msg_; /* Unlikely name to avoid shadowing variables */ \ try { \ @@ -139,6 +150,7 @@ template std::string FormatStringFromLogArgs(const char *fmt, LogPrintf(__VA_ARGS__); \ } \ } while(0) +#endif template bool error(const char* fmt, const Args&... args) From 405b86a92aee4f2ddb6710bfe07ff714f2afcfa2 Mon Sep 17 00:00:00 2001 From: Andrew Chow Date: Tue, 6 Jun 2017 17:26:56 -0700 Subject: [PATCH 2/3] Replace lcov -r commands with faster way Instead of using lcov -r (which is extremely slow), first use a python script to perform bulk cleanup of the /usr/include/* coverage. Then use lcov -a to remove the duplicate entries. This has the same effect of lcov -r but runs significantly faster --- Makefile.am | 12 ++++++++---- contrib/filter-lcov.py | 24 ++++++++++++++++++++++++ 2 files changed, 32 insertions(+), 4 deletions(-) create mode 100755 contrib/filter-lcov.py diff --git a/Makefile.am b/Makefile.am index 27d9f5af8..88879b9e4 100644 --- a/Makefile.am +++ b/Makefile.am @@ -171,13 +171,15 @@ baseline.info: $(LCOV) -c -i -d $(abs_builddir)/src -o $@ baseline_filtered.info: baseline.info - $(LCOV) -r $< "/usr/include/*" $(LCOV_OPTS) -o $@ + $(abs_builddir)/contrib/filter-lcov.py "/usr/include/" $< $@ + $(LCOV) -a $@ $(LCOV_OPTS) -o $@ leveldb_baseline.info: baseline_filtered.info $(LCOV) -c -i -d $(abs_builddir)/src/leveldb -b $(abs_builddir)/src/leveldb -o $@ leveldb_baseline_filtered.info: leveldb_baseline.info - $(LCOV) -r $< "/usr/include/*" $(LCOV_OPTS) -o $@ + $(abs_builddir)/contrib/filter-lcov.py "/usr/include/" $< $@ + $(LCOV) -a $@ $(LCOV_OPTS) -o $@ baseline_filtered_combined.info: leveldb_baseline_filtered.info baseline_filtered.info $(LCOV) -a $(LCOV_OPTS) leveldb_baseline_filtered.info -a baseline_filtered.info -o $@ @@ -189,7 +191,8 @@ test_bitcoin.info: baseline_filtered_combined.info $(LCOV) -z $(LCOV_OPTS) -d $(abs_builddir)/src/leveldb test_bitcoin_filtered.info: test_bitcoin.info - $(LCOV) -r $< "/usr/include/*" $(LCOV_OPTS) -o $@ + $(abs_builddir)/contrib/filter-lcov.py "/usr/include/" $< $@ + $(LCOV) -a $@ $(LCOV_OPTS) -o $@ functional_test.info: test_bitcoin_filtered.info -@TIMEOUT=15 test/functional/test_runner.py $(EXTENDED_FUNCTIONAL_TESTS) @@ -198,7 +201,8 @@ functional_test.info: test_bitcoin_filtered.info $(LCOV) -z $(LCOV_OPTS) -d $(abs_builddir)/src/leveldb functional_test_filtered.info: functional_test.info - $(LCOV) -r $< "/usr/include/*" $(LCOV_OPTS) -o $@ + $(abs_builddir)/contrib/filter-lcov.py "/usr/include/" $< $@ + $(LCOV) -a $@ $(LCOV_OPTS) -o $@ test_bitcoin_coverage.info: baseline_filtered_combined.info test_bitcoin_filtered.info $(LCOV) -a $(LCOV_OPTS) baseline_filtered.info -a leveldb_baseline_filtered.info -a test_bitcoin_filtered.info -o $@ diff --git a/contrib/filter-lcov.py b/contrib/filter-lcov.py new file mode 100755 index 000000000..ce2966c43 --- /dev/null +++ b/contrib/filter-lcov.py @@ -0,0 +1,24 @@ +#!/usr/bin/env python3 + +import argparse + +parser = argparse.ArgumentParser(description='Remove the coverage data from a tracefile for all files matching the pattern.') +parser.add_argument('pattern', help='the pattern of files to remove') +parser.add_argument('tracefile', help='the tracefile to remove the coverage data from') +parser.add_argument('outfile', help='filename for the output to be written to') + +args = parser.parse_args() +tracefile = args.tracefile +pattern = args.pattern +outfile = args.outfile + +in_remove = False +with open(tracefile, 'r') as f: + with open(outfile, 'w') as wf: + for line in f: + if line.startswith("SF:") and pattern in line: + in_remove = True + if not in_remove: + wf.write(line) + if line == 'end_of_record\n': + in_remove = False From d5711f4a2d59adc45755b13e3776b9d36e1c55f5 Mon Sep 17 00:00:00 2001 From: Andrew Chow Date: Thu, 8 Jun 2017 18:15:55 -0700 Subject: [PATCH 3/3] Filter subtrees and and benchmarks from coverage report Remove leveldb baseline coverage gathering. Added filter rules to remove all of the subtress (leveldb, secp256k1, ctaes, univalue) and benchmarking from the coverage report. These items are unnecessary as we do not test for any of the subtrees and benchmark coverage is unneeded. --- Makefile.am | 35 ++++++++++++----------------------- contrib/filter-lcov.py | 7 ++++--- 2 files changed, 16 insertions(+), 26 deletions(-) diff --git a/Makefile.am b/Makefile.am index 88879b9e4..8216b7d60 100644 --- a/Makefile.am +++ b/Makefile.am @@ -59,10 +59,10 @@ OSX_PACKAGING = $(OSX_DEPLOY_SCRIPT) $(OSX_FANCY_PLIST) $(OSX_INSTALLER_ICONS) \ $(top_srcdir)/contrib/macdeploy/detached-sig-apply.sh \ $(top_srcdir)/contrib/macdeploy/detached-sig-create.sh -COVERAGE_INFO = baseline_filtered_combined.info baseline.info \ - leveldb_baseline.info test_bitcoin_filtered.info total_coverage.info \ +COVERAGE_INFO = baseline.info \ + test_bitcoin_filtered.info total_coverage.info \ baseline_filtered.info functional_test.info functional_test_filtered.info \ - leveldb_baseline_filtered.info test_bitcoin_coverage.info test_bitcoin.info + test_bitcoin_coverage.info test_bitcoin.info dist-hook: -$(GIT) archive --format=tar HEAD -- src/clientversion.cpp | $(AMTAR) -C $(top_distdir) -xf - @@ -166,49 +166,38 @@ $(BITCOIN_CLI_BIN): FORCE $(MAKE) -C src $(@F) if USE_LCOV +LCOV_FILTER_PATTERN=-p "/usr/include/" -p "src/leveldb/" -p "src/bench/" -p "src/univalue" -p "src/crypto/ctaes" -p "src/secp256k1" baseline.info: $(LCOV) -c -i -d $(abs_builddir)/src -o $@ baseline_filtered.info: baseline.info - $(abs_builddir)/contrib/filter-lcov.py "/usr/include/" $< $@ + $(abs_builddir)/contrib/filter-lcov.py $(LCOV_FILTER_PATTERN) $< $@ $(LCOV) -a $@ $(LCOV_OPTS) -o $@ -leveldb_baseline.info: baseline_filtered.info - $(LCOV) -c -i -d $(abs_builddir)/src/leveldb -b $(abs_builddir)/src/leveldb -o $@ - -leveldb_baseline_filtered.info: leveldb_baseline.info - $(abs_builddir)/contrib/filter-lcov.py "/usr/include/" $< $@ - $(LCOV) -a $@ $(LCOV_OPTS) -o $@ - -baseline_filtered_combined.info: leveldb_baseline_filtered.info baseline_filtered.info - $(LCOV) -a $(LCOV_OPTS) leveldb_baseline_filtered.info -a baseline_filtered.info -o $@ - -test_bitcoin.info: baseline_filtered_combined.info +test_bitcoin.info: baseline_filtered.info $(MAKE) -C src/ check $(LCOV) -c $(LCOV_OPTS) -d $(abs_builddir)/src -t test_bitcoin -o $@ $(LCOV) -z $(LCOV_OPTS) -d $(abs_builddir)/src - $(LCOV) -z $(LCOV_OPTS) -d $(abs_builddir)/src/leveldb test_bitcoin_filtered.info: test_bitcoin.info - $(abs_builddir)/contrib/filter-lcov.py "/usr/include/" $< $@ + $(abs_builddir)/contrib/filter-lcov.py $(LCOV_FILTER_PATTERN) $< $@ $(LCOV) -a $@ $(LCOV_OPTS) -o $@ functional_test.info: test_bitcoin_filtered.info -@TIMEOUT=15 test/functional/test_runner.py $(EXTENDED_FUNCTIONAL_TESTS) $(LCOV) -c $(LCOV_OPTS) -d $(abs_builddir)/src --t functional-tests -o $@ $(LCOV) -z $(LCOV_OPTS) -d $(abs_builddir)/src - $(LCOV) -z $(LCOV_OPTS) -d $(abs_builddir)/src/leveldb functional_test_filtered.info: functional_test.info - $(abs_builddir)/contrib/filter-lcov.py "/usr/include/" $< $@ + $(abs_builddir)/contrib/filter-lcov.py $(LCOV_FILTER_PATTERN) $< $@ $(LCOV) -a $@ $(LCOV_OPTS) -o $@ -test_bitcoin_coverage.info: baseline_filtered_combined.info test_bitcoin_filtered.info - $(LCOV) -a $(LCOV_OPTS) baseline_filtered.info -a leveldb_baseline_filtered.info -a test_bitcoin_filtered.info -o $@ +test_bitcoin_coverage.info: baseline_filtered.info test_bitcoin_filtered.info + $(LCOV) -a $(LCOV_OPTS) baseline_filtered.info -a test_bitcoin_filtered.info -o $@ -total_coverage.info: baseline_filtered_combined.info test_bitcoin_filtered.info functional_test_filtered.info - $(LCOV) -a $(LCOV_OPTS) baseline_filtered.info -a leveldb_baseline_filtered.info -a test_bitcoin_filtered.info -a functional_test_filtered.info -o $@ | $(GREP) "\%" | $(AWK) '{ print substr($$3,2,50) "/" $$5 }' > coverage_percent.txt +total_coverage.info: test_bitcoin_filtered.info functional_test_filtered.info + $(LCOV) -a $(LCOV_OPTS) baseline_filtered.info -a test_bitcoin_filtered.info -a functional_test_filtered.info -o $@ | $(GREP) "\%" | $(AWK) '{ print substr($$3,2,50) "/" $$5 }' > coverage_percent.txt test_bitcoin.coverage/.dirstamp: test_bitcoin_coverage.info $(GENHTML) -s $(LCOV_OPTS) $< -o $(@D) diff --git a/contrib/filter-lcov.py b/contrib/filter-lcov.py index ce2966c43..299377d69 100755 --- a/contrib/filter-lcov.py +++ b/contrib/filter-lcov.py @@ -3,7 +3,7 @@ import argparse parser = argparse.ArgumentParser(description='Remove the coverage data from a tracefile for all files matching the pattern.') -parser.add_argument('pattern', help='the pattern of files to remove') +parser.add_argument('--pattern', '-p', action='append', help='the pattern of files to remove', required=True) parser.add_argument('tracefile', help='the tracefile to remove the coverage data from') parser.add_argument('outfile', help='filename for the output to be written to') @@ -16,8 +16,9 @@ in_remove = False with open(tracefile, 'r') as f: with open(outfile, 'w') as wf: for line in f: - if line.startswith("SF:") and pattern in line: - in_remove = True + for p in pattern: + if line.startswith("SF:") and p in line: + in_remove = True if not in_remove: wf.write(line) if line == 'end_of_record\n':