From 535ed9223dcb32bf90ead5b2c95052838b780620 Mon Sep 17 00:00:00 2001 From: Gavin Andresen Date: Thu, 24 Sep 2015 13:13:38 -0400 Subject: [PATCH 1/2] Simple benchmarking framework Benchmarking framework, loosely based on google's micro-benchmarking library (https://github.com/google/benchmark) Wny not use the Google Benchmark framework? Because adding Even More Dependencies isn't worth it. If we get a dozen or three benchmarks and need nanosecond-accurate timings of threaded code then switching to the full-blown Google Benchmark library should be considered. The benchmark framework is hard-coded to run each benchmark for one wall-clock second, and then spits out .csv-format timing information to stdout. It is left as an exercise for later (or maybe never) to add command-line arguments to specify which benchmark(s) to run, how long to run them for, how to format results, etc etc etc. Again, see the Google Benchmark framework for where that might end up. See src/bench/MilliSleep.cpp for a sanity-test benchmark that just benchmarks 'sleep 100 milliseconds.' To compile and run benchmarks: cd src; make bench Sample output: Benchmark,count,min,max,average Sleep100ms,10,0.101854,0.105059,0.103881 --- configure.ac | 6 ++++ src/Makefile.am | 5 +++ src/Makefile.bench.include | 45 ++++++++++++++++++++++++ src/bench/MilliSleep.cpp | 16 +++++++++ src/bench/bench.cpp | 60 ++++++++++++++++++++++++++++++++ src/bench/bench.h | 69 +++++++++++++++++++++++++++++++++++++ src/bench/bench_bitcoin.cpp | 21 +++++++++++ 7 files changed, 222 insertions(+) create mode 100644 src/Makefile.bench.include create mode 100644 src/bench/MilliSleep.cpp create mode 100644 src/bench/bench.cpp create mode 100644 src/bench/bench.h create mode 100644 src/bench/bench_bitcoin.cpp diff --git a/configure.ac b/configure.ac index f0e0a74fe..24ce0f177 100644 --- a/configure.ac +++ b/configure.ac @@ -91,6 +91,11 @@ AC_ARG_ENABLE(tests, [use_tests=$enableval], [use_tests=yes]) +AC_ARG_ENABLE(bench, + AS_HELP_STRING([--enable-bench],[compile benchmarks (default is yes)]), + [use_bench=$enableval], + [use_bench=yes]) + AC_ARG_WITH([comparison-tool], AS_HELP_STRING([--with-comparison-tool],[path to java comparison tool (requires --enable-tests)]), [use_comparison_tool=$withval], @@ -881,6 +886,7 @@ AM_CONDITIONAL([ENABLE_WALLET],[test x$enable_wallet = xyes]) AM_CONDITIONAL([ENABLE_TESTS],[test x$use_tests = xyes]) AM_CONDITIONAL([ENABLE_QT],[test x$bitcoin_enable_qt = xyes]) AM_CONDITIONAL([ENABLE_QT_TESTS],[test x$use_tests$bitcoin_enable_qt_test = xyesyes]) +AM_CONDITIONAL([ENABLE_BENCH],[test x$use_bench = xyes]) AM_CONDITIONAL([USE_QRCODE], [test x$use_qr = xyes]) AM_CONDITIONAL([USE_LCOV],[test x$use_lcov = xyes]) AM_CONDITIONAL([USE_COMPARISON_TOOL],[test x$use_comparison_tool != xno]) diff --git a/src/Makefile.am b/src/Makefile.am index 67e848be3..45346df95 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -61,6 +61,7 @@ endif bin_PROGRAMS = TESTS = +BENCHMARKS = if BUILD_BITCOIND bin_PROGRAMS += bitcoind @@ -445,6 +446,10 @@ if ENABLE_TESTS include Makefile.test.include endif +if ENABLE_BENCH +include Makefile.bench.include +endif + if ENABLE_QT include Makefile.qt.include endif diff --git a/src/Makefile.bench.include b/src/Makefile.bench.include new file mode 100644 index 000000000..126e3a102 --- /dev/null +++ b/src/Makefile.bench.include @@ -0,0 +1,45 @@ +bin_PROGRAMS += bench/bench_bitcoin +BENCH_SRCDIR = bench +BENCH_BINARY = bench/bench_bitcoin$(EXEEXT) + + +bench_bench_bitcoin_SOURCES = \ + bench/bench_bitcoin.cpp \ + bench/bench.cpp \ + bench/bench.h \ + bench/MilliSleep.cpp + +bench_bench_bitcoin_CPPFLAGS = $(BITCOIN_INCLUDES) $(EVENT_CLFAGS) $(EVENT_PTHREADS_CFLAGS) -I$(builddir)/bench/ +bench_bench_bitcoin_LDADD = \ + $(LIBBITCOIN_SERVER) \ + $(LIBBITCOIN_COMMON) \ + $(LIBBITCOIN_UNIVALUE) \ + $(LIBBITCOIN_UTIL) \ + $(LIBBITCOIN_CRYPTO) \ + $(LIBLEVELDB) \ + $(LIBMEMENV) \ + $(LIBSECP256K1) + +if ENABLE_ZMQ +bench_bench_bitcoin_LDADD += $(LIBBITCOIN_ZMQ) $(ZMQ_LIBS) +endif + +if ENABLE_WALLET +bench_bench_bitcoin_LDADD += $(LIBBITCOIN_WALLET) +endif + +bench_bench_bitcoin_LDADD += $(BOOST_LIBS) $(BDB_LIBS) $(SSL_LIBS) $(CRYPTO_LIBS) $(MINIUPNPC_LIBS) $(EVENT_PTHREADS_LIBS) $(EVENT_LIBS) +bench_bench_bitcoin_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) + + +CLEAN_BITCOIN_BENCH = bench/*.gcda bench/*.gcno + +CLEANFILES += $(CLEAN_BITCOIN_BENCH) + +bitcoin_bench: $(BENCH_BINARY) + +bench: $(BENCH_BINARY) FORCE + $(BENCH_BINARY) + +bitcoin_bench_clean : FORCE + rm -f $(CLEAN_BITCOIN_BENCH) $(bench_bench_bitcoin_OBJECTS) $(BENCH_BINARY) diff --git a/src/bench/MilliSleep.cpp b/src/bench/MilliSleep.cpp new file mode 100644 index 000000000..991397a23 --- /dev/null +++ b/src/bench/MilliSleep.cpp @@ -0,0 +1,16 @@ +// Copyright (c) 2015 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include "bench.h" +#include "main.h" +#include "utiltime.h" + +static void Sleep100ms(benchmark::State& state) +{ + while (state.KeepRunning()) { + MilliSleep(100); + } +} + +BENCHMARK(Sleep100ms); diff --git a/src/bench/bench.cpp b/src/bench/bench.cpp new file mode 100644 index 000000000..572080760 --- /dev/null +++ b/src/bench/bench.cpp @@ -0,0 +1,60 @@ +// Copyright (c) 2015 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. +#include "bench.h" +#include +#include + +using namespace benchmark; + +std::map BenchRunner::benchmarks; + +static double gettimedouble(void) { + struct timeval tv; + gettimeofday(&tv, NULL); + return tv.tv_usec * 0.000001 + tv.tv_sec; +} + +BenchRunner::BenchRunner(std::string name, BenchFunction func) +{ + benchmarks.insert(std::make_pair(name, func)); +} + +void +BenchRunner::RunAll(double elapsedTimeForOne) +{ + std::cout << "Benchmark" << "," << "count" << "," << "min" << "," << "max" << "," << "average" << "\n"; + + for (std::map::iterator it = benchmarks.begin(); + it != benchmarks.end(); ++it) { + + State state(it->first, elapsedTimeForOne); + BenchFunction& func = it->second; + func(state); + } +} + +bool State::KeepRunning() +{ + double now = gettimedouble(); + if (count == 0) { + beginTime = now; + } + else { + double elapsedOne = now - lastTime; + if (elapsedOne < minTime) minTime = elapsedOne; + if (elapsedOne > maxTime) maxTime = elapsedOne; + } + lastTime = now; + ++count; + + if (now - beginTime < maxElapsed) return true; // Keep going + + --count; + + // Output results + double average = (now-beginTime)/count; + std::cout << name << "," << count << "," << minTime << "," << maxTime << "," << average << "\n"; + + return false; +} diff --git a/src/bench/bench.h b/src/bench/bench.h new file mode 100644 index 000000000..fee9b8c38 --- /dev/null +++ b/src/bench/bench.h @@ -0,0 +1,69 @@ +// Copyright (c) 2015 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. +#ifndef BITCOIN_BENCH_H +#define BITCOIN_BENCH_H + +// Simple micro-benchmarking framework; API mostly matches a subset of the Google Benchmark +// framework (see https://github.com/google/benchmark) +// Wny not use the Google Benchmark framework? Because adding Yet Another Dependency +// (that uses cmake as its build system and has lots of features we don't need) isn't +// worth it. + +/* + * Usage: + +static void CODE_TO_TIME(benchmark::State& state) +{ + ... do any setup needed... + while (state.KeepRunning()) { + ... do stuff you want to time... + } + ... do any cleanup needed... +} + +BENCHMARK(CODE_TO_TIME); + + */ + + +#include +#include +#include +#include +#include + +namespace benchmark { + + class State { + std::string name; + double maxElapsed; + double beginTime; + double lastTime, minTime, maxTime; + int64_t count; + public: + State(std::string _name, double _maxElapsed) : name(_name), maxElapsed(_maxElapsed), count(0) { + minTime = std::numeric_limits::max(); + maxTime = std::numeric_limits::min(); + } + bool KeepRunning(); + }; + + typedef boost::function BenchFunction; + + class BenchRunner + { + static std::map benchmarks; + + public: + BenchRunner(std::string name, BenchFunction func); + + static void RunAll(double elapsedTimeForOne=1.0); + }; +} + +// BENCHMARK(foo) expands to: benchmark::BenchRunner bench_11foo("foo", foo); +#define BENCHMARK(n) \ + benchmark::BenchRunner BOOST_PP_CAT(bench_, BOOST_PP_CAT(__LINE__, n))(BOOST_PP_STRINGIZE(n), n); + +#endif // BITCOIN_BENCH_H diff --git a/src/bench/bench_bitcoin.cpp b/src/bench/bench_bitcoin.cpp new file mode 100644 index 000000000..db1402216 --- /dev/null +++ b/src/bench/bench_bitcoin.cpp @@ -0,0 +1,21 @@ +// Copyright (c) 2015 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include "bench.h" + +#include "key.h" +#include "main.h" +#include "util.h" + +int +main(int argc, char** argv) +{ + ECC_Start(); + SetupEnvironment(); + fPrintToDebugLog = false; // don't want to write to debug.log file + + benchmark::BenchRunner::RunAll(); + + ECC_Stop(); +} From 7072c544b52774ac5a22835121e8e2747ad61158 Mon Sep 17 00:00:00 2001 From: Gavin Andresen Date: Tue, 29 Sep 2015 17:17:24 -0400 Subject: [PATCH 2/2] Support very-fast-running benchmarks Avoid calling gettimeofday every time through the benchmarking loop, by keeping track of how long each loop takes and doubling the number of iterations done between time checks when they take less than 1/16'th of the total elapsed time. --- src/Makefile.bench.include | 2 +- src/bench/{MilliSleep.cpp => Examples.cpp} | 18 ++++++++++++++++++ src/bench/bench.cpp | 14 +++++++++++--- src/bench/bench.h | 2 ++ 4 files changed, 32 insertions(+), 4 deletions(-) rename src/bench/{MilliSleep.cpp => Examples.cpp} (50%) diff --git a/src/Makefile.bench.include b/src/Makefile.bench.include index 126e3a102..61fe9e287 100644 --- a/src/Makefile.bench.include +++ b/src/Makefile.bench.include @@ -7,7 +7,7 @@ bench_bench_bitcoin_SOURCES = \ bench/bench_bitcoin.cpp \ bench/bench.cpp \ bench/bench.h \ - bench/MilliSleep.cpp + bench/Examples.cpp bench_bench_bitcoin_CPPFLAGS = $(BITCOIN_INCLUDES) $(EVENT_CLFAGS) $(EVENT_PTHREADS_CFLAGS) -I$(builddir)/bench/ bench_bench_bitcoin_LDADD = \ diff --git a/src/bench/MilliSleep.cpp b/src/bench/Examples.cpp similarity index 50% rename from src/bench/MilliSleep.cpp rename to src/bench/Examples.cpp index 991397a23..b6b020a97 100644 --- a/src/bench/MilliSleep.cpp +++ b/src/bench/Examples.cpp @@ -6,6 +6,8 @@ #include "main.h" #include "utiltime.h" +// Sanity test: this should loop ten times, and +// min/max/average should be close to 100ms. static void Sleep100ms(benchmark::State& state) { while (state.KeepRunning()) { @@ -14,3 +16,19 @@ static void Sleep100ms(benchmark::State& state) } BENCHMARK(Sleep100ms); + +// Extremely fast-running benchmark: +#include + +volatile double sum = 0.0; // volatile, global so not optimized away + +static void Trig(benchmark::State& state) +{ + double d = 0.01; + while (state.KeepRunning()) { + sum += sin(d); + d += 0.000001; + } +} + +BENCHMARK(Trig); diff --git a/src/bench/bench.cpp b/src/bench/bench.cpp index 572080760..89c3b0cc2 100644 --- a/src/bench/bench.cpp +++ b/src/bench/bench.cpp @@ -36,14 +36,22 @@ BenchRunner::RunAll(double elapsedTimeForOne) bool State::KeepRunning() { - double now = gettimedouble(); + double now; if (count == 0) { - beginTime = now; + beginTime = now = gettimedouble(); } else { - double elapsedOne = now - lastTime; + // timeCheckCount is used to avoid calling gettime most of the time, + // so benchmarks that run very quickly get consistent results. + if ((count+1)%timeCheckCount != 0) { + ++count; + return true; // keep going + } + now = gettimedouble(); + double elapsedOne = (now - lastTime)/timeCheckCount; if (elapsedOne < minTime) minTime = elapsedOne; if (elapsedOne > maxTime) maxTime = elapsedOne; + if (elapsedOne*timeCheckCount < maxElapsed/16) timeCheckCount *= 2; } lastTime = now; ++count; diff --git a/src/bench/bench.h b/src/bench/bench.h index fee9b8c38..bf591a2be 100644 --- a/src/bench/bench.h +++ b/src/bench/bench.h @@ -41,10 +41,12 @@ namespace benchmark { double beginTime; double lastTime, minTime, maxTime; int64_t count; + int64_t timeCheckCount; public: State(std::string _name, double _maxElapsed) : name(_name), maxElapsed(_maxElapsed), count(0) { minTime = std::numeric_limits::max(); maxTime = std::numeric_limits::min(); + timeCheckCount = 1; } bool KeepRunning(); };