performance test framework updates: allow custom test parameters, better error reporting

This commit is contained in:
koe 2022-12-01 17:03:53 -06:00
parent 9367b432f6
commit 5c505bda29
2 changed files with 81 additions and 30 deletions

View file

@ -101,12 +101,14 @@ int main(int argc, char** argv)
const std::string filter = tools::glob_to_regex(command_line::get_arg(vm, arg_filter)); const std::string filter = tools::glob_to_regex(command_line::get_arg(vm, arg_filter));
const std::string timings_database = command_line::get_arg(vm, arg_timings_database); const std::string timings_database = command_line::get_arg(vm, arg_timings_database);
Params p; Params core_params;
if (!timings_database.empty()) if (!timings_database.empty())
p.td = TimingsDatabase(timings_database); core_params.td = TimingsDatabase(timings_database);
p.verbose = command_line::get_arg(vm, arg_verbose); core_params.verbose = command_line::get_arg(vm, arg_verbose);
p.stats = command_line::get_arg(vm, arg_stats); core_params.stats = command_line::get_arg(vm, arg_stats);
p.loop_multiplier = command_line::get_arg(vm, arg_loop_multiplier); core_params.loop_multiplier = command_line::get_arg(vm, arg_loop_multiplier);
ParamsShuttle p{core_params};
performance_timer timer; performance_timer timer;
timer.start(); timer.start();

View file

@ -31,6 +31,8 @@
#pragma once #pragma once
#include <iostream> #include <iostream>
#include <memory>
#include <type_traits>
#include <stdint.h> #include <stdint.h>
#include <boost/chrono.hpp> #include <boost/chrono.hpp>
@ -41,7 +43,7 @@
#include "common/perf_timer.h" #include "common/perf_timer.h"
#include "common/timings.h" #include "common/timings.h"
class performance_timer class performance_timer final
{ {
public: public:
typedef boost::chrono::high_resolution_clock clock; typedef boost::chrono::high_resolution_clock clock;
@ -67,7 +69,7 @@ private:
clock::time_point m_start; clock::time_point m_start;
}; };
struct Params struct Params final
{ {
TimingsDatabase td; TimingsDatabase td;
bool verbose; bool verbose;
@ -75,45 +77,79 @@ struct Params
unsigned loop_multiplier; unsigned loop_multiplier;
}; };
template <typename T> struct ParamsShuttle
class test_runner {
Params core_params;
ParamsShuttle() = default;
ParamsShuttle(Params &params) : core_params{params}
{}
virtual ~ParamsShuttle() = default; // virtual for non-final type
};
template <typename T, typename ParamsT,
typename std::enable_if<!std::is_same<ParamsT, ParamsShuttle>::value, bool>::type = true>
bool init_test(T &test, ParamsT &params_shuttle)
{
// assume if the params shuttle isn't the base shuttle type, then the test must take the shuttle as an input on init
if (!test.init(params_shuttle))
return false;
return true;
}
template <typename T, typename ParamsT,
typename std::enable_if<std::is_same<ParamsT, ParamsShuttle>::value, bool>::type = true>
bool init_test(T &test, ParamsT &params_shuttle)
{
if (!test.init())
return false;
return true;
}
template <typename T, typename ParamsT>
class test_runner final
{ {
public: public:
test_runner(const Params &params) test_runner(const ParamsT &params_shuttle)
: m_elapsed(0) : m_elapsed(0)
, m_params(params) , m_params_shuttle(params_shuttle)
, m_per_call_timers(T::loop_count * params.loop_multiplier, {true}) , m_core_params(params_shuttle.core_params)
, m_per_call_timers(T::loop_count * params_shuttle.core_params.loop_multiplier, {true})
{ {
} }
bool run() int run()
{ {
static_assert(0 < T::loop_count, "T::loop_count must be greater than 0"); static_assert(0 < T::loop_count, "T::loop_count must be greater than 0");
T test; T test;
if (!test.init()) if (!init_test(test, m_params_shuttle))
return false; return -1;
performance_timer timer; performance_timer timer;
timer.start(); timer.start();
warm_up(); warm_up();
if (m_params.verbose) if (m_core_params.verbose)
std::cout << "Warm up: " << timer.elapsed_ms() << " ms" << std::endl; std::cout << "Warm up: " << timer.elapsed_ms() << " ms" << std::endl;
timer.start(); timer.start();
for (size_t i = 0; i < T::loop_count * m_params.loop_multiplier; ++i) for (size_t i = 0; i < T::loop_count * m_core_params.loop_multiplier; ++i)
{ {
if (m_params.stats) if (m_core_params.stats)
m_per_call_timers[i].resume(); m_per_call_timers[i].resume();
if (!test.test()) if (!test.test())
return false; return i + 1;
if (m_params.stats) if (m_core_params.stats)
m_per_call_timers[i].pause(); m_per_call_timers[i].pause();
} }
m_elapsed = timer.elapsed_ms(); m_elapsed = timer.elapsed_ms();
m_stats.reset(new Stats<tools::PerformanceTimer, uint64_t>(m_per_call_timers)); m_stats.reset(new Stats<tools::PerformanceTimer, uint64_t>(m_per_call_timers));
return true; return 0;
} }
int elapsed_time() const { return m_elapsed; } int elapsed_time() const { return m_elapsed; }
@ -122,7 +158,7 @@ public:
int time_per_call(int scale = 1) const int time_per_call(int scale = 1) const
{ {
static_assert(0 < T::loop_count, "T::loop_count must be greater than 0"); static_assert(0 < T::loop_count, "T::loop_count must be greater than 0");
return m_elapsed * scale / (T::loop_count * m_params.loop_multiplier); return m_elapsed * scale / (T::loop_count * m_core_params.loop_multiplier);
} }
uint64_t get_min() const { return m_stats->get_min(); } uint64_t get_min() const { return m_stats->get_min(); }
@ -156,20 +192,25 @@ private:
private: private:
volatile uint64_t m_warm_up; ///<! This field is intended for preclude compiler optimizations volatile uint64_t m_warm_up; ///<! This field is intended for preclude compiler optimizations
int m_elapsed; int m_elapsed;
Params m_params; Params m_core_params;
ParamsT m_params_shuttle;
std::vector<tools::PerformanceTimer> m_per_call_timers; std::vector<tools::PerformanceTimer> m_per_call_timers;
std::unique_ptr<Stats<tools::PerformanceTimer, uint64_t>> m_stats; std::unique_ptr<Stats<tools::PerformanceTimer, uint64_t>> m_stats;
}; };
template <typename T> template <typename T, typename ParamsT>
void run_test(const std::string &filter, Params &params, const char* test_name) bool run_test(const std::string &filter, ParamsT &params_shuttle, const char* test_name)
{ {
static_assert(std::is_base_of<ParamsShuttle, ParamsT>::value, "Must use a ParamsShuttle.");
Params &params = params_shuttle.core_params;
boost::smatch match; boost::smatch match;
if (!filter.empty() && !boost::regex_match(std::string(test_name), match, boost::regex(filter))) if (!filter.empty() && !boost::regex_match(std::string(test_name), match, boost::regex(filter)))
return; return true;
test_runner<T> runner(params); test_runner<T, ParamsT> runner(params_shuttle);
if (runner.run()) int run_result{runner.run()};
if (run_result == 0)
{ {
if (params.verbose) if (params.verbose)
{ {
@ -227,16 +268,24 @@ void run_test(const std::string &filter, Params &params, const char* test_name)
double pc = fabs(100. * (prev_instance.mean - runner.get_mean()) / prev_instance.mean); double pc = fabs(100. * (prev_instance.mean - runner.get_mean()) / prev_instance.mean);
cmp = ", " + std::to_string(pc) + "% " + (mean > prev_instance.mean ? "slower" : "faster"); cmp = ", " + std::to_string(pc) + "% " + (mean > prev_instance.mean ? "slower" : "faster");
} }
cmp += " -- " + std::to_string(prev_instance.mean); cmp += " -- " + std::to_string(prev_instance.mean);
} }
std::cout << " (min " << mins << " " << unit << ", 90th " << p95s << " " << unit << ", median " << meds << " " << unit << ", std dev " << stddevs << " " << unit << ")" << cmp; std::cout << " (min " << mins << " " << unit << ", 90th " << p95s << " " << unit << ", median " << meds << " " << unit << ", std dev " << stddevs << " " << unit << ")" << cmp;
} }
std::cout << std::endl; std::cout << std::endl;
} }
else if (run_result == -1)
{
std::cout << test_name << " - FAILED ON INIT" << std::endl;
return false;
}
else else
{ {
std::cout << test_name << " - FAILED" << std::endl; std::cout << test_name << " - FAILED ON TEST LOOP " << run_result << std::endl;
return false;
} }
return true;
} }
#define QUOTEME(x) #x #define QUOTEME(x) #x