From 8ecd3f15928a6b4ede86fa19aff5d64d8e8f9637 Mon Sep 17 00:00:00 2001 From: Patrick Pelissier Date: Mon, 23 Dec 2019 23:05:48 +0100 Subject: [PATCH] Add best_within and variance --- bench/common.c | 124 +++++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 115 insertions(+), 9 deletions(-) diff --git a/bench/common.c b/bench/common.c index a68ec957..4858b921 100644 --- a/bench/common.c +++ b/bench/common.c @@ -26,6 +26,9 @@ #include #include "common.h" +// Maximum supported value for repeat argument +#define MAX_REPEAT 10000 + // The global result unsigned long g_result; @@ -46,8 +49,9 @@ void compiler_barrier(void *p) struct parse_opt_s { int test_function; double from, to, step, grow; + double within; unsigned repeat; - bool graph, best, average, quiet; + bool graph, best, best_within, average, quiet; }; static void @@ -62,9 +66,11 @@ parse_config(struct parse_opt_s *opt, int argc, const char *argv[]) opt->grow = 1.1; opt->graph = false; opt->best = false; + opt->best_within = false; opt->average = false; opt->quiet = false; opt->repeat = 1; + opt->within = .05; for(int i = 1; i < argc ; i++) { @@ -88,14 +94,23 @@ parse_config(struct parse_opt_s *opt, int argc, const char *argv[]) } else if (strcmp(argv[i], "--repeat") == 0) { i++; opt->repeat = strtol(argv[i], &end, 10); + opt->repeat = opt->repeat > MAX_REPEAT ? MAX_REPEAT : opt->repeat; } else if (strcmp(argv[i], "--graph") == 0) { opt->graph = true; } else if (strcmp(argv[i], "--best") == 0) { opt->best = true; opt->average = false; + opt->best_within = false; + } else if (strcmp(argv[i], "--best-within") == 0) { + opt->best_within = true; + opt->best = false; + opt->average = false; + i++; + opt->within = strtod(argv[i], &end) / 100.0; } else if (strcmp(argv[i], "--average") == 0) { opt->average = true; opt->best = false; + opt->best_within = false; } else if (strcmp(argv[i], "--quiet") == 0) { opt->quiet = true; } else { @@ -135,10 +150,72 @@ select_config(int func, size_t n, const config_func_t functions[]) exit(-1); } +// Utility function for qsort +static int cmp_double(const void *pa, const void *pb) +{ + const double *a = (const double *) pa, *b = (const double *)pb; + return *a < *b ? -1 : *a > *b; +} + +/* Get the longest sequence of values within 'f'% */ +static int +get_sequence(int i, int n, double tab[], double f) +{ + int j; + for(j = i + 1; j < n; j++) { + if (tab[j] > f * tab[i]) + break; + } + return j-i; +} + +/* Return the minimum of the longest sequence of values with 'f'% */ +static double +get_best_within(int n, double tab[], double f) +{ + qsort(tab, n, sizeof(double), cmp_double); + int best_i = 0; + int best_seq = get_sequence(0, n, tab, f); + for(int i = 1; i < n && (n-i) > best_seq ; i++) { + int seq = get_sequence(i, n, tab, f); + if (seq > best_seq) { + best_i = i; + best_seq = seq; + } + } + return tab[best_i]; +} + +// Integer sqrt function (to avoid depending on libm) +static unsigned integer_sqrt(unsigned n) +{ + // Find greatest shift. + int shift = 2; + unsigned nShifted = n >> shift; + // We check for nShifted being n, since some implementations + // perform shift operations modulo the word size. + while (nShifted != 0 && nShifted != n){ + shift += 2; + nShifted = n >> shift; + } + shift -= 2; + // Find digits of result. + unsigned result = 0; + while (shift >= 0) { + result = result << 1; + unsigned candidateResult = result + 1; + if (candidateResult*candidateResult <= n >> shift) { + result = candidateResult; + } + shift -= 2; + } + return result; +} + void test(const char library[], size_t n, const config_func_t functions[], int argc, const char *argv[]) { - // options + // Init & parse arguments // NUMBER // --from NUMBER --to NUMBER --step NUMBER --grow NUMBER // --graph --best --average --quiet @@ -147,7 +224,9 @@ test(const char library[], size_t n, const config_func_t functions[], int argc, parse_config(&arg, argc, argv); int i = select_config(arg.test_function, n, functions); double from = arg.from == 0 ? functions[i].default_n : arg.from; - double to = arg.from == 0 ? functions[i].default_n : arg.to; + double to = arg.to == 0 ? functions[i].default_n : arg.to; + + // Open plot file if needed FILE *graph_file = NULL; if (arg.graph) { char filename[100]; @@ -159,32 +238,59 @@ test(const char library[], size_t n, const config_func_t functions[], int argc, } fprintf(graph_file, "# plotting %s-%d : %s\n# N T", library, arg.test_function, functions[i].funcname); } + // Do the bench for(double n = from; n <= to ; n = (arg.grow == 0 ? n + arg.step : n * arg.grow)) { double best = (1.0/0.0); double avg = 0.0; + double variance = 0.0; + static double measure[MAX_REPEAT]; + double ba = 0.0; + if (functions[i].init != NULL) functions[i].init(n); + + // Measure the time of the test_function for(unsigned r = 0; r < arg.repeat; r++) { double t0 = test_function(arg.graph|arg.best|arg.average|arg.quiet ? NULL : functions[i].funcname, (size_t) n, functions[i].func); best = t0 < best ? t0 : best; avg += t0; + variance += t0 * t0; + measure[r] = t0; } + + // Closure of the tests if (functions[i].clear != NULL) functions[i].clear(); - avg /= arg.repeat; + + // Finish computing the different times + if (arg.repeat > 1) { + avg /= arg.repeat; + variance = (variance - arg.repeat * avg * avg) / (arg.repeat - 1); + ba = get_best_within(arg.repeat, measure, 1. + arg.within); + } + + // The average value shall be around best_within (and greater) + // otherwise the result doesn't seem reliable. + bool reliable = (ba <= avg && ba * (1.0+arg.within/2) + 0.5 > avg); + + // Print the result if (arg.graph) { - fprintf(graph_file, "%f %f\n", n, arg.average ? avg : best); + fprintf(graph_file, "%f %f\n", n, arg.average ? avg : arg.best ? best : ba); } else if (arg.quiet) { - printf("%lu\n", (unsigned long)(arg.average ? avg : best)); + printf("%lu\n", (unsigned long)(arg.average ? avg : arg.best ? best : ba)); } else if (arg.repeat > 1) { - if (arg.average == false) + if (arg.average == false && arg.best_within == false) printf ("%20.20s time %lu ms for n = %lu *** BEST ***\n", functions[i].funcname, (unsigned long) best, (unsigned long) n); - if (arg.best == false) - printf ("%20.20s time %lu ms for n = %lu *** AVERAGE ***\n", functions[i].funcname, (unsigned long) avg, (unsigned long) n); + if (arg.best == false && arg.best_within == false) + printf ("%20.20s time %lu ms +/- %u ms for n = %lu *** AVERAGE ***\n", functions[i].funcname, (unsigned long) avg, 2*integer_sqrt(variance), (unsigned long) n); + if (arg.best == false && arg.average == false) + printf ("%20.20s time %lu ms for n = %lu *** BEST within %d%% *** %s\n", functions[i].funcname, (unsigned long) ba, (unsigned long) n, (int)(100*arg.within), reliable ? "" : "(unreliable result)"); } } + + // Close the FILE if needed if (arg.graph) { fclose(graph_file); printf("File plot-%s-%d.dat generated.\n"