Add best_within and variance

bench/common.c

@@ -26,6 +26,9 @@
 #include <string.h>
 #include "common.h"
 
+// Maximum supported value for repeat argument
+#define MAX_REPEAT 10000
+
 // The global result
 unsigned long g_result;
 
@@ -46,8 +49,9 @@ void compiler_barrier(void *p)
 struct parse_opt_s {
   int test_function;
   double from, to, step, grow;
+  double within;
   unsigned repeat;
-  bool graph, best, average, quiet;
+  bool graph, best, best_within, average, quiet;
 };
 
 static void
@@ -62,9 +66,11 @@ parse_config(struct parse_opt_s *opt, int argc, const char *argv[])
   opt->grow = 1.1;
   opt->graph = false;
   opt->best = false;
+  opt->best_within = false;
   opt->average = false;
   opt->quiet = false;
   opt->repeat = 1;
+  opt->within = .05;
   
   for(int i = 1; i < argc ; i++)
     {
@@ -88,14 +94,23 @@ parse_config(struct parse_opt_s *opt, int argc, const char *argv[])
 	} else if (strcmp(argv[i], "--repeat") == 0) {
 	  i++;
 	  opt->repeat = strtol(argv[i], &end, 10);
+	  opt->repeat = opt->repeat > MAX_REPEAT ? MAX_REPEAT : opt->repeat;
 	} else if (strcmp(argv[i], "--graph") == 0) {
 	  opt->graph = true;
 	} else if (strcmp(argv[i], "--best") == 0) {
 	  opt->best = true;
 	  opt->average = false;
+	  opt->best_within = false;
+	} else if (strcmp(argv[i], "--best-within") == 0) {
+	  opt->best_within = true;
+	  opt->best = false;
+	  opt->average = false;
+	  i++;
+	  opt->within = strtod(argv[i], &end) / 100.0;
 	} else if (strcmp(argv[i], "--average") == 0) {
 	  opt->average = true;
 	  opt->best = false;
+	  opt->best_within = false;
 	} else if (strcmp(argv[i], "--quiet") == 0) {
 	  opt->quiet = true;
 	} else {
@@ -135,10 +150,72 @@ select_config(int func, size_t n, const config_func_t functions[])
   exit(-1);
 }
 
+// Utility function for qsort
+static int cmp_double(const void *pa, const void *pb)
+{
+  const double *a = (const double *) pa, *b = (const double *)pb;
+  return *a < *b ? -1 : *a > *b;
+}
+
+/* Get the longest sequence of values within 'f'% */
+static int
+get_sequence(int i, int n, double tab[], double f)
+{
+  int j;
+  for(j = i + 1; j < n; j++) {
+    if (tab[j] > f * tab[i])
+      break;
+  }
+  return j-i;
+}
+
+/* Return the minimum of the longest sequence of values with 'f'% */
+static double
+get_best_within(int n, double tab[], double f)
+{
+  qsort(tab, n, sizeof(double), cmp_double);
+  int best_i   = 0;
+  int best_seq = get_sequence(0, n, tab, f);
+  for(int i = 1; i < n && (n-i) > best_seq ; i++) {
+    int seq = get_sequence(i, n, tab, f);
+    if (seq > best_seq) {
+      best_i = i;
+      best_seq = seq;
+    }
+  }
+  return tab[best_i];
+}
+
+// Integer sqrt function (to avoid depending on libm)
+static unsigned integer_sqrt(unsigned n)
+{
+  // Find greatest shift.
+  int shift = 2;
+  unsigned nShifted = n >> shift;
+  // We check for nShifted being n, since some implementations
+  // perform shift operations modulo the word size.
+  while (nShifted != 0 && nShifted != n){
+    shift += 2;
+    nShifted = n >> shift;
+  }
+  shift -= 2;
+  // Find digits of result.
+  unsigned result = 0;
+  while (shift >= 0) {
+    result = result << 1;
+    unsigned candidateResult = result + 1;
+    if (candidateResult*candidateResult <= n >> shift) {
+      result = candidateResult;
+    }
+    shift -= 2;
+  }
+  return result;
+}
+
 void
 test(const char library[], size_t n, const config_func_t functions[], int argc, const char *argv[])
 {
-  // options
+  // Init & parse arguments
   // NUMBER
   // --from NUMBER --to NUMBER --step NUMBER --grow NUMBER
   // --graph --best --average --quiet
@@ -147,7 +224,9 @@ test(const char library[], size_t n, const config_func_t functions[], int argc,
   parse_config(&arg, argc, argv);
   int i = select_config(arg.test_function, n, functions);
   double from = arg.from == 0 ? functions[i].default_n : arg.from;
-  double to   = arg.from == 0 ? functions[i].default_n : arg.to;
+  double to   = arg.to   == 0 ? functions[i].default_n : arg.to;
+
+  // Open plot file if needed
   FILE *graph_file = NULL;
   if (arg.graph) {
     char filename[100];
@@ -159,32 +238,59 @@ test(const char library[], size_t n, const config_func_t functions[], int argc,
     }
     fprintf(graph_file, "# plotting %s-%d : %s\n# N T", library, arg.test_function, functions[i].funcname);
   }
+  
   // Do the bench
   for(double n = from; n <= to ; n = (arg.grow == 0 ? n + arg.step : n * arg.grow))
     {
       double best = (1.0/0.0);
       double avg  = 0.0;
+      double variance = 0.0;
+      static double measure[MAX_REPEAT];
+      double ba = 0.0;
+      
       if (functions[i].init != NULL)
 	functions[i].init(n);
+
+      // Measure the time of the test_function
       for(unsigned r = 0; r < arg.repeat; r++) {
 	double t0 = test_function(arg.graph|arg.best|arg.average|arg.quiet ? NULL : functions[i].funcname, (size_t) n, functions[i].func);
 	best = t0 < best ? t0 : best;
 	avg += t0;
+	variance += t0 * t0;
+	measure[r] = t0;
       }
+
+      // Closure of the tests
       if (functions[i].clear != NULL)
 	functions[i].clear();
-      avg /= arg.repeat;
+
+      // Finish computing the different times
+      if (arg.repeat > 1) {
+	avg /= arg.repeat;
+	variance = (variance - arg.repeat * avg * avg) / (arg.repeat - 1);
+	ba = get_best_within(arg.repeat, measure, 1. + arg.within);
+      }
+      
+      // The average value shall be around best_within (and greater)
+      // otherwise the result doesn't seem reliable.
+      bool reliable = (ba <= avg && ba * (1.0+arg.within/2) + 0.5 > avg);
+      
+      // Print the result
       if (arg.graph) {
-	fprintf(graph_file, "%f %f\n", n, arg.average ? avg : best);
+	fprintf(graph_file, "%f %f\n", n, arg.average ? avg : arg.best ? best : ba);
       } else if (arg.quiet) {
-	printf("%lu\n", (unsigned long)(arg.average ? avg : best));
+	printf("%lu\n", (unsigned long)(arg.average ? avg : arg.best ? best : ba));
       } else if (arg.repeat > 1) {
-	if (arg.average == false)
+	if (arg.average == false && arg.best_within == false)
 	  printf ("%20.20s time %lu ms for n = %lu ***   BEST  ***\n", functions[i].funcname, (unsigned long) best, (unsigned long) n);
-	if (arg.best == false)
+	if (arg.best == false  && arg.best_within == false)
-	  printf ("%20.20s time %lu ms for n = %lu *** AVERAGE ***\n", functions[i].funcname, (unsigned long) avg, (unsigned long) n);
+	  printf ("%20.20s time %lu ms +/- %u ms for n = %lu *** AVERAGE ***\n", functions[i].funcname, (unsigned long) avg, 2*integer_sqrt(variance), (unsigned long) n);
+	if (arg.best == false  && arg.average == false)
+	  printf ("%20.20s time %lu ms for n = %lu *** BEST within %d%% *** %s\n", functions[i].funcname, (unsigned long) ba, (unsigned long) n, (int)(100*arg.within), reliable ? "" : "(unreliable result)");
       }
     }
+  
+  // Close the FILE if needed
   if (arg.graph) {
     fclose(graph_file);
     printf("File plot-%s-%d.dat generated.\n"