From ed27bedcb450933d318913eabfad7ca36a473c2a Mon Sep 17 00:00:00 2001 From: Patrick Pelissier Date: Sat, 20 Aug 2022 23:29:02 +0200 Subject: [PATCH] Make coverage perfoms a check that the computed coverage is at least not lower than the reference coverage, ensuring no regression on the coverage of the tests can occur. NOTE: some tests (m-i-shared) perform some dynamic test cases where the exact coverage depend on the exact dynamic behavior of the test. And this behavior is not always reproducible (for exact the calls to m_core_backoff_wait may not occur if the threads are not trying to grab the resource at the exact same time). As such, the reference of such tests is taken from the worst case of execution of the coverage (with the lowest number of lines covered). --- tests/Makefile | 13 +++++++++---- tests/synthesis.ref | 27 +++++++++++++++++++++++++++ 2 files changed, 36 insertions(+), 4 deletions(-) create mode 100644 tests/synthesis.ref diff --git a/tests/Makefile b/tests/Makefile index c60374bf..66ce4277 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -166,8 +166,8 @@ gene-serial: ##################################################################################################### coverage: - $(MAKE) `ls test-*.c|$(SED) 's/\.c/\.gcov/g'` - $(MAKE) synthesis + $(MAKE) `ls test-*.c|$(SED) 's/\.c/\.gcov/g'` CC="cc -std=c11" + $(MAKE) synthesis.synt check-synthesis SYNTHESIS_DATA= M-ALGO test-malgo.c.c test-malgo.synt \ M-ARRAY test-marray.c.c test-marray.synt \ @@ -197,7 +197,7 @@ SYNTHESIS_DATA= M-ALGO test-malgo.c.c test-malgo.synt \ M-VARIANT test-mvariant.c.c test-mvariant.synt \ M-WORKER ../m-worker.h test-mworker.synt -synthesis: +synthesis.synt: @printf "+-------------------------------+-------------------------------+\n" @for i in $(SYNTHESIS_DATA) ; do \ if test "$$j" = "" ; then j=$$i ; continue ; fi; \ @@ -205,9 +205,14 @@ synthesis: printf "|\t%16.16s\t|\t%16.16s\t|\n" "$$j" "`grep -v Creating $$i|grep $$k -F -A1 |tail -1|cut -f2 -d':'`"; \ j=""; \ k=""; \ - done + done > synthesis.synt + @cat synthesis.synt @printf "+-------------------------------+-------------------------------+\n" +# Compare the computed coverage against the reference coverage (synthesis.ref), allowing only increase of coverage +check-synthesis: synthesis.synt + @paste synthesis.ref synthesis.synt |awk '($$2 != $$9) {err=1} { n1=$$4*$$6/100 ; n2=$$11*$$13/100; if (n1 > n2) { printf "ERROR for %s: new coverage (%d) is lower than reference (%d)\n",$$2,n2,n1 ; err=2 } } END {exit err}' + ##################################################################################################### # Code analyzer tools diff --git a/tests/synthesis.ref b/tests/synthesis.ref new file mode 100644 index 00000000..9e2d20a7 --- /dev/null +++ b/tests/synthesis.ref @@ -0,0 +1,27 @@ +| M-ALGO | 100.00% of 1160 | +| M-ARRAY | 98.24% of 908 | +| M-BITSET | 98.31% of 413 | +| M-BBPTREE | 99.60% of 998 | +| M-BUFFER | 98.86% of 877 | +| M-CONCURRENT | 100.00% of 102 | +| M-CORE | 100.00% of 268 | +| M-DEQUE | 99.47% of 943 | +| M-DICT | 93.00% of 1528 | +| M-FUNCOBJ | 100.00% of 108 | +| M-GENINT | 97.40% of 77 | +| M-I-LIST | 98.10% of 474 | +| M-I-SHARED | 100.00% of 112 | +| M-LIST | 99.12% of 1368 | +| M-MEMPOOL | 97.98% of 99 | +| M-MUTEX | 100.00% of 19 | +| M-PRIOQUEUE | 99.17% of 600 | +| M-RBTREE | 99.04% of 937 | +| M-SERIAL-BIN | 70.87% of 254 | +| M-SERIAL-JSON | 100.00% of 494 | +| M-SHARED | 99.07% of 324 | +| M-SNAPSHOT | 99.54% of 439 | +| M-STRING | 99.40% of 831 | +| M-TREE | 98.46% of 1232 | +| M-TUPLE | 100.00% of 186 | +| M-VARIANT | 99.25% of 266 | +| M-WORKER | 98.73% of 79 |