Software Testing Trying Hard to Make Software Fail Ludovic Court`es Service d'exp´erimentation& d´eveloppement 7 June 2011 1 Introduction 2 Practicalities Running Tests What to Test? When Write Tests? 3 Beyond to find & isolate bugs to make sure bugs get fixed to make sure bugs don't come back Want your software to be studied for its bugs? Why Test? \Faults in Linux: Ten Years Later", Palix et. al, 2010 http://faultlinux.lip6.fr/ to find & isolate bugs to make sure bugs get fixed to make sure bugs don't come back Why Test? Want your software to be studied for its bugs? \Faults in Linux: Ten Years Later", Palix et. al, 2010 http://faultlinux.lip6.fr/ to make sure bugs get fixed to make sure bugs don't come back Want your software to be studied for its bugs? \Faults in Linux: Ten Years Later", Palix et. al, 2010 http://faultlinux.lip6.fr/ Why Test? to find & isolate bugs to make sure bugs don't come back Want your software to be studied for its bugs? \Faults in Linux: Ten Years Later", Palix et. al, 2010 http://faultlinux.lip6.fr/ Why Test? to find & isolate bugs to make sure bugs get fixed Want your software to be studied for its bugs? \Faults in Linux: Ten Years Later", Palix et. al, 2010 http://faultlinux.lip6.fr/ Why Test? to find & isolate bugs to make sure bugs get fixed to make sure bugs don't come back 1 Introduction 2 Practicalities Running Tests What to Test? When Write Tests? 3 Beyond \Determining success or failure of tests must be an automatic process." | B. Meyer, Principle #4 1 Introduction 2 Practicalities Running Tests What to Test? When Write Tests? 3 Beyond #ifdef CAN RUN TEST #else /* Skip this test. */ return 77; #endif $foo.c make checkrecheck FAIL#includeCCLD: bar foo <stdlib.h> =====================CC bar.o 1intCC of main 1 lib.otest () failed ====================={ CCLD bar CCLD baz $PASS viif bar.c(it: fooworks ()) FAILreturn: bar EXIT SUCCESS; $SKIP makeelse: recheck baz check PROGRAMS = $(TESTS) =====================CCLDreturn bar EXIT FAILURE; 1PASS of: 3 bar tests failed TESTS ENVIRONMENT=\ (1===================== test was not run) valgrind -q \ See1 test test-suite.log passed --leak-check=full \ --error-exitcode=1 ===================== } $ make check-html Automake Rules http://gnu.org/software/automake/ configure.ac AM INIT AUTOMAKE([ \ color-tests \ parallel-tests]) Makefile.am TESTS = foo bar baz #ifdef CAN RUN TEST #else /* Skip this test. */ return 77; #endif $foo.c make checkrecheck FAIL#includeCCLD: bar foo <stdlib.h> =====================CC bar.o 1intCC of main 1 lib.otest () failed ====================={ CCLD bar CCLD baz $PASS viif bar.c(it: fooworks ()) FAILreturn: bar EXIT SUCCESS; $SKIP makeelse: recheck baz =====================CCLDreturn bar EXIT FAILURE; 1PASS of: 3 bar tests failed TESTS ENVIRONMENT=\ (1===================== test was not run) valgrind -q \ See1 test test-suite.log passed --leak-check=full \ --error-exitcode=1 ===================== } $ make check-html Automake Rules http://gnu.org/software/automake/ configure.ac AM INIT AUTOMAKE([ \ color-tests \ parallel-tests]) Makefile.am TESTS = foo bar baz check PROGRAMS = $(TESTS) #ifdef CAN RUN TEST #else /* Skip this test. */ return 77; #endif $foo.c make checkrecheck FAIL#includeCCLD: bar foo <stdlib.h> =====================CC bar.o 1intCC of main 1 lib.otest () failed ====================={ CCLD bar CCLD baz $PASS viif bar.c(it: fooworks ()) FAILreturn: bar EXIT SUCCESS; $SKIP makeelse: recheck baz =====================CCLDreturn bar EXIT FAILURE; 1PASS of: 3 bar tests failed (1===================== test was not run) See1 test test-suite.log passed ===================== } $ make check-html Automake Rules http://gnu.org/software/automake/ configure.ac AM INIT AUTOMAKE([ \ color-tests \ parallel-tests]) Makefile.am TESTS = foo bar baz check PROGRAMS = $(TESTS) TESTS ENVIRONMENT=\ valgrind -q \ --leak-check=full \ --error-exitcode=1 $ make checkrecheck FAILCCLD: bar foo =====================CC bar.o 1CC of 1 lib.otest failed =====================CCLD bar #ifdefCCLD bazCAN RUN TEST $PASS vi bar.c: foo FAIL: bar $SKIP make: recheck baz =====================CCLD bar 1PASS#else of: 3 bar tests failed (1=====================/* test Skip was this not test. run) */ See1 testreturn test-suite.log passed 77; =====================#endif $ make check-html Automake Rules http://gnu.org/software/automake/ foo.c configure.ac #include <stdlib.h> AM INIT AUTOMAKE([ \ color-tests \ int main () parallel-tests]) { Makefile.am if (it works ()) TESTS = foo bar baz return EXIT SUCCESS; else check PROGRAMS = $(TESTS) return EXIT FAILURE; TESTS ENVIRONMENT=\ valgrind -q \ --leak-check=full \ --error-exitcode=1 } $ make checkrecheck FAILCCLD: bar foo =====================CC bar.o 1CC of 1 lib.otest failed =====================CCLD bar CCLD baz $PASS vi bar.c: foo FAIL: bar $SKIP make: recheck baz =====================CCLD bar 1PASS of: 3 bar tests failed (1===================== test was not run) See1 test test-suite.log passed ===================== $ make check-html Automake Rules http://gnu.org/software/automake/ foo.c configure.ac #include <stdlib.h> AM INIT AUTOMAKE([ \ color-tests \ int main () parallel-tests]) { #ifdef CAN RUN TEST Makefile.am if (it works ()) TESTS = foo bar baz return EXIT SUCCESS; else check PROGRAMS = $(TESTS) return EXIT FAILURE; #else TESTS ENVIRONMENT=\ /* Skip this test. */ valgrind -q \ return 77; --leak-check=full \ #endif --error-exitcode=1 } $foo.c make recheck FAIL#includeCCLD: bar foo <stdlib.h> =====================CC bar.o 1intCC of main 1 lib.otest () failed ====================={ CCLD bar #ifdefCCLD bazCAN RUN TEST $PASS viif bar.c(it: fooworks ()) FAILreturn: bar EXIT SUCCESS; $SKIP makeelse: recheck baz =====================CCLDreturn bar EXIT FAILURE; 1PASS#else of: 3 bar tests failed (1=====================/* test Skip was this not test. run) */ See1 testreturn test-suite.log passed 77; =====================#endif } $ make check-html Automake Rules http://gnu.org/software/automake/ $ make check configure.ac AM INIT AUTOMAKE([ \ color-tests \ parallel-tests]) Makefile.am TESTS = foo bar baz check PROGRAMS = $(TESTS) TESTS ENVIRONMENT=\ valgrind -q \ --leak-check=full \ --error-exitcode=1 $foo.c make recheck FAIL#include: bar <stdlib.h> ===================== 1int of main 1 test () failed ====================={ #ifdef CAN RUN TEST $ viif bar.c(it works ()) return EXIT SUCCESS; $ makeelse recheck CCLDreturn bar EXIT FAILURE; PASS#else: bar =====================/* Skip this test. */ 1 testreturn passed 77; =====================#endif } $ make check-html Automake Rules http://gnu.org/software/automake/ $ make check configure.ac CCLD foo AM INIT AUTOMAKE([ \ CC bar.o color-tests \ CC lib.o parallel-tests]) CCLD bar CCLD baz Makefile.am PASS: foo TESTS = foo bar baz FAIL: bar SKIP: baz check PROGRAMS = $(TESTS) ===================== 1 of 3 tests failed TESTS ENVIRONMENT=\ (1 test was not run) valgrind -q \ See test-suite.log --leak-check=full \ --error-exitcode=1 ===================== $foo.c make check #includeCCLD foo <stdlib.h> CC bar.o intCC main lib.o () { CCLD bar #ifdefCCLD bazCAN RUN TEST $PASS viif bar.c(it: fooworks ()) FAILreturn: bar EXIT SUCCESS; $SKIP makeelse: recheck baz =====================CCLDreturn bar EXIT FAILURE; 1PASS#else of: 3 bar tests failed (1=====================/* test Skip was this not test. run) */ See1 testreturn test-suite.log passed 77; =====================#endif } $ make check-html Automake Rules http://gnu.org/software/automake/ $ make recheck configure.ac FAIL: bar AM INIT AUTOMAKE([ \ ===================== color-tests \ 1 of 1 test failed parallel-tests]) ===================== Makefile.am TESTS = foo bar baz check PROGRAMS = $(TESTS) TESTS ENVIRONMENT=\ valgrind -q \ --leak-check=full \ --error-exitcode=1 $foo.c make check #includeCCLD foo <stdlib.h> CC bar.o intCC main lib.o () { CCLD bar #ifdefCCLD bazCAN RUN TEST PASSif (it: fooworks ()) FAILreturn: bar EXIT SUCCESS; $SKIP makeelse: recheck baz =====================CCLDreturn bar EXIT FAILURE; 1PASS#else of: 3 bar tests failed (1=====================/* test Skip was this not test. run) */ See1 testreturn test-suite.log passed 77; =====================#endif } $ make check-html Automake Rules http://gnu.org/software/automake/ $ make recheck configure.ac FAIL: bar AM INIT AUTOMAKE([ \ ===================== color-tests \ 1 of 1 test failed parallel-tests]) ===================== Makefile.am $ vi bar.c TESTS = foo bar baz check PROGRAMS = $(TESTS) TESTS ENVIRONMENT=\ valgrind -q \ --leak-check=full \ --error-exitcode=1 $foo.c make check #includeCCLD foo <stdlib.h> CC bar.o intCC main lib.o () { CCLD bar #ifdefCCLD bazCAN RUN TEST PASSif (it: fooworks ()) FAILreturn: bar EXIT SUCCESS; SKIPelse: baz =====================return EXIT FAILURE; 1#else of 3 tests failed (1/* test Skip was this not test. run) */ Seereturn test-suite.log 77; =====================#endif } $ make check-html Automake Rules http://gnu.org/software/automake/ $ make recheck configure.ac FAIL: bar AM INIT AUTOMAKE([ \ ===================== color-tests \ 1 of 1 test failed parallel-tests]) ===================== Makefile.am $ vi bar.c TESTS = foo bar baz $ make recheck check PROGRAMS = $(TESTS) CCLD bar PASS: bar TESTS ENVIRONMENT=\ ===================== valgrind -q \ 1 test passed --leak-check=full \ --error-exitcode=1 ===================== $foo.c make check #includeCCLD foo <stdlib.h> CC bar.o intCC main lib.o () { CCLD bar #ifdefCCLD bazCAN RUN TEST PASSif (it: fooworks ()) FAILreturn: bar EXIT SUCCESS; SKIPelse: baz =====================return EXIT FAILURE; 1#else