Today, QA is mostly testing “50% of my company employees are testers, Modern Symbolic Execution: and the rest spends 50% of their time testing!” DART, EGT, CUTE, jCUTE, EXE, Bill Gates 1995 KLEE, CREST, CATG Cristian Cadar Koushik Sen Department of Computing EECS Department Imperial College London University of California, Berkeley 1 2 A Familiar Program: QuickSort A Familiar Program: QuickSort void quicksort (int[] a, int lo, int hi) { void quicksort (int[] a, int lo, int hi) { n Test QuickSort int i=lo, j=hi, h; int i=lo, j=hi, h; q int x=a[(lo+hi)/2]; int x=a[(lo+hi)/2]; Create an array q Initialize the elements of // partition // partition the array do { do { q Execute the program on while (a[i]<x) i++; while (a[i]<x) i++; this array while (a[j]>x) j--; if (i<=j) { while (a[j]>x) j--; if (i<=j) { n How much confidence h=a[i]; h=a[i]; a[i]=a[j]; a[i]=a[j]; do I have in this testing a[j]=h; a[j]=h; method? i++; i++; n Is my test suite j--; j--; *Complete*? } } } while (i<=j); } while (i<=j); n Can someone generate a small and *Complete* // recursion // recursion test suite for me? if (lo<j) quicksort(a, lo, j); if (lo<j) quicksort(a, lo, j); if (i<hi) quicksort(a, i, hi); if (i<hi) quicksort(a, i, hi); } } 3 4 1 Automated Test Generation Automated Test Generation n Studied since 70’s n Studied since 70’s q King 76, Myers 79 q King 76, Myers 79 n 30 years have passed, and yet no effective n 30 years have passed, and yet no effective solution solution n What Happened??? n What Happened??? q Program-analysis techniques were expensive q Automated theorem proving and constraint solving techniques were not efficient 5 6 Automated Test Generation Automated Test Generation n Studied since 70’s n Studied since 70’s q King 76, Myers 79 q King 76, Myers 79 n 30 years have passed, and yet no effective solution n 30 Question:years have passed,Can we anduse yet teachniques no effective n What Happened??? solutionfrom axiomatic semantics in q Program-analysis techniques were expensive Automated Test Generation? q Automated theorem proving and constraint solving n What Happened??? techniques were not efficient q Program-analysis techniques were expensive n In the recent years we have seen remarkable q Automated theorem proving and constraint solving progress in static program-analysis and techniques were not efficient constraint solving q SLAM, BLAST, ESP, Bandera, Saturn, MAGIC 7 8 2 Goal n Automated Unit Testing of real-world C and Java Programs q Generate test inputs Symbolic Execution q Execute unit under test on generated test inputs n so that all reachable statements are executed q Any assertion violation gets caught 9 10 Goal Execution Paths of a Program n Automated Unit Testing of real-world C and n Can be seen as a binary Java Programs tree with possibly infinite depth Y q Generate test inputs N q Computation tree q Execute unit under test on generated test inputs n Each node represents the Y n so that all reachable statements are executed execution of a “if then else” N Y N q Any assertion violation gets caught statement n Each edge represents the n Our Approach: N Y execution of a sequence of Y q Explore all execution paths of an Unit for all non-conditional statements possible inputs n Each path in the tree Y n Exploring all execution paths ensure that all reachable represents an equivalence statements are executed class of inputs N Y 11 12 3 Example of Computation Tree Example of Computation Tree int double (int v) { int double (int v) { return 2*v; return 2*v; } } void testMe (int x, int y) { void testMe (int x, int y) { N z==y Y N z==y Y z = double (y); z = double (y); x=0, y=1 if (z == x) { if (z == x) { if (x > y+10) { N x>y+10 Y if (x > y+10) { N x>y+10 Y x=0, y=0 ERROR; ERROR; } } x=22, y=11 } } ERROR ERROR } } 13 14 Computation Tree Computation Tree void testMe1(int x) { void testMe1(int x) { for (int j=0; j < 2; j++) { for (int j=0; j < 2; j++) { j < 2 Y if (x==j) { if (x==j) { printf(“Good\n”); printf(“Good\n”); x==j } } N Y } } } } j < 2 j < 2 How many feasible execution paths do you have in this Y Y program? ☐ 3 x==j x==j ☐ 4 N Y N ☐ 5 j < 2 j < 2 j < 2 ☐ more than 100 15 16 4 Computation Tree Computation Tree Repeated many times void testMe2(int x, unsigned int N) { void testMe2(int x, unsigned int N) { for (int j=0; j < N; j++) { for (int j=0; j < N; j++) { j < 2 if (x==j) { if (x==j) { printf(“Good\n”); printf(“Good\n”); x==j } } } } } } j < 2 j < 2 How many feasible execution paths do you have in this program? ☐ 3 x==j x==j ☐ 4 ☐ 5 j < 2 j < 2 j < 2 ☐ more than 100 17 18 Existing Approach I Random Testing Approach n Random testing testMe(int x){ int double (int v) { q generate random inputs if(x == 94389){ return 2*v; q execute the program on ERROR; } n Random Test Driver: generated inputs q random value for x and y } void testMe (int x, int y) { n Probability of reaching } an error can be z = double (y); astronomically small if (z == x) { n Probability of hitting Probability of reaching ERROR = 1/232 if (x > y+10) { ERROR is extremely low ERROR; } } } 19 20 5 Existing Approach II Concrete Execution n Symbolic Execution int double (int v) { x = 30, y = 15 q use symbolic values for input variables return 2*v; q execute the program } symbolically on symbolic input values void testMe (int x, int y) { q collect symbolic path z = double (y); constraints q use theorem prover to if (z == x) { check if a branch can be taken if (x > y+10) { ERROR; } } } 21 22 Concrete Execution Concrete Execution int double (int v) { x = 30, y = 15 int double (int v) { x = 30, y = 15 return 2*v; return 2*v; } x = 30, y = 15 } x = 30, y = 15 z = 30 z = 30 void testMe (int x, int y) { void testme (int x, int y) { z = double (y); z = double (y); z == x if (z == x) { if (z == x) { true if (x > y+10) { if (x > y+10) { ERROR; ERROR; } } } } } } 24 6 Concrete Execution Concrete Execution int double (int v) { x = 30, y = 15 int double (int v) { x = 30, y = 15 return 2*v; return 2*v; } x = 30, y = 15 } x = 30, y = 15 z = 30 z = 30 void testMe (int x, int y) { void testMe (int x, int y) { z = double (y); z == x z = double (y); z == x if (z == x) { true if (z == x) { true if (x > y+10) { if (x > y+10) { x > y + 10 x > y + 10 ERROR; ERROR; true true } } } } } } ERROR 25 26 Symbolic Execution Symbolic Execution int double (int v) { x = x0, y = y0 int double (int v) { x = x0, y = y0 true return 2*v; return 2*v; } } x = x0, y = y0 z = 2y0 void testMe (int x, int y) { void testMe (int x, int y) { z = double (y); z = double (y); if (z == x) { if (z == x) { if (x > y+10) { if (x > y+10) { ERROR; ERROR; } } } } } } 27 7 Symbolic Execution Symbolic Execution int double (int v) { x = x0, y = y0 int double (int v) { x = x0, y = y0 true true return 2*v; return 2*v; Check path feasibility } x = x0, y = y0 } x = x0, y = y0 using a SMT solver z = 2y0 z = 2y0 void testMe (int x, int y) { void testMe (int x, int y) { z = double (y); z = double (y); 2y0== x0 2y0== x0 2y0== x0 2y0== x0 if (z == x) { 2y0!= x0 if (z == x) { 2y0!= x0 if (x > y+10) { if (x > y+10) { ERROR; ERROR; } } } } } } Symbolic Execution Symbolic Execution int double (int v) { x = x0, y = y0 int double (int v) { x = x0, y = y0 true true return 2*v; return 2*v; Check path feasibility } x = x0, y = y0 } x = x0, y = y0 using a SMT solver z = 2y0 z = 2y0 void testMe (int x, int y) { void testMe (int x, int y) { z = double (y); z = double (y); 2y0== x0 2y0== x0 2y0== x0 2y0== x0 if (z == x) { 2y0!= x0 if (z == x) { 2y0!= x0 if (x > y+10) { if (x > y+10) { x = x0, y = y0 x = x0, y = y0 x0 > y0 + 10 x0 > y0 + 10 z = 2y0 z = 2y0 ERROR; 2y0== x0 && ERROR; 2y0== x0 && 2y == x && 2y == x && } 0 0 x0 > y0+10 } 0 0 x0 > y0+10 x <= y +10 x <= y +10 } 0 0 } 0 0 } } 31 32 8 Symbolic Execution Symbolic Execution int double (int v) { x = x0, y = y0 int double (int v) { x = x0, y = y0 Solve these constraints: true true return 2*v; return 2*v; (a.k.a. path constraints) } x = x0, y = y0 } x = x0, y = y0 to generate inputs for each path z = 2y0 z = 2y0 void testMe (int x, int y) { void testMe (int x, int y) { z = double (y); z = double (y); 2y0== x0 2y0== x0 2y0== x0 2y0== x0 if (z == x) { 2y0!= x0 if (z == x) { 2y0!= x0 if (x > y+10) { if (x > y+10) { x = x0, y = y0 x = x0, y = y0 x0 > y0 + 10 x0 > y0 + 10 z = 2y0 z = 2y0 ERROR; 2y0== x0 && ERROR; 2y0== x0 && 2y == x && 2y == x && } 0 0 x0 > y0+10 } 0 0 x0 > y0+10 x <= y +10 x <= y +10 } 0 0 } 0 0 x = x , y = y x = x , y = y 0 0 ERROR 0 0 ERROR } z = 2y0 } z = 2y0 33 34 Symbolic Execution Existing Approach II n Symbolic Execution testMe(int x){ int double (int v) { x = x0, y = y0 Solve these constraints: q use symbolic values for true if(pickEven(x) == 17) { return 2*v; (a.k.a.