Project

General

Profile

Statistics
| Branch: | Tag: | Revision:

lustrec / scripts / gen_test_suite.sh @ 6423979f

History | View | Annotate | Download (12 KB)

1
eval set -- $(getopt -n $0 -o "-ahstbrmkcnv:" -- "$@")
2

    
3
declare h s b m k n a v t c r
4
declare -a files
5

    
6
node="top"
7
inputxsl="/home/ploc/Repositories/svn/cavale.enseeiht.fr/lustrec/lustre_compiler/branches/mutation/scripts/input.xsl"
8
outputxsl="/home/ploc/Repositories/svn/cavale.enseeiht.fr/lustrec/lustre_compiler/branches/mutation/scripts/output.xsl"
9
inputcsv_xsl="/home/ploc/Repositories/svn/cavale.enseeiht.fr/lustrec/lustre_compiler/branches/mutation/scripts/input_cvs.xsl"
10
xsl_inter_to_pkind="/home/ploc/Repositories/svn/cavale.enseeiht.fr/lustrec/lustre_compiler/branches/mutation/scripts/input-kind-inter.xsl"
11

    
12
xmlcombinepy="/home/ploc/Repositories/svn/cavale.enseeiht.fr/lustrec/lustre_compiler/branches/mutation/scripts/xmlcombine.py"
13

    
14
mkdir -p tmp
15
mkdir -p summaries
16

    
17

    
18
base_suite () {
19
  mkdir -p ${conds_dir}
20
  mkdir -p ${suite_dir}
21
  echo Generating base test suite: $suite_file
22
  rm -f ${suite_file}.no_mcdc tmp/tmp_${name}.res
23
  centralized_file=${conds_dir}/${name}.ec
24
  lus2ec $file $node -o $centralized_file 
25
  timeout 10m lustrem -verbose 0 -mcdc_cond -node $node $centralized_file > $cond_file
26
  while read cond; do
27
    echo .. $cond 
28
    cat $centralized_file | grep -v "^--" | xargs | sed "s/tel/tel\n/g" | grep -v "node ${node}" | sed "s/; /;\n/g" > tmp/tmp_${name}
29
    cat $centralized_file | grep -v "^--" | xargs | sed "s/tel/tel\n/g" | grep "node ${node}" | sed "s/tel/--!PROPERTY: not ($cond); tel/;s/; /;\n/g" >> tmp/tmp_${name}
30
    timeout $timeout pkind -tg tmp/tmp_${name} | grep -v local > tmp/tmp_${name}_last.res 
31
    [ ${PIPESTATUS[1]} -ne 0 ] && echo $cond >> ${suite_file}.no_mcdc
32

    
33
    # Careful, we may produce duplicate
34
    if [ -s tmp/tmp_${name}.res ]; then
35
	if [ -s tmp/tmp_${name}_last.res ]; then
36
	    python ${xmlcombinepy} tmp/tmp_${name}.res tmp/tmp_${name}_last.res > tmp/tmp_${name}.res2
37
	    mv tmp/tmp_${name}.res2 tmp/tmp_${name}.res
38
	    rm tmp/tmp_${name}_last.res
39
	fi
40
    else
41
	mv tmp/tmp_${name}_last.res tmp/tmp_${name}.res
42
    fi
43
#xsltproc ${inputxsl} - 2> /dev/null| grep -v xml | tee -a tmp/tmp_${name}.res 
44
  done < "$cond_file"
45
  # Remove duplicates
46
  cat tmp/tmp_${name}.res | grep -v Results | tr -d '\n' | sed "s/<\/testTrace>/<\/testTrace>\n/g" | sort -u > $suite_file
47
#  sort -u tmp/tmp_${name}.res > $suite_file
48
#  mv tmp/tmp_${name}.res $suite_file
49
  #rm tmp.res tmp 
50
}
51

    
52
gen_ref_bin () {
53
  echo Generating reference binary
54
  mkdir -p ${ref_bin_dir}
55
  lustrec -d ${ref_bin_dir} -node $node -verbose 0 $file
56
  make -C ${ref_bin_dir} -f $name.makefile > /dev/null
57
}
58
    
59
gen_mutants () {
60
  echo Generating mutants
61
  mkdir -p ${mutants_dir}/${name}
62
  timeout 10m lustrem -nb 200 -d ${mutants_dir}/${name} -verbose $verbose $file
63
}
64

    
65
compile_mutants () {
66
  pushd ${mutants_dir}/${name}
67
  rm -f ${name}.mutants_compile
68
  for mutant in ${name}.mutant*.lus; do
69
      echo "echo Compiling mutant ${mutant} && lustrec -node ${node}_mutant -verbose 0 $mutant && make -f `basename $mutant .lus`.makefile > /dev/null" >> ${name}.mutants_compile
70
  done
71
  parallel < ${name}.mutants_compile
72
  popd
73
}
74
    
75
# echo Running test
76
# while read test; do
77
#   echo .. test $test >&2 
78
#   echo $test | sed "s/ /\n/g" > test_tmp
79
#   ./$binary < test_tmp >> oracle
80
# done < $suite_file
81
    
82
run_mutants () {
83
  echo Trying to kill mutants: running test on mutants $name
84
  echo "Summary for file ${file}" > $summary
85
  echo "# tests: `wc -l ${suite_file}`" >>$summary 
86
  all_killed=1
87
# rm -f log_tmp; touch log_tmp
88
  rm -f tmp/log_tmp_${name}
89
  for mutant in mutants/${name}/${name}.mutant.*_${node}_mutant; do
90
    killed=0
91
    while read test; do
92
      test=`echo $test | xsltproc  ${inputxsl} - | grep -v xml`
93
      echo $test | sed "s/ /\n/g" > tmp/test_tmp_${name}
94
      $binary < tmp/test_tmp_${name} > tmp/oracle_tmp_${name}
95
      $mutant < tmp/test_tmp_${name} > tmp/run_tmp_${name}
96
      diff tmp/oracle_tmp_${name} tmp/run_tmp_${name} > /dev/null
97
      [ $? -eq 1 ] && killed=1
98
      [ $killed -eq 1 ] && echo $mutant killed by test $test >> tmp/log_tmp_${name} 
99
      [ $killed -eq 1 ] && [ $verbose -ge 2 ] && echo $mutant killed by test $test  
100
      [ $killed -eq 1 ] && break
101
    done < "$suite_file"
102
    [ $killed -eq 0 ] && all_killed=0 && echo "$mutant unkilled" >> tmp/log_tmp_${name}
103
    [ $killed -eq 0 ] && all_killed=0 && [ $verbose -ge 2 ] && echo "$mutant unkilled"
104
  done
105
  [ $all_killed -eq 1 ] && echo "All mutants killed" | tee -a $summary && echo "" >> $summary && cat tmp/log_tmp_${name} >> $summary
106
  [ $all_killed -eq 0 ] && echo "`cat tmp/log_tmp_${name} | grep unkilled | wc -l` mutant(s) unkilled" >> $summary && echo "" >> $summary && cat tmp/log_tmp_${name} >> $summary
107
# rm -f tmp/test_tmp_${name} tmp/oracle_tmp_${name} tmp/run_tmp_${name} tmp/log_tmp_${name}
108
}
109

    
110

    
111
kill_mutants () {
112
# Bulding the flat version of file
113
     lus2ec $file top -o ${file_flat}
114

    
115

    
116
  rm -f tmp/tmp_${name}_new.res ${summary}_new
117
  unkilleds=`grep ${name} $summary | grep unkilled | sed "s/ .*//"`
118
  echo unkilleds: $unkilleds
119
  rm -f tmp/tmp_${name}_new.res tmp/tmp_${name}_last_new.res
120
  touch ${summary}_new tmp/tmp_${name}_new.res
121
  
122
  inputs=`cat $file | grep -v "^[' ']*--" | xargs | sed "s/.*node ${node}[' ']*(\([^')']*\))[' ']*returns[' ']*(\([^')']*\))[' ']*;.*/\1/"`
123
  outputs=`cat $file | grep -v "^[' ']*--" | xargs | sed "s/.*node ${node}[' ']*(\([^')']*\))[' ']*returns[' ']*([' ']*\([^')']*\))[' ']*;.*/\2/;s/ //g"`
124
  outputs_mutant=`echo $outputs | sed "s/ /_mutant /;s/\([a-zA-Z]\):/\1_mutant:/"`
125
  echo outputs: /$outputs/ outputs mutant: /$outputs_mutant/
126
  top_call=`echo $outputs | sed "s/:[^';']*;/,/g;s/:.*//"`
127
  mutant_call=`echo $outputs_mutant | sed "s/:[^';']*;/,/g;s/:.*//"`
128
  input_args=`echo $inputs | sed "s/:[^';']*;/,/g;s/:.*//"`
129
  for unkilled in $unkilleds; do
130
    mutant=mutants/${name}/`basename $unkilled _${node}_mutant`.lus
131
    echo "Trying to create a new test to kill $mutant"
132
    kill_main=mutants/${name}/`basename $unkilled _${node}_mutant`.kill_lus
133
    cat $file > ${kill_main}
134
    echo " " >> ${kill_main}
135
    cat $mutant >> ${kill_main}
136
    echo "node kill_main ($inputs) returns (kill_the_mutant: bool);" >> ${kill_main}
137
    echo "var $outputs; $outputs_mutant;" >>  ${kill_main}
138
    echo "let" >>  ${kill_main}
139
    echo "  $top_call = ${node} ($input_args);" >>  ${kill_main}
140
    echo "  $mutant_call = ${node}_mutant ($input_args);" >>  ${kill_main}
141

    
142
    IFS="," 
143
    tmp_uid=v$(echo `date +%N`)
144
    tmp_vars=tmp/$tmp_uid
145
    touch ${tmp_vars} 
146
    cat $file | grep -v "^[' ']*--" | xargs | sed "s/.*node ${node}[' ']*(\([^')']*\))[' ']*returns[' ']*(\([^')']*\))[' ']*;.*/\1/" | sed "s/:[^';']*;/\n/g;s/:.*/\n/;s/,/\n/g" | xargs | sed "s/ /\n/g" > tmp/inputs
147
    while read i; do
148
    	echo "($i=$i)" >> ${tmp_vars}; 
149
    done < tmp/inputs
150
    tautology=`cat $tmp_vars | xargs | sed "s/) (/) and (/g"`
151
    rm ${tmp_vars}
152
    echo "  kill_the_mutant = ($top_call = $mutant_call) and ${tautology};" >>  ${kill_main}
153
    echo "tel" >>  ${kill_main}
154
    lus2ec ${kill_main} kill_main -o ${kill_main}.flat.ec  | grep -v Pollux
155
    cat ${kill_main}.flat.ec | xargs | sed "s/tel/--!PROPERTY: kill_the_mutant; tel/;s/;/;\n/g" > ${kill_main}.flat_lus
156
    test_ok=1
157
    prove_ok=1
158
    # echo "timeout $timeout pkind -tg ${kill_main}.flat_lus | grep -v local | grep -v output > tmp/tmp_${name}_last_input_new.res"
159
    timeout $timeout pkind -tg ${kill_main}.flat_lus | grep -v local | grep -v output > tmp/tmp_${name}_last_input_new.res
160
    [ ${PIPESTATUS[1]} -eq 0 ] && test_ok=0 
161
    if [ -s tmp/tmp_${name}_last_input_new.res ]; then
162
	
163
	# We call kind-inter to rebuild the expected output
164
	xsltproc ${inputcsv_xsl} tmp/tmp_${name}_last_input_new.res| grep -v xml | sed "s/ //g" | grep -v "^$" > tmp/input_test
165
	timeout $timeout kind-inter -xml -f tmp/input_test -xml ${file_flat} > tmp/tmp_${name}_last_new.res
166
	# cat tmp/${file_flat}.xml
167

    
168
	new_tests=""
169
        # Careful, we may produce duplicate
170
	if [ -s tmp/tmp_${name}_new.res ]; then
171
	    if [ -s tmp/tmp_${name}_last_new.res ]; then
172
		new_test=`cat tmp/tmp_${name}_last_new.res | xsltproc  ${inputxsl} - | grep -v xml`
173
		python ${xmlcombinepy} tmp/tmp_${name}_new.res tmp/tmp_${name}_last_new.res > tmp/tmp_${name}_new.res2
174
		mv tmp/tmp_${name}_new.res2 tmp/tmp_${name}_new.res
175
		rm tmp/tmp_${name}_last_new.res
176
	    fi
177
	else
178
	    mv tmp/tmp_${name}_last_new.res tmp/tmp_${name}_new.res
179
	fi
180
    fi
181
    if [ $verbose -ge 2 ]; then
182
    	[ $test_ok -eq 0 ]  && echo "$mutant killed by new test ${new_test}" | tee -a ${summary}_new	
183
    	[ $test_ok -ne 0 ] && echo "Unable to create a test to kill mutant $mutant" | tee -a ${summary}_new
184
    else
185
    	[ $test_ok -eq 0 ]  && echo "$mutant killed by new test ${new_test}"  >> ${summary}_new
186
    	[ $test_ok -ne 0 ] && echo "Unable to create a test to kill mutant: $mutant" >> ${summary}_new
187
    fi
188
    [ $test_ok -ne 0 ] && timeout $timeout pkind ${kill_main}.flat_lus 2> /dev/null | grep -q Valid \
189
    	&&  [ ${PIPESTATUS[1]} -eq 0 ] && prove_ok=0
190
    [ $test_ok -ne 0 ] && [ $prove_ok -eq 0 ] && { \
191
    	if [ $verbose -ge 2 ]; then \
192
    	echo "Unobservable mutation: beware of dead code" | tee -a ${summary}_new; \
193
    	else \
194
    	echo "Unobservable mutation: beware of dead code" >> ${summary}_new; \
195
    	fi \
196
    	}
197
    [ $test_ok -ne 0 ] && [ $prove_ok -ne 0 ] && { \
198
     	if [ $verbose -ge 2 ]; then \
199
     	echo "Inconclusive analysis" | tee -a ${summary}_new; \
200
     	else \
201
     	echo "Inconclusive analysis">> ${summary}_new; \
202
     	fi \
203
     	}
204
  #rm tmp.res tmp                                             
205

    
206
  done
207
  cat tmp/tmp_${name}_new.res | grep -v Results | tr -d '\n' | sed "s/<\/testTrace>/<\/testTrace>\n/g" | sort -u > ${suite_file}_new
208
}
209

    
210
run_tests () {
211
    for suitef in ${suite_file} ${suite_file}_new ; do 
212
	echo $suitef
213
	while read test; do
214
	    inputs=`echo $test | xsltproc ${inputxsl} - | grep -v xml`; 
215
	    echo $inputs | sed "s/ /\n/g" > tmp/test_tmp_${name}
216
	    echo "" >> tmp/test_tmp_${name}
217
	    $binary < tmp/test_tmp_${name} | sed "s/^.*: '\([^']*\)'/\1/g" | xargs > tmp/result_tmp_${name}
218
	    echo $test | xsltproc ${outputxsl} - | grep -v xml | xargs > tmp/oracle_tmp_${name}
219
	    diff tmp/oracle_tmp_${name} tmp/result_tmp_${name} > /dev/null
220
	    [ $? -eq 1 ] && echo "Failure:" && echo "inputs: $inputs" && echo "outputs: \"`cat tmp/result_tmp_${name}`\"" && echo "expected outputs: \"`cat tmp/oracle_tmp_${name}`\"" && echo $test && exit 1
221
	done < "$suitef"
222
    done
223
}
224

    
225
usage () {
226
  echo "usage: $0 [-hsbmka] [-n nodename] lustre_file"
227
  echo "-n node: by default node is main"
228
  echo "-a: perform all steps"
229
  echo "-s: generate base test suite"
230
  echo "-b: generate and compile main binary"
231
  echo "-m: generate mutants"
232
  echo "-c: compile mutants"
233
  echo "-k: try to kill mutants"
234
  echo "-t: create new tests to kill unkilled mutants"
235
  echo "-r: run the tests on the binaries"
236
  echo "-v: verbose level (default 1)"
237
}
238

    
239
nobehavior=1
240
verbose=1
241

    
242
while [ $# -gt 0 ] ; do
243
  case "$1" in
244
      -v) shift ; verbose="$1"; shift ;;
245
      -a) nobehavior=0; s=1 ; b=1; m=1; c=1; k=1; t=1; r=1; shift ;;
246
      -h) nobehavior=0; h=1 ; shift ;;
247
      -t) nobehavior=0; t=1 ; shift ;;
248
      -s) nobehavior=0; s=1 ; shift ;;
249
      -b) nobehavior=0; b=1 ; shift ;;
250
      -m) nobehavior=0; m=1 ; shift ;;
251
      -c) nobehavior=0; c=1 ; shift ;;
252
      -k) nobehavior=0; k=1 ; shift ;;
253
      -r) nobehavior=0; r=1 ; shift ;;
254
      -n) shift ; node="$1" ; shift ;;
255
      --) shift ;;
256
      -*) echo "bad option '$1'" ; exit 1 ;;
257
      *) files=("${files[@]}" "$1") ; shift ;;
258
  esac
259
done
260

    
261
file=${files[0]}
262
file_flat=${file}_top_flatten.lus
263
ref_bin_dir="./ref_bin"
264
mutants_dir="./mutants"
265
conds_dir="./conds"
266
suite_dir="./tests"
267
name=`basename $file .lus`
268
binary=${ref_bin_dir}/${name}_${node}
269
cond_file=${conds_dir}/$name.conds
270
suite_file=${suite_dir}/$name.test_suite
271
summary=summaries/$name.summary
272
timeout=2
273

    
274
if [ ${#files} -eq 0 ] ; then
275
  echo input lustre file required
276
  exit 1
277
fi
278

    
279
[ ! -z "$h" ] && usage && exit 0
280
[ ! -z "$s" ] && base_suite 
281
[ ! -z "$b" ] && gen_ref_bin
282
[ ! -z "$m" ] && gen_mutants
283
[ ! -z "$c" ] && compile_mutants
284
[ ! -z "$k" ] && run_mutants
285
[ ! -z "$t" ] && kill_mutants
286
[ ! -z "$r" ] && run_tests
287
[ "$nobehavior" -eq 1 ] && echo "Must provide an argument in [sbmkta]" && usage
288