Commit feced78e authored by Houtan Bastani's avatar Houtan Bastani
Browse files

make check: support xfail tests

parent 76ccaac9
...@@ -148,6 +148,8 @@ MODFILES = \ ...@@ -148,6 +148,8 @@ MODFILES = \
deterministic_simulations/rbc_det5.mod \ deterministic_simulations/rbc_det5.mod \
walsh.mod walsh.mod
XFAIL_MODFILES =
# Dependencies # Dependencies
example1_use_dll.m.trs: example1.m.trs example1_use_dll.m.trs: example1.m.trs
example1_use_dll.o.trs: example1.o.trs example1_use_dll.o.trs: example1.o.trs
...@@ -215,18 +217,21 @@ dsge-var/dsgevar_forward_calibrated_lambda.o.trs: dsge-var/simul_hybrid.o.trs ...@@ -215,18 +217,21 @@ dsge-var/dsgevar_forward_calibrated_lambda.o.trs: dsge-var/simul_hybrid.o.trs
dsge-var/dsgevar_forward_estimated_lambda.o.trs: dsge-var/simul_hybrid.o.trs dsge-var/dsgevar_forward_estimated_lambda.o.trs: dsge-var/simul_hybrid.o.trs
# Matlab TRS Files # Matlab TRS Files
MTRSFILES = $(patsubst %.mod, %.m.trs, $(MODFILES)) M_TRS_FILES = $(patsubst %.mod, %.m.trs, $(MODFILES))
MTRSFILES += run_block_byte_tests_matlab.m.trs M_TRS_FILES += run_block_byte_tests_matlab.m.trs
M_XFAIL_TRS_FILES = $(patsubst %.mod, %.m.trs, $(XFAIL_MODFILES))
# Octave TRS Files # Octave TRS Files
OTRSFILES = $(patsubst %.mod, %.o.trs, $(MODFILES)) O_TRS_FILES = $(patsubst %.mod, %.o.trs, $(MODFILES))
OTRSFILES += run_block_byte_tests_octave.o.trs O_TRS_FILES += run_block_byte_tests_octave.o.trs
O_XFAIL_TRS_FILES = $(patsubst %.mod, %.o.trs, $(XFAIL_MODFILES))
EXTRA_DIST = \ EXTRA_DIST = \
read_trs_files.sh \ read_trs_files.sh \
run_test_matlab.m \ run_test_matlab.m \
run_test_octave.m \ run_test_octave.m \
$(MODFILES) \ $(MODFILES) \
$(XFAIL_MODFILES) \
run_block_byte_tests_matlab.m \ run_block_byte_tests_matlab.m \
run_block_byte_tests_octave.m \ run_block_byte_tests_octave.m \
homotopy/common.mod \ homotopy/common.mod \
...@@ -292,19 +297,19 @@ endif ...@@ -292,19 +297,19 @@ endif
check-local: $(TARGETS) check-local: $(TARGETS)
if HAVE_OCTAVE if HAVE_OCTAVE
./read_trs_files.sh "$(OTRSFILES)" ./read_trs_files.sh "$(O_TRS_FILES)" "$(O_XFAIL_TRS_FILES)"
endif endif
if HAVE_CMD_LINE_MATLAB if HAVE_CMD_LINE_MATLAB
./read_trs_files.sh "$(MTRSFILES)" ./read_trs_files.sh "$(M_TRS_FILES)" "$(M_XFAIL_TRS_FILES)"
endif endif
@cat run_test_matlab_output.txt @cat run_test_matlab_output.txt
@cat run_test_octave_output.txt @cat run_test_octave_output.txt
check-matlab: $(MTRSFILES) check-matlab: $(M_XFAIL_TRS_FILES) $(M_TRS_FILES)
@echo 'Matlab Tests Done' @echo 'Matlab Tests Done'
check-octave: $(OTRSFILES) check-octave: $(O_XFAIL_TRS_FILES) $(O_TRS_FILES)
@echo 'Octave Tests Done' @echo 'Octave Tests Done'
%.m.trs %.m.log: %.mod %.m.trs %.m.log: %.mod
...@@ -324,10 +329,14 @@ check-octave: $(OTRSFILES) ...@@ -324,10 +329,14 @@ check-octave: $(OTRSFILES)
$(OCTAVE) --no-init-file --silent --no-history $< > $*.o.log 2>&1 $(OCTAVE) --no-init-file --silent --no-history $< > $*.o.log 2>&1
clean-local: clean-local:
rm -f $(MTRSFILES) \ rm -f $(M_TRS_FILES) \
$(OTRSFILES) \ $(M_XFAIL_TRS_FILES) \
$(patsubst %.trs, %.log, $(MTRSFILES)) \ $(O_TRS_FILES) \
$(patsubst %.trs, %.log, $(OTRSFILES)) $(O_XFAIL_TRS_FILES) \
$(patsubst %.trs, %.log, $(M_TRS_FILES)) \
$(patsubst %.trs, %.log, $(M_XFAIL_TRS_FILES)) \
$(patsubst %.trs, %.log, $(O_TRS_FILES)) \
$(patsubst %.trs, %.log, $(O_XFAIL_TRS_FILES))
rm -f $(patsubst %.mod, %.m, $(MODFILES)) \ rm -f $(patsubst %.mod, %.m, $(MODFILES)) \
$(patsubst %.mod, %_static.*, $(MODFILES)) \ $(patsubst %.mod, %_static.*, $(MODFILES)) \
......
#!/bin/bash #!/bin/bash
declare -i total=0; declare -i total=0;
declare -i total_xfail=0;
declare -i failed=0; declare -i failed=0;
declare -i xpassed=0;
declare -a failed_tests=(""); declare -a failed_tests=("");
declare -a xpassed_tests=("");
# Parse TRS Files # Parse TRS Files
for file in $1 ; do for file in $1 ; do
...@@ -20,6 +23,24 @@ for file in $1 ; do ...@@ -20,6 +23,24 @@ for file in $1 ; do
done done
((passed=$total-$failed)); ((passed=$total-$failed));
# Parse XFAIL TRS Files
for file in $2 ; do
# Find number of tests run in xfail trs file
((xfail = `grep number-tests $file | cut -d: -f3`))
((total_xfail += $xfail))
# Find number of tests failed in trs file
numpassed=`grep number-failed-tests $file | cut -d: -f3`
if [ $numpassed -eq 0 ] ; then
((xpassed += (($xfail - $numpassed))))
for xpassedfile in `grep list-of-passed-tests $file | cut -d: -f3` ; do
xpassed_tests=("${xpassed_tests[@]}" "$xpassedfile");
done
fi
done
((xfailed=$total_xfail-$xpassed));
((total+=$total_xfail));
# Determine if we are parsing Matlab or Octave trs files # Determine if we are parsing Matlab or Octave trs files
if [ `grep -c '.m.trs' <<< $1` -eq 0 ]; then if [ `grep -c '.m.trs' <<< $1` -eq 0 ]; then
prg='OCTAVE'; prg='OCTAVE';
...@@ -36,6 +57,8 @@ echo '================================' >> $outfile ...@@ -36,6 +57,8 @@ echo '================================' >> $outfile
echo '| TOTAL: '$total >> $outfile echo '| TOTAL: '$total >> $outfile
echo '| PASS: '$passed >> $outfile echo '| PASS: '$passed >> $outfile
echo '| FAIL: '$failed >> $outfile echo '| FAIL: '$failed >> $outfile
echo '| XFAIL: '$xfailed >> $outfile
echo '| XPASS: '$xpassed >> $outfile
if [ $failed -gt 0 ] ; then if [ $failed -gt 0 ] ; then
echo '| LIST OF FAILED TESTS:' >> $outfile echo '| LIST OF FAILED TESTS:' >> $outfile
for file in ${failed_tests[@]} ; do for file in ${failed_tests[@]} ; do
...@@ -47,4 +70,15 @@ if [ $failed -gt 0 ] ; then ...@@ -47,4 +70,15 @@ if [ $failed -gt 0 ] ; then
echo '| * '$modfile >> $outfile echo '| * '$modfile >> $outfile
done done
fi fi
if [ $xpassed -gt 0 ] ; then
echo '| LIST OF XPASSED TESTS:' >> $outfile
for file in ${xpassed_tests[@]} ; do
if [ "$prg" == "MATLAB" ]; then
modfile=`sed 's/\.m\.trs/\.mod/g' <<< $file` >> $outfile
else
modfile=`sed 's/\.o\.trs/\.mod/g' <<< $file` >> $outfile
fi
echo '| * '$modfile >> $outfile
done
fi
echo >> $outfile echo >> $outfile
...@@ -56,6 +56,7 @@ else ...@@ -56,6 +56,7 @@ else
fprintf(fid,':test-result: PASS\n'); fprintf(fid,':test-result: PASS\n');
fprintf(fid,':number-tests: 1\n'); fprintf(fid,':number-tests: 1\n');
fprintf(fid,':number-failed-tests: 0\n'); fprintf(fid,':number-failed-tests: 0\n');
fprintf(fid,':list-of-passed-tests: %s\n', [name '.mod']);
end end
fclose(fid); fclose(fid);
exit; exit;
...@@ -62,6 +62,7 @@ else ...@@ -62,6 +62,7 @@ else
fprintf(fid,':test-result: PASS\n'); fprintf(fid,':test-result: PASS\n');
fprintf(fid,':number-tests: 1\n'); fprintf(fid,':number-tests: 1\n');
fprintf(fid,':number-failed-tests: 0\n'); fprintf(fid,':number-failed-tests: 0\n');
fprintf(fid,':list-of-passed-tests: %s\n', [name '.mod']);
end end
fclose(fid); fclose(fid);
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment