Testsuite:

- no longer use the 'noclearall' option when calling Dynare
- test the simulation path for the various stack_solve_algo options
- simplify code

tests/.gitignore

@@ -11,7 +11,10 @@
 *.prn
 *_simul
 
-/block_bytecode/wsOct
+wsOct
+/run_test_octave_output.txt
+/run_test_matlab_output.txt
+
 /block_bytecode/ls2003_tmp.mod
 /partial_information/PItest3aHc0PCLsimModPiYrVarobsAll_PCL*
 /partial_information/PItest3aHc0PCLsimModPiYrVarobsCNR_PCL*

tests/Makefile.am

-DYNARE_ROOT = $(abs_top_srcdir)/matlab
-
 MODFILES = \
 	ramst.mod \
 	ramst_a.mod \
@@ -132,22 +130,11 @@ endif
 
 check-local: $(TARGETS)
 
-check-test-file: $(MODFILES)
-	@set -e; \
-	echo "function name=filesToTest()" > filesToTest.m; \
-	echo "name = { ..." >> filesToTest.m; \
-	for modfile in $(MODFILES); do \
-		echo "    '"$$modfile"' ..." >> filesToTest.m; \
-	done
-	@set -e; \
-	echo "    };"  >> filesToTest.m; \
-	echo "end"  >> filesToTest.m;
-
 check-matlab:
-	$(MATLAB)/bin/matlab -nodesktop -nosplash -r run_test_matlab
+	DYNARE_VERSION="$(PACKAGE_VERSION)" MODFILES="$(MODFILES)" $(MATLAB)/bin/matlab -nodesktop -nosplash -r run_test_matlab
 
 check-octave:
-	$(OCTAVE) --norc --silent --no-history --path $(DYNARE_ROOT) --eval dynver=\"$(PACKAGE_VERSION)\" run_test_octave.m
+	DYNARE_VERSION="$(PACKAGE_VERSION)" MODFILES="$(MODFILES)" $(OCTAVE) --norc --silent --no-history run_test_octave.m
 
 clean-local:
 	rm -f $(patsubst %.mod, %.m, $(MODS)) \
@@ -189,6 +176,9 @@ clean-local:
 
 	rm -rf partial_information/PItest3aHc0PCLsimModPiYrVarobsAll_PCL* partial_information/PItest3aHc0PCLsimModPiYrVarobsCNR_PCL*
 
-	rm -rf block_bytecode/ws block_bytecode/ls2003_tmp*
+	rm -rf block_bytecode/ls2003_tmp*
+
+	rm -f $(shell find -name wsOct) \
+		$(shell find -name wsMat.mat)
 
-	rm -f run_test_matlab_output.txt run_test_octave_output.txt
\ No newline at end of file
+	rm -f run_test_matlab_output.txt run_test_octave_output.txt

tests/printMakeCheckOctaveErrMsg.m

@@ -5,7 +5,7 @@ function printMakeCheckOctaveErrMsg(modfilename, err)
     printf("********************************************\n");
     printf("  WHILE RUNNING MODFILE: %s\n", modfilename);
     printf("                    MSG: %s\n", err.message);
-    if (size(err.stack, 1) > 0)
+    if (isfield(err, 'stack'))
         printf("                IN FILE: %s\n", err.stack(1).file);
         printf("            IN FUNCTION: %s\n", err.stack(1).name);
         printf("     ON LINE and COLUMN: %d and %d\n",err.stack(1).line,err.stack(1).column);

tests/run_test_matlab.m

@@ -15,38 +15,56 @@
 % You should have received a copy of the GNU General Public License
 % along with Dynare.  If not, see <http://www.gnu.org/licenses/>.
 
-% List of files to be tested
-name=filesToTest();
-
-% BASE TESTS
-failedBase = {};
+% Implementation notes:
+%
+% Before every call to Dynare, the contents of the workspace is saved in
+% 'wsMat.mat', and reloaded after Dynare has finished (this is necessary since
+% Dynare does a 'clear -all').  Also note that we take care of clearing the
+% 'exception' variable in all 'catch' block, because otherwise the next 'load
+% wsMat' within a 'catch' block will overwrite the last exception.
 
 top_test_dir = pwd;
 addpath(top_test_dir);
 addpath([top_test_dir '/../matlab']);
-for i=1:size(name,2)
+
+% Test Dynare Version
+if ~strcmp(dynare_version(), getenv('DYNARE_VERSION'))
+  error('Incorrect version of Dynare is being tested')
+end
+
+% Test MOD files listed in Makefile.am
+name=getenv('MODFILES');
+num_modfiles = 0;
+
+failedBase = {};
+
+while ~isempty(name)
+    [modfile, name] = strtok(name);
+    num_modfiles = num_modfiles + 1;
+    [directory, testfile, ext] = fileparts([top_test_dir '/' modfile]);
+    cd(directory);
+    disp('');
+    disp(['***  TESTING: ' modfile ' ***']);
     try
-        [directory, testfile, ext] = fileparts([top_test_dir '/' name{i}]);
-        cd(directory);
-        disp(['***  TESTING: ' name{i} ' ***']);
-        dynare([testfile ext], 'noclearall')
+        save wsMat
+        dynare([testfile ext])
+        load wsMat
     catch exception
-        failedBase{size(failedBase,2)+1} = name{i};
-        printMakeCheckMatlabErrMsg(name{i}, exception);
+        load wsMat
+        failedBase{size(failedBase,2)+1} = modfile;
+        printMakeCheckMatlabErrMsg(modfile, exception);
+        clear exception
     end
+    delete('wsMat.mat')
 
     cd(top_test_dir);
-    save('makeCheckMatlabBase.mat', 'name', 'failedBase', 'i', 'top_test_dir');
-    clear -all;
-    load('makeCheckMatlabBase.mat');
 end
 
-% BLOCK TEST
-clear i name;
+% Test block_bytecode/ls2003.mod with various combinations of
+% block/bytecode/solve_algo/stack_solve_algo
 failedBlock = {};
 num_block_tests = 0;
 cd([top_test_dir '/block_bytecode']);
-save('makeCheckBlockByteMatlab.mat', 'failedBlock', 'top_test_dir');
 for blockFlag = 0:1
     for bytecodeFlag = 0:1
         default_solve_algo = 2;
@@ -64,72 +82,72 @@ for blockFlag = 0:1
 
         for i = 1:length(solve_algos)
             num_block_tests = num_block_tests + 1;
-            save wsMat
             if ~blockFlag && ~bytecodeFlag && (i == 1)
+                % This is the reference simulation path against which all
+                % other simulations will be tested
                 try
+                    save wsMat
                     run_ls2003(blockFlag, bytecodeFlag, solve_algos(i), default_stack_solve_algo)
+                    load wsMat
                     y_ref = oo_.endo_simul;
                     save('test.mat','y_ref');
                 catch exception
                     load wsMat
-                    load('makeCheckBlockByteMatlab.mat', 'failedBlock', 'top_test_dir');
                     failedBlock{size(failedBlock,2)+1} = ['block_bytecode/run_ls2003.m(' num2str(blockFlag) ', ' num2str(bytecodeFlag) ', ' num2str(solve_algos(i)) ', ' num2str(default_stack_solve_algo) ')'];
                     printMakeCheckMatlabErrMsg(['block_bytecode/run_ls2003.m(' num2str(blockFlag) ', ' num2str(bytecodeFlag) ', ' num2str(solve_algos(i)) ', ' num2str(default_stack_solve_algo) ')'], exception);
-                    save('makeCheckBlockByteMatlab.mat', 'failedBlock', 'top_test_dir');
+                    clear exception
                 end
             else
                 try
+                    save wsMat
                     run_ls2003(blockFlag, bytecodeFlag, solve_algos(i), default_stack_solve_algo)
+                    load wsMat
+                    % Test against the reference simulation path
                     load('test.mat','y_ref');
                     diff = oo_.endo_simul - y_ref;
                     if(abs(diff) > options_.dynatol)
-                        load wsMat
-                        load('makeCheckBlockByteMatlab.mat', 'failedBlock', 'top_test_dir');
                         failedBlock{size(failedBlock,2)+1} = ['block_bytecode/run_ls2003.m(' num2str(blockFlag) ', ' num2str(bytecodeFlag) ', ' num2str(solve_algos(i)) ', ' num2str(default_stack_solve_algo) ')'];
-                        exception = MException('ERROR: diff fails for block code in run_test_octave.m', ['makecheck found error: if (abs(diff) <= options_.dynatol) FAILS. line 85, col 1.' ]);
+                        exception = MException('ERROR: simulation path differs from the reference path');
                         printMakeCheckMatlabErrMsg(['block_bytecode/run_ls2003.m(' num2str(blockFlag) ', ' num2str(bytecodeFlag) ', ' num2str(solve_algos(i)) ', ' num2str(default_stack_solve_algo) ')'], exception);
-                        save('makeCheckBlockByteMatlab.mat', 'failedBlock', 'top_test_dir');
+                        clear exception
                     end
                 catch exception
                     load wsMat
-                    load('makeCheckBlockByteMatlab.mat', 'failedBlock', 'top_test_dir');
                     failedBlock{size(failedBlock,2)+1} = ['block_bytecode/run_ls2003.m(' num2str(blockFlag) ', ' num2str(bytecodeFlag) ', ' num2str(solve_algos(i)) ', ' num2str(default_stack_solve_algo) ')'];
                     printMakeCheckMatlabErrMsg(['block_bytecode/run_ls2003.m(' num2str(blockFlag) ', ' num2str(bytecodeFlag) ', ' num2str(solve_algos(i)) ', ' num2str(default_stack_solve_algo) ')'], exception);
-                    save('makeCheckBlockByteMatlab.mat', 'failedBlock', 'top_test_dir');
+                    clear exception
                 end
             end
-            load wsMat
         end
         for i = 1:length(stack_solve_algos)
             num_block_tests = num_block_tests + 1;
-            save wsMat
             try
+                save wsMat
                 run_ls2003(blockFlag, bytecodeFlag, default_solve_algo, stack_solve_algos(i))
+                load wsMat
+                % Test against the reference simulation path
+                load('test.mat','y_ref');
+                diff = oo_.endo_simul - y_ref;
+                if(abs(diff) > options_.dynatol)
+                    failedBlock{size(failedBlock,2)+1} = ['block_bytecode/run_ls2003.m(' num2str(blockFlag) ', ' num2str(bytecodeFlag) ', ' num2str(default_solve_algo) ', ' num2str(stack_solve_algos(i)) ')'];
+                    exception = MException('ERROR: simulation path difers from the reference path');
+                    printMakeCheckMatlabErrMsg(['block_bytecode/run_ls2003.m(' num2str(blockFlag) ', ' num2str(bytecodeFlag) ', ' num2str(default_solve_algo) ', ' num2str(stack_solve_algos(i)) ')'], exception);
+                    clear exception
+                end
             catch exception
                 load wsMat
-                load('makeCheckBlockByteMatlab.mat', 'failedBlock', 'top_test_dir');
                 failedBlock{size(failedBlock,2)+1} = ['block_bytecode/run_ls2003.m(' num2str(blockFlag) ', ' num2str(bytecodeFlag) ', ' num2str(solve_algos(i)) ', ' num2str(default_stack_solve_algo) ')'];
                 printMakeCheckMatlabErrMsg(['block_bytecode/run_ls2003.m(' num2str(blockFlag) ', ' num2str(bytecodeFlag) ', ' num2str(solve_algos(i)) ', ' num2str(default_stack_solve_algo) ')'], exception);
-                save('makeCheckBlockByteMatlab.mat', 'failedBlock', 'top_test_dir');
+                clear exception
             end
-            load wsMat
         end
     end
 end
-
-load('makeCheckBlockByteMatlab.mat');
-save('makeCheckBlockByteMatlab.mat', 'failedBlock', 'top_test_dir', 'num_block_tests');
-delete('wsMat.mat');
-clear -all;
-
-load('makeCheckBlockByteMatlab.mat');
-delete('makeCheckBlockByteMatlab.mat');
+delete('wsMat.mat')
 
 cd(top_test_dir);
-load('makeCheckMatlabBase.mat');
-delete('makeCheckMatlabBase.mat');
 
-total_tests = size(name,2)+num_block_tests;
+total_tests = num_modfiles+num_block_tests;
 
 % print output to screen and to file
 fid = fopen('run_test_matlab_output.txt', 'w');

tests/run_test_octave.m

@@ -15,47 +15,53 @@
 ## You should have received a copy of the GNU General Public License
 ## along with Dynare.  If not, see <http://www.gnu.org/licenses/>.
 
+## Implementation notes:
+##
+## Before every call to Dynare, the contents of the workspace is saved in
+## 'wsOct', and reloaded after Dynare has finished (this is necessary since
+## Dynare does a 'clear -all').
+
+top_test_dir = pwd;
+addpath(top_test_dir);
+addpath([top_test_dir '/../matlab']);
+
 ## Test Dynare Version
-if !strcmp(dynare_version(), dynver)
+if !strcmp(dynare_version(), getenv("DYNARE_VERSION"))
   error("Incorrect version of Dynare is being tested")
 endif
 
-## List of files to be tested
-name = filesToTest();
-
 ## Ask gnuplot to create graphics in text mode
 ## Note that setenv() was introduced in Octave 3.0.2, for compatibility
 ## with MATLAB
 putenv("GNUTERM", "dumb")
 
-## BASE TESTS
+## Test MOD files listed in Makefile.am
+name = strsplit(getenv("MODFILES"), " ");
+
 failedBase = {};
 
-top_test_dir = pwd;
-addpath(top_test_dir);
 for i=1:size(name,2)
+  [directory, testfile, ext] = fileparts([top_test_dir '/' name{i}]);
+  cd(directory);
+  printf("\n***  TESTING: %s ***\n", name{i});
   try
-    [directory, testfile, ext] = fileparts([top_test_dir '/' name{i}]);
-    cd(directory);
-    printf("***  TESTING: %s ***\n", name{i});
-    dynare([testfile ext], 'noclearall')
+    save wsOct
+    dynare([testfile ext])
+    load wsOct
   catch
+    load wsOct
     failedBase{size(failedBase,2)+1} = name{i};
     printMakeCheckOctaveErrMsg(name{i}, lasterror);
   end_try_catch
-
+  delete('wsOct');
   cd(top_test_dir);
-  save('makeCheckOctaveBase.mat', 'name', 'failedBase', 'i', 'top_test_dir');
-  clear -all;
-  load('makeCheckOctaveBase.mat');
 end
 
-## BLOCK TEST
-clear i name;
+## Test block_bytecode/ls2003.mod with various combinations of
+## block/bytecode/solve_algo/stack_solve_algo
 failedBlock = {};
 num_block_tests = 0;
 cd([top_test_dir '/block_bytecode']);
-save('makeCheckOctaveBlockByte.mat', 'failedBlock', 'top_test_dir');
 for blockFlag = 0:1
   for bytecodeFlag = 0:1
     ## Recall that solve_algo=7 and stack_solve_algo=2 are not supported
@@ -75,74 +81,66 @@ for blockFlag = 0:1
 
     for i = 1:length(solve_algos)
       num_block_tests = num_block_tests + 1;
-      save wsOct
       if !blockFlag && !bytecodeFlag && (i == 1)
+        ## This is the reference simulation path against which all
+        ## other simulations will be tested
         try
+          save wsOct
           run_ls2003(blockFlag, bytecodeFlag, solve_algos(i), default_stack_solve_algo)
+          load wsOct
           y_ref = oo_.endo_simul;
           save('test.mat','y_ref');
         catch
           load wsOct
-          load('makeCheckOctaveBlockByte.mat', 'failedBlock', 'top_test_dir');
           failedBlock{size(failedBlock,2)+1} = ['block_bytecode/run_ls2003.m(' num2str(blockFlag) ', ' num2str(bytecodeFlag) ', ' num2str(solve_algos(i)) ', ' num2str(default_stack_solve_algo) ')'];
           printMakeCheckOctaveErrMsg(['block_bytecode/run_ls2003.m(' num2str(blockFlag) ', ' num2str(bytecodeFlag) ', ' num2str(solve_algos(i)) ', ' num2str(default_stack_solve_algo) ')'], lasterror);
-          save('makeCheckOctaveBlockByte.mat', 'failedBlock', 'top_test_dir');
         end_try_catch
       else
         try
+          save wsOct
           run_ls2003(blockFlag, bytecodeFlag, solve_algos(i), default_stack_solve_algo)
+          load wsOct
+          ## Test against the reference simulation path
           load('test.mat','y_ref');
           diff = oo_.endo_simul - y_ref;
           if(abs(diff) > options_.dynatol)
-            load wsOct
-            load('makeCheckOctaveBlockByte.mat', 'failedBlock', 'top_test_dir');
             failedBlock{size(failedBlock,2)+1} = ['block_bytecode/run_ls2003.m(' num2str(blockFlag) ', ' num2str(bytecodeFlag) ', ' num2str(solve_algos(i)) ', ' num2str(default_stack_solve_algo) ')'];
-            differr.message = ["makecheck found error: if (abs(diff) <= options_.dynatol) FAILS." ];
-            differr.stack(1).file = "run_test_octave.m";
-            differr.stack(1).name = "run_test_octave.m";
-            differr.stack(1).line = 96;
-            differr.stack(1).column = 1;
+            differr.message = ["ERROR: simulation path differs from the reference path" ];
             printMakeCheckOctaveErrMsg(['block_bytecode/run_ls2003.m(' num2str(blockFlag) ', ' num2str(bytecodeFlag) ', ' num2str(solve_algos(i)) ', ' num2str(default_stack_solve_algo) ')'], differr);
-            save('makeCheckOctaveBlockByte.mat', 'failedBlock', 'top_test_dir');
           endif
         catch
           load wsOct
-          load('makeCheckOctaveBlockByte.mat', 'failedBlock', 'top_test_dir');
           failedBlock{size(failedBlock,2)+1} = ['block_bytecode/run_ls2003.m(' num2str(blockFlag) ', ' num2str(bytecodeFlag) ', ' num2str(solve_algos(i)) ', ' num2str(default_stack_solve_algo) ')'];
           printMakeCheckOctaveErrMsg(['block_bytecode/run_ls2003.m(' num2str(blockFlag) ', ' num2str(bytecodeFlag) ', ' num2str(solve_algos(i)) ', ' num2str(default_stack_solve_algo) ')'], lasterror);
-          save('makeCheckOctaveBlockByte.mat', 'failedBlock', 'top_test_dir');
         end_try_catch
       endif
-      load wsOct
     endfor
     for i = 1:length(stack_solve_algos)
       num_block_tests = num_block_tests + 1;
-      save wsOct
       try
+        save wsOct
         run_ls2003(blockFlag, bytecodeFlag, default_solve_algo, stack_solve_algos(i))
+        load wsOct
+        ## Test against the reference simulation path
+        load('test.mat','y_ref');
+        diff = oo_.endo_simul - y_ref;
+        if(abs(diff) > options_.dynatol)
+          failedBlock{size(failedBlock,2)+1} = ['block_bytecode/run_ls2003.m(' num2str(blockFlag) ', ' num2str(bytecodeFlag) ', ' num2str(default_solve_algo) ', ' num2str(stack_solve_algos(i)) ')'];
+          differr.message = ["ERROR: simulation path differs from the reference path" ];
+          printMakeCheckOctaveErrMsg(['block_bytecode/run_ls2003.m(' num2str(blockFlag) ', ' num2str(bytecodeFlag) ', ' num2str(default_solve_algo) ', ' num2str(stack_solve_algos(i)) ')'], differr);
+        endif
       catch
         load wsOct
-        load('makeCheckOctaveBlockByte.mat', 'failedBlock', 'top_test_dir');
         failedBlock{size(failedBlock,2)+1} = ['block_bytecode/run_ls2003.m(' num2str(blockFlag) ', ' num2str(bytecodeFlag) ', ' num2str(solve_algos(i)) ', ' num2str(default_stack_solve_algo) ')'];
         printMakeCheckOctaveErrMsg(['block_bytecode/run_ls2003.m(' num2str(blockFlag) ', ' num2str(bytecodeFlag) ', ' num2str(solve_algos(i)) ', ' num2str(default_stack_solve_algo) ')'], lasterror);
-        save('makeCheckOctaveBlockByte.mat', 'failedBlock', 'top_test_dir');
       end_try_catch
-      load wsOct
     endfor
   endfor
 endfor
 
-load('makeCheckOctaveBlockByte.mat');
-save('makeCheckOctaveBlockByte.mat', 'failedBlock', 'top_test_dir', 'num_block_tests');
 delete('wsOct');
-clear -all;
-
-load('makeCheckOctaveBlockByte.mat');
-delete('makeCheckOctaveBlockByte.mat');
 
 cd(top_test_dir);
-load('makeCheckOctaveBase.mat');
-delete('makeCheckOctaveBase.mat');
 
 total_tests = size(name,2)+num_block_tests;
 
@@ -180,7 +178,6 @@ if size(failedBase,2) > 0 || size(failedBlock,2) > 0
   error("make check-octave FAILED");
 end
 fclose(fid);
-clear -all
 
 ## Local variables:
 ## mode: Octave