v4 parser:

* added macro processor skeleton
* added "savemacro" option, which saves the result of macro expansion in a file named "filename-macroexp.mod"; this file can be fed back to Dynare
* implemented @include directive; note that nested includes are possible; error messages in Dynare pre-processor will point to the filename effectively edited by the user
* split DynareMain.cc in two files, because of a Bison restriction: it is impossible to include both MacroDriver.hh and ParsingDriver.hh from the same source file


git-svn-id: https://www.dynare.org/svn/dynare/dynare_v4@1672 ac1d8469-bf42-47a9-8791-bf33cf982152

parser.src/DynareBison.yy

@@ -27,6 +27,24 @@ using namespace std;
 class ParsingDriver;
 
 #include "ExprNode.hh"
+
+/* Little hack: we redefine the macro which computes the locations, because
+   we need to access the location from within the parsing driver for error
+   and warning messages. */
+#define YYLLOC_DEFAULT(Current, Rhs, N)                 \
+  do {                                                  \
+    if (N)                                              \
+      {                                                 \
+        (Current).begin = (Rhs)[1].begin;               \
+        (Current).end   = (Rhs)[N].end;                 \
+      }                                                 \
+    else                                                \
+      {                                                 \
+        (Current).begin = (Current).end = (Rhs)[0].end;	\
+      }                                                 \
+    driver.location = (Current);                        \
+  } while(false)
+
 %}
 
 %name-prefix="Dynare"
@@ -35,11 +53,6 @@ class ParsingDriver;
 %lex-param { ParsingDriver &driver }
 
 %locations
-%initial-action
-{
-  // Initialize the location filenames
-  @$.begin.filename = @$.end.filename = &driver.file;
-};
 
 %debug
 %error-verbose

parser.src/DynareFlex.ll

@@ -24,12 +24,12 @@ using namespace std;
 #include "DynareBison.hh"
 
 // Announce to Flex the prototype we want for lexing function
-#define YY_DECL                                            \
+#define YY_DECL                                                \
   Dynare::parser::token_type                                   \
-  DynareFlex::lex(Dynare::parser::semantic_type *yylval,       \
-                  Dynare::parser::location_type *yylloc,       \
-                  ParsingDriver &driver)
-
+    DynareFlex::lex(Dynare::parser::semantic_type *yylval,     \
+                    Dynare::parser::location_type *yylloc,     \
+                    ParsingDriver &driver)
+ 
 // Shortcut to access tokens defined by Bison
 typedef Dynare::parser::token token;
 
@@ -38,7 +38,7 @@ typedef Dynare::parser::token token;
    not of token_type.  */
 #define yyterminate() return Dynare::parser::token_type (0);
 
-int comment_caller;
+int comment_caller, line_caller;
 /* Particular value : when sigma_e command is found
  this flag is set to 1, when command finished it is set to 0
  */
@@ -49,12 +49,15 @@ int sigma_e = 0;
 
 %option prefix="Dynare"
 
-%option case-insensitive noyywrap nounput batch debug never-interactive yylineno
+%option case-insensitive noyywrap nounput batch debug never-interactive
 
 %x COMMENT
 %x DYNARE_STATEMENT
 %x DYNARE_BLOCK
 %x NATIVE
+%x LINE1
+%x LINE2
+%x LINE3
 
 %{
 // Increments location counter for every token read
@@ -67,6 +70,20 @@ int sigma_e = 0;
   yylloc->step();
 %}
 
+ /* Rules for matching @line directives */
+<*>^@line\ \"   { line_caller = YYSTATE; BEGIN(LINE1); }
+<LINE1>[^\"\n]* {
+                  if (yylloc->begin.filename)
+                    delete yylloc->begin.filename;
+                  yylloc->begin.filename = yylloc->end.filename = new string(yytext);
+                  BEGIN(LINE2);
+                }
+<LINE2>\"       BEGIN(LINE3);
+<LINE3>[0-9]+   {
+                  yylloc->begin.line = yylloc->end.line = atoi(yytext) - 1;
+                  BEGIN(line_caller);
+                }
+
  /* spaces, tabs and EOL are ignored */
 <*>[ \t\r\f]+  { yylloc->step(); }
 <*>[\n]+       { yylloc->lines(yyleng); yylloc->step(); }
@@ -385,7 +402,7 @@ int sigma_e = 0;
  /* Add the native statement */
 <NATIVE>.* { driver.add_native(yytext); BEGIN INITIAL; }
 
-<*>.      { driver.error("Unrecognized character: '" + string(yytext) + "'"); }
+<*>.      { driver.error(*yylloc, "character unrecognized by lexer"); }
 %%
 
 DynareFlex::DynareFlex(istream* in, ostream* out)
@@ -395,10 +412,10 @@ DynareFlex::DynareFlex(istream* in, ostream* out)
 
 /* This implementation of DynareFlexLexer::yylex() is required to fill the
  * vtable of the class DynareFlexLexer. We define the scanner's main yylex
- * function via YY_DECL to reside in the Scanner class instead. */
+ * function via YY_DECL to reside in the DynareFlex class instead. */
 
 #ifdef yylex
-#undef yylex
+# undef yylex
 #endif
 
 int

parser.src/DynareMain.cc

@@ -19,62 +19,80 @@
 
 using namespace std;
 
-#include "ParsingDriver.hh"
-#include "ModFile.hh"
+#include <iostream>
+#include <sstream>
+#include <fstream>
 
-/*!
-  \brief Main function of Dynare.
-  \param argc Number of command line argumetns from runtime system
-  \param argv Command line arguments from runtime system
+#include <cctype>    // for tolower()
+#include <algorithm> // for transform()
+
+#include "macro/MacroDriver.hh"
+
+/* Prototype for second part of main function
+   Splitting main() in two parts was necessary because ParsingDriver.h and MacroDriver.h can't be
+   included simultaneously (because of Bison limitations).
 */
+void main2(stringstream &in, string &basename, bool trace_scanning, bool trace_parsing,
+           bool clear_all);
+
 int
 main(int argc, char** argv)
 {
   if (argc < 2)
     {
-      cerr << "Missing model file" << endl;
-      cerr << "Dynare usage: dynare model_file [debug]" << endl;
+      cerr << "Missing model file!" << endl;
+      cerr << "Dynare usage: dynare mod_file [debug] [noclearall] [savemacro]" << endl;
       exit(-1);
     }
 
-  ParsingDriver p;
-
   bool clear_all = true;
+  bool save_macro = false;
+  bool trace_scanning = false;
+  bool trace_parsing = false;
 
   // Parse options
   for (int arg = 2; arg < argc; arg++)
     {
       if (string(argv[arg]) == string("debug"))
         {
-          p.trace_scanning = true;
-          p.trace_parsing = true;
+          trace_scanning = true;
+          trace_parsing = true;
         }
-      else
-        if (string(argv[arg]) == string("noclearall"))
-          clear_all = false;
+      else if (string(argv[arg]) == string("noclearall"))
+        clear_all = false;
+      else if (string(argv[arg]) == string("savemacro"))
+        save_macro = true;
     }
 
   cout << "Starting Dynare ..." << endl;
   cout << "Parsing your model file ..." << endl;
 
-  // Launch parsing
-  ModFile *mod_file = p.parse(argv[1]);
-
-  // Run checking pass
-  mod_file->checkPass();
-
-  // Do computations
-  mod_file->computingPass();
-
-  // FIXME
+  // Construct basename (check file extension is correct then remove it)
   string basename = argv[1];
+  string ext = basename.substr(basename.size() - 4);
+  transform(ext.begin(), ext.end(), ext.begin(), (int(*)(int)) tolower); // Convert ext to lowercase
+  if (ext != string(".mod") && ext != string(".dyn"))
+    {
+      cerr << "mod_file extension must be .mod or .dyn!" << endl;
+      exit(-1);
+    }
   basename.erase(basename.size() - 4, 4);
 
-  mod_file->writeOutputFiles(basename, clear_all);
+  // Do macro processing
+  MacroDriver m;
+  m.trace_scanning = trace_scanning;
+  m.trace_parsing = trace_parsing;
+  stringstream macro_output;
+  m.parse(argv[1], macro_output);
+  if (save_macro)
+    {
+      ofstream macro_output_file((basename + "-macroexp.mod").c_str());
+      macro_output_file << macro_output.str();
+      macro_output_file.close();
+    }
 
-  delete mod_file;
+  // Do the rest
+  main2(macro_output, basename, trace_scanning, trace_parsing, clear_all);
 
-  cout << "Parsing done" << endl;
-  cout << "Starting Matlab computing ..." << endl;
   return 0;
 }

parser.src/DynareMain2.cc

+/*
+ * Copyright (C) 2008 Dynare Team
+ *
+ * This file is part of Dynare.
+ *
+ * Dynare is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * Dynare is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with Dynare.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+using namespace std;
+
+#include <iostream>
+
+#include "ParsingDriver.hh"
+#include "ModFile.hh"
+
+void
+main2(stringstream &in, string &basename, bool trace_scanning, bool trace_parsing, bool clear_all)
+{
+  ParsingDriver p;
+  p.trace_scanning = trace_scanning;
+  p.trace_parsing = trace_parsing;
+
+  // Do parsing and construct internal representation of mod file
+  ModFile *mod_file = p.parse(in);
+
+  // Run checking pass
+  mod_file->checkPass();
+
+  // Do computations
+  mod_file->computingPass();
+
+  // Write outputs
+  mod_file->writeOutputFiles(basename, clear_all);
+
+  delete mod_file;
+
+  cout << "Parsing done" << endl;
+  cout << "Starting Matlab computing ..." << endl;
+}

parser.src/Makefile

-CPP = g++
-
-CPPFLAGS = -Wall
+include Makefile.include
 
 ifeq ($(shell uname -o), Cygwin)
-	# Detection of uninitialized variables is buggy in Cygwin and generates spurious warnings
-	CPPFLAGS += -Wno-uninitialized
-	CPPFLAGS += -mno-cygwin
 	DYNARE_M = dynare_m.exe
 	DYNARE_S = dynare_s.exe
 else 
@@ -14,38 +9,23 @@ else
 endif
 
 ifeq ($(CROSS_WIN32), yes)
-	CPP = i586-mingw32msvc-g++
-	# Detection of uninitialized variables is buggy in MinGW and generates spurious warnings
-	CPPFLAGS += -Wno-uninitialized
 	DYNARE_M = dynare_m.exe
 	DYNARE_S = dynare_s.exe
 endif
 
-ifeq ($(DEBUG),yes)
-	CPPFLAGS += -ggdb
-else
-	CPPFLAGS += -O3
-endif
-
-ifeq ($(VALGRIND), yes)
-	CPPFLAGS = -Wall -O -g -fno-inline
-endif
-
-
-COMMON_OBJ=\
-	DynareFlex.o\
-	DynareBison.o\
-	ComputingTasks.o\
-	DynareMain.o\
-	ModelTree.o\
-	NumericalConstants.o\
-	NumericalInitialization.o\
-	Shocks.o\
-	SigmaeInitialization.o\
-	SymbolTable.o\
-	TmpSymbolTable.o\
-	VariableTable.o\
-	ParsingDriver.o\
+COMMON_OBJ = \
+	DynareFlex.o \
+	DynareBison.o \
+	ComputingTasks.o \
+	ModelTree.o \
+	NumericalConstants.o \
+	NumericalInitialization.o \
+	Shocks.o \
+	SigmaeInitialization.o \
+	SymbolTable.o \
+	TmpSymbolTable.o \
+	VariableTable.o \
+	ParsingDriver.o \
 	DataTree.o \
 	ModFile.o \
 	Statement.o \
@@ -54,38 +34,32 @@ COMMON_OBJ=\
 	ModelBlocks.o \
 	BlockTriangular.o \
 	Model_Graph.o \
-	SymbolGaussElim.o
+	SymbolGaussElim.o \
+	DynareMain.o \
+	DynareMain2.o
 
 MATLAB_OBJ = InterfaceMatlab.o
 
 SCILAB_OBJ = InterfaceScilab.o
 
 
-################################################################################
-### Build ######################################################################
-################################################################################
+# Build rules
+
+all: all-recursive $(DYNARE_M) $(DYNARE_S)
 
-all: $(DYNARE_M) $(DYNARE_S)
+all-recursive:
+	make -C macro
 
-$(DYNARE_M): $(COMMON_OBJ) $(MATLAB_OBJ)
-	$(CPP) $(CPPFLAGS) -o $(DYNARE_M) $(COMMON_OBJ) $(MATLAB_OBJ)
+$(DYNARE_M): $(COMMON_OBJ) $(MATLAB_OBJ) macro/libmacro.a
+	$(CPP) $(CPPFLAGS) -o $(DYNARE_M) $(COMMON_OBJ) $(MATLAB_OBJ) -Lmacro -lmacro
 	cp $(DYNARE_M) ../matlab/
 
-$(DYNARE_S): $(COMMON_OBJ) $(SCILAB_OBJ)
-	$(CPP) $(CPPFLAGS) -o $(DYNARE_S) $(COMMON_OBJ) $(SCILAB_OBJ)
+$(DYNARE_S): $(COMMON_OBJ) $(SCILAB_OBJ) macro/libmacro.a
+	$(CPP) $(CPPFLAGS) -o $(DYNARE_S) $(COMMON_OBJ) $(SCILAB_OBJ) -Lmacro -lmacro
 	cp $(DYNARE_S) ../scilab/
 
 
-################################################################################
-### Compile ####################################################################
-################################################################################
-
-%.o : %.cc
-	$(CPP) $(CPPFLAGS) -MD -I include -c $<
-	@cp $*.d $*.P; \
-	  sed -e 's/#.*//' -e 's/^[^:]*: *//' -e 's/ *\\$$//' \
-	      -e '/^$$/ d' -e 's/$$/ :/' < $*.d >> $*.P; \
-	  rm -f $*.d
+# Dependencies
 
 -include $(COMMON_OBJ:.o=.P) $(MATLAB_OBJ:.o=.P) $(SCILAB_OBJ:.o=.P)
 
@@ -97,11 +71,9 @@ DynareBison.cc include/DynareBison.hh: DynareBison.yy include/ParsingDriver.hh
 	mv DynareBison.hh location.hh stack.hh position.hh include/
 
 
-################################################################################
-### Clean ######################################################################
-################################################################################
+# Clean
 
-clean:
+clean: clean-recursive
 	rm -f *.o *.P \
 		*~ include/*~ \
 		DynareBison.output \
@@ -109,10 +81,17 @@ clean:
 		$(DYNARE_M) \
 		$(DYNARE_S)
 
-distclean: clean
+clean-recursive:
+	make -C macro clean
+
+distclean: clean distclean-recursive
 	rm -f DynareBison.cc \
 		include/position.hh \
 		include/stack.hh \
 		include/location.hh \
 		include/DynareBison.hh
 
+distclean-recursive:
+	make -C macro distclean
+
+.PHONY: all all-recursive clean clean-recursive distclean distclean-recursive

parser.src/Makefile.include

+CPP = g++
+
+CPPFLAGS = -Wall
+
+ifeq ($(shell uname -o), Cygwin)
+	# Detection of uninitialized variables is buggy in Cygwin and generates spurious warnings
+	CPPFLAGS += -Wno-uninitialized
+	CPPFLAGS += -mno-cygwin
+endif
+
+ifeq ($(CROSS_WIN32), yes)
+	CPP = i586-mingw32msvc-g++
+	# Detection of uninitialized variables is buggy in MinGW and generates spurious warnings
+	CPPFLAGS += -Wno-uninitialized
+endif
+
+ifeq ($(DEBUG),yes)
+	CPPFLAGS += -ggdb
+else
+	CPPFLAGS += -O3
+endif
+
+ifeq ($(VALGRIND), yes)
+	CPPFLAGS = -Wall -O -g -fno-inline
+endif
+
+# General rule for compilation
+%.o : %.cc
+	$(CPP) $(CPPFLAGS) -MD -I include -c $<
+	@cp $*.d $*.P; \
+	  sed -e 's/#.*//' -e 's/^[^:]*: *//' -e 's/ *\\$$//' \
+	      -e '/^$$/ d' -e 's/$$/ :/' < $*.d >> $*.P; \
+	  rm -f $*.d
+
+# Local variables:
+# mode: makefile
+# End:

parser.src/ParsingDriver.cc

@@ -17,6 +17,9 @@
  * along with Dynare.  If not, see <http://www.gnu.org/licenses/>.
  */
 
+#include <fstream>
+#include <iostream>
+
 #include "ParsingDriver.hh"
 #include "Statement.hh"
 
@@ -58,7 +61,7 @@ ParsingDriver::reset_data_tree()
 }
 
 ModFile *
-ParsingDriver::parse(const string &f)
+ParsingDriver::parse(istream &in)
 {
   mod_file = new ModFile();
 
@@ -66,10 +69,6 @@ ParsingDriver::parse(const string &f)
 
   reset_data_tree();
 
-  file = f;
-
-  ifstream in(f.c_str());
-
   lexer = new DynareFlex(&in);
   lexer->set_debug(trace_scanning);
 
@@ -92,14 +91,13 @@ ParsingDriver::error(const Dynare::parser::location_type &l, const string &m)
 void
 ParsingDriver::error(const string &m)
 {
-  cerr << "ERROR: " << file << ":" << lexer->lineno() << ": " << m << endl;
-  exit(-1);
+  error(location, m);
 }
 
 void
 ParsingDriver::warning(const string &m)
 {
-  cerr << "WARNING: " << file << ":" << lexer->lineno() << ": " << m << endl;
+  cerr << "WARNING: " << location << ": " << m << endl;
 }
 
 void

parser.src/include/DynareBison.hh

@@ -56,9 +56,27 @@ class ParsingDriver;
 
 #include "ExprNode.hh"
 
+/* Little hack: we redefine the macro which computes the locations, because
+   we need to access the location from within the parsing driver for error
+   and warning messages. */
+#define YYLLOC_DEFAULT(Current, Rhs, N)                 \
+  do {                                                  \
+    if (N)                                              \
+      {                                                 \
+        (Current).begin = (Rhs)[1].begin;               \
+        (Current).end   = (Rhs)[N].end;                 \
+      }                                                 \
+    else                                                \
+      {                                                 \
+        (Current).begin = (Current).end = (Rhs)[0].end;	\
+      }                                                 \
+    driver.location = (Current);                        \
+  } while(false)
+
+
 
 /* Line 35 of lalr1.cc.  */
-#line 62 "DynareBison.hh"
+#line 80 "DynareBison.hh"
 
 #include "location.hh"
 
@@ -109,13 +127,13 @@ namespace Dynare
     /// Symbol semantic values.
 #ifndef YYSTYPE
     union semantic_type
-#line 48 "DynareBison.yy"
+#line 61 "DynareBison.yy"
 {
   string *string_val;
   NodeID node_val;
 }
 /* Line 35 of lalr1.cc.  */
-#line 119 "DynareBison.hh"
+#line 137 "DynareBison.hh"
 	;
 #else
     typedef YYSTYPE semantic_type;

parser.src/include/ParsingDriver.hh

@@ -20,8 +20,13 @@
 #ifndef _PARSING_DRIVER_HH
 #define _PARSING_DRIVER_HH
 
-#include <iostream>
+#ifdef _MACRO_DRIVER_HH
+# error Impossible to include both ParsingDriver.hh and MacroDriver.hh
+#endif
+
+#include <string>
 #include <vector>
+#include <istream>
 
 #include "ModFile.hh"
 #include "TmpSymbolTable.hh"
@@ -36,9 +41,9 @@ using namespace std;
 
 // Declare DynareFlexLexer class
 #ifndef __FLEX_LEXER_H
-#define yyFlexLexer DynareFlexLexer
-#include <FlexLexer.h>
-#undef yyFlexLexer
+# define yyFlexLexer DynareFlexLexer
+# include <FlexLexer.h>
+# undef yyFlexLexer
 #endif
 
 //! The lexer class
@@ -139,18 +144,15 @@ public:
   virtual ~ParsingDriver();
 
   //! Starts parsing, and constructs the MOD file representation
-  /*! \param f Name of file to parse
-
-      The returned pointer should be deleted after use.
-   */
-  ModFile *parse(const string &f);
-
-  //! Name of file being parsed
-  string file;
+  /*! The returned pointer should be deleted after use */
+  ModFile *parse(istream &in);
 
   //! Reference to the lexer
   class DynareFlex *lexer;
 
+  //! Copy of parsing location, maintained by YYLLOC_DEFAULT macro in DynareBison.yy
+  Dynare::parser::location_type location;
+
   //! Trace scanning ?
   /*! If set to true before calling parse(), the flex scanner will dump a lot of debugging information. Defaults to false.
   */
@@ -163,11 +165,11 @@ public:
   //! Estimation parameters
   EstimationParams estim_params;
 
-  //! Error handler with location
+  //! Error handler with explicit location
   void error(const Dynare::parser::location_type &l, const string &m);
-  //! Error handler without location
+  //! Error handler using saved location
   void error(const string &m);
-  //! Warning handler
+  //! Warning handler using saved location
   void warning(const string &m);
 
   //! Check if a given symbol exists in the parsing context, and is not a mod file local variable