macro processor 2.0

.gitignore

@@ -33,9 +33,9 @@ TAGS
 /src/location.hh
 /src/position.hh
 /src/stack.hh
-/src/macro/MacroBison.cc
-/src/macro/MacroBison.hh
-/src/macro/MacroFlex.cc
+/src/macro/Parser.cc
+/src/macro/Parser.hh
+/src/macro/Tokenizer.cc
 /src/macro/location.hh
 /src/macro/position.hh
 /src/macro/stack.hh

src/DynareMain1.cc

@@ -20,7 +20,7 @@
 #include <sstream>
 #include <fstream>
 
-#include "macro/MacroDriver.hh"
+#include "macro/Driver.hh"
 
 bool compareNewline (int i, int j) {
   return i == '\n' && j == '\n';
@@ -28,12 +28,13 @@ bool compareNewline (int i, int j) {
 
 void
 main1(const string &filename, const string &basename, istream &modfile, bool debug, bool save_macro, string &save_macro_file,
-      bool no_line_macro, bool no_empty_line_macro, const vector<pair<string, string>> &defines, const vector<string> &path, stringstream &macro_output)
+      bool no_line_macro_arg, bool no_empty_line_macro, const vector<pair<string, string>> &defines, const vector<string> &path,
+      stringstream &macro_output)
 {
   // Do macro processing
-  MacroDriver m;
-
-  m.parse(filename, basename, modfile, macro_output, debug, no_line_macro, defines, path);
+  macro::Environment env = macro::Environment();
+  macro::Driver m(env, no_line_macro_arg);
+  m.parse(filename, basename, modfile, macro_output, debug, defines, path);
   if (save_macro)
     {
       if (save_macro_file.empty())

src/ParsingDriver.hh

@@ -21,7 +21,7 @@
 #define _PARSING_DRIVER_HH
 
 #ifdef _MACRO_DRIVER_HH
-# error Impossible to include both ParsingDriver.hh and MacroDriver.hh
+# error Impossible to include both ParsingDriver.hh and macro/Driver.hh
 #endif
 
 #include <string>

src/macro/Directives.cc

+/*
+ * Copyright © 2019 Dynare Team
+ *
+ * This file is part of Dynare.
+ *
+ * Dynare is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * Dynare is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with Dynare.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "Directives.hh"
+
+using namespace macro;
+
+void
+Eval::interpret(ostream &output, bool no_line_macro)
+{
+  try
+    {
+      output << expr->eval()->to_string();
+    }
+  catch (StackTrace &ex)
+    {
+      ex.push("Evaluation", location);
+      error(ex);
+    }
+  catch (exception &e)
+    {
+      error(StackTrace("Evaluation", e.what(), location));
+    }
+}
+
+void
+Include::interpret(ostream &output, bool no_line_macro)
+{
+  try
+    {
+      StringPtr msp = dynamic_pointer_cast<String>(expr->eval());
+      if (!msp)
+        throw StackTrace("File name does not evaluate to a string");
+      name = *msp;
+    }
+  catch (StackTrace &ex)
+    {
+      ex.push("@#include", location);
+      error(ex);
+    }
+  catch (exception &e)
+    {
+      error(StackTrace("@#include", e.what(), location));
+    }
+}
+
+void
+IncludePath::interpret(ostream &output, bool no_line_macro)
+{
+  try
+    {
+      StringPtr msp = dynamic_pointer_cast<String>(expr->eval());
+      if (!msp)
+        throw StackTrace("File name does not evaluate to a string");
+      path = *msp;
+    }
+  catch (StackTrace &ex)
+    {
+      ex.push("@#includepath", location);
+      error(ex);
+    }
+  catch (exception &e)
+    {
+      error(StackTrace("@#includepath", e.what(), location));
+    }
+}
+
+void
+Define::interpret(ostream &output, bool no_line_macro)
+{
+  try
+    {
+      if (var)
+        env.define(var, value);
+      else if (func)
+        env.define(func, value);
+      else
+        throw StackTrace("LHS of can be either a variable or a function");
+    }
+  catch (StackTrace &ex)
+    {
+      ex.push("@#define", location);
+      error(ex);
+    }
+  catch (exception &e)
+    {
+      error(StackTrace("@#define", e.what(), location));
+    }
+}
+
+void
+Echo::interpret(ostream &output, bool no_line_macro)
+{
+  try
+    {
+      cout << "@#echo (" << getLocation() << "): " << expr->eval()->to_string() << endl;
+    }
+  catch (StackTrace &ex)
+    {
+      ex.push("@#echo", location);
+      error(ex);
+    }
+  catch (exception &e)
+    {
+      error(StackTrace("@#echo", e.what(), location));
+    }
+  printLineInfo(output, no_line_macro);
+}
+
+void
+Error::interpret(ostream &output, bool no_line_macro)
+{
+  try
+    {
+      throw StackTrace(expr->eval()->to_string());
+    }
+  catch (StackTrace &ex)
+    {
+      ex.push("@#error", location);
+      error(ex);
+    }
+  catch (exception &e)
+    {
+      error(StackTrace("@#error", e.what(), location));
+    }
+}
+
+void
+For::interpret(ostream &output, bool no_line_macro)
+{
+  ArrayPtr ap;
+  try
+    {
+      ap = dynamic_pointer_cast<Array>(index_vals->eval());
+      if (!ap)
+        throw StackTrace("The index must loop through an array");
+    }
+  catch (StackTrace &ex)
+    {
+      ex.push("@#for", location);
+      error(ex);
+    }
+  catch (exception &e)
+    {
+      error(StackTrace("@#for", e.what(), location));
+    }
+
+  for (size_t i = 0; i < ap->size(); i++)
+    {
+      if (index_vec.size() == 1)
+        env.define(index_vec.at(0), ap->at(i));
+      else
+        {
+          BaseTypePtr btp = dynamic_pointer_cast<BaseType>(ap->at(i));
+          if (!btp)
+            error(StackTrace("@#for", "Unexpected error encountered in for loop", location));
+
+          if (btp->getType() == codes::BaseType::Tuple)
+            {
+              TuplePtr mtp = dynamic_pointer_cast<Tuple>(btp);
+              if (index_vec.size() != mtp->size())
+                error(StackTrace("@#for", "Encountered tuple of size " + to_string(mtp->size())
+                                 + " but only have " + to_string(index_vec.size()) + " index variables", location));
+              else
+                for (size_t j = 0; j < index_vec.size(); j++)
+                  env.define(index_vec.at(j), mtp->at(j));
+            }
+        }
+
+      bool printLine = true;
+      for (auto & statement : statements)
+        {
+          if (printLine)
+            {
+              statement->printLineInfo(output, no_line_macro);
+              printLine = false;
+            }
+          statement->interpret(output, no_line_macro);
+        }
+    }
+  printEndLineInfo(output, no_line_macro);
+}
+
+void
+If::interpret(ostream &output, bool no_line_macro)
+{
+  DoublePtr dp;
+  BoolPtr bp;
+  try
+    {
+      dp = dynamic_pointer_cast<Double>(condition->eval());
+      bp = dynamic_pointer_cast<Bool>(condition->eval());
+      if (!bp && !dp)
+        error(StackTrace("@#if", "The condition must evaluate to a boolean or a double", location));
+    }
+  catch (StackTrace &ex)
+    {
+      ex.push("@#if", location);
+      error(ex);
+    }
+  catch (exception &e)
+    {
+      error(StackTrace("@#if", e.what(), location));
+    }
+
+  if ((bp && *bp) || (dp && *dp))
+    loopIf(output, no_line_macro);
+  else
+    loopElse(output, no_line_macro);
+  printEndLineInfo(output, no_line_macro);
+}
+
+void
+If::loopIf(ostream &output, bool no_line_macro)
+{
+  bool printLine = !no_line_macro;
+  for (auto & statement : if_statements)
+    {
+      if (printLine)
+        {
+          statement->printLineInfo(output, no_line_macro);
+          printLine = false;
+        }
+      statement->interpret(output, no_line_macro);
+    }
+}
+
+void
+If::loopElse(ostream &output, bool no_line_macro)
+{
+  bool printLine = !no_line_macro;
+  for (auto & statement : else_statements)
+    {
+      if (printLine)
+        {
+          statement->printLineInfo(output, no_line_macro);
+          printLine = false;
+        }
+      statement->interpret(output, no_line_macro);
+    }
+}
+
+void
+Ifdef::interpret(ostream &output, bool no_line_macro)
+{
+  BaseTypePtr btnp = dynamic_pointer_cast<BaseType>(condition);
+  VariablePtr vp = dynamic_pointer_cast<Variable>(condition);
+  if (btnp || (vp && env.isVariableDefined(vp->getName())))
+    loopIf(output, no_line_macro);
+  else
+    loopElse(output, no_line_macro);
+  printEndLineInfo(output, no_line_macro);
+}
+
+void
+Ifndef::interpret(ostream &output, bool no_line_macro)
+{
+  BaseTypePtr btnp = dynamic_pointer_cast<BaseType>(condition);
+  VariablePtr vp = dynamic_pointer_cast<Variable>(condition);
+  if (!(btnp || (vp && env.isVariableDefined(vp->getName()))))
+    loopIf(output, no_line_macro);
+  else
+    loopElse(output, no_line_macro);
+  printEndLineInfo(output, no_line_macro);
+}
+

src/macro/Directives.hh

+/*
+ * Copyright (C) 2019 Dynare Team
+ *
+ * This file is part of Dynare.
+ *
+ * Dynare is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * Dynare is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with Dynare.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef _DIRECTIVES_HH
+#define _DIRECTIVES_HH
+
+#include "Expressions.hh"
+
+using namespace std;
+
+namespace macro
+{
+  class Directive : public Node
+  {
+    // A Parent class just for clarity
+  public:
+    Directive(Environment &env_arg, const Tokenizer::location location_arg) : Node(env_arg, move(location_arg)) { }
+    // Directives can be interpreted
+    virtual void interpret(ostream &output, bool no_line_macro) = 0;
+  };
+
+  using DirectivePtr = shared_ptr<Directive>;
+
+
+  class TextNode : public Directive
+  {
+    // Class for text not interpreted by macroprocessor
+    // Not a real directive node
+    // Treated as such as the output is only to be interpreted
+  private:
+    const string text;
+  public:
+    TextNode(const string text_arg, Environment &env_arg, const Tokenizer::location location_arg) :
+      Directive(env_arg, move(location_arg)), text{move(text_arg)} { }
+    inline void interpret(ostream &output, bool no_line_macro) override { output << text; }
+  };
+
+
+  class Eval : public Directive
+  {
+    // Class for @{} statements
+    // Not a real directive node
+    // Treated as such as the output is only to be interpreted
+  private:
+    const ExpressionPtr expr;
+  public:
+    Eval(const ExpressionPtr expr_arg, Environment &env_arg, const Tokenizer::location location_arg) :
+      Directive(env_arg, move(location_arg)), expr{move(expr_arg)} { }
+    void interpret(ostream &output, bool no_line_macro) override;
+  };
+
+
+  class Include : public Directive
+  {
+  private:
+    const ExpressionPtr expr;
+    string name;
+  public:
+    Include(const ExpressionPtr expr_arg, Environment &env_arg, const Tokenizer::location location_arg) :
+      Directive(env_arg, move(location_arg)), expr{move(expr_arg)} { }
+    void interpret(ostream &output, bool no_line_macro) override;
+    inline string getName() const { return name; }
+  };
+
+
+  class IncludePath : public Directive
+  {
+  private:
+    const ExpressionPtr expr;
+    string path;
+  public:
+    IncludePath(const ExpressionPtr expr_arg, Environment &env_arg, const Tokenizer::location location_arg) :
+      Directive(env_arg, move(location_arg)), expr{move(expr_arg)} { }
+    void interpret(ostream &output, bool no_line_macro) override;
+    inline string getPath() const { return path; }
+  };
+
+
+  class Define : public Directive
+  {
+  private:
+    const VariablePtr var;
+    const FunctionPtr func;
+    const ExpressionPtr value;
+  public:
+    Define(const VariablePtr var_arg,
+           const ExpressionPtr value_arg,
+           Environment &env_arg, const Tokenizer::location location_arg) :
+      Directive(env_arg, move(location_arg)), var{move(var_arg)}, value{move(value_arg)} { }
+    Define(const FunctionPtr func_arg,
+           const ExpressionPtr value_arg,
+           Environment &env_arg, const Tokenizer::location location_arg) :
+      Directive(env_arg, move(location_arg)), func{move(func_arg)}, value{move(value_arg)} { }
+    void interpret(ostream &output, bool no_line_macro) override;
+  };
+
+
+  class Echo : public Directive
+  {
+  private:
+    const ExpressionPtr expr;
+  public:
+    Echo(const ExpressionPtr expr_arg,
+         Environment &env_arg, const Tokenizer::location location_arg) :
+      Directive(env_arg, move(location_arg)), expr{move(expr_arg)} { }
+    void interpret(ostream &output, bool no_line_macro) override;
+  };
+
+
+  class Error : public Directive
+  {
+  private:
+    const ExpressionPtr expr;
+  public:
+    Error(const ExpressionPtr expr_arg,
+          Environment &env_arg, const Tokenizer::location location_arg) :
+      Directive(env_arg, move(location_arg)), expr{move(expr_arg)} { }
+    void interpret(ostream &output, bool no_line_macro) override;
+  };
+
+
+  class EchoMacroVars : public Directive
+  {
+  public:
+    EchoMacroVars(Environment &env_arg, const Tokenizer::location location_arg) : Directive(env_arg, move(location_arg)) { }
+    inline void interpret(ostream &output, bool no_line_macro) override { env.print(); }
+  };
+
+
+  class For : public Directive
+  {
+  private:
+    const vector<VariablePtr> index_vec;
+    const ExpressionPtr index_vals;
+    vector<DirectivePtr> statements;
+  public:
+    For(const vector<VariablePtr> index_vec_arg,
+        const ExpressionPtr index_vals_arg,
+        const vector<DirectivePtr> statements_arg,
+        Environment &env_arg, const Tokenizer::location location_arg) :
+      Directive(env_arg, move(location_arg)), index_vec{move(index_vec_arg)},
+      index_vals{move(index_vals_arg)}, statements{statements_arg} { }
+    void interpret(ostream &output, bool no_line_macro) override;
+  };
+
+
+  class If : public Directive
+  {
+  protected:
+    const ExpressionPtr condition;
+    vector<DirectivePtr> if_statements;
+    vector<DirectivePtr> else_statements;
+  public:
+    If(const ExpressionPtr condition_arg,
+       const vector<DirectivePtr> if_statements_arg,
+       Environment &env_arg, const Tokenizer::location location_arg) :
+      Directive(env_arg, move(location_arg)), condition{move(condition_arg)}, if_statements{if_statements_arg} { }
+    If(const ExpressionPtr condition_arg,
+       const vector<DirectivePtr> if_statements_arg,
+       const vector<DirectivePtr> else_statements_arg,
+       Environment &env_arg, const Tokenizer::location location_arg) :
+      Directive(env_arg, move(location_arg)),
+      condition{move(condition_arg)}, if_statements{if_statements_arg}, else_statements{else_statements_arg} { }
+    void interpret(ostream &output, bool no_line_macro) override;
+  protected:
+    void loopIf(ostream &output, bool no_line_macro);
+    void loopElse(ostream &output, bool no_line_macro);
+  };
+
+
+  class Ifdef : public If
+  {
+  public:
+    Ifdef(const ExpressionPtr condition_arg,
+          const vector<DirectivePtr> if_statements_arg,
+          Environment &env_arg, const Tokenizer::location location_arg) :
+      If(move(condition_arg), move(if_statements_arg), env_arg, move(location_arg)) { }
+    Ifdef(const ExpressionPtr condition_arg,
+          const vector<DirectivePtr> if_statements_arg,
+          const vector<DirectivePtr> else_statements_arg,
+          Environment &env_arg, const Tokenizer::location location_arg) :
+      If(move(condition_arg), move(if_statements_arg), move(else_statements_arg), env_arg, move(location_arg)) { }
+    void interpret(ostream &output, bool no_line_macro) override;
+  };
+
+
+  class Ifndef : public If
+  {
+  public:
+    Ifndef(const ExpressionPtr condition_arg,
+           const vector<DirectivePtr> if_statements_arg,
+           Environment &env_arg, const Tokenizer::location location_arg) :
+      If(move(condition_arg), move(if_statements_arg), env_arg, move(location_arg)) { }
+    Ifndef(const ExpressionPtr condition_arg,
+           const vector<DirectivePtr> if_statements_arg,
+           const vector<DirectivePtr> else_statements_arg,
+           Environment &env_arg, const Tokenizer::location location_arg) :
+      If(move(condition_arg), move(if_statements_arg), move(else_statements_arg), env_arg, move(location_arg)) { }
+    void interpret(ostream &output, bool no_line_macro) override;
+  };
+}
+#endif

src/macro/Driver.cc

+/*
+ * Copyright © 2019 Dynare Team
+ *
+ * This file is part of Dynare.
+ *
+ * Dynare is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * Dynare is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with Dynare.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "Driver.hh"
+
+#include <fstream>
+#include <sstream>
+
+using namespace macro;
+
+void
+Driver::parse(const string &file_arg, const string &basename_arg, istream &modfile,
+              ostream &output, bool debug, const vector<pair<string, string>> &defines,
+              vector<string> paths_arg)
+{
+#ifdef _WIN32
+  string FILESEP = "\\";
+#else
+  string FILESEP = "/";
+#endif
+
+  file = file_arg;
+  basename = basename_arg;
+  paths = move(paths_arg);
+
+  if (!defines.empty())
+    {
+      stringstream command_line_defines_with_endl;
+      for (auto & define : defines)
+        try
+          {
+            stoi(define.second);
+            command_line_defines_with_endl << "@#define " << define.first << " = " << define.second << endl;
+          }
+        catch (const invalid_argument &)
+          {
+            if (!define.second.empty() && define.second.at(0) == '[' && define.second.at(define.second.length()-1) == ']')
+              // If the input is an array. Issue #1578
+              command_line_defines_with_endl << "@#define " << define.first << " = " << define.second << endl;
+            else
+              command_line_defines_with_endl << "@#define " << define.first << " = \"" << define.second << "\"" << endl;
+          }
+      Driver m(env, true);
+      istream is(command_line_defines_with_endl.rdbuf());
+      m.parse("command_line_defines", "command_line_defines", is, output, debug, vector<pair<string, string>>{}, paths);
+    }
+  stringstream file_with_endl;
+  file_with_endl << modfile.rdbuf() << endl;
+
+  lexer = make_unique<TokenizerFlex>(&file_with_endl);
+  lexer->set_debug(debug);
+
+  Tokenizer::parser parser(*this);
+  parser.set_debug_level(debug);
+
+  // Launch macro-processing
+  parser.parse();
+
+  // Interpret parsed statements
+  bool printLine = true;
+  for (auto & statement : statements)
+    {
+      if (printLine)
+        {
+          statement->printLineInfo(output, no_line_macro);
+          printLine = false;
+        }
+      statement->interpret(output, no_line_macro);
+
+      auto ipp = dynamic_pointer_cast<IncludePath>(statement);
+      if (ipp)
+        paths.emplace_back(ipp->getPath());
+
+      auto ip = dynamic_pointer_cast<Include>(statement);
+      if (ip)
+        {
+          string filename = ip->getName();
+          ifstream incfile(filename, ios::binary);
+          if (incfile.fail())
+            {
+              ostringstream dirs;
+              dirs << "." << FILESEP << endl;
+              for (const auto & path : paths)
+                {
+                  string testfile = path + FILESEP + filename;
+                  incfile = ifstream(testfile, ios::binary);
+                  if (incfile.good())
+                    break;
+                  dirs << path << endl;
+                }
+              if (incfile.fail())
+                error(statement->getLocation(), "Could not open " + filename +
+                      ". The following directories were searched:\n" + dirs.str());
+            }
+
+          string basename = filename;
+          size_t pos = basename.find_last_of('.');
+          if (pos != string::npos)
+            basename.erase(pos);
+
+          Driver m(env, no_line_macro);
+          m.parse(filename, basename, incfile, output, debug, vector<pair<string, string>>{}, paths);
+        }
+    }
+}
+
+void
+Driver::error(const Tokenizer::parser::location_type &location, const string &message) const
+{
+  cerr << "ERROR in macro-processor: " << location << ": " << message << endl;
+  exit(EXIT_FAILURE);
+}

src/macro/Driver.hh

+/*
+ * Copyright © 2019 Dynare Team
+ *
+ * This file is part of Dynare.
+ *
+ * Dynare is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * Dynare is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with Dynare.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef _MACRO_DRIVER_HH
+#define _MACRO_DRIVER_HH
+
+#ifdef _PARSING_DRIVER_HH
+# error Impossible to include both ../ParsingDriver.hh and Driver.hh
+#endif
+
+#include "Parser.hh"
+#include "Environment.hh"
+#include "Expressions.hh"
+
+#include <stack>
+
+using namespace std;
+
+// Declare TokenizerFlexLexer class
+#ifndef __FLEX_LEXER_H
+# define yyFlexLexer TokenizerFlexLexer
+# include <FlexLexer.h>
+# undef yyFlexLexer
+#endif
+
+namespace macro
+{
+  /* The lexer class
+   * It was necessary to subclass the TokenizerFlexLexer class generated by Flex,