From 7b629a6c16b5d7a64693158f5b268281beca1ae6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?S=C3=A9bastien=20Villemot?= <sebastien@dynare.org>
Date: Fri, 15 Nov 2024 17:18:00 +0100
Subject: [PATCH] =?UTF-8?q?=F0=9F=90=9B=20Flex=20scanner:=20fix=20column?=
=?UTF-8?q?=20location=20counter=20in=20the=20presence=20of=20dates?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
Dates handling inside Dynare statements was done in the Flex scanner using a
hack that would introduce some extra characters in the character stream (in
order to return a call to the “dates('…')†constructor to Bison). But this
would break the column location counter, because the latter would count
characters that do not exist in the user file.
This commit fixes it by implementing a more natural handling of dates, adding
the “dates('…')†constructor directly at the level of the semantic value.
---
src/DynareBison.yy | 5 +++--
src/DynareFlex.ll | 44 ++++++++------------------------------------
src/ParsingDriver.hh | 3 ---
3 files changed, 11 insertions(+), 41 deletions(-)
diff --git a/src/DynareBison.yy b/src/DynareBison.yy
index ac49f71e..a679f333 100644
--- a/src/DynareBison.yy
+++ b/src/DynareBison.yy
@@ -106,7 +106,7 @@ str_tolower(string s)
%token END ENDVAL EQUAL ESTIMATION ESTIMATED_PARAMS ESTIMATED_PARAMS_BOUNDS ESTIMATED_PARAMS_INIT EXTENDED_PATH ENDOGENOUS_PRIOR EXPRESSION
%token FILENAME DIRNAME FILTER_STEP_AHEAD FILTERED_VARS FIRST_OBS FIRST_SIMULATION_PERIOD LAST_SIMULATION_PERIOD LAST_OBS
%token SET_TIME OSR_PARAMS_BOUNDS KEEP_KALMAN_ALGO_IF_SINGULARITY_IS_DETECTED
-%token <string> FALSE FLOAT_NUMBER DATES
+%token <string> FALSE FLOAT_NUMBER DATE
%token DEFAULT FIXED_POINT FLIP OPT_ALGO COMPILATION_SETUP COMPILER ADD_FLAGS SUBSTITUTE_FLAGS ADD_LIBS SUBSTITUTE_LIBS
%token FORECAST K_ORDER_SOLVER INSTRUMENTS SHIFT MEAN STDEV VARIANCE MODE INTERVAL SHAPE DOMAINN
%token GAMMA_PDF GRAPH GRAPH_FORMAT CONDITIONAL_VARIANCE_DECOMPOSITION NOCHECK STD
@@ -2093,7 +2093,8 @@ prior_pdf : BETA_PDF
{ $$ = PriorDistributions::weibull; }
;
-date_expr : DATES
+date_expr : DATE
+ { $$ = "dates('" + $1 + "')"; }
| date_expr PLUS INT_NUMBER
{ $$ = $1 + '+' + $3; }
;
diff --git a/src/DynareFlex.ll b/src/DynareFlex.ll
index 244255b9..1c670266 100644
--- a/src/DynareFlex.ll
+++ b/src/DynareFlex.ll
@@ -60,7 +60,6 @@ string eofbuff;
%x VERBATIM_BLOCK
%x NATIVE
%x NATIVE_COMMENT
-%x DATES_STATEMENT
%x LINE1
%x LINE2
%x LINE3
@@ -92,12 +91,12 @@ DATE -?[0-9]+([ya]|m([1-9]|1[0-2])|q[1-4])
}
/* spaces, tabs and carriage returns are ignored */
-<INITIAL,DYNARE_STATEMENT,DYNARE_BLOCK,COMMENT,DATES_STATEMENT,LINE1,LINE2,LINE3>[[:space:]]+ { yylloc->step(); }
+<INITIAL,DYNARE_STATEMENT,DYNARE_BLOCK,COMMENT,LINE1,LINE2,LINE3>[[:space:]]+ { yylloc->step(); }
/* Comments */
-<INITIAL,DYNARE_STATEMENT,DYNARE_BLOCK,DATES_STATEMENT>%.*
-<INITIAL,DYNARE_STATEMENT,DYNARE_BLOCK,DATES_STATEMENT>"//".*
-<INITIAL,DYNARE_STATEMENT,DYNARE_BLOCK,DATES_STATEMENT>"/*" {comment_caller = YY_START; BEGIN COMMENT;}
+<INITIAL,DYNARE_STATEMENT,DYNARE_BLOCK>%.*
+<INITIAL,DYNARE_STATEMENT,DYNARE_BLOCK>"//".*
+<INITIAL,DYNARE_STATEMENT,DYNARE_BLOCK>"/*" {comment_caller = YY_START; BEGIN COMMENT;}
<COMMENT>"*/" {BEGIN comment_caller;}
<COMMENT>.
@@ -258,25 +257,9 @@ DATE -?[0-9]+([ya]|m([1-9]|1[0-2])|q[1-4])
/* Inside of a Dynare statement */
<DYNARE_STATEMENT>{DATE} {
- /* If a date is found within a statement, substitute it with a call to
- the dates() constructor in the input character stream. Then it will
- be handled by the rule that follows the present one. */
- char* yycopy = strdup(yytext);
- char* uput = yycopy + yyleng;
- unput(')');
- unput('\'');
- while (uput > yycopy)
- unput(*--uput);
- unput('\'');
- unput('(');
- unput('s');
- unput('e');
- unput('t');
- unput('a');
- unput('d');
- free( yycopy );
- }
-<DYNARE_STATEMENT>dates {dates_parens_nb=0; BEGIN DATES_STATEMENT; yylval->build<string>("dates");}
+ yylval->emplace<string>(yytext);
+ return token::DATE;
+}
<DYNARE_STATEMENT>file {return token::FILE;}
<DYNARE_STATEMENT>datafile {return token::DATAFILE;}
<DYNARE_STATEMENT>dirname {return token::DIRNAME;}
@@ -1080,17 +1063,6 @@ DATE -?[0-9]+([ya]|m([1-9]|1[0-2])|q[1-4])
return token::INT_NUMBER;
}
-<DATES_STATEMENT>\( { yylval->as<string>().append(yytext); dates_parens_nb++; }
-<DATES_STATEMENT>\) {
- yylval->as<string>().append(yytext);
- if (--dates_parens_nb == 0)
- {
- BEGIN DYNARE_STATEMENT;
- return token::DATES;
- }
- }
-<DATES_STATEMENT>. { yylval->as<string>().append(yytext); }
-
<DYNARE_BLOCK>\|e { return token::PIPE_E; }
<DYNARE_BLOCK>\|x { return token::PIPE_X; }
<DYNARE_BLOCK>\|p { return token::PIPE_P; }
@@ -1222,7 +1194,7 @@ DATE -?[0-9]+([ya]|m([1-9]|1[0-2])|q[1-4])
<NATIVE_COMMENT>"*/"[[:space:]]*\n { BEGIN NATIVE; }
<NATIVE_COMMENT>.
-<INITIAL,DYNARE_STATEMENT,DYNARE_BLOCK,COMMENT,DATES_STATEMENT,LINE1,LINE2,LINE3,NATIVE_COMMENT><<EOF>> { yyterminate(); }
+<INITIAL,DYNARE_STATEMENT,DYNARE_BLOCK,COMMENT,LINE1,LINE2,LINE3,NATIVE_COMMENT><<EOF>> { yyterminate(); }
<*>. { driver.error(*yylloc, "character unrecognized by lexer"); }
%%
diff --git a/src/ParsingDriver.hh b/src/ParsingDriver.hh
index d1b7a1b1..ffb2f4cb 100644
--- a/src/ParsingDriver.hh
+++ b/src/ParsingDriver.hh
@@ -73,9 +73,6 @@ public:
//! Increment the location counter given a token
static void location_increment(Dynare::parser::location_type* yylloc, const char* yytext);
-
- //! Count parens in dates statement
- int dates_parens_nb;
};
//! Drives the scanning and parsing of the .mod file, and constructs its abstract representation
--
GitLab