From 67b8ef1a1937e8434bf252a616eeda2e3d7e5c4b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?S=C3=A9bastien=20Villemot?= <sebastien@dynare.org>
Date: Fri, 19 Nov 2021 14:13:30 +0100
Subject: [PATCH] Macroprocessor: fix line counter inside @{} constructs
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
There were two bugs:
– with the LF convention, newlines were counted twice
– with the CR+LF convention, they were counted four times (because the CR was
included in yyleng, alongside the LF)
The fix consists in implementing a location_increment() method similar to the
one used for the Dynare parser. This is the most robust solution.
By the way, mark the method DynareFlex::location_increment() method static.
(cherry picked from commit 1de3476546da22eb6bc11bfa23d78b3576f83217)
---
src/ParsingDriver.hh | 2 +-
src/macro/Driver.hh | 3 ++-
src/macro/Tokenizer.ll | 27 +++++++++++++++------------
3 files changed, 18 insertions(+), 14 deletions(-)
diff --git a/src/ParsingDriver.hh b/src/ParsingDriver.hh
index 0c21db3b..a4aee28b 100644
--- a/src/ParsingDriver.hh
+++ b/src/ParsingDriver.hh
@@ -75,7 +75,7 @@ public:
string filename;
//! Increment the location counter given a token
- void location_increment(Dynare::parser::location_type *yylloc, const char *yytext);
+ static void location_increment(Dynare::parser::location_type *yylloc, const char *yytext);
//! Count parens in dates statement
int dates_parens_nb;
diff --git a/src/macro/Driver.hh b/src/macro/Driver.hh
index 5f1367f3..4d600a80 100644
--- a/src/macro/Driver.hh
+++ b/src/macro/Driver.hh
@@ -1,5 +1,5 @@
/*
- * Copyright © 2019-2020 Dynare Team
+ * Copyright © 2019-2021 Dynare Team
*
* This file is part of Dynare.
*
@@ -56,6 +56,7 @@ namespace macro
Tokenizer::parser::token_type lex(Tokenizer::parser::semantic_type *yylval,
Tokenizer::parser::location_type *yylloc,
macro::Driver &driver);
+ static void location_increment(Tokenizer::parser::location_type *yylloc, const char *yytext);
};
//! Implements the macro expansion using a Flex scanner and a Bison parser
diff --git a/src/macro/Tokenizer.ll b/src/macro/Tokenizer.ll
index c72561e4..c930cc54 100644
--- a/src/macro/Tokenizer.ll
+++ b/src/macro/Tokenizer.ll
@@ -50,7 +50,7 @@ using token = Tokenizer::parser::token;
%{
// Increments location counter for every token read
- # define YY_USER_ACTION yylloc->columns(yyleng);
+ #define YY_USER_ACTION location_increment(yylloc, yytext);
%}
SPC [ \t]+
@@ -175,22 +175,15 @@ CONT \\\\{SPC}*
}
<expr,eval>{SPC}+ { }
-<eval>{EOL}+ { yylloc->lines(yyleng); yylloc->lines(yyleng); }
+<eval>{EOL}+ { }
<eval>\} { BEGIN(INITIAL); return token::END_EVAL; }
-<expr,end_line>{CONT}("//".*)?{SPC}*{EOL} { yylloc->lines(1); yylloc->step(); }
-<expr,end_line>{SPC}*("//".*)?{EOL} {
- yylloc->lines(1);
- BEGIN(INITIAL);
- return token::EOL;
- }
+<expr,end_line>{CONT}("//".*)?{SPC}*{EOL} { yylloc->step(); }
+<expr,end_line>{SPC}*("//".*)?{EOL} { BEGIN(INITIAL); return token::EOL; }
<INITIAL>^{SPC}*@#{SPC}* { BEGIN(directive); }
<INITIAL>@\{ { BEGIN(eval); return token::BEGIN_EVAL; }
-<INITIAL>{EOL} {
- yylloc->lines(1);
- return token::EOL;
- }
+<INITIAL>{EOL} { return token::EOL; }
<INITIAL><<EOF>> { yyterminate(); }
<directive,expr,eval,end_line><<EOF>> { driver.error(*yylloc, "unexpected end of file"); }
@@ -200,6 +193,16 @@ CONT \\\\{SPC}*
%%
+void
+TokenizerFlex::location_increment(Tokenizer::parser::location_type *yylloc, const char *yytext)
+{
+ while (*yytext != 0)
+ if (*yytext++ == '\n')
+ yylloc->lines(1);
+ else
+ yylloc->columns(1);
+}
+
/* This implementation of TokenizerFlexLexer::yylex() is required to fill the
* vtable of the class TokenizerFlexLexer. We define the scanner's main yylex
* function via YY_DECL to reside in the TokenizerFlex class instead. */
--
GitLab