From f2cc9d389954f16dbb88c3401e90c7981e3599d6 Mon Sep 17 00:00:00 2001
From: Houtan Bastani <houtan@dynare.org>
Date: Mon, 17 Aug 2015 17:08:33 +0200
Subject: [PATCH] preprocessor: add utf-8 tokenizing
---
preprocessor/DynareFlex.ll | 6 +++---
preprocessor/macro/MacroFlex.ll | 2 +-
2 files changed, 4 insertions(+), 4 deletions(-)
diff --git a/preprocessor/DynareFlex.ll b/preprocessor/DynareFlex.ll
index 5948febc00..307adf59d0 100644
--- a/preprocessor/DynareFlex.ll
+++ b/preprocessor/DynareFlex.ll
@@ -777,7 +777,7 @@ DATE -?[0-9]+([YyAa]|[Mm]([1-9]|1[0-2])|[Qq][1-4]|[Ww]([1-9]{1}|[1-4][0-9]|5[0-2
<DYNARE_STATEMENT>max_dim_cova_group {return token::MAX_DIM_COVA_GROUP;}
<DYNARE_STATEMENT>gsa_sample_file {return token::GSA_SAMPLE_FILE;}
-<DYNARE_STATEMENT,DYNARE_BLOCK>[A-Za-z_][A-Za-z0-9_]* {
+<DYNARE_STATEMENT,DYNARE_BLOCK>[A-Za-z_\x80-\xf3][A-Za-z0-9_\x80-\xf3]* {
yylval->string_val = new string(yytext);
return token::NAME;
}
@@ -839,7 +839,7 @@ DATE -?[0-9]+([YyAa]|[Mm]([1-9]|1[0-2])|[Qq][1-4]|[Ww]([1-9]{1}|[1-4][0-9]|5[0-2
element in initval (in which case Dynare recognizes the matrix name as an external
function symbol), and may want to modify the matrix later with Matlab statements.
*/
-<INITIAL>[A-Za-z_][A-Za-z0-9_]* {
+<INITIAL>[A-Za-z_\x80-\xf3][A-Za-z0-9_\x80-\xf3]* {
if (driver.symbol_exists_and_is_not_modfile_local_or_external_function(yytext))
{
BEGIN DYNARE_STATEMENT;
@@ -857,7 +857,7 @@ DATE -?[0-9]+([YyAa]|[Mm]([1-9]|1[0-2])|[Qq][1-4]|[Ww]([1-9]{1}|[1-4][0-9]|5[0-2
/* For joint prior statement, match [symbol, symbol, ...]
If no match, begin native and push everything back on stack
*/
-<INITIAL>\[([[:space:]]*[A-Za-z_][A-Za-z0-9_]*[[:space:]]*,{1}[[:space:]]*)*([[:space:]]*[A-Za-z_][A-Za-z0-9_]*[[:space:]]*){1}\] {
+<INITIAL>\[([[:space:]]*[A-Za-z_\x80-\xf3][A-Za-z0-9_\x80-\xf3]*[[:space:]]*,{1}[[:space:]]*)*([[:space:]]*[A-Za-z_\x80-\xf3][A-Za-z0-9_\x80-\xf3]*[[:space:]]*){1}\] {
string yytextcpy = string(yytext);
yytextcpy.erase(remove(yytextcpy.begin(), yytextcpy.end(), '['), yytextcpy.end());
yytextcpy.erase(remove(yytextcpy.begin(), yytextcpy.end(), ']'), yytextcpy.end());
diff --git a/preprocessor/macro/MacroFlex.ll b/preprocessor/macro/MacroFlex.ll
index 0b5ff7a21e..772a8fdabb 100644
--- a/preprocessor/macro/MacroFlex.ll
+++ b/preprocessor/macro/MacroFlex.ll
@@ -201,7 +201,7 @@ CONT \\\\
<STMT>echo { return token::ECHO_DIR; }
<STMT>error { return token::ERROR; }
-<STMT,EXPR>[A-Za-z_][A-Za-z0-9_]* {
+<STMT,EXPR>[A-Za-z_\x80-\xf3][A-Za-z0-9_\x80-\xf3]* {
yylval->string_val = new string(yytext);
return token::NAME;
}
--
GitLab