diff git a/doc/preprocessor/Makefile.am b/doc/preprocessor/Makefile.am
index afdbb73a5c594673e4e03f1b731d9ddd521811e6..65ee1437338e28a11d6078fef82f0af87d98ea5d 100644
 a/doc/preprocessor/Makefile.am
+++ b/doc/preprocessor/Makefile.am
@@ 4,7 +4,7 @@ pdflocal: preprocessor.pdf
endif
endif
SRC = preprocessor.tex expr.png exprsharing.png matrices.png overview.png
+SRC = preprocessor.tex expr.png exprsharing.png matrices.png overview.png jsonpreprocessor.png
EXTRA_DIST = $(SRC)
diff git a/doc/preprocessor/exprsharing.dia b/doc/preprocessor/exprsharing.dia
deleted file mode 100644
index 93da37ab9f9b2ffbde15c09502764159c2fb85eb..0000000000000000000000000000000000000000
Binary files a/doc/preprocessor/exprsharing.dia and /dev/null differ
diff git a/doc/preprocessor/exprsharing.png b/doc/preprocessor/exprsharing.png
index d2d2d9002f66bc6f7c7cccdf77788d493fa1fab8..10661b38bb6c7918ffdf8dcedf32a979a9bdabdf 100644
Binary files a/doc/preprocessor/exprsharing.png and b/doc/preprocessor/exprsharing.png differ
diff git a/doc/preprocessor/exprsharing.xml b/doc/preprocessor/exprsharing.xml
new file mode 100644
index 0000000000000000000000000000000000000000..4cd581ec09036d9c913a679b0a07a8fa49e490c9
 /dev/null
+++ b/doc/preprocessor/exprsharing.xml
@@ 0,0 +1 @@
+7VlJc5swFP41HNsB4QUfE2fpoZ3JTA5NjirIoEZGVMg2zq+vhMSOWhI7uPWYgy19T+v3Fp6E5S7X2T2DSfSNBohYwA4yy72xAJiDqfiVwF4BM9tVQMhwoCCnAh7xK9KgrdENDlDaaMgpJRwnTdCncYx83sAgY3TXbLaipDlrAkPUAR59SLrodxzwSKHe1K7wLwiHUTGzY2vJD+i/hIxuYj2fBdxV/ijxGhZj6fZpBAO6q0HureUuGaVcldbZEhFJbUGb6ndnkJbrZijmQzp4qsMWkg0qVjwjout1gLeSdcw1J7NfG7mo6wdJHbDdRQWJUqj/q67vgRKG2tCKio0Invi+tQ4p+JTmlnMl12MnmXlFaQLjErsFludZi3khFOw05BWsJu/APcsUaM9+DkFBY9dAqE0Yv6hc7yLM0WMCfSnZCfcTWMTXRNQcUYRpojxihTMUyL1zRl/QkhLK8qFcO38khZiQGn6XPwIPGQywMKBCFtMYlWuq25Y2ty1iHGU1SNvaPaJrxNleNNFSd67tXseFia7uKicrfCyq+VeBQe3WYTlwZdqioK3b4BqzLqeBcHpd1XvMXVfSdiMJ8uka+7pcoxhlmD9pWJafCzwWa3qqV551q9LNZeUn4nyvYx7ccCogynhEQxpD8pXSRI9gVpxREyndMF/vbqHjJmQhKuOUwuTG/6guhgjkeNuMhgexPz86+46JfftfZR+civ2FMcofGl7N0fIs4xfwmvHLcUYMYN7xA9jnqdGJlOjUblTw2/Aj92RRzDY60gGpC5j0+Rah4RszkbP0uMnshB7njKdu56JsEV5BS9mLEZUNxlP260XZ4iwwOaGy3Y9Qdn+StL8ouyeMj6nsSY+y2yzHwZW8NxI1n8A0lVlTndxG7mMm9u0nw0FZkNdNghYDc6CB/A5OlfQMDxTnPmEI3K7T0pvaje5Vv49qD9S6IACz1kCKg85AuQ2U2x5mFtPRzWL4kfW9ZlHwfrGLd9tFyfTZXFW0jOHvh66ey4tFvxkd/YjV54T/9zXdMdivfRv44BOu+aro6FlwIqg+j9Sokwf1KNmYGpWp0Ph34kV0HkPb4KJr+Xoe7TZDVKvPiOo1WH2qdW9/Aw==
\ No newline at end of file
diff git a/doc/preprocessor/expr.dia b/doc/preprocessor/expr.dia
deleted file mode 100644
index f99a39825f2ad22c0f789696da718c0ad6a55d5c..0000000000000000000000000000000000000000
Binary files a/doc/preprocessor/expr.dia and /dev/null differ
diff git a/doc/preprocessor/expr.png b/doc/preprocessor/expr.png
index 5cab996b25e46e90a2edb6aaea66b0519da0b649..8167c05d331db3ffe5804dc80fc0bea46200bbe2 100644
Binary files a/doc/preprocessor/expr.png and b/doc/preprocessor/expr.png differ
diff git a/doc/preprocessor/expr.xml b/doc/preprocessor/expr.xml
new file mode 100644
index 0000000000000000000000000000000000000000..c29edf636e80a4f8ee1140227b4fb36e1faa9754
 /dev/null
+++ b/doc/preprocessor/expr.xml
@@ 0,0 +1 @@
+7VhLc5swEP41HNsBKXbwMXGS9tDOZCaHJkcVZFAjIypkG+fXd4XEGzJOnJI2Yw629K0eu98+EHLwcp1/kSSNv4uQcge5Ye7gKwehczSDXw3sDTB3sQEiyUIDeTVwx56oBV2LblhIs9ZAJQRXLG2DgUgSGqgWRqQUu/awleDtXVMS0R5wFxDeR3+wUMUG9WdujX+lLIrLnT3XSn6S4DGSYpPY/RyEV8VjxGtSrmXHZzEJxa4B4WsHL6UQyrTW+ZJyTW1Jm5l3MyKt9JY0UYdM8M2ELeEbWmo85zD1MmRbzTpTlpP5741W6vJWU4dcvKghaEX2v576GiiVtAutBBgCPKl9Rw8t+JQVkXOh9XHTfFyjLCVJhV0jx/edxXkpBHZa8ho2m/fgATUBHbDnGBS1rEbgNgh+6FzuYqboXUoCLdlB+gEWqzWHngdNkqUmI1Ysp6G2XUnxSJeCC1kshd3i0RQyzhv4TfEAHkkSMgigUpaIhFY6balUNB8NN68KYqgNVKypknsYYiegMxv3ti5g293VSVbmWNzIrxIjNq2jauE6tKFho3skNeZ9TkNIetu1Nhapq2m70gQFYs0C225QTHOm7i2s2w8lnoBO983Ogx1Vpbnu/KJK7W3NIxslABJSxSISCeHfhEjtCuOOM5Zo9Z/3A1grNjKwoxa2kBIZ0apwDbtLUk4U27aXP4r98zdn3xtj3/1v2EcTsb8YrfLHltfxavkh65c3a9ev6r0/RQHz376AfZ6NJpER/XNpVJ4QW3mEp6pi7mgiHXF0QWdDucVF9MKTyIfMOIzfMeO86dztnZwN76mOr/0JfY2m8/XTydeoLqbv4Wz8N5w9fEban5w9UMWndPbZgLO7LCfhhb42gl7ASZbpQ1OT3NbRZ5zYl38Yvu4Q5PfPQIth/g/k9+CTkt3hVrAiJ4brNnY7bjO620nN26jOOl7neqDnf2Nxb6EiBCqrD4uK2eRRcfgH65tFhec6p7B4JiygW9+1muH1fTa+/gM=
\ No newline at end of file
diff git a/doc/preprocessor/jsonpreprocessor.png b/doc/preprocessor/jsonpreprocessor.png
new file mode 100644
index 0000000000000000000000000000000000000000..80c987eb8d443ff76fa6f506bd6f61e1219227be
Binary files /dev/null and b/doc/preprocessor/jsonpreprocessor.png differ
diff git a/doc/preprocessor/jsonpreprocessor.xml b/doc/preprocessor/jsonpreprocessor.xml
new file mode 100644
index 0000000000000000000000000000000000000000..5bf10918a18095f8e7df27452d473fdd03ba6d91
 /dev/null
+++ b/doc/preprocessor/jsonpreprocessor.xml
@@ 0,0 +1 @@
+7Vxtc5s4EP41/tgMCPPij7ET96bTXDKTm2n7UcaKTYORR8iJc7/+BEgYJNnGiQw4vUxnCosEaPfZZ1cr4YEzWW2/Erhe3uE5igfAmm8Hzs0AANsCFvsvk7wVEm/kFoIFiea80U7wGP2LRE8u3URzlNYaUoxjGq3rwhAnCQppTQYJwa/1Zk84rj91DRdIETyGMFalP6I5XRbSwLV28r9QtFiKJ9sWvzKD4fOC4E3CnzcAzlP+V1xeQXEv3j5dwjl+rYic24EzIRjT4mi1naA4061QW9Fvuudq+d4EJbRJBxAA/iL0TQwezZku+GmCE/bfOB8RyvrY7CzEqyhkxxY7XtJVzMVoG9GfleNfWZMrNztLKHn7yXvkJ7trpQaya78RpW8cDnBDMRNhQpd4gRMYf8d4zW+fUoKf0QTHmOTv7Fj5H7uiKoDrJMUbEvIR8jFTSBaItwK+Uwiz0Vc6crV9RXiF2HuzBgTFkEYvdaRADrhF2W6ndHbA9b7HBsUtXmC84Te9WuF5BtuI2QB4MXvF8YywowUtR1i1Vxwzv8jM9LqMKHpcw3ycr8w16wZStDad2hN7ckhrL4hQtD2oD351yHtwj3c4wF937mMPuWxZcR3hUh9RYKAocAVDgr+g7RrmsAXWJWrU863OVApcrwkvHGEC7v2cC+waE+yI4Re/gWEmqHNWM14QJFBjBtc1zQx512tC4FulwRpHCU0rd37IBBUHA3U4iJgz3dPetuxD7dlB8Qb63o5bf5rv1odTqI13knBVqqQR1OzhYajtzH67k1asayQKNYVbIxjZvooiO+gsvASjVkP82R1bq8h3GQUEXhee7AwlT/b9g54stxdp8r72ABxsX/f893isrwTcyRKFz0z0ANO0QYA9Nax642t7yl5/zAJ4XJHz3NpIuAUWqFvF9dV4CzTxdmgk3p6Yh1ufIQ8XpFgjylFnRDmyTrPBJyVKjVE6IkrXOo0o5fadE6U6M/mHwCR9wmR12WTpeJJl2iVL508ky5HGL63uyLLdwlFfyVJnlG7I0pNd8ghZyu07J8uRQpYTvFpvaJQsLpssXSDNxNsly6BNR7X7QZZCm1W/9Ixnlu/y08A5zU/l9sCzjfodcFutD/QYIMB4jaYTgDicSY0BxGt1XtRngPifAyB80c8YQCwlcv8gLAwz0f2Gsgh+oXHbC7qc5KhFtjtIYzhjHe9DChm4wPTbJo7goFjXapId9XRhS6fXs61sqXNy1mec/xMEcOkKBaNhewr11Lx9FiUwf78LV6PjtYnLofEoa9WiLGinqjGd8jA7h+myfNPs5AFSikiSS4DVeDktcDWRuBeBWCyEcLzwhWJTYVWMu+JYpSsJTxIhwcoey8ItDwz8VISH4uzq6kr0Z4LZzhmPuicDxjo7DDez9/homXhJsXqa/zH5gsB5xCAgrnGsC/FNRFBII5zkl0jmiuO0QObIjOdLhhwCNSAF9pkCvS2Wfip2XjPUZkXiL+kahdET8/HMiLKdCFoTlLJRw1w5wMJPg8vc6lJmACIjGLYYwJgZNY4m6ymZX2cbCzMviGGaZrRbU0+VO2vMt58g91U338GXtRU0Aaij7FjRrntAuaeRqGbjyh7vOrKXRF3eke4zku5TKEG5z+kbbaTnWLXp1cenS0BH7G3gTbstyzASa9VxgcRa6Ob1q77A064XZz2v2U6no7ARZZhj8HxXyc5XAfP5d+26mkTQNl6SaZ6wt1pY7+sKmNYqHS2B2XLB5khhTG7f9RJYqctKZHiAJM1XwC6yhCbtCm+3giYCTVWbmsR6ghM2oLRIoZnjnJpC91Hvjoi80qJRTfG6oGpG8Wrt8mMJzn7S0pc6DjFn893gnia5br4bvJ3sRd6n7ViSAZumL7YtcVsgvYuh7Freli6eY261Si3wGgaf43hec/D5Stw2VRgTldcqPoN+wdOTduSV+8ZOzq4leA5H0o0MwdMTk7NzrbZryuWfFJ5As5G0Z/CUtzG6cvjrOzyNr/WLJZA/AJ6aD0L+h6dheJreaeCr7KlL6b9HM5KvQB5f2niNVjHMZ+oVBFczcz6P12X5h+tux6sbJ+Tz8swTqEuTQLdCIVaCP1bpUEnh5i1h42Gyh2wVAocoTdm4gXW7ReGGwlmjmVMbyk8ZtCmviwDnPNZwVVtYGlsErgFbiE9Udba4yz57rlijD1PXHXiaFw2Of/d8tpqBmPX8WZVV3XfPHX6wegDiEt08Lpl43jvGV0gnhjMUj8sfB5GeZIKSpMqnrTqNH2icpmz4sY9BTvwgp8nelUO/HQDOVAsvc9PGSebJ5fLA07ia25preYprfXu8/3vQ/n4BBdoa1e1Fuy1Xvc64X4Cd7n6Pp8hZdz965Nz+Bw==
\ No newline at end of file
diff git a/doc/preprocessor/matrices.dia b/doc/preprocessor/matrices.dia
deleted file mode 100644
index e4af95f8c369ca0b6274c8e03bd3be9f34e4f6b1..0000000000000000000000000000000000000000
Binary files a/doc/preprocessor/matrices.dia and /dev/null differ
diff git a/doc/preprocessor/matrices.png b/doc/preprocessor/matrices.png
index bff4e417300f7a33710df9b31b1c7fbbb6bf5092..b3710ca005015035c79bbcd3c3c991e6aff518bb 100644
Binary files a/doc/preprocessor/matrices.png and b/doc/preprocessor/matrices.png differ
diff git a/doc/preprocessor/matrices.xml b/doc/preprocessor/matrices.xml
new file mode 100644
index 0000000000000000000000000000000000000000..38fd30cacbe4fa3904c89a4c4072394c52e401f3
 /dev/null
+++ b/doc/preprocessor/matrices.xml
@@ 0,0 +1 @@
+5ZhNj5swEIZ/DXeMycdem/3oYbetlErdPbowMe4aJnLMQvrra2BIQr2pKnXFqoRDYr8e2+PHk1coAV/l9Z0R2+wBU9BBFKZ1wK+DKFouQvfZCPtOWES8E6RRaSexo7BWP4FEmidLlcJuEGgRtVXboZhgUUBiB5owBqth2Ab1cNetkOAJ60RoX/2mUpvRsWbhUf8ISmb9ziykke8ieZYGy4L2CyK+aZ9uOBf9WhS/y0SK1YnEbwK+Moi2a+X1CnSDtsfWzbs9M3rI20Bh/2bCFaVh9/3RIXUkqFtg4b4+tOeBZkboegnmKqF2ZnPtmsw1oVb2keSm/dTrhTX7x9POE0Udzt50foC1eyoEUVp0EhqbocRC6HvELa2wswafYYUaTZsvD9vHjXQnadI/y4KkHZYmoaiIqksYCRTFD/fgyhswB5ezCzGghVUvw9UFFZo8xB1huwbxPnNZ4ZvDZ+fgh/8L/Hgk+LT1i9AlLfogrFG1fyVaO9dpbqLKlIX1VrTZV873hndwHs5GaX2i37aP06URqXJ8+jG6ccoMjIX6zzR9TjQh6g2Y/Le3m+poZmxJWnZiZL3B/QtZ7pG9xwqMk74aJQpZatF0poT7akibsRFxM+YzvCwLj30XmY1l4dGlW/gr8OcjwY89o/mEJhd6YuYSz97TXfilF/jML/DlWO4SX7q1vwJ/MRL8mecu633erOLwTstgfn99GfVtce5h/nworIlxnrN35LzwOH/BnWp/MVF4DRtVOI4TA86iIXAejwh86b+dgBTTBh7PxwPuuse/ydqxk78i+c0v
\ No newline at end of file
diff git a/doc/preprocessor/overview.dia b/doc/preprocessor/overview.dia
deleted file mode 100644
index ea635011a0f588660e1e4ed2ccf1cf225b8941d2..0000000000000000000000000000000000000000
Binary files a/doc/preprocessor/overview.dia and /dev/null differ
diff git a/doc/preprocessor/overview.png b/doc/preprocessor/overview.png
index a49d4f77766f2b3b59f9e261d321885db715e871..e3bac90a6187e2dddeb380996ff0021381a84d0e 100644
Binary files a/doc/preprocessor/overview.png and b/doc/preprocessor/overview.png differ
diff git a/doc/preprocessor/overview.xml b/doc/preprocessor/overview.xml
new file mode 100644
index 0000000000000000000000000000000000000000..bfbc09c2e5cc38dc3fd823b1dbe123a2802e8912
 /dev/null
+++ b/doc/preprocessor/overview.xml
@@ 0,0 +1 @@
+7Vtbc9o4FP41PG7HkrGBx0JK9yGZZoadafuo2AJ7a1seWQTor98jW/JNpCWNsYFsJjORjo9uR5/OTcrIXsT7z5ykwQPzaTTClr8f2XcjjJGFLfgjKYeC4rqzgrDhoa+YKsIq/El1S0Xdhj7NGoyCsUiEaZPosSShnmjQCOds12Rbs6g5ako21CCsPBKZ1K+hL4KCOnWsiv43DTeBHhlZ6ssT8X5sONsmarwRttf5T/E5JrovxZ8FxGe7Gsn+NLIXnDFRlOL9gkZStlpsRbvlC1/LeXOaiFMauE7R4plEW7X2u0MCzYH2yGnKmUezjPERdiPocP4kSxtZUvMXBy2zbBfGEUmgNg9EHAERQXHNErFSTBbUvSCM/HtyYFs5v0yAvHRtHjAe/gR+ohvDZy4UOrArewujaMEimI99l7BirLLRSnamhuE0g2aPWg6oJN2TTOipsCgiaRY+5ZOTLDHhmzCZMyFYrJj0qpbmyD7JAuqXU+XsB9UcsO1W/lMKqr4lapeeKRd0XyOpLfpMWUwFPwCL/jpVcDk0j8muwuZYswQ1XNpInwl1HjZl1xUmoKBgcRwiU2xsNfXhtKiqEkaO+VIaHotDT0mwBga6D8W3Wvm7ZPngyFoCs/qmWuSV6lt5RuS3f6kQBwUJshUMSIyLgG1YQqJ7xtI/34+MbbmnVqjWDPjbUMU1sQuaXPwvN43TiIjwualK3rID2DijH2LmS70Wwhb8/mRCb6A45S7tglDQVUryZe5Adzf3xxDacokWaNENiO0Whh0TxGh8BMSa700YNiQYE4+zv+g+JTlssXUDInWc/iQ6cTvXCkoTKL2AGlqhUhLftWIeQCugI2oBOUOpBWTqBTB4WZhsTgDwa2Hrzj+i5bJlgivvphs4O008u0c0BD6C53EXeJ68Rzw7R/A8GQzPpi8aJoJyWHI+XAoeHKwGBmWJiXC27lGJW9Zy2ZVzZ49bhhH3aBgn0/cI+8kR2E8Hg/3EgP0ioBDJQARGsuxGdDma9anMnc5Rffkhi0ZwI2ZBXaM6b/qRc3KoMaQMFHVW6/lREiowjGdNMOBpKyvxG/6p09r9YgIVFsqVnHbizHDgH06SbM14fFunrlcXSmcg3pctmZmnDo8HsyUz05awON2KPCi4ZmQ7uGVP7D6R/R7tiRZmHdk6jT+wPXGnr7MnbX4069agTLtPhlwnPnDnXvQw+MAd46P75MKV4qPz5MIg+HCtTuGhJVWz2l85mGAgfdkKsN43YrP79UaRIaPb90a159k4c1bXZ+7ke6uxgetXJPTIGlhH8o5fOrA50ynO62Uk9RzU9jh6TOphM6P0QEREnqDhF0+QZ6lZZI70vPLs8qZrYjXlafd41XXk9hDazPNfa1WcPmwtr1qe5QOWPgTqmlHqPExIPr8rF+Okz9z97B1aOJ3RbOTuO49KT94CjZg3WTihk48v8F2JjTvm3Z3tQYd9Zuxbv8A+Ggr7ton9ywi4y71vZedeCqhQW2/a3UZU2tv6Hx7DqUbbUI3ly9IH+fjqNW9LLzG6xW0MH3t+da7odtZ9dHv5GSUN5obtx0MBfGp6sbC+4qIFCpTIu8T8Vcpgbw27tPaodSd7xseGUK1ewBfav/o3A/vTfw==
\ No newline at end of file
diff git a/doc/preprocessor/preprocessor.tex b/doc/preprocessor/preprocessor.tex
index fbe62c0bddac134848744a2bf74425866c2ffbd7..4643440e76130a4d83a48d28ea66713dccd35db6 100644
 a/doc/preprocessor/preprocessor.tex
+++ b/doc/preprocessor/preprocessor.tex
@@ 26,11 +26,11 @@
\title{The Dynare Preprocessor}
\author[S. Villemot]{Sébastien Villemot}
+\author[S. Villemot, H.Bastani]{Sébastien Villemot \and Houtan Bastani}
\institute{CEPREMAP}
\date{October 19, 2007}
+\date{1 February 2017}
\AtBeginSection[]
{
@@ 46,7 +46,7 @@
\end{frame}
\begin{frame}
 \frametitle{General overview}
+ \frametitle{Overview}
\begin{center}
\includegraphics[width=11cm]{overview.png}
\end{center}
@@ 56,6 +56,21 @@
\tableofcontents
\end{frame}
+\section{Invoking the preprocessor}
+
+\begin{frame}
+ \frametitle{Calling Dynare}
+ \begin{itemize}
+ \item Dynare is called from the host language platform with the syntax \texttt{dynare <>.mod}
+ \item This call can be followed by certain options:
+ \begin{itemize}
+ \item Some of these options impact host language platform functionality, \textit{e.g.} \texttt{nograph} prevents graphs from being displayed in Matlab
+ \item Some cause differences in the output created by default, \textit{e.g.} \texttt{notmpterms} prevents temporary terms from being written to the static/dynamic files
+ \item While others impact the functionality of the macroprocessor or the preprocessor, \textit{e.g.} \texttt{nostrict} shuts off certain checks that the preprocessor does by defalut
+ \end{itemize}
+ \end{itemize}
+\end{frame}
+
\section{Parsing}
\begin{frame}
@@ 75,9 +90,9 @@
\frametitle{Lexical analysis}
\begin{itemize}
\item The lexical analyzer recognizes the ``words'' (or \alert{lexemes}) of the language
\item Lexical analyzer is described in \texttt{DynareFlex.ll}. This file is transformed into C++ source code by the program \texttt{flex}
\item This file gives the list of the known lexemes (described by regular expressions), and gives the associated \alert{token} for each of them
\item For punctuation (semicolon, parentheses, ...), operators (+, , ...) or fixed keywords (\textit{e.g.} \texttt{model}, \texttt{varexo}, ...), the token is simply an integer uniquely identifying the lexeme
+\item Defined in \texttt{DynareFlex.ll}, it is transformed into C++ source code by the program \texttt{flex}
+\item This file details the list of known lexemes (described by regular expressions) and the associated \alert{token} for each of them
+\item For punctuation (semicolon, parentheses, \ldots), operators (+, , \ldots) or fixed keywords (\textit{e.g.} \texttt{model}, \texttt{varexo}, \ldots), the token is simply an integer uniquely identifying the lexeme
\item For variable names or numbers, the token also contains the associated string for further processing
%\item \textit{Note:} the list of tokens can be found at the beginning of \texttt{DynareBison.yy}
\item When invoked, the lexical analyzer reads the next characters of the input, tries to recognize a lexeme, and either produces an error or returns the associated token
@@ 115,6 +130,23 @@ SEMICOLON
\end{itemize}
\end{frame}
+\begin{frame}
+\frametitle{Syntax analysis}
+\framesubtitle{In Dynare}
+\begin{itemize}
+\item The \texttt{mod} file grammar is described in \texttt{DynareBison.yy}, which is transformed into C++ source code by the program \texttt{bison}
+\item The grammar tells a story which looks like:
+ \begin{itemize}
+ \item A \texttt{mod} file is a list of statements
+ \item A statement can be a \texttt{var} statement, a \texttt{varexo} statement, a \texttt{model} block, an \texttt{initval} block, \ldots
+ \item A \texttt{var} statement begins with the token \texttt{VAR}, then a list of \texttt{NAME}s, then a semicolon
+ \item A \texttt{model} block begins with the token \texttt{MODEL}, then a semicolon, then a list of equations separated by semicolons, then an \texttt{END} token
+ \item An equation can be either an expression, or an expression followed by an \texttt{EQUAL} token and another expression
+ \item An expression can be a \texttt{NAME}, or a \texttt{FLOAT\_NUMBER}, or an expression followed by a \texttt{PLUS} and another expression, \ldots
+ \end{itemize}
+\end{itemize}
+\end{frame}
+
\begin{frame}[fragile]
\frametitle{Syntax analysis}
Using the list of tokens produced by lexical analysis, the syntax analyzer determines which ``sentences'' are valid in the language, according to a \alert{grammar} composed of \alert{rules}.
@@ 139,33 +171,14 @@ expression := expression PLUS expression
\end{itemize}
\end{frame}
\begin{frame}
\frametitle{Syntax analysis}
\framesubtitle{In Dynare}
\begin{itemize}
\item The \texttt{mod} file grammar is described in \texttt{DynareBison.yy}
\item The grammar is transformed into C++ source code by the program \texttt{bison}
\item The grammar tells a story which looks like:
 \begin{itemize}
 \item A \texttt{mod} file is a list of statements
 \item A statement can be a \texttt{var} statement, a \texttt{varexo} statement, a \texttt{model} block, an \texttt{initval} block, ...
 \item A \texttt{var} statement begins with the token \texttt{VAR}, then a list of \texttt{NAME}s, then a semicolon
 \item A \texttt{model} block begins with the token \texttt{MODEL}, then a semicolon, then a list of equations separated by semicolons, then an \texttt{END} token
 \item An equation can be either an expression, or an expression followed by an \texttt{EQUAL} token and another expression
 \item An expression can be a \texttt{NAME}, or a \texttt{FLOAT\_NUMBER}, or an expression followed by a \texttt{PLUS} and another expression, ...
 \end{itemize}
\end{itemize}
\end{frame}


\begin{frame}
\frametitle{Semantic actions}
\begin{itemize}
\item So far we have only described how to accept valid \texttt{mod} files and to reject others
\item But validating is not enough: one need to do something about what has been parsed
\item Each rule of the grammar can have a \alert{semantic action} associated to it: C/C++ code enclosed in curly braces
\item Each rule can return a semantic value (referenced to by \texttt{\$\$} in the action)
\item In the action, it is possible to refer to semantic values returned by components of the rule (using \texttt{\$1}, \texttt{\$2}, ...)
+\item But validating is not enough: one needs to do something with the parsed \texttt{mod} file
+\item Every grammar rule can have a \alert{semantic action} associated with it: C/C++ code enclosed by curly braces
+\item Every rule can return a semantic value (referenced by \texttt{\$\$} in the action)
+\item In the action, it is possible to refer to semantic values returned by components of the rule (using \texttt{\$1}, \texttt{\$2}, \ldots)
\end{itemize}
\end{frame}
@@ 179,9 +192,9 @@ expression := expression PLUS expression
%type expression
expression_list := expression SEMICOLON
 { cout << $1; }
+ { cout << $1 << endl; }
 expression_list expression SEMICOLON
 { cout << $2; };
+ { cout << $2 << endl; };
expression := expression PLUS expression
{ $$ = $1 + $3; }
@@ 201,10 +214,10 @@ expression := expression PLUS expression
The class \texttt{ParsingDriver} has the following roles:
\begin{itemize}
\item Given the \texttt{mod} filename, it opens the file and launches the lexical and syntaxic analyzers on it
+\item It opens the \texttt{mod} file and launches the lexical and syntaxic analyzers on it
\item It implements most of the semantic actions of the grammar
\item By doing so, it creates an object of type \texttt{ModFile}, which is the data structure representing the \texttt{mod} file
\item Or, if there is a parsing error (unknown keyword, undeclared symbol, syntax error), it displays the line and column numbers where the error occurred, and exits
+\item Or, if there is a parsing error (unknown keyword, undeclared symbol, syntax error), it displays the line and column numbers where the error occurred and exits
\end{itemize}
\end{frame}
@@ 213,15 +226,15 @@ The class \texttt{ParsingDriver} has the following roles:
\begin{frame}
\frametitle{The \texttt{ModFile} class}
\begin{itemize}
 \item This class is the internal data structure used to store all the informations contained in a \texttt{mod} file
+ \item This class is the internal data structure used to store all the information contained in a \texttt{mod} file
\item One instance of the class represents one \texttt{mod} file
\item The class contains the following elements (as class members):
\begin{itemize}
 \item a symbol table
 \item a numerical constants table
 \item two trees of expressions: one for the model, and one for the expressions outside the model
 \item the list of the statements (parameter initializations, shocks block, \texttt{check}, \texttt{steady}, \texttt{simul}, ...)
 \item an evaluation context
+ \item a symbol table, numerical constants table, external functions table
+ \item trees of expressions: dynamic model, static model, original model, ramsey dynamic model, steady state model, trend dynamic model, \ldots
+ \item the list of the statements (parameter initializations, \texttt{shocks} block, \texttt{check}, \texttt{steady}, \texttt{simul}, \ldots)
+ \item modelspecification and userpreference variables: \texttt{block}, \texttt{bytecode}, \texttt{use\_dll}, \texttt{no\_static}, \ldots
+ \item an evaluation context (containing \texttt{initval} and parameter values)
\end{itemize}
\item An instance of \texttt{ModFile} is the output of the parsing process (return value of \texttt{ParsingDriver::parse()})
\end{itemize}
@@ 230,13 +243,13 @@ The class \texttt{ParsingDriver} has the following roles:
\begin{frame}
\frametitle{The symbol table (1/3)}
\begin{itemize}
 \item A \alert{symbol} is simply the name of a variable, of a parameter or of a function unknown to the preprocessor: actually everything that is not recognized as a Dynare keyword
 \item The \alert{symbol table} is a simple structure used to maintain the list of the symbols used in the \texttt{mod} file
 \item For each symbol, stores:
+ \item A \alert{symbol} is simply the name of a variable (endogenous, exogenous, local, auxiliary, etc), parameter, external function, \ldots basically everything that is not recognized as a Dynare keyword
+ \item \alert{SymbolTable} is a simple class used to maintain the list of the symbols used in the \texttt{mod} file
+ \item For each symbol, it stores:
\begin{itemize}
 \item its name (a string)
 \item its type (an integer)
 \item a unique integer identifier (unique for a given type, but not across types)
+ \item its name, tex\_name, and long\_name (strings, some of which can be empty)
+ \item its type (an enumerator defined in \texttt{CodeInterpreter.hh})
+ \item a unique integer identifier (also has a unique identifier by type)
\end{itemize}
\end{itemize}
\end{frame}
@@ 250,19 +263,23 @@ The class \texttt{ParsingDriver} has the following roles:
\item Exogenous deterministic variables
\item Parameters
\item Local variables inside model: declared with a pound sign (\#) construction
 \item Local variables outside model: no declaration needed, not interpreted by the preprocessor (\textit{e.g.} Matlab loop indexes)
 \item Names of functions unknown to the preprocessor: no declaration needed, not interpreted by the preprocessor, only allowed outside model (until we create an interface for providing custom functions with their derivatives)
+ \item Local variables outside model: no declaration needed (\textit{e.g.} lhs symbols in equations from \texttt{steady\_state\_model} block, expression outside of model block, \ldots)
+ \item External functions
+ \item Trend variables
+ \item Log Trend variables
+ \item Unused Endogenous variables (created when \texttt{nostrict} option is passed)
\end{itemize}
\end{frame}
\begin{frame}
 \frametitle{The symbol table (2/3)}
+ \frametitle{The symbol table (3/3)}
\begin{itemize}
\item Symbol table filled in:
\begin{itemize}
 \item using the \texttt{var}, \texttt{varexo}, \texttt{varexo\_det}, \texttt{parameter} declarations
+ \item using the \texttt{var}, \texttt{varexo}, \texttt{varexo\_det}, \texttt{parameter}, \texttt{external\_function}, \texttt{trend\_var}, and \texttt{log\_trend\_var} declarations
\item using pound sign (\#) constructions in the model block
\item on the fly during parsing: local variables outside models or unknown functions when an undeclared symbol is encountered
+ \item during the creation of auxiliary variables in the transform pass
\end{itemize}
\item Roles of the symbol table:
\begin{itemize}
@@ 273,49 +290,65 @@ The class \texttt{ParsingDriver} has the following roles:
\end{frame}
\begin{frame}
 \frametitle{Expression trees (1/2)}
+ \frametitle{Expression trees (1/3)}
\begin{itemize}
\item The data structure used to store expressions is essentially a \alert{tree}
\item Graphically, the tree representation of $(1+z)*\log(y)$ is:
\begin{center}
 \includegraphics[width=4cm]{expr.png}
+ \includegraphics[width=6cm]{expr.png}
\end{center}
\item No need to store parentheses
\item Each circle represents a \alert{node}
 \item A node has at most one parent and at most two children
+ \item A non external function node has at most one parent and at most three children (an external function node has as many children as arguments)
\end{itemize}
\end{frame}
\begin{frame}
 \frametitle{Expression trees (2/2)}
+ \frametitle{Expression trees (2/3)}
\begin{itemize}
 \item In Dynare preprocessor, a tree node is a represented by an instance of the abstract class \texttt{ExprNode}
 \item This class has 5 subclasses, corresponding to the 5 types of nodes:
+ \item A tree node is represented by an instance of the abstract class \texttt{ExprNode}
+ \item This class has 5 subclasses, corresponding to the 5 types of nonexternalfunction nodes:
\begin{itemize}
\item \texttt{NumConstNode} for constant nodes: contains the identifier of the numerical constants it represents
\item \texttt{VariableNode} for variable/parameters nodes: contains the identifier of the variable or parameter it represents
 \item \texttt{UnaryOpNode} for unary operators (\textit{e.g.} unary minus, $\log$, $\sin$): contains an integer representing the operator, and a pointer to its child
 \item \texttt{BinaryOpNode} for binary operators (\textit{e.g.} $+$, $*$, pow): contains an integer representing the operator, and pointers to its two children
 \item \texttt{UnknownFunctionNode} for functions unknown to the parser (\textit{e.g.} user defined functions): contains the identifier of the function name, and a vector containing an arbitrary number of children (the function arguments)
+ \item \texttt{UnaryOpNode} for unary operators (\textit{e.g.} unary minus, $\log$, $\sin$): contains an enumerator representing the operator, and a pointer to its child
+ \item \texttt{BinaryOpNode} for binary operators (\textit{e.g.} $+$, $*$, pow): contains an enumerator representing the operator, and pointers to its two children
+ \item \texttt{TrinaryOpNode} for trinary operators (\textit{e.g.} $normcdf$, $normpdf$): contains an enumerator representing the operator and pointers to its three children
\end{itemize}
\end{itemize}
\end{frame}
+\begin{frame}
+ \frametitle{Expression trees (3/3)}
+ \begin{itemize}
+ \item The abstract class \texttt{ExprNode} has an abstract subclass called \texttt{AbstractExternalFunctionNode}
+ \item This abstract subclass has 3 subclasses, corresponding to the 3 types of external function nodes:
+ \begin{itemize}
+ \item \texttt{ExternalFunctionNode} for external functions. Contains the identifier of the external function and a vector of its arguments
+ \item \texttt{FirstDerivExternalFunctionNode} for the first derivative of an external function. In addition to the information contained in \texttt{ExternalFunctionNode}, contains the index w.r.t. which this node is the derivative.
+ \item \texttt{SecondDerivExternalFunctionNode} for the second derivative of an external function. In addition to the information contained in \texttt{FirstDerivExternalFunctionNode}, contains the index w.r.t. which this node is the second derivative.
+ \end{itemize}
+ \end{itemize}
+\end{frame}
+
+
\begin{frame}
\frametitle{Classes \texttt{DataTree} and \texttt{ModelTree}}
\begin{itemize}
\item Class \texttt{DataTree} is a container for storing a set of expression trees
 \item Class \texttt{ModelTree} is a subclass of \texttt{DataTree}, specialized for storing a set of model equations (among other things, contains symbolic derivation algorithm)
+ \item Class \texttt{ModelTree} is a subclass container of \texttt{DataTree}, specialized for storing a set of model equations.
+ \item In the code, we use \texttt{ModelTree}derived classes: \texttt{DynamicModel} (the model with lags) and \texttt{StaticModel} (the model without lags)
\item Class \texttt{ModFile} contains:
\begin{itemize}
 \item one instance of \texttt{ModelTree} for storing the equations of model block
\item one instance of \texttt{DataTree} for storing all expressions outside model block
+ \item several instances of \texttt{DynamicModel}, one each for storing the equations of the model block for the original model, modified model, original Ramsey model, the Ramsey FOCs, etc.
+ \item one instance of \texttt{StaticModel} for storing the equations of model block without lags
\end{itemize}
\item Expression storage is optimized through three mechanisms:
\begin{itemize}
 \item precomputing of numerical constants
\item symbolic simplification rules
\item subexpression sharing
+ \item precomputing of numerical constants
\end{itemize}
\end{itemize}
\end{frame}
@@ 335,15 +368,16 @@ The class \texttt{ParsingDriver} has the following roles:
\begin{itemize}
\item from \texttt{ParsingDriver} in the semantic actions associated to the parsing of expressions
\item during symbolic derivation, to create derivatives expressions
+ \item when creating the static model from the dynamic model
+ \item \ldots
\end{itemize}
 \item Note that \texttt{NodeID} is an alias (typedef) for \texttt{ExprNode*}
\end{itemize}
\end{frame}
\begin{frame}
\frametitle{Reduction of constants and symbolic simplifications}
\begin{itemize}
 \item The construction methods compute constants whenever it is possible
+ \item The construction methods compute constants whenever possible
\begin{itemize}
\item Suppose you ask to construct the node $1+1$
\item The \texttt{AddPlus()} method will return a pointer to a constant node containing 2
@@ 369,7 +403,7 @@ The class \texttt{ParsingDriver} has the following roles:
\item Expressions share a common subexpression: $1+z$
\item The internal representation of these expressions is:
\begin{center}
 \includegraphics[width=6cm]{exprsharing.png}
+ \includegraphics[width=7cm]{exprsharing.png}
\end{center}
\end{itemize}
\end{frame}
@@ 379,14 +413,14 @@ The class \texttt{ParsingDriver} has the following roles:
\begin{itemize}
\item Construction methods implement a simple algorithm which achieves maximal expression sharing
\item Algorithm uses the fact that each node has a unique memory address (pointer to the corresponding instance of \texttt{ExprNode})
 \item It maintains 5 tables which keep track of the already constructed nodes: one table by type of node (constants, variables, unary ops, binary ops, unknown functions)
+ \item It maintains 9 tables which keep track of the alreadyconstructed nodes: one table by type of node (constants, variables, unary ops, binary ops, trinary ops, external functions, first deriv of external functions, second deriv of external functions, local variables)
\item Suppose you want to create the node $e_1+e_2$ (where $e_1$ and $e_2$ are subexpressions):
\begin{itemize}
\item the algorithm searches the binary ops table for the tuple equal to (address of $e_1$, address of $e_2$, op code of +) (it is the \alert{search key})
 \item if the tuple is found in the table, the node already exists, and its memory address is returned
 \item otherwise, the node is created, and is added to the table with its search key
+ \item if the tuple is found in the table, the node already exists and its memory address is returned
+ \item otherwise, the node is created and is added to the table with its search key
\end{itemize}
 \item Maximum sharing is achieved, because expression trees are constructed bottomup
+ \item Maximum sharing is achieved because expression trees are constructed bottomup
\end{itemize}
\end{frame}
@@ 401,9 +435,9 @@ The class \texttt{ParsingDriver} has the following roles:
\end{itemize}
\item Widely used constants
\begin{itemize}
 \item class \texttt{DataTree} has attributes containing pointers to one, zero, and minus one constants
 \item these constants are used in many places (in simplification rules, in derivation algorithm...)
 \item subexpression sharing algorithm ensures that those constants will never be duplicated
+ \item class \texttt{DataTree} has attributes containing pointers to constants: $0$, $1$, $2$, $1$, \texttt{NaN}, $\infty$, $\infty$, and $\pi$
+ \item these constants are used in many places (in simplification rules, in derivation algorithm\ldots)
+ \item subexpression sharing algorithm ensures that these constants will never be duplicated
\end{itemize}
\end{itemize}
\end{frame}
@@ 414,11 +448,11 @@ The class \texttt{ParsingDriver} has the following roles:
\item A statement is represented by an instance of a subclass of the abstract class \texttt{Statement}
\item Three groups of statements:
\begin{itemize}
 \item initialization statements (parameter initialization with $p = \ldots$, \texttt{initval}, \texttt{histval} or \texttt{endval} block)
 \item shocks blocks
 \item computing tasks (\texttt{check}, \texttt{simul}, ...)
+ \item initialization statements (parameter initialization with $p = \ldots$, \texttt{initval}, \texttt{histval}, or \texttt{endval} block)
+ \item shocks blocks (\texttt{shocks}, \texttt{mshocks}, \ldots)
+ \item computing tasks (\texttt{steady}, \texttt{check}, \texttt{simul}, \ldots)
\end{itemize}
 \item Each type of statement has its own class (\textit{e.g.} \texttt{InitValStatement}, \texttt{SimulStatement}, ...)
+ \item Each type of statement has its own class (\textit{e.g.} \texttt{InitValStatement}, \texttt{SimulStatement}, \ldots)
\item The class \texttt{ModFile} stores a list of pointers of type \texttt{Statement*}, corresponding to the statements of the \texttt{mod} file, in their order of declaration
\item Heavy use of polymorphism in the check pass, computing pass, and when writing outputs: abstract class \texttt{Statement} provides a virtual method for these 3 actions
\end{itemize}
@@ 429,8 +463,9 @@ The class \texttt{ParsingDriver} has the following roles:
\begin{itemize}
\item The \texttt{ModFile} class contains an \alert{evaluation context}
\item It is a map associating a numerical value to some symbols
 \item Filled in with \texttt{initval} block, and with parameters initializations
+ \item Filled in with \texttt{initval} block values and parameter initializations
\item Used during equation normalization (in the block decomposition), for finding nonzero entries in the jacobian
+ \item Used in testing that trends are compatible with a balanced growth path, for finding nonzero cross partials of equations with respect to trend variables and endogenous varibales
\end{itemize}
\end{frame}
@@ 439,67 +474,95 @@ The class \texttt{ParsingDriver} has the following roles:
\begin{frame}
\frametitle{Error checking during parsing}
\begin{itemize}
 \item Some errors in the \texttt{mod} file can be detected during the parsing:
+ \item Some errors in the \texttt{mod} file can be detected during parsing:
\begin{itemize}
\item syntax errors
 \item use of undeclared symbol in model block, initval block...
+ \item use of undeclared symbols in model block, initval block\ldots
\item use of a symbol incompatible with its type (\textit{e.g.} parameter in initval, local variable used both in model and outside model)
 \item multiple shocks declaration for the same variable
+ \item multiple shock declarations for the same variable
\end{itemize}
 \item But some other checks can only be done when parsing is completed
+ \item But some other checks can only be done when parsing is completed\ldots
\end{itemize}
\end{frame}
\begin{frame}
\frametitle{Check pass}
\begin{itemize}
 \item The check pass is implemented through method \texttt{ModFile::checkPass()}
 \item Does the following checks:
+ \item The check pass is implemented through the method \texttt{ModFile::checkPass()}
+ \item Performs many checks. Examples include:
\begin{itemize}
\item check there is at least one equation in the model (except if doing a standalone BVAR estimation)
 \item check there is not both a \texttt{simul} and a \texttt{stoch\_simul} (or another command triggering local approximation)
+ \item checks for coherence in statements (\textit{e.g.} options passed to statements do not conflict with each other, required options have been passed)
+ \item checks for coherence among statements (\textit{e.g.} if \texttt{osr} statement is present, ensure \texttt{osr\_params} and \texttt{optim\_weights} statements are present)
+ \item checks for coherence between statements and attributes of \texttt{mod} file (\textit{e.g.} \texttt{use\_dll} is not used with \texttt{block} or \texttt{bytecode})
+ \end{itemize}
+ \end{itemize}
+\end{frame}
+
+\section{Transform pass}
+
+\begin{frame}
+ \frametitle{Transform pass (1/2)}
+ \begin{itemize}
+ \item The transform pass is implemented through the method \texttt{ModFile::transformPass(bool nostrict)}
+ \item It makes necessary transformations (notably to the dynamic model, symbol table, and statements list) preparing the \texttt{ModFile} object for the computing pass. Examples of transformations include:
+ \begin{itemize}
+ \item creation of auxiliary variables and equations for leads, lags, expectation operator, differentiated forward variables, etc.
+ \item detrending of model equations if nonstationary variables are present
+ \item decreasing leads/lags of predetermined variables by one period
+ \item addition of FOCs of Langrangian for Ramsey problem
+ \item addition of \texttt{dsge\_prior\_weight} initialization before all other statements if estimating a DSGEVAR where the weight of the DSGE prior of the VAR is calibrated
\end{itemize}
 \item Other checks could be added in the future, for example:
+ \end{itemize}
+\end{frame}
+
+\begin{frame}
+ \frametitle{Transform pass (2/2)}
+ \begin{itemize}
+ \item It then freezes the symbol table, meaning that no more symbols can be created on the \texttt{ModFile} object
+ \item Finally checks are performed on the transformed model. Examples include:
\begin{itemize}
 \item check that every endogenous variable is used at least once in current period
 \item check there is a single \texttt{initval} (or \texttt{histval}, \texttt{endval}) block
 \item check that \texttt{varobs} is used if there is an estimation
+ \item same number of endogenous varibables as equations (not done in certain situations, \textit{e.g.} Ramsey, discretionary policy, etc.)
+ \item correspondence among variables and statements, \textit{e.g.} Ramsey policy, identification, perfect foresight solver, and simul are incompatible with deterministic exogenous variables
+ \item correspondence among statements, \textit{e.g.} for DSGEVAR without \texttt{bayesian\_irf} option, the number of shocks must be greater than or equal to the number of observed variables
\end{itemize}
\end{itemize}
\end{frame}
+
\section{Computing pass}
\begin{frame}
\frametitle{Overview of the computing pass}
\begin{itemize}
\item Computing pass implemented in \texttt{ModFile::computingPass()}
 \item Begins with a determination of which derivatives to compute
 \item Then, calls \texttt{ModelTree::computingPass()}, which computes:
+ \item Creates Static model from Dynamic (by removing leads/lags)
+ \item Determines which derivatives to compute
+ \item Then calls \texttt{DynamicModel::computingPass()} which computes:
\begin{itemize}
\item leag/lag variable incidence matrix
 \item symbolic derivatives
 \item equation normalization + block decomposition (only in \texttt{sparse\_dll} mode)
+ \item symbolic derivatives w.r.t. endogenous, exogenous, and parameters, if needed
+ \item equation normalization + block decomposition
\item temporary terms
 \item symbolic gaussian elimination (only in \texttt{sparse\_dll} mode) \textit{(actually this is done in the output writing pass, but should be moved to the computing pass)}
+ \item computes equation cross references, if desired
\end{itemize}
+ \item NB: analagous operations for Static model are performed by \texttt{StaticModel::computingPass()}
+ \item Asserts that equations declared linear are indeed linear (by checking that Hessian == 0)
\item Finally, calls \texttt{Statement::computingPass()} on all statements
\end{itemize}
\end{frame}
\begin{frame}
 \frametitle{The variable table}
+ \frametitle{Model Variables}
\begin{itemize}
 \item In the context of class \texttt{ModelTree}, a \alert{variable} is a pair (symbol, lead/lag)
 \item The symbol must correspond to an endogenous or exogenous variable (in the sense of the model)
 \item The class \texttt{VariableTable} keeps track of those pairs
 \item An instance of \texttt{ModelTree} contains an instance of \texttt{VariableTable}
 \item Each pair (\texttt{symbol\_id}, lead/lag) is given a unique \texttt{variable\_id}
 \item After the computing pass, the class \texttt{VariableTable} writes the leag/lag incidence matrix:
+ \item In the context of class \texttt{ModelTree}, a \alert{variable} is a pair (symbol, lag)
+ \item The symbol must correspond to a variable of type endogenous, exogenous, deterministic exogenous variable, or parameter
+ \item The \texttt{SymbolTable} class keeps track of valid symbols while the \texttt{variable\_node\_map} keeps track of model variables (symbol, lag pairs stored in \texttt{VariableNode} objects)
+ \item After the computing pass, the \texttt{DynamicModel} class writes the leag/lag incidence matrix:
\begin{itemize}
 \item endogenous symbols in row
 \item leads/lags in column
 \item elements of the matrix are either 0 or correspond to a variable ID, depending on whether the pair (symbol, lead/lag) is used or not in the model
+ \item three rows: the first row indicates $t1$, the second row $t$, and the third row $t+1$
+ \item one column for every endogenous symbol in order of declaration; NB: includes endogenous auxiliary variables created during the transform pass
+ \item elements of the matrix are either 0 (if the variable does not appear in the model) or correspond to the variable's column in the Jacobian of the dynamic model
\end{itemize}
\end{itemize}
\end{frame}
@@ 507,56 +570,49 @@ The class \texttt{ParsingDriver} has the following roles:
\begin{frame}
\frametitle{Static versus dynamic model}
\begin{itemize}
 \item The static model is simply the (dynamic) model from which the leads/lags have been omitted
+ \item The static model is simply the dynamic model without leads and lags
\item Static model used to characterize the steady state
\item The jacobian of the static model is used in the (Matlab) solver for determining the steady state
 \item No need to derive static and dynamic models independently: \\
 static derivatives can be easily deduced from dynamic derivatives
\end{itemize}
\begin{block}{Example}
\begin{itemize}
 \item suppose dynamic model is $2x \cdot x_{1} = 0$
 \item static model is $2x^2 = 0$, whose derivative w.r. to $x$ is $4x$
 \item dynamic derivative w.r. to $x$ is $2x_{1}$, and w.r. to $x_{1}$ is $2x$
 \item removing leads/lags from dynamic derivatives and summing over the two partial derivatives w.r. to $x$ and $x_{1}$ gives $4x$
+ \item suppose dynamic model is $2x_t \cdot x_{t1} = 0$
+ \item static model is $2x^2 = 0$, whose derivative w.r.t. $x$ is $4x$
+ \item dynamic derivative w.r.t. $x_t$ is $2x_{t1}$, and w.r.t. $x_{t1}$ is $2x_t$
+ \item removing leads/lags from dynamic derivatives and summing over the two partial derivatives w.r.t. $x_t$ and $x_{t1}$ gives $4x$
\end{itemize}
\end{block}
\end{frame}
\begin{frame}
 \frametitle{Which derivatives to compute ?}
+ \frametitle{Which derivatives to compute?}
\begin{itemize}
\item In deterministic mode:
\begin{itemize}
 \item static jacobian (w.r. to endogenous variables only)
 \item dynamic jacobian (w.r. to endogenous variables only)
+ \item static jacobian w.r.t. endogenous variables only
+ \item dynamic jacobian w.r.t. endogenous variables only
\end{itemize}
\item In stochastic mode:
\begin{itemize}
 \item static jacobian (w.r. to endogenous variables only)
 \item dynamic jacobian (w.r. to all variables)
 \item possibly dynamic hessian (if \texttt{order} option $\geq 2$)
+ \item static jacobian w.r.t. endogenous variables only
+ \item dynamic jacobian w.r.t. endogenous, exogenous, and deterministic exogenous variables
+ \item dynamic hessian w.r.t. endogenous, exogenous, and deterministic exogenous variables
\item possibly dynamic 3rd derivatives (if \texttt{order} option $\geq 3$)
+ \item possibly dynamic jacobian and/or hessian w.r.t. parameters (if \texttt{identification} or analytic derivs needed for \texttt{estimation} and \texttt{params\_derivs\_order} $>0$)
\end{itemize}
 \item For ramsey policy: the same as above, but with one further order of derivation than declared by the user with \texttt{order} option (the derivation order is determined in the check pass, see \texttt{RamseyPolicyStatement::checkPass()})
+ \item For Ramsey policy: the same as above, but with one further order of derivation than declared by the user with \texttt{order} option (the derivation order is determined in the check pass, see \texttt{RamseyPolicyStatement::checkPass()})
\end{itemize}
\end{frame}
\begin{frame}
\frametitle{Derivation algorithm (1/2)}
\begin{itemize}
 \item Derivation of the model implemented in \texttt{ModelTree::derive()}
 \item Simply calls \texttt{ExprNode::getDerivative(varID)} on each equation node
+ \item Derivation of the model implemented in \texttt{ModelTree::computeJacobian()}, \texttt{ModelTree::computeHessian()}, \texttt{ModelTree::computeThirdDerivatives()}, and \texttt{ModelTree::computeParamsDerivatives()}
+ \item Simply call \texttt{ExprNode::getDerivative(deriv\_id)} on each equation node
\item Use of polymorphism:
\begin{itemize}
 \item for a constant or variable node, derivative is straightforward (0 or 1)
 \item for a unary or binary op node, recursively calls method \texttt{getDerivative()} on children to construct derivative of parent, using usual derivation rules, such as:
 \begin{itemize}
 \item $(log(e))' = \frac{e'}{e}$
 \item $(e_1 + e_2)' = e'_1 + e'_2$
 \item $(e_1 \cdot e_2)' = e'_1\cdot e_2 + e_1\cdot e'_2$
 \item $\ldots$
 \end{itemize}
+ \item for a constant or variable node, derivative is straightforward ($0$ or $1$)
+ \item for a unary, binary, trinary op nodes and external function nodes, recursively calls method \texttt{computeDerivative()} on children to construct derivative
\end{itemize}
\end{itemize}
\end{frame}
@@ 567,18 +623,17 @@ The class \texttt{ParsingDriver} has the following roles:
\begin{itemize}
\item Caching of derivation results
\begin{itemize}
 \item method \texttt{ExprNode::getDerivative(varID)} memorizes its result in a member attribute the first time it is called
 \item so that the second time it is called (with the same argument), simply returns the cached value without recomputation
+ \item method \texttt{ExprNode::getDerivative(deriv\_id)} memorizes its result in a member attribute (\texttt{derivatives}) the first time it is called
+ \item the second time it is called (with the same argument), it simply returns the cached value without recomputation
\item caching is useful because of subexpression sharing
\end{itemize}
 \pause
 \item Symbolic \textit{a priori}
+ \item Efficiently finds symbolic derivatives equal to $0$
\begin{itemize}
\item consider the expression $x+y^2$
 \item without any computation, you know its derivative w.r. to $z$ is zero
 \item each node stores in an attribute the set of variables which appear in the expression it represents ($\{x,y\}$ in the example)
 \item that set is computed in the constructor (straigthforwardly for a variable or a constant, recursively for other nodes, using the sets of the children)
 \item when \texttt{getDerivative(varID)} is called, immediately returns zero if \texttt{varID} is not in that set
+ \item without any computation, you know its derivative w.r.t. $z$ is zero
+ \item each node stores in an attribute (\texttt{non\_null\_derivatives}) the set of variables which appear in the expression it represents ($\{x,y\}$ in the example)
+ \item this set is computed in \texttt{prepareForDerivation()}
+ \item when \texttt{getDerivative(deriv\_id)} is called, immediately returns zero if \texttt{deriv\_id} is not in that set
\end{itemize}
\end{itemize}
\end{frame}
@@ 588,8 +643,8 @@ The class \texttt{ParsingDriver} has the following roles:
\begin{itemize}
\item When the preprocessor writes equations and derivatives in its outputs, it takes advantage of subexpression sharing
\item In Matlab static and dynamic output files, equations are preceded by a list of \alert{temporary terms}
 \item Those terms are temporary variables containing expressions shared by several equations or derivatives
 \item Doing so greatly enhances the computing speed of model residual, jacobian or hessian
+ \item These terms are variables containing expressions shared by several equations or derivatives
+ \item Using these terms greatly enhances the computing speed of the model residual, jacobian, hessian, or third derivative
\end{itemize}
\begin{block}{Example}
\begin{columns}[t]
@@ 603,9 +658,9 @@ residual(1)=3*(x+y^2)+1;
\begin{column}{4.8cm}
Can be optimized in:
\begin{verbatim}
T01=x+y^2;
residual(0)=T01z^3;
residual(1)=3*T01+1;
+T1=x+y^2;
+residual(0)=T1z^3;
+residual(1)=3*T1+1;
\end{verbatim}
\end{column}
\end{columns}
@@ 615,8 +670,7 @@ residual(1)=3*T01+1;
\begin{frame}
\frametitle{Temporary terms (2/2)}
\begin{itemize}
 \item Expression storage in the preprocessor implements maximal sharing...
 \item ...but it is not optimal for the Matlab output files, because creating a temporary variable also has a cost (in terms of CPU and of memory)
+ \item Expression storage in the preprocessor implements maximal sharing but this is not optimal for the Matlab output files, because creating a temporary variable also has a cost (in terms of CPU and of memory)
\item Computation of temporary terms implements a tradeoff between:
\begin{itemize}
\item cost of duplicating subexpressions
@@ 630,9 +684,9 @@ residual(1)=3*T01+1;
\begin{frame}
\frametitle{The special case of Ramsey policy}
\begin{itemize}
 \item For most statements, the method \texttt{computingPass()} is a noop...
 \item ...except for \texttt{planner\_objective} statement, which serves to declare planner objective when doing optimal policy under commitment
 \item Class \texttt{PlannerObjectiveStatement} contains an instance of \texttt{ModelTree}: used to store the objective (only one equation in the tree)
+ \item For most statements, the method \texttt{computingPass()} is a noop\ldots
+ \item \ldots except for \texttt{planner\_objective} statement, which serves to declare planner objective when doing optimal policy under commitment
+ \item Class \texttt{PlannerObjectiveStatement} contains an instance of \texttt{ModelTree}, which stores the objective function (\texttt{i.e.} only one equation in the tree)
\item During the computing pass, triggers the computation of the first and second order (static) derivatives of the objective
\end{itemize}
\end{frame}
@@ 648,24 +702,26 @@ residual(1)=3*T01+1;
\begin{itemize}
\item general initialization commands
\item symbol table output (from \texttt{SymbolTable::writeOutput()})
 \item lead/lag incidence matrix (from \texttt{ModelTree::writeOutput()})
+ \item lead/lag incidence matrix (from \texttt{DynamicModel::writeDynamicMFile()})
\item call to Matlab functions corresponding to the statements of the \texttt{mod} file (written by calling \texttt{Statement::writeOutput()} on all statements through polymorphism)
\end{itemize}
\item Subsidiary output files:
\begin{itemize}
\item one for the static model
\item one for the dynamic model
 \item and one for the planner objective (if relevant)
 \item written through \texttt{ModelTree} methods: \texttt{writeStaticFile()} and \texttt{writeDynamicFile()}
+ \item one for the auxiliary variables
+ \item one for the steady state file (if relevant)
+ \item one for the planner objective (if relevant)
\end{itemize}
\end{itemize}
\end{frame}
\begin{frame}
\frametitle{Model output files}
 Three possibles modes for \texttt{ModelTree} (see \texttt{mode} attribute):
+ Three possible output types:
\begin{itemize}
 \item Standard mode: static and dynamic files in Matlab
+ \item Matlab/Octave mode: static and dynamic files in Matlab
+ \item Julia mode: static and dynamic files in Julia
\item DLL mode:
\begin{itemize}
\item static and dynamic files in C++ source code (with corresponding headers)
@@ 684,34 +740,69 @@ residual(1)=3*T01+1;
\end{itemize}
\end{frame}
\section{Conclusion}
+\section{Proposed Changes}
+
+\newcounter{sauvegardeenumi}
+\newcommand{\asuivre}{\setcounter{sauvegardeenumi}{\theenumi}}
+\newcommand{\suite}{\setcounter{enumi}{\thesauvegardeenumi}}
\begin{frame}
 \frametitle{Future work (1/2)}
 \framesubtitle{Enhancements, optimizations}
 \begin{itemize}
 \item Refactor and reorganize some portions of the code
 \item Create a testsuite (with unitary tests)
 \item Separate computation of temporary terms between static and dynamic outputs
 \item Enhance subexpression sharing algorithm (using associativity, commutativity and factorization rules)
 \item Add many checks on the structure of the \texttt{mod} file
 \end{itemize}
+ \frametitle{Proposed changes with addition of Julia support (1/2)}
+ \begin{enumerate}
+ \item Julia output is provided upon parsing of \texttt{mod} file, everything else done in Julia
+ \begin{itemize}
+ \item Pros: very few changes to the preprocessor
+ \item Cons: repeated code (same checks, transformations, computations done in preprocessor and Julia); potential code divergence/two parallel projects
+ \end{itemize}
+ \item Dump preprocessor altogether: do everything with Julia
+ \begin{itemize}
+ \item Pros: simple to distribute, move away from C++ (no contributions, requires more expertise)
+ \item Cons: Matlab/Octave users must also download Julia, a big project, speed (?)
+ \end{itemize}
+ \asuivre
+ \end{enumerate}
\end{frame}
\begin{frame}
 \frametitle{Future work (2/2)}
 \framesubtitle{Features}
 \begin{itemize}
 \item Add precompiler macros (\#include, \#define, \#if)
 \item Add handling for several (sub)models
 \item Add indexed variables and control statements (if, loops) both in models and command language
 \item Add sum, diff, prod operators
 \item For unknown functions in the model: let user provide a derivative, or trigger numerical derivation
 \item Generalize binary code output
 \item Generalize block decomposition ?
 \end{itemize}
+ \frametitle{Proposed changes with addition of Julia support (2/2)}
+ \begin{enumerate}
+ \suite
+ \item Create libraries out of the preprocessor
+ \begin{itemize}
+ \item Pros: Dynare interaction similar across HLPs, preprocessor used as is
+ \item Cons: difficult for outsiders to contribute, big project, not much benefit in speed when compared to\ldots
+ \end{itemize}
+ \item Write \texttt{mod} file from HLP then call preprocessor; option to output JSON file representing \texttt{ModFile} object at every step of the preprocessor
+ \begin{itemize}
+ \item Pros: Dynare interaction similar across HLPs, preprocessor used as is, minimal amount of work, easy incremental step, allows users to support any given HPL given the JSON output
+ \item Cons: unnecessary processing when certain changes made in host language, keeps defaults of current preprocessor, speed (?)
+ \end{itemize}
+ \item Other ideas?
+ \end{enumerate}
\end{frame}
+\begin{frame}
+ \frametitle{Using HLP \texttt{mod} file objects (1/2)}
+ \begin{center}
+ \includegraphics[width=11cm]{jsonpreprocessor.png}
+ \end{center}
+\end{frame}
+\begin{frame}
+ \frametitle{Using HLP \texttt{mod} file objects (2/2)}
+ \begin{itemize}
+ \item Allows interactivity for all HLPs; requires only
+ \begin{itemize}
+ \item A definition of a mod file class in the HLP
+ \item A library function that converts an HLP mod file object to a \texttt{mod} file
+ \end{itemize}
+ \item Allows users to use Dynare with any HPL. Standard JSON output can be read in any HPL; user can use it construct desired HPL objects and work with model in their language of preference
+ \item Easy first step
+ \item No divergence of codebase: don't need to repeat code (checks, transformations, etc.) across platforms
+ \item Creates \texttt{mod} files that can be used on other host language platforms
+ \item Adds one more HLP library to distribute
+ \item Need to design/implement classes that will store processed dynare \texttt{mod} file in various HLPs
+ \end{itemize}
+\end{frame}
\end{document}
diff git a/doc/preprocessor/readme b/doc/preprocessor/readme
new file mode 100644
index 0000000000000000000000000000000000000000..a84b96c794e8bb3572d83bcce8d05bcfce56ef7f
 /dev/null
+++ b/doc/preprocessor/readme
@@ 0,0 +1,2 @@
+NB: .png files made on https://www.draw.io. They can be edited by loading the .xml file with the same name
+