Commit 471451ca authored by MichelJuillard's avatar MichelJuillard
Browse files

reporting + various fixes

parent 1a4c3b15
This diff is collapsed.
[dynare-preprocessor]
git-tree-sha1 = "02b6b7dd187eab53400ddc820c4c32be55a570a7"
using Pkg.Artifacts
# This is the path to the Artifacts.toml we will manipulate
artifact_toml = joinpath(@__DIR__, "Artifacts.toml")
# Query the `Artifacts.toml` file for the hash bound to the name "dynare-preprocessor"
# (returns `nothing` if no such binding exists)
preprocessor_hash = artifact_hash("dynare-preprocessor", artifact_toml)
@show preprocessor_hash
if preprocessor_hash == nothing || !artifact_exists(preprocessor_hash)
# create_artifact() returns the content-hash of the artifact directory once we're finished creating it
preprocessor_hash = create_artifact() do artifact_dir
@show artifact_dir
# We create the artifact by simply downloading a few files into the new artifact directory
preprocessor_url_base = "https://git.dynare.org/julia-packages//-/tree/master/"
download("$(preprocessor_url_base)/linux/64/preprocess.tar.gz", joinpath(artifact_dir, "preprocessor.tar.gz"))
end
# Now bind that hash within our `Artifacts.toml`. `force = true` means that if it already exists,
# just overwrite with the new content-hash. Unless the source files change, we do not expect
# the content hash to change, so this should not cause unnecessary version control churn.
bind_artifact!(artifact_toml, "dynare-preprocessor", preprocessor_hash)
end
# Get the path of the iris dataset, either newly created or previously generated.
# this should be something like `~/.julia/artifacts/dbd04e28be047a54fbe9bf67e934be5b5e0d357a`
dynare_preprocessor_path = artifact_path(preprocessor_hash)
@show dynare_preprocessor_path
......@@ -4,6 +4,7 @@ using GR
gr()
include("dynare_containers.jl")
include("model.jl")
export get_abc, get_de
include("symboltable.jl")
......@@ -18,7 +19,8 @@ export steady_state!
include("Utils.jl")
export get_power_deriv
include("dynare_table.jl")
include("reporting/report.jl")
export @dynare
macro dynare(modfilename, args...)
......
using LinearRationalExpectations
using TimeDataFrames
export Context, DynareSymbol, Model, ModelResults, Results, Simulation, SymbolType, Work, Trends
struct Model
endogenous_nbr
exogenous_nbr
lagged_exogenous_nbr
exogenous_deterministic_nbr
parameter_nbr
lead_lag_incidence
n_static
n_fwrd
n_bkwrd
n_both
n_states
DErows1
DErows2
n_dyn
i_static
i_dyn::Array{Int64,1}
i_bkwrd
i_bkwrd_b
i_bkwrd_ns
i_fwrd
i_fwrd_b
i_fwrd_ns
i_both
i_non_states
p_static
p_bkwrd
p_bkwrd_b
p_fwrd
p_fwrd_b
p_both_b
p_both_f
i_current
p_current
n_current
i_current_ns
p_current_ns
n_current_ns
icolsD
jcolsD
icolsE
jcolsE
colsUD
colsUE
i_cur_fwrd
n_cur_fwrd
p_cur_fwrd
i_cur_bkwrd
n_cur_bkwrd
p_cur_bkwrd
i_cur_both
n_cur_both
p_cur_both
gx_rows
hx_rows
i_current_exogenous
i_lagged_exogenous
serially_correlated_exogenous
Sigma_e
maximum_endo_lag
maximum_endo_lead
maximum_exo_lag
maximum_exo_lead
maximum_exo_det_lag
maximum_exo_det_lead
maximum_lag
maximum_lead
orig_maximum_endo_lag
orig_maximum_endo_lead
orig_maximum_exo_lag
orig_maximum_exo_lead
orig_maximum_exo_det_lag
orig_maximum_exo_det_lead
orig_maximum_lag
orig_maximum_lead
dynamic_indices
current_dynamic_indices
forward_indices_d
backward_indices_d
current_dynamic_indices_d
exogenous_indices
dynamic!
static!
steady_state!
end
function Model(modfilename, endogenous_nbr, lead_lag_incidence,
exogenous_nbr, lagged_exogenous_nbr, exogenous_deterministic_nbr,
parameter_nbr, maximum_endo_lag, maximum_endo_lead,
maximum_exo_lag, maximum_exo_lead, maximum_exo_det_lag,
maximum_exo_det_lead, maximum_lag, maximum_lead,
orig_maximum_endo_lag, orig_maximum_endo_lead,
orig_maximum_exo_lag, orig_maximum_exo_lead,
orig_maximum_exo_det_lag, orig_maximum_exo_det_lead,
orig_maximum_lag, orig_maximum_lead)
i_static = findall((lead_lag_incidence[1,:] .== 0) .& (lead_lag_incidence[3,:] .== 0))
p_static = lead_lag_incidence[2,i_static]
i_dyn = findall((lead_lag_incidence[1,:] .> 0) .| (lead_lag_incidence[3,:] .> 0))
n_static = length(i_static)
i_bkwrd = findall((lead_lag_incidence[1,:] .> 0) .& (lead_lag_incidence[3,:] .== 0))
i_bkwrd_b = findall((lead_lag_incidence[1,:] .> 0))
i_bkwrd_ns = findall(lead_lag_incidence[1,i_dyn] .> 0)
p_bkwrd = lead_lag_incidence[1,i_bkwrd]
p_bkwrd_b = lead_lag_incidence[1,i_bkwrd_b]
n_bkwrd = length(i_bkwrd)
i_fwrd = findall((lead_lag_incidence[3,:] .> 0) .& (lead_lag_incidence[1,:] .== 0))
i_fwrd_b = findall((lead_lag_incidence[3,:] .> 0))
i_fwrd_ns = findall(lead_lag_incidence[3,i_dyn] .> 0)
p_fwrd = lead_lag_incidence[3,i_fwrd]
p_fwrd_b = lead_lag_incidence[3,i_dyn[i_fwrd_ns]]
n_fwrd = length(i_fwrd)
i_both = findall((lead_lag_incidence[1,:] .> 0) .& (lead_lag_incidence[3,:] .> 0))
i_non_states = union(i_fwrd, i_static)
p_both_b = lead_lag_incidence[1,i_both]
p_both_f = lead_lag_incidence[3,i_both]
n_both = length(i_both)
n_states = n_bkwrd + n_both
i_current = findall(lead_lag_incidence[2,:] .> 0 )
p_current = lead_lag_incidence[2,i_current]
n_current = count(i->(i > 0),lead_lag_incidence[2,:])
i_current_ns = findall(lead_lag_incidence[2,i_dyn] .> 0 )
p_current_ns = lead_lag_incidence[2,i_dyn[i_current_ns]]
n_current_ns = count(i->(i > 0),lead_lag_incidence[2,i_dyn])
i_cur_fwrd = findall(lead_lag_incidence[2,i_fwrd] .> 0)
n_cur_fwrd = length(i_cur_fwrd)
p_cur_fwrd = lead_lag_incidence[2,i_fwrd[i_cur_fwrd]]
i_cur_bkwrd = findall(lead_lag_incidence[2,i_bkwrd] .> 0)
n_cur_bkwrd = length(i_cur_bkwrd)
p_cur_bkwrd = lead_lag_incidence[2,i_bkwrd[i_cur_bkwrd]]
i_cur_both = findall(lead_lag_incidence[2,i_both] .> 0)
n_cur_both = length(i_cur_both)
p_cur_both = lead_lag_incidence[2,i_both[i_cur_both]]
icolsD = [1:n_cur_bkwrd; n_bkwrd + n_both .+ (1:(n_fwrd+n_both))]
jcolsD = [p_cur_bkwrd; p_fwrd; p_both_f]
# derivatives of current values of variables that are both
# forward and backward are included in the E matrix
icolsE = [1:(n_bkwrd + n_both); n_bkwrd + n_both .+ (1:(n_fwrd+n_both))]
jcolsE = [p_bkwrd; p_both_b; p_cur_fwrd; p_cur_both]
colsUD = n_bkwrd .+ (1:n_both)
colsUE = n_both + n_fwrd .+ colsUD
n_dyn = endogenous_nbr - n_static + n_both
DErows1 = 1:(n_dyn-n_both)
DErows2 = (n_dyn-n_both) .+ (1:n_both)
gx_rows = n_bkwrd .+ (1:(n_fwrd+n_both))
hx_rows = 1:(n_bkwrd + n_both)
i_current_exogenous = maximum(lead_lag_incidence) .+ (1:exogenous_nbr)
i_lagged_exogenous = 0:-1
Sigma_e = zeros(exogenous_nbr, exogenous_nbr)
serially_correlated_exogenous = false
static_indices = i_static
current_indices = i_current
forward_indices = i_fwrd
both_indices = i_both
backward_indices = i_bkwrd
backward_number = n_bkwrd
forward_number = n_fwrd
current_number = n_current
dynamic_indices = setdiff(collect(1:endogenous_nbr), static_indices)
current_dynamic_indices = setdiff(current_indices, static_indices)
purely_forward_indices = setdiff(forward_indices, both_indices)
forward_indices_d = findall(in(forward_indices), dynamic_indices)
backward_indices_d = findall(in(backward_indices), dynamic_indices)
current_dynamic_indices_d = findall(in(current_dynamic_indices), dynamic_indices)
exogenous_indices = (backward_number + current_number
+ forward_number .+ (1:exogenous_nbr))
dynamic! = load_dynare_function(modfilename*"Dynamic.jl")
static! = load_dynare_function(modfilename*"Static.jl")
if isfile(modfilename*"SteadyState2.jl")
steady_state! = load_dynare_function(modfilename*"SteadyState2.jl")
else
steady_state! = nothing
end
Model(endogenous_nbr, exogenous_nbr, lagged_exogenous_nbr,
exogenous_deterministic_nbr, parameter_nbr,
lead_lag_incidence, n_static, n_fwrd, n_bkwrd, n_both,
n_states, DErows1, DErows2, n_dyn, i_static, i_dyn, i_bkwrd,
i_bkwrd_b, i_bkwrd_ns, i_fwrd, i_fwrd_b, i_fwrd_ns, i_both,
i_non_states, p_static, p_bkwrd, p_bkwrd_b, p_fwrd,
p_fwrd_b, p_both_b, p_both_f, i_current, p_current,
n_current, i_current_ns, p_current_ns, n_current_ns, icolsD,
jcolsD, icolsE, jcolsE, colsUD, colsUE, i_cur_fwrd,
n_cur_fwrd, p_cur_fwrd, i_cur_bkwrd, n_cur_bkwrd,
p_cur_bkwrd, i_cur_both, n_cur_both, p_cur_both, gx_rows,
hx_rows, i_current_exogenous, i_lagged_exogenous,
serially_correlated_exogenous, Sigma_e, maximum_endo_lag,
maximum_endo_lead, maximum_exo_lag, maximum_exo_lead,
maximum_exo_det_lag, maximum_exo_det_lead, maximum_lag,
maximum_lead, orig_maximum_endo_lag, orig_maximum_endo_lead,
orig_maximum_exo_lag, orig_maximum_exo_lead,
orig_maximum_exo_det_lag, orig_maximum_exo_det_lead,
orig_maximum_lag, orig_maximum_lead, dynamic_indices,
current_dynamic_indices, forward_indices_d,
backward_indices_d, current_dynamic_indices_d,
exogenous_indices, dynamic!, static!, steady_state!)
end
struct Simulation
name::String
statement::String
options::Dict{String, Any}
data::TimeDataFrame
end
struct Trends
endogenous_steady_state::Vector{Float64}
endogenous_linear_trend::Vector{Float64}
endogenous_quadratic_trend::Vector{Float64}
exogenous_steady_state::Vector{Float64}
exogenous_linear_trend::Vector{Float64}
exogenous_quadratic_trend::Vector{Float64}
exogenous_det_steady_state::Vector{Float64}
exogenous_det_linear_trend::Vector{Float64}
exogenous_det_quadratic_trend::Vector{Float64}
function Trends(ny, nx, nxd)
endogenous_steady_state = Vector{Float64}(undef, ny)
endogenous_linear_trend = Vector{Float64}(undef, ny)
endogenous_quadratic_trend = Vector{Float64}(undef, ny)
exogenous_steady_state = Vector{Float64}(undef, nx)
exogenous_linear_trend = Vector{Float64}(undef, nx)
exogenous_quadratic_trend = Vector{Float64}(undef, nx)
exogenous_det_steady_state = Vector{Float64}(undef, nxd)
exogenous_det_linear_trend = Vector{Float64}(undef, nxd)
exogenous_det_quadratic_trend = Vector{Float64}(undef, nxd)
new(endogenous_steady_state, endogenous_linear_trend,
endogenous_quadratic_trend, exogenous_steady_state,
exogenous_linear_trend, exogenous_quadratic_trend,
exogenous_det_steady_state, exogenous_det_linear_trend,
exogenous_det_quadratic_trend)
end
end
mutable struct ModelResults
endogenous_steady_state::Vector{Float64}
trends::Trends
endogenous_variance::Matrix{Float64}
stationary_variables::Vector{Bool}
exogenous_steady_state::Vector{Float64}
exogenous_deterministic_steady_state::Vector{Float64}
linearrationalexpectations::LinearRationalExpectationsResults
simulations::Vector{Simulation}
smoother::Dict{String, Any}
end
struct Results
model_results::Vector{ModelResults}
end
mutable struct Work
params::Vector{Float64}
residuals::Vector{Float64}
temporary_values::Vector{Float64}
dynamic_variables::Vector{Float64}
exogenous_variables::Matrix{Float64}
jacobian::Matrix{Float64}
qr_jacobian::Matrix{Float64}
model_has_trend::Bool
histval::Matrix{Float64}
end
@enum SymbolType Endogenous Exogenous ExogenousDeterministic Parameter DynareFunction
struct DynareSymbol
longname::String
texname::String
type::SymbolType
orderintype::Integer
end
mutable struct Context
symboltable::Dict{String, DynareSymbol}
models::Vector{Model}
options::Dict
results::Results
work::Work
end
struct ModelInfo
lead_lag_incidence::Array{Int64}
nstatic
nfwrd
npred
nboth
nsfwrd
nspred
ndynamic
maximum_endo_lag
maximum_endo_lead
maximum_exo_lag
maximum_exo_lead
maximum_exo_det_lag
maximum_exo_det_lead
maximum_lag
maximum_lead
orig_maximum_endo_lag
orig_maximum_endo_lead
orig_maximum_exo_lag
orig_maximum_exo_lead
orig_maximum_exo_det_lag
orig_maximum_exo_det_lead
orig_maximum_lag
orig_maximum_lead
end
using PrettyTables
function dynare_table(data, title, column_header, row_header, note;
fmt="%10.4f")
title = ""
function dynare_table_text(data, title, column_header, row_header, note, fmt)
if length(title) > 0
pretty_table([title],
noheader = true,
tf = borderless,
highlighters = hl_row(1, crayon"bold"))
highlighters = (hl_row(1, crayon"bold"),
hl_col(1, crayon"bold")),
backend = :text)
end
formatter = (v, i, j) -> (i > 1 && j > 1) ? round(v, digits=4) : v
pretty_table(data,
noheader = true,
formatters = ft_printf(fmt, 1:size(data,1)+1),
formatters = formatter,
cell_alignment = Dict( (1, i) => :c
for i = 1 :size(data,1)+1),
highlighters = (hl_row(1, crayon"bold"),
hl_col(1, crayon"bold")),
hlines = [:begin, 1, :end],
body_hlines_format = Tuple('─' for _ = 1:4),
vlines = [1])
vlines = [1],
backend = :text)
if length(note) > 0
pretty_table([note],
noheader = true,
tf = borderless)
tf = borderless,
backend = :text)
end
end
function dynare_table_latex(data, title, column_header, row_header, note, fmt)
io = IOBuffer()
write(io, "\\begin{table}[h]\n")
write(io, "\\centering\n")
write(io, "\\begin{threeparttable}\n")
if length(title) > 0
write(io, "\\caption{$title}\n")
end
println(" ")
pretty_table(io,
data,
noheader = true,
# formatters = ft_printf(fmt, 1:size(data,1)+1),
# cell_alignment = Dict( (1, i) => :c
# for i = 1 :size(data,1)+1),
# tf = latex_simple,
# highlighters = (LatexHighlighter((data, i, j) -> (i == 1), ["textbf"]),
# LatexHighlighter((data, i, j) -> (j == 1), ["textbf"])),
# vlines = [1],
backend = :latex)
if length(note) > 0
write(io, "\\begin{tablenotes}\n")
write(io, "\\item $note\n")
write(io, "\\end{tablenotes}\n")
end
write(io, "\\end{threeparttable}\n")
write(io, "\\end{table}\n")
return String(take!(io))
end
function dynare_table(data, title, column_header, row_header, note;
fmt="%10.4f", backend=:text)
if backend == :text
dynare_table_text(data, title, column_header, row_header, note, fmt)
else
dynare_table_latex(data, title, column_header, row_header, note, fmt)
end
end
################
# LaTeX-engine #
################
@enum(LaTeXEngine, LUALATEX, PDFLATEX)
"""
The active LaTeX engine. Initialized the first time [`latexengine`](@ref) is called.
"""
const ACTIVE_LATEX_ENGINE = Ref{Union{Nothing, LaTeXEngine}}(nothing)
function latexengine()
if ACTIVE_LATEX_ENGINE[] === nothing
for (engine, enum) in zip(("lualatex", "pdflatex"), (LUALATEX, PDFLATEX))
@debug "latexengine: looking for latex engine $engine"
if Sys.which(engine) !== nothing
@debug "latexengine: found latex engine $engine, using it"
return ACTIVE_LATEX_ENGINE[] = enum
end
end
throw(MissingExternalProgramError("No LaTeX installation found, figures will not be generated. ",
"Make sure either pdflatex or lualatex are installed and that ",
"the PATH variable is correctly set."))
end
return ACTIVE_LATEX_ENGINE[]
end
latexengine!(eng::LaTeXEngine) = ACTIVE_LATEX_ENGINE[] = eng
_engine_cmd(eng::LaTeXEngine) = `$(lowercase(string(eng)))`
"""
succ, log, cmd = $SIGNATURES
Compile `filename` with LaTeX engine `eng` using the given `flags`.
Return the result of `success`, the contents of the logfile as a string, and the
`Cmd` object that was run (the latter two useful for diagnostics and informative
error messages).
Temporary files (`.aux`, `.log`) are cleaned up.
!!! NOTE
Changing the working directory is required because of external tools like
`gnuplot`, which don't respect `--output-directory`.
"""
function run_latex_once(filename::AbstractString, eng::LaTeXEngine, flags)
dir, file = splitdir(filename)
cmd = `$(_engine_cmd(eng)) $flags $file`
@debug "running latex command $cmd in dir $dir"
succ = cd(dir) do
success(cmd)
end
logfile = _replace_fileext(filename, ".log")
log = if !isfile(logfile)
@warn "failed to find logfile at $(repr(logfile))"
""
else
read(logfile, String)
end
succ, log, cmd
end
function rm_tmpfiles(filename::AbstractString)
logfile = _replace_fileext(filename, ".log")
auxfile = _replace_fileext(filename, ".aux")
rm(logfile; force = true)
rm(auxfile; force = true)
rm(filename; force = true)
end
DEFAULT_FLAGS = Union{String}[] # no default flags currently
"""
Custom flags to the engine can be used in the latex command by `push!`-ing them
into the global variable `CUSTOM_FLAGS`.
"""
CUSTOM_FLAGS = Union{String}[]
############
# Preamble #
############
"""
A vector of stings, added after [`DEFAULT_PREAMBLE`](@ref).
Use this for additional definitions `\\usepackage` statements required by the LaTeX
code you include into plots.
"""
CUSTOM_PREAMBLE = String[]
"""
The default preamble for LaTeX documents. Don't change this, customize
[`CUSTOM_PREAMBLE`](@ref) instead.
"""
DEFAULT_PREAMBLE =
String[
"\\usepackage{pgfplots}",
"\\pgfplotsset{compat=newest}",
"\\usepgfplotslibrary{groupplots}",
"\\usepgfplotslibrary{polar}",
"\\usepgfplotslibrary{smithchart}",
"\\usepgfplotslibrary{statistics}",
"\\usepgfplotslibrary{dateplot}",
]
# Collects the full preamble from the different sources, default and custom
function _default_preamble()
preamble = []
push!(preamble, "% Default preamble")
append!(preamble, DEFAULT_PREAMBLE)
# Collect custom preambles
if !isempty(CUSTOM_PREAMBLE)
push!(preamble, "% Custom preamble from global variable:")
append!(preamble, CUSTOM_PREAMBLE)
end
if isfile(CUSTOM_PREAMBLE_PATH)
str = read(CUSTOM_PREAMBLE_PATH, String)
if !isempty(str)
push!(preamble, "% Custom preamble from custom_preamble.tex:")
push!(preamble, str, "\n")
end
end
if haskey(ENV, "PGFPLOTSX_PREAMBLE_PATH") && isfile(ENV["PGFPLOTSX_PREAMBLE_PATH"])
push!(preamble, "% Custom preamble from ENV path:")
push!(preamble, read(ENV["PGFPLOTSX_PREAMBLE_PATH"], String), "\n")
end
return preamble
end
#######
# PNG #
#######
@enum(PNGEngine, NO_PNG_ENGINE, PNG_PDF_TO_CAIRO, PDF_TO_PPM)
const ACTIVE_PNG_ENGINE = Ref{Union{Nothing, PNGEngine}}(nothing)
function png_engine()
if ACTIVE_PNG_ENGINE[] === nothing
for (engine, enum) in zip(("pdftocairo", "pdftoppm"), (PNG_PDF_TO_CAIRO, PDF_TO_PPM))
@debug "png_engine: looking for png engine $engine"
if Sys.which(engine) !== nothing
@debug "png_engine: found png engine $engine, using it"
return ACTIVE_PNG_ENGINE[] = enum
end
end
return ACTIVE_PNG_ENGINE[] = NO_PNG_ENGINE
end
return ACTIVE_PNG_ENGINE[]
end
function convert_pdf_to_png(pdf::String, png::String; engine::PNGEngine=png_engine(), dpi=150::Integer)
if engine == NO_PNG_ENGINE
throw(MissingExternalProgramError("No PDF to PNG converter found, we looked for `pdftocairo` and `pdftoppm`. ",
"Make sure one of these are installed and available at PATH and restart Julia."))
end
if engine == PNG_PDF_TO_CAIRO
cmd = `pdftocairo -png -r $dpi -singlefile $pdf $png`
elseif engine == PDF_TO_PPM
cmd = `pdftoppm -png -r $dpi -singlefile $pdf $png`
else
error("unreachable reached")
end
@debug "convert_pdf_to_png: running $cmd"
return run(cmd)
end
#######
# SVG #
#######
@enum(SVGEngine, NO_SVG_ENGINE, SVG_PDF_TO_CAIRO, PDF_TO_SVG)
const ACTIVE_SVG_ENGINE = Ref{Union{Nothing, SVGEngine}}(nothing)
function svg_engine()
if ACTIVE_SVG_ENGINE[] === nothing
for (engine, enum) in zip(("pdftocairo", "pdf2svg"), (SVG_PDF_TO_CAIRO, PDF_TO_SVG))
@debug "svg_engine: looking for svg engine $engine"