diff --git a/matlab/+occbin/DSGE_smoother.m b/matlab/+occbin/DSGE_smoother.m index 364b328862aaf3b86735f10fcbe5c26117419457..f8b2ea94dd99faa726546b57067dca8642746592 100644 --- a/matlab/+occbin/DSGE_smoother.m +++ b/matlab/+occbin/DSGE_smoother.m @@ -61,6 +61,9 @@ function [alphahat,etahat,epsilonhat,ahat0,SteadyState,trend_coeff,aKK,T0,R0,P,P smoother_field_list = {'SmoothedVariables', 'UpdatedVariables', 'SmoothedShocks'}; +if not(isempty(xparam1)) + M_ = set_all_parameters(xparam1,estim_params_,M_); +end regime_history=[]; if options_.occbin.smoother.linear_smoother && nargin==12 %% linear smoother @@ -68,7 +71,7 @@ if options_.occbin.smoother.linear_smoother && nargin==12 [alphahat,etahat,epsilonhat,ahat,SteadyState,trend_coeff,aK,T0,R0,P,PK,decomp,Trend,state_uncertainty,oo_,bayestopt_,alphahat0,state_uncertainty0] = ... DsgeSmoother(xparam1,gend,Y,data_index,missing_value,M_,oo_,options_,bayestopt_,estim_params_); tmp_smoother=store_smoother_results(M_,oo_,options_,bayestopt_,dataset_,dataset_info,alphahat,etahat,epsilonhat,ahat,SteadyState,trend_coeff,... - aK,P,PK,decomp,Trend,state_uncertainty); + aK,P,PK,decomp,Trend,state_uncertainty,alphahat0,state_uncertainty0); for jf=1:length(smoother_field_list) oo_.occbin.linear_smoother.(smoother_field_list{jf}) = tmp_smoother.(smoother_field_list{jf}); end @@ -135,10 +138,12 @@ catch ME for iter = 1:numel(ME.stack) ME.stack(iter) end - end +end if error_indicator || isempty(alphahat0) + if options_.occbin.smoother.linear_smoother etahat= oo_.occbin.linear_smoother.etahat; alphahat0= oo_.occbin.linear_smoother.alphahat0; + state_uncertainty0 = oo_.occbin.linear_smoother.state_uncertainty0; base_regime = struct(); if M_.occbin.constraint_nbr==1 base_regime.regime = 0; @@ -157,6 +162,13 @@ if error_indicator || isempty(alphahat0) oo_.occbin.smoother.regime_history(jper) = base_regime; end end + else + oo_.occbin.smoother.error_flag=322; + ahat0 = ahat; + aKK=aK; + PKK=PK; + return; + end end oo_.occbin.smoother.realtime_regime_history = oo_.occbin.smoother.regime_history; @@ -186,6 +198,11 @@ if out.error_flag print_info(out.error_flag, options_.noprint, options_) oo_.occbin.smoother.error_flag=321; return; +elseif not(isequal(out.regime_history(1:options_.occbin.likelihood.first_period_binding_regime_allowed-1),regime_history(1:options_.occbin.likelihood.first_period_binding_regime_allowed-1))) + fprintf('Occbin smoother:: simulation violates first_period_binding_regime_allowed.\n') + print_info(out.error_flag, options_.noprint, options_) + oo_.occbin.smoother.error_flag=322; + return; end regime_history = out.regime_history; if options_.smoother_redux @@ -246,9 +263,14 @@ while is_changed && maxiter>iter && ~is_periodic [~, out, ss] = occbin.solver(M_,options_,oo_.dr,oo_.steady_state,oo_.exo_steady_state,oo_.exo_det_steady_state); if out.error_flag fprintf('Occbin smoother:: simulation within smoother did not converge.\n') - print_info(out.error_flag, false, options_) + print_info(out.error_flag, options_.noprint, options_) oo_.occbin.smoother.error_flag=321; return; + elseif not(isequal(out.regime_history(1:options_.occbin.likelihood.first_period_binding_regime_allowed-1),regime_history(1:options_.occbin.likelihood.first_period_binding_regime_allowed-1))) + fprintf('Occbin smoother:: simulation violates first_period_binding_regime_allowed.\n') + print_info(out.error_flag, options_.noprint, options_) + oo_.occbin.smoother.error_flag=322; + return; end regime_history = out.regime_history; TT = ss.T(oo_.dr.order_var,oo_.dr.order_var,:); @@ -416,24 +438,26 @@ if (~is_changed || occbin_smoother_debug) && nargin==12 TT = sto_TT; oo_.occbin.smoother.regime_history=regime_history0(end-1,:); end - tmp_smoother=store_smoother_results(M_,oo_,options_,bayestopt_,dataset_,dataset_info,alphahat,etahat,epsilonhat,ahat0,SteadyState,trend_coeff,aKK,P,PKK,decomp,Trend,state_uncertainty); - for jf=1:length(smoother_field_list) - oo_.occbin.smoother.(smoother_field_list{jf}) = tmp_smoother.(smoother_field_list{jf}); + if options_.occbin.smoother.store_results + tmp_smoother=store_smoother_results(M_,oo_,options_,bayestopt_,dataset_,dataset_info,alphahat,etahat,epsilonhat,ahat0,SteadyState,trend_coeff,aKK,P,PKK,decomp,Trend,state_uncertainty,alphahat0,state_uncertainty0); + for jf=1:length(smoother_field_list) + oo_.occbin.smoother.(smoother_field_list{jf}) = tmp_smoother.(smoother_field_list{jf}); + end + oo_.occbin.smoother.alphahat=alphahat; + oo_.occbin.smoother.etahat=etahat; + oo_.occbin.smoother.epsilonhat=epsilonhat; + oo_.occbin.smoother.ahat=ahat0; + oo_.occbin.smoother.SteadyState=SteadyState; + oo_.occbin.smoother.trend_coeff=trend_coeff; + oo_.occbin.smoother.aK=aKK; + oo_.occbin.smoother.T0=TT; + oo_.occbin.smoother.R0=RR; + oo_.occbin.smoother.C0=CC; + oo_.occbin.smoother.simul.piecewise = out.piecewise(1:end-1,:); + if ~options_.occbin.simul.piecewise_only + oo_.occbin.smoother.simul.linear = out.linear(1:end-1,:); + end end - oo_.occbin.smoother.alphahat=alphahat; - oo_.occbin.smoother.etahat=etahat; - oo_.occbin.smoother.epsilonhat=epsilonhat; - oo_.occbin.smoother.ahat=ahat0; - oo_.occbin.smoother.SteadyState=SteadyState; - oo_.occbin.smoother.trend_coeff=trend_coeff; - oo_.occbin.smoother.aK=aKK; - oo_.occbin.smoother.T0=TT; - oo_.occbin.smoother.R0=RR; - oo_.occbin.smoother.C0=CC; - oo_.occbin.smoother.simul.piecewise = out.piecewise(1:end-1,:); - if ~options_.occbin.simul.piecewise_only - oo_.occbin.smoother.simul.linear = out.linear(1:end-1,:); - end if options_.occbin.smoother.plot GraphDirectoryName = CheckPath('graphs',M_.fname); latexFolder = CheckPath('latex',M_.dname); diff --git a/matlab/+occbin/check_regime_incertainty.m b/matlab/+occbin/check_regime_incertainty.m new file mode 100644 index 0000000000000000000000000000000000000000..c04cb6bc02f7d6c480e44de1fa3c70e67a67d6e4 --- /dev/null +++ b/matlab/+occbin/check_regime_incertainty.m @@ -0,0 +1,67 @@ +%% check state and shock uncertainty +nstate = size(P1x,1); +nshock = size(Qt,1); + +Kgain=[P1x(:,:,2)*Z'; P0*Tx(:,:,1)'*Z'; Qt(:,:,2)*Rx(:,:,1)'*Z']*inv(Z*P1x(:,:,2)*Z'); + +axo = [a1x(:,2); a0; zeros(nshock,1)]+Kgain*vx(:,2); + + +max(abs(axo(1:nstate)-alphahat(:,2))), +max(abs(axo(nstate+1:nstate*2)-alphahat(:,1))), +max(abs(axo(nstate*2+1:end)-etahat(:,1))), + +Paxo = (eye(size(axo,1))-Kgain*[Z zeros(nshock,nstate) zeros(nshock,nshock)])*[[P1x(:,:,2); P0*Tx(:,:,1)'; Qt(:,:,2)*Rx(:,:,1)'] [Tx(:,:,1)*P0; P0; zeros(nshock,nstate)] [Rx(:,:,1)*Qt(:,:,2); zeros(nstate,nshock); Qt(:,:,2)]]; + +max(max(abs(Paxo(1:nstate,1:nstate)-Px(:,:,1)))), + +if options_.occbin.filter.state_covariance + max(max(abs(Paxo(1:nstate,1:nstate)-V(:,:,2)))), + max(max(abs(Paxo(nstate+1:nstate*2,nstate+1:nstate*2)-V(:,:,1)))), +end + +Pix = Paxo(nstate+1:end, nstate+1:end); % covariance between lagged state and shocks given the data +Pix = 0.5*(Pix+Pix'); + +[UU,S,VV] = svd(Pix); % X=U*S*V'; S = U'*X*V; +is = find(diag(S)>1.e-12); +StateVectorVarianceSquareRoot = chol(S(is,is))';%reduced_rank_cholesky(ReducedForm.StateVectorVariance)'; + +% Get the rank of StateVectorVarianceSquareRoot +state_variance_rank = size(StateVectorVarianceSquareRoot,2); + +% % Factorize the covariance matrix of the structural innovations +% Q_lower_triangular_cholesky = chol(Q)'; +number_of_particles = 31; + +% Initialization of the weights across particles. +xx = UU(:,is)*StateVectorVarianceSquareRoot*norminv(qmc_scrambled(state_variance_rank,number_of_particles,1))'; + +for k=1:number_of_particles + aho(:,k)=Tx(:,:,1)*(xx(1:nstate,k)+alphahat(:,1))+Rx(:,:,1)*(xx(nstate+1:end,k)+etahat); +end + +%% +opts_simul = occbin_options.opts_simul; +for k=1:number_of_particles + opts_simul.SHOCKS = zeros(3,M_.exo_nbr); + opts_simul.exo_pos=1:M_.exo_nbr; + opts_simul.SHOCKS(1,:) = (etahat+xx(nstate+1:end,k))'; + if opts_simul.restrict_state_space + tmp=zeros(M_.endo_nbr,1); + tmp(dr.restrict_var_list,1)=xx(1:nstate,k)+alphahat(:,1); + opts_simul.endo_init = tmp(dr.inv_order_var,1); + my_order_var = dr.order_var(dr.restrict_var_list); + else + tmp = xx(1:nstate,k)+alphahat(:,1); + opts_simul.endo_init = tmp(dr.inv_order_var,1); + my_order_var = dr.order_var; + end + options_.occbin.simul=opts_simul; + [~, out, ss] = occbin.solver(M_,options_,dr,endo_steady_state,exo_steady_state,exo_det_steady_state); + % identity with aho + % [aho(:,k)- (out.piecewise(1,dr.order_var(dr.restrict_var_list))'-out.ys(dr.order_var(dr.restrict_var_list)))] + ako(:,k) = (out.piecewise(1,dr.order_var(dr.restrict_var_list))'-out.ys(dr.order_var(dr.restrict_var_list))); + success(k) = isequal(regx,out.regime_history); + regvec(k) = out.regime_history(1); +end diff --git a/matlab/+occbin/kalman_update_algo_1.m b/matlab/+occbin/kalman_update_algo_1.m index 83cbd587ba8f3ae9e440bd46ce7e1743d54a5f1e..e00a43bba3a00be3ab772397f4024db5d0bf0714 100644 --- a/matlab/+occbin/kalman_update_algo_1.m +++ b/matlab/+occbin/kalman_update_algo_1.m @@ -148,6 +148,7 @@ else end if error_flag etahat=NaN(size(QQQ,1),1); + lik=inf; warning(orig_warning_state); return; end @@ -310,6 +311,8 @@ end if ~error_flag a = out.piecewise(1:2,my_order_var)' - repmat(out.ys(my_order_var),1,2); regimes_=regimes_(1:3); +else + lik=inf; end T = ss.T(my_order_var,my_order_var,1:2); R = ss.R(my_order_var,:,1:2); @@ -328,7 +331,6 @@ end function [a, a1, P, P1, v, alphahat, etahat, lik, V, error_flag] = occbin_kalman_update0(a,a1,P,P1,data_index,Z,v,Y,H,QQQ,TT,RR,CC,iF,L,mm, rescale_prediction_error_covariance, IF_likelihood, state_uncertainty_flag) alphahat=NaN(size(a)); etahat=NaN(size(QQQ,1),2); -lik=Inf; error_flag=0; if state_uncertainty_flag @@ -343,7 +345,7 @@ if nargin<18 IF_likelihood=0; end t=2; -lik=0; +lik=inf; %% forward pass % given updated variables and covarnace in t=1, we make the step to t=2 T = TT(:,:,t); @@ -361,6 +363,7 @@ if isempty(di) a(:,t) = a1(:,t); L(:,:,t) = eye(mm); P1(:,:,t+1) = T*P(:,:,t)*T' + QQ; %p. 111, DK(2012) + lik=0; else ZZ = Z(di,:); v(di,t) = Y(di,t) - ZZ*a(:,t); diff --git a/matlab/+occbin/kalman_update_algo_3.m b/matlab/+occbin/kalman_update_algo_3.m index efffc9badbb50593472ba8681a4f0d2133733efc..31c1f133147965433b5f25c30a631832ec9f51e7 100644 --- a/matlab/+occbin/kalman_update_algo_3.m +++ b/matlab/+occbin/kalman_update_algo_3.m @@ -182,6 +182,7 @@ end if out.error_flag error_flag = out.error_flag; etahat=etahat(:,2); + lik=inf; return; end @@ -288,6 +289,8 @@ end if ~error_flag a = out.piecewise(1:nk+1,my_order_var)' - repmat(out.ys(my_order_var),1,nk+1); regimes_=regimes_(1:3); +else + lik = inf; end T = ss.T(my_order_var,my_order_var,:); R = ss.R(my_order_var,:,:); diff --git a/matlab/+occbin/kalman_update_engine.m b/matlab/+occbin/kalman_update_engine.m index 3459a26712cdb504bbc9852ee227ee886914a9fd..8e1e804e060beb9b1db5df18e7f975bdd19e05c5 100644 --- a/matlab/+occbin/kalman_update_engine.m +++ b/matlab/+occbin/kalman_update_engine.m @@ -38,8 +38,13 @@ if is_multivariate else [ax, a1x, Px, P1x, vx, Fix, Kix, Tx, Rx, Cx, regx, info, M_, likx, alphahat, etahat,TTx,RRx,CCx] = occbin.kalman_update_algo_3(a0,a1,P0,P1,data_index,Z,vv,Fi,Ki,Y,H,Qt,T0,R0,TT,RR,CC,struct(),M_,dr,endo_steady_state,exo_steady_state,exo_det_steady_state,options_,occbin_options,kalman_tol,nk); end +if info==0 likvec = likx; regvec = regx(1); +else + likx = inf; + likvec=[]; +end info0=info; if info if ~isequal(regimes_(1:2),[base_regime base_regime]) @@ -49,8 +54,12 @@ if info [ax, a1x, Px, P1x, vx, Fix, Kix, Tx, Rx, Cx, regx, info, M_, likx, alphahat, etahat,TTx,RRx,CCx] = occbin.kalman_update_algo_3(a0,a1,P0,P1,data_index,Z,vv,Fi,Ki,Y,H,Qt,T0,R0,TT,RR,CC,regimes_(1:2),M_,dr,endo_steady_state,exo_steady_state,exo_det_steady_state,options_,occbin_options,kalman_tol,nk); end end + if info==0 likvec = likx; regvec = regx(1); + else + likx = inf; + end info1=info; else if ~isequal(regimes_(1:2),[base_regime base_regime]) @@ -102,6 +111,12 @@ else end end +if t<options_.occbin.likelihood.first_period_binding_regime_allowed + % I do not search further since I started guessing with base regime + return +end + + diffstart=0; if info==0 if M_.occbin.constraint_nbr==1 @@ -123,7 +138,7 @@ if info==0 end end end -if options_.occbin.filter.use_relaxation && diffstart>options_.occbin.filter.use_relaxation +if options_.occbin.filter.use_relaxation && diffstart>options_.occbin.filter.use_relaxation_tol_period guess_regime = [base_regime base_regime]; options_.occbin.filter.guess_regime = true; guess_regime(1) = regx(1); @@ -183,7 +198,9 @@ if options_.occbin.filter.use_relaxation && diffstart>options_.occbin.filter.use options_.occbin.likelihood.loss_function_regime_guess = false; end -if options_.occbin.likelihood.brute_force_regime_guess && (info0 || info1) %|| (info==0 && ~isequal(regx(1),base_regime)) +% if options_.occbin.likelihood.brute_force_regime_guess && (info0 || info1) %|| (info==0 && ~isequal(regx(1),base_regime)) +if (options_.occbin.likelihood.brute_force_regime_guess && (info0 && info1)) ... + || (options_.occbin.likelihood.brute_force_extra_regime_guess && (info0 || info1)) %|| (info==0 && ~isequal(regx(1),base_regime)) guess_regime = [base_regime base_regime]; options_.occbin.filter.guess_regime = true; @@ -327,9 +344,13 @@ if options_.occbin.likelihood.brute_force_regime_guess && (info0 || info1) %|| ( end end if isnew - likvec = [likvec likx2{use_index}]; + if isempty(likvec) + regvec = regx2{use_index}(1); + else regvec = [regvec; regx2{use_index}(1)]; end + likvec = [likvec likx2{use_index}]; + end end if info2==0 && likx2{use_index}<likx ax=ax2{use_index}; @@ -379,8 +400,12 @@ if options_.occbin.likelihood.loss_function_regime_guess && (info0 || info1) %|| end end if isnew - likvec = [likvec likx2]; + if isempty(likvec) + regvec = regx2(1); + else regvec = [regvec; regx2(1)]; + end + likvec = [likvec likx2]; end if info2==0 && likx2<likx ax=ax2; @@ -411,7 +436,7 @@ if options_.occbin.likelihood.loss_function_regime_guess && (info0 || info1) %|| end if length(likvec)>1 % sum the likelihood of multiple solutions - likx = -2*log(sum(exp(-likvec./2))); + likx = -2*log(sum(exp(-(likvec-min(likvec))./2)))+min(likvec); end if info(1)==0 diff --git a/matlab/+occbin/local_EnKF_iteration.m b/matlab/+occbin/local_EnKF_iteration.m new file mode 100644 index 0000000000000000000000000000000000000000..fa436980ed3d6c1ea1f5bdf5f682a0a9e75ad0bb --- /dev/null +++ b/matlab/+occbin/local_EnKF_iteration.m @@ -0,0 +1,60 @@ +function [liky, StateVectors, output] = local_occbin_EnKF_iteration(yhat,data_index,Z,Y,H,QQQ,t,M_,oo_,options_,occbin_options) +% OUTPUT +% out: structure + +if t==54 %41 %56 %58 %75 + do_nothing = true; +end + +output = struct(); +base_regime = struct(); +if M_.occbin.constraint_nbr==1 + base_regime.regime = 0; + base_regime.regimestart = 1; +else + base_regime.regime1 = 0; + base_regime.regimestart1 = 1; + base_regime.regime2 = 0; + base_regime.regimestart2 = 1; +end + +number_of_particles = size(yhat,2); +success= false(number_of_particles,1); + +di=data_index{1}; +% would be useful to add cov of alphahaty, so that we sample from t-1|t, to +% evaluate uncertainty and precision of PKF around its best estimate!! +% uncertainty of estimated shock etahaty +ZZ = Z(di,:); +opts_simul = occbin_options.opts_simul; +if opts_simul.restrict_state_space + my_order_var = oo_.dr.order_var(oo_.dr.restrict_var_list); +else + my_order_var = oo_.dr.order_var; +end + +% % random shocks +[US,XS] = eig(QQQ(:,:,2)); +% P= U*X*U'; +ishock = find(sqrt(abs(diag(XS)))>1.e-10); +ShockVarianceSquareRoot = chol(XS(ishock,ishock))';%reduced_rank_cholesky(ReducedForm.StateVectorVariance)'; +% Get the rank of ShockVarianceSquareRoot +shock_variance_rank = size(ShockVarianceSquareRoot,2); +ShockVectors = bsxfun(@plus,US(:,ishock)*ShockVarianceSquareRoot*transpose(norminv(qmc_scrambled(shock_variance_rank,number_of_particles,1))),zeros(length(US),1)); + +% Ensemble Particle filter step ... +[liky, StateVectors, sim_regimes, sim_sample] = ... + occbin.ensemble_kalman_filter_step(number_of_particles,yhat,ShockVectors, di, H, my_order_var, Y, ZZ, base_regime, M_, oo_, options_, opts_simul); + + +%% estimate empirical density of states + + + + +if any(not(success)) + % keyboard +end + +end + diff --git a/matlab/+occbin/local_iteration.m b/matlab/+occbin/local_iteration.m new file mode 100644 index 0000000000000000000000000000000000000000..58788242f80b71a52a9e1b4ae1829f249ab5d5a9 --- /dev/null +++ b/matlab/+occbin/local_iteration.m @@ -0,0 +1,62 @@ +function ysim = local_occbin_iteration(yhat, epsilon, DynareOutput, Model, DynareOptions) + +nshocks = size(epsilon,2); +success= false(nshocks,1); +% isimul=0; +ysim = nan(length(DynareOutput.dr.restrict_var_list),nshocks); +DynareOptions.occbin.simul.waitbar=false; +% opts_simul = occbin_options.opts_simul; +% opts_simul.SHOCKS = zeros(3,M_.exo_nbr); +% opts_simul.exo_pos=1:M_.exo_nbr; +% opts_simul.SHOCKS(1,:) = etahat(:,2)'; +my_order_var = DynareOutput.dr.order_var(DynareOutput.dr.restrict_var_list); +% my_order_var = DynareOutput.dr.order_var(DynareOutput.dr.restrict_var_list(DynareOutput.dr.restrict_columns)); +% options_.occbin.simul=opts_simul; +% [~, out, ss] = occbin.solver(M_,DynareOutput,options_); +endo_nbr = Model.endo_nbr; +restrict_var_list = DynareOutput.dr.restrict_var_list; +restrict_columns = DynareOutput.dr.restrict_columns; +restrict_column_var_list = restrict_var_list(restrict_columns); +inv_order_var=DynareOutput.dr.inv_order_var; + +% for k=1:nshocks +parfor k=1:nshocks + opts = DynareOptions; % clearly classify opts as a temporary variable for parfor + tmp=zeros(endo_nbr,1); + tmp(restrict_column_var_list,1)=yhat(:,k); + opts.occbin.simul.endo_init=tmp(inv_order_var,1); + opts.occbin.simul.SHOCKS = epsilon(:,k)'; + [~, outs0] = occbin.solver(Model,DynareOutput,opts); + + if outs0.error_flag + success(k)=false; + else + success(k)=true; + outs1=outs0; +% oos1=occbin.unpack_simulations(Model,oos0,DynareOptions); + % the last successful simul!!! + end + if success(k) +% isimul=isimul+1; + ysim(:,k)=outs1.piecewise(1,my_order_var); +% regime_history(k)=outs1.regime_history; +% for iendo=1:Model.endo_nbr +% oo.occbin.endo_piecewise.(Model.endo_names{iendo})(k,1)=oos1.occbin.endo_piecewise.(Model.endo_names{iendo})(1); +% oo.occbin.endo_linear.(Model.endo_names{iendo})(k,1)=oos1.occbin.endo_linear.(Model.endo_names{iendo})(1); +% end +% for iexo=1:Model.exo_nbr +% oo.occbin.exo.(Model.exo_names{iexo})(k,1)=oos1.occbin.exo.(Model.exo_names{iexo})(1); +% end + + end + +end + +if any(any(isnan(ysim))) + keyboard +end + +if any(not(success)) + keyboard +end + diff --git a/matlab/+occbin/local_pkf_iteration.m b/matlab/+occbin/local_pkf_iteration.m new file mode 100644 index 0000000000000000000000000000000000000000..9149cac333ced915722e98a14c26464b328a19a9 --- /dev/null +++ b/matlab/+occbin/local_pkf_iteration.m @@ -0,0 +1,1362 @@ +function [ay, a1y, Py, P1y, vy, Ty, Ry, Cy, regimesy, info, M_, liky, StateVectors, output] = local_pkf_iteration(yhat,a,a1,P,P1,data_index,Z,v,Y,H,QQQ,T0,R0,TT,RR,CC,regimes0,isqvec,t,M_,dr, endo_steady_state, exo_steady_state, exo_det_steady_state,options_,occbin_options) +% OUTPUT +% out: structure + +if t==187 %202 %203 %207 %54 %41 %56 %58 %75 + do_nothing = true; +end + +base_regime = struct(); +if M_.occbin.constraint_nbr==1 + base_regime.regime = 0; + base_regime.regimestart = 1; +else + base_regime.regime1 = 0; + base_regime.regimestart1 = 1; + base_regime.regime2 = 0; + base_regime.regimestart2 = 1; +end + +if isempty(yhat) + [U,X] = svd(0.5*(P+P')); + % P= U*X*U'; % symmetric matrix! + is = find(diag(X)>1.e-12); + StateVectorVarianceSquareRoot = chol(X(is,is))';%reduced_rank_cholesky(ReducedForm.StateVectorVariance)'; + + % Get the rank of StateVectorVarianceSquareRoot + state_variance_rank = size(StateVectorVarianceSquareRoot,2); + + % % Factorize the covariance matrix of the structural innovations + % Q_lower_triangular_cholesky = chol(Q)'; + + % Initialization of the weights across particles. + number_of_particles = options_.occbin.filter.particle.number_of_particles; + yhat = U(:,is)*StateVectorVarianceSquareRoot*transpose(norminv(qmc_scrambled(state_variance_rank,number_of_particles,1)))+repmat(a,[1 number_of_particles]); +end + + +number_of_particles = size(yhat,2); +number_of_shocks_per_particle =options_.occbin.filter.particle.number_of_shocks_per_particle; +epsilon = nan(M_.exo_nbr,number_of_particles); +regimes = cell(number_of_particles,1); +simul_regimes = cell(number_of_particles,1); +success= false(number_of_particles,1); +is_constrained= false(number_of_particles,M_.occbin.constraint_nbr); +is_constrained_in_expectation= false(number_of_particles,M_.occbin.constraint_nbr); +is_simul_constrained= false(number_of_particles,M_.occbin.constraint_nbr); +is_simul_constrained_in_expectation= false(number_of_particles,M_.occbin.constraint_nbr); +regime_exit = nan(number_of_particles,M_.occbin.constraint_nbr); +simul_regime_exit = nan(number_of_particles,M_.occbin.constraint_nbr); + +di=data_index{2}; +[ay, a1y, Py, P1y, vy, Ty, Ry, Cy, regimesy, info, M_, liky, etahaty, alphahaty, Vy] = occbin.kalman_update_engine(a,a1,P,P1,t,data_index,Z,v,Y,H,QQQ,T0,R0,TT,RR,CC,regimes0,base_regime,di,M_,dr, endo_steady_state, exo_steady_state, exo_det_steady_state,options_,occbin_options); +if info + options_.occbin.filter.particle.empirical_data_density.status = true; +end +% would be useful to add cov of alphahaty, so that we sample from t-1|t, to +% evaluate uncertainty and precision of PKF around its best estimate!! +% uncertainty of estimated shock etahaty +ZZ = Z(di,:); +opts_simul = occbin_options.opts_simul; +if opts_simul.restrict_state_space + my_order_var = dr.order_var(dr.restrict_var_list); +else + my_order_var = dr.order_var; +end +if options_.debug + % START block computing uncertainty of shocks t|t. not used so far + ishock = find(sqrt(diag(QQQ(:,:,2)))>1.e-8); + if rank(P) + Ve0 = inv(inv(QQQ(ishock,ishock,2))+Ry(:,ishock,2)'*ZZ'*((ZZ*Ty(:,:,2)*P*Ty(:,:,2)'*ZZ')\(ZZ*Ry(:,ishock,2)))); + e0 = Ve0*Ry(:,ishock,2)'*ZZ'*((ZZ*Ty(:,:,2)*P*Ty(:,:,2)'*ZZ')\(Y(di,2)-ZZ*(Cy(:,2)+Ty(:,:,2)*a))); + disp_verbose(['check shock t|t: ' num2str(max(abs(e0-etahaty)))],options_.debug) + else + Ve0 = zeros(length(ishock),length(ishock)); + e0 = etahaty(ishock); + end + % END block computing uncertainty of shocks t|t. not used so far +end + +if options_.debug + % % updated state mean and covariance t-1|t + [U,X] = svd(Vy(:,:,1)); + % P= U*X*U'; + is = find(diag(X)>1.e-12); + StateVectorVarianceSquareRoot = chol(X(is,is))';%reduced_rank_cholesky(ReducedForm.StateVectorVariance)'; + StateVectorMean = alphahaty(:,1); + + % Get the rank of StateVectorVarianceSquareRoot + state_variance_rank = size(StateVectorVarianceSquareRoot,2); + if state_variance_rank + StateVectors = bsxfun(@plus,U(:,is)*StateVectorVarianceSquareRoot*transpose(norminv(qmc_scrambled(state_variance_rank,number_of_particles,1))),StateVectorMean); + else + StateVectors = bsxfun(@plus,zeros(length(StateVectorMean),number_of_particles),StateVectorMean); + end + + % % % yhat0=yhat; + % % % yhat1 = StateVectors; +end + +if ~(options_.occbin.filter.particle.likelihood_only && ~options_.occbin.filter.particle.empirical_data_density.status) + % % random shocks + [US,XS] = svd(QQQ(:,:,2)); + % P= U*X*U'; + ishock = find(sqrt(abs(diag(XS)))>1.e-10); + ShockVarianceSquareRoot = chol(XS(ishock,ishock))';%reduced_rank_cholesky(ReducedForm.StateVectorVariance)'; + % Get the rank of ShockVarianceSquareRoot + shock_variance_rank = size(ShockVarianceSquareRoot,2); + ShockVectors = bsxfun(@plus,US(:,ishock)*ShockVarianceSquareRoot*transpose(norminv(qmc_scrambled(shock_variance_rank,number_of_particles,1))),zeros(length(US),1)); + + if info + % particle filter step ... + [sim_llik, sim_regimes, sim_sample, hp_sim_regimes] = ... + occbin.ppf_simulated_density(number_of_particles, 1,yhat,ShockVectors, di, H, my_order_var, QQQ, Y, ZZ, base_regime, regimesy, M_, dr, endo_steady_state, exo_steady_state, exo_det_steady_state, options_, opts_simul); + [yhat1,liky1, updated_regimes, updated_sample, updated_mode, use_pkf_distribution] = ... + occbin.pkf_conditional_density(yhat, liky, a, a1, P, P1,ay,Py, t, ... + data_index,Z,v,Y,H,QQQ,T0,R0,TT,RR,CC,info,regimes0,base_regime,regimesy,isqvec, ... + M_,dr, endo_steady_state, exo_steady_state, exo_det_steady_state,options_,occbin_options); + options_.occbin.filter.guess_regime = true; + guess_regime=sim_sample.regime{hp_sim_regimes.index(1)}; + guess_regime = [guess_regime base_regime]; + [ay, a1y, Py, P1y, vy, Ty, Ry, Cy, regimesy, info, M_, liky, etahaty, alphahaty, Vy] = occbin.kalman_update_algo_1(a,a1,P,P1,data_index,Z,v,Y,H,QQQ,T0,R0,TT,RR,CC,guess_regime,M_,dr, endo_steady_state, exo_steady_state, exo_det_steady_state,options_,occbin_options); + if info == 330 % periodic solution + options_.occbin.filter.periodic_solution=true; + [ay, a1y, Py, P1y, vy, Ty, Ry, Cy, regimesy, info, M_, liky, etahaty, alphahaty, Vy] = occbin.kalman_update_algo_1(a,a1,P,P1,data_index,Z,v,Y,H,QQQ,T0,R0,TT,RR,CC,guess_regime,M_,dr, endo_steady_state, exo_steady_state, exo_det_steady_state,options_,occbin_options); + options_.occbin.filter.periodic_solution=false; + end + if info + options_.occbin.filter.guess_regime_nocheck = true; + [ay, a1y, Py, P1y, vy, Ty, Ry, Cy, regimesy, info, M_, liky, etahaty, alphahaty, Vy] = occbin.kalman_update_algo_1(a,a1,P,P1,data_index,Z,v,Y,H,QQQ,T0,R0,TT,RR,CC,guess_regime,M_,dr, endo_steady_state, exo_steady_state, exo_det_steady_state,options_,occbin_options); + options_.occbin.filter.guess_regime_nocheck = false; + end + options_.occbin.filter.guess_regime = false; + + end + if number_of_shocks_per_particle>1 + ShockVectorsPerParticle = bsxfun(@plus,US(:,ishock)*ShockVarianceSquareRoot*transpose(norminv(qmc_scrambled(shock_variance_rank,number_of_shocks_per_particle,1))),zeros(shock_variance_rank,1)); + end +end + + +a10=a1; +P10 = P1; +P0=P; +try + chol(P0); +catch + P0 = 0.5*(P0+P0'); +end +a0=a; +P=P*0; +P1(:,:,1)=P; +QQ = RR(:,:,2)*QQQ(:,:,2)*transpose(RR(:,:,2)); + +a(:,1) = alphahaty(:,1); +a1(:,1) = alphahaty(:,1); +a1(:,2) = TT(:,:,2)*a; +P1(:,:,2) = QQ; +options_.occbin.filter.state_covariance=false; +my_data_index = data_index; +my_data_index{1} = []; +my_data = Y; +my_data(:,1) = nan; + +if options_.debug + + use_the_engine = true; + if info==0 + % start with PKF regime to speed-up search! + guess_regime = regimesy(1:2); + options_.occbin.filter.guess_regime = true; + [ayy, a1yy, Pyy, P1yy, vyy, Tyy, Ryy, Cyy, regimesyy, infoyy, M_, likyy, etahatyy, alphahatyy] = occbin.kalman_update_algo_1(a,a1,P,P1,my_data_index,Z,v,my_data,H,QQQ,T0,R0,TT,RR,CC,guess_regime,M_,dr, endo_steady_state, exo_steady_state, exo_det_steady_state,options_,occbin_options); + options_.occbin.filter.guess_regime = false; + if infoyy == 0 + use_the_engine = false; + end + end + if use_the_engine + [ayy, a1yy, Pyy, P1yy, vyy, Tyy, Ryy, Cyy, regimesyy, infoyy, M_, likyy, etahatyy, alphahatyy] = occbin.kalman_update_engine(a,a1,P,P1,t,my_data_index,Z,v,my_data,H,QQQ,T0,R0,TT,RR,CC,regimes0,base_regime,my_data_index,M_,dr, endo_steady_state, exo_steady_state, exo_det_steady_state,options_,occbin_options); + end + + % check consistency + % distribution of state in 0 (mean a0, variance P0), smoothed state is + % a=alphahaty(:,1)=alphahatyy(:,1) + [U,X,W] = svd(0.5*(P0+P0')); + % P0= U*X*U'; + is = find(diag(X)>1.e-12); + + dy = W(:,is)'*(a0 - a(:,1)); + F0 = W(:,is)'*P0*W(:,is); + sig0=sqrt(diag(F0)); + log_dF0 = log(det(F0./(sig0*sig0')))+2*sum(log(sig0)); + iF = inv(F0./(sig0*sig0'))./(sig0*sig0'); + lik0 = log_dF0 + transpose(dy)*iF*dy + length(is)*log(2*pi); + + di = data_index{2}; + ZZ = Z(di,:); + % optimal smoothed state in 0 is distributed as alphaty(:,1), Vy(:,:,1), hence dy0=0 + [m, Vm, likm, log_dF00, iF00] = get_the_smoothed_density(a0, alphahatyy(:,1), Y(di,2), U, W, X, is, QQQ, Cyy, Ryy, Tyy, ZZ); + if abs((likyy+lik0-likm)-liky)>1.e-10 + disp_verbose(['t =' int2str(t) '. Consistency check of probabilities of filtered states and observables larger than tolerance'],options_.debug) + disp_verbose(['Tolerance: ' num2str(1.e-10), ' vs Check: ' num2str(abs((likyy+lik0-likm)-liky))],options_.debug) + end + + iU=inv(U); + % % % F00 = iU(is,:)*Vy(:,:,1)*U(:,is); + % % % % % % % F00 = U(:,is)'*Vy(:,:,1)*U(:,is); + % % % sig00=sqrt(diag(F00)); + % % % log_dF00 = log(det(F00./(sig00*sig00')))+2*sum(log(sig00)); + % % % iF00 = inv(F00./(sig00*sig00'))./(sig00*sig00'); + % % % dy0 = U(:,is)'*( a(:,1)- a(:,1)); + % % % lik00 = log_dF00 + transpose(dy0)*iF00*dy0 + length(is)*log(2*pi); + % % % + % % % if abs((likyy+lik0-lik00)-liky)>1.e-10 + % % % disp('consistency of probabilities of filtered states and observables is missing!') + % % % end +end + +if ~(options_.occbin.filter.particle.likelihood_only && ~options_.occbin.filter.particle.empirical_data_density.status) + if ~options_.occbin.filter.particle.nograph + + % observed Y mean and covariance given0 + PZ0 = ZZ*P10(:,:,2)*ZZ'+H(di,di); + ObsVectorMean0 = ZZ*a10(:,2); + if isempty(reduced_rank_cholesky(PZ0)') + obs_variance_rank=0; + else + ObsVectorVarianceSquareRoot0 = chol(PZ0)';%reduced_rank_cholesky(ReducedForm.StateVectorVariance)'; + + % Get the rank of ObsVectorVarianceSquareRoot + obs_variance_rank = size(ObsVectorVarianceSquareRoot0,2); + end + % ObsVectors0 = zeros(size(ObsVectorMean0,1),number_of_particles); + % % % ObsVectors0 = bsxfun(@plus,ObsVectorVarianceSquareRoot0*transpose(norminv(qmc_scrambled(obs_variance_rank,number_of_particles,1))),ObsVectorMean0); + end + % observed Y mean and covariance + PZ = ZZ*P1y(:,:,2)*ZZ'+H(di,di); + ObsVectorMean = ZZ*a1y(:,2); + ObsVectorVarianceSquareRoot = chol(PZ)';%reduced_rank_cholesky(ReducedForm.StateVectorVariance)'; + + % Get the rank of ObsVectorVarianceSquareRoot + obs_variance_rank = size(ObsVectorVarianceSquareRoot,2); + + % ObsVectors = zeros(size(ObsVectorMean,1),number_of_particles); + % % % ObsVectors = bsxfun(@plus,ObsVectorVarianceSquareRoot*transpose(norminv(qmc_scrambled(obs_variance_rank,number_of_particles,1))),ObsVectorMean); +end + + +%% estimate empirical density of states +if options_.debug + nd=length(is); + if nd>0 + if nd>2 + bw = nan(nd,1); + for jd=1:nd + ss=std(yhat(is(jd),:)); + bw(jd) = ss*(4/(nd+2)/number_of_particles)^(1/(nd+4)); + end + f1 = mvksdensity(yhat(is,:)',[yhat(is,:) alphahaty(is,1)]','bandwidth',bw,'Function','pdf'); + else + f1 = ksdensity(yhat(is,:)',[yhat(is,:) alphahaty(is,1)]'); + end + fm = f1(end); + f1 = f1(1:end-1); + end +end + +P1(:,:,1)=P; +P1(:,:,2) = QQ; +number_of_successful_particles = 0; +simulated_regimes = []; +number_of_simulated_regimes = 0; +updated_regimes = []; +number_of_updated_regimes = 0; +out.error_flag=0; +for k=1:number_of_particles + % parfor k=1:number_of_particles + a(:,1) = yhat(:,k); + a1(:,1) = yhat(:,k); + a1(:,2) = TT(:,:,2)*yhat(:,k); + + + if ~(options_.occbin.filter.particle.likelihood_only && ~options_.occbin.filter.particle.empirical_data_density.status) + % run particles + opts_simul.SHOCKS(1,:) = ShockVectors(:,k)'; + if opts_simul.restrict_state_space + tmp=zeros(M_.endo_nbr,1); + tmp(dr.restrict_var_list,1)=yhat(:,k); + opts_simul.endo_init = tmp(dr.inv_order_var,1); + else + opts_simul.endo_init = yhat(dr.inv_order_var,k); + end + options_.occbin.simul=opts_simul; + options_.occbin.simul.piecewise_only = false; + options_.noprint = true; + [~, out, ss] = occbin.solver(M_,options_,dr, endo_steady_state, exo_steady_state, exo_det_steady_state); + end + + % conditional likelihood + options_.occbin.filter.state_covariance=true; + use_the_engine = true; + if info==0 + % start with PKF regime to speed-up search! + guess_regime = regimesy(1:2); + options_.occbin.filter.guess_regime = true; + [ax, a1x, Px, P1x, vx, Tx, Rx, Cx, regimesx, infox, ~, likxc, etahatx,alphahatx, Vxc] = occbin.kalman_update_algo_1(a,a1,P,P1,my_data_index,Z,v,my_data,H,QQQ,T0,R0,TT,RR,CC,guess_regime,M_,dr, endo_steady_state, exo_steady_state, exo_det_steady_state,options_,occbin_options); + options_.occbin.filter.guess_regime = false; + if infox == 0 + use_the_engine = false; + end + end + if use_the_engine + [ax, a1x, Px, P1x, vx, Tx, Rx, Cx, regimesx, infox, ~, likxc, etahatx,alphahatx, Vxc] = occbin.kalman_update_engine(a,a1,P,P1,t,my_data_index,Z,v,my_data,H,QQQ,T0,R0,TT,RR,CC,regimes0,base_regime,my_data_index{2},M_,dr, endo_steady_state, exo_steady_state, exo_det_steady_state,options_,occbin_options); + end + + if (out.error_flag==0 && infox ==0) || (out.error_flag==0 && info) + + if infox ==0 + % handle conditional likelihood and updated regime + [~, tmp, tmp_str]=occbin.backward_map_regime(regimesx(1)); + if M_.occbin.constraint_nbr==1 + tmp_str = tmp_str(2,:); + else + tmp_str = [tmp_str(2,:) tmp_str(4,:)]; + end + if number_of_updated_regimes>0 + this_updated_regime = find(ismember(all_updated_regimes,{tmp_str})); + end + if number_of_updated_regimes==0 || isempty(this_updated_regime) + number_of_updated_regimes = number_of_updated_regimes+1; + updated_regimes(number_of_updated_regimes).index = k; + if isequal(regimesy(1),regimesx(1)) + updated_regimes(number_of_updated_regimes).is_pkf_regime = true; + else + updated_regimes(number_of_updated_regimes).is_pkf_regime = false; + end + updated_regimes(number_of_updated_regimes).regime = tmp_str; + all_updated_regimes = {updated_regimes.regime}; + if ~options_.occbin.filter.particle.likelihood_only + updated_regimes(number_of_updated_regimes).obsvar = ZZ*(Tx(:,:,1)*P0*Tx(:,:,1)'+Rx(:,:,1)*QQQ(:,:,2)*Rx(:,:,1)')*ZZ' + H(di,di); + updated_regimes(number_of_updated_regimes).obsmean = ZZ*(Tx(:,:,1)*a0+Cx(:,1)); + updated_regimes(number_of_updated_regimes).ss.C = Cx(:,1); + updated_regimes(number_of_updated_regimes).ss.R = Rx(:,:,1); + updated_regimes(number_of_updated_regimes).ss.T = Tx(:,:,1); + uF = updated_regimes(number_of_updated_regimes).obsvar; + sig=sqrt(diag(uF)); + vv = Y(di,2) - updated_regimes(number_of_updated_regimes).obsmean; + if options_.rescale_prediction_error_covariance + log_duF = log(det(uF./(sig*sig')))+2*sum(log(sig)); + iuF = inv(uF./(sig*sig'))./(sig*sig'); + else + log_duF = log(det(uF)); + iuF = inv(uF); + end + updated_regimes(number_of_updated_regimes).lik = ... + log_duF + transpose(vv)*iuF*vv + length(di)*log(2*pi); + end + else + updated_regimes(this_updated_regime).index = [updated_regimes(this_updated_regime).index k]; + end + + number_of_successful_particles = number_of_successful_particles + 1; + + likxx(k) = likxc; + + if options_.debug + + % % conditional update info + dy = W(:,is)'*(a0 - a(:,1)); + likx0(k,1) = log_dF0 + transpose(dy)*iF*dy + length(is)*log(2*pi); + [mx(:,k), Vmx(:,:,k), likmx00(k,1)] = get_the_smoothed_density(a0, a(:,1), Y(di,2), U, W, X, is, QQQ, Cx, Rx, Tx, ZZ); + dy0 = iU(is,:)*( alphahaty(:,1)- a(:,1)); + % dy0 = U(:,is)'*( alphahaty(:,1)- a(:,1)); + likx00(k,1) = log_dF00 + transpose(dy0)*iF00*dy0 + length(is)*log(2*pi); + + likx(k) = (likxx(k)+likx0(k)-likx00(k)); + likmx(k) = (likxx(k)+likx0(k)-likmx00(k)); + lik(k)=(likxx(k)-likx00(k)); + likj(k)=(likxx(k)+likx0(k)); + end + + epsilon(:,k)=etahatx; + regimes{k} = regimesx; + success(k)=true; + y01(:,k) = alphahatx(:,1); %smoothed state 0|1! + y11(:,k) = ax(:,1); %updated state! + if M_.occbin.constraint_nbr==1 + if logical(regimesx(1).regime(1)) + do_nothing=true; + end + regime_exit(k,1) = max(regimesx(1).regimestart); + is_constrained(k,1) = logical(regimesx(1).regime(1)); + is_constrained_in_expectation(k,1) = any(regimesx(1).regime); + else + regime_exit(k,1) = max(regimesx(1).regimestart1); + regime_exit(k,2) = max(regimesx(1).regimestart2); + is_constrained(k,1) = logical(regimesx(1).regime1(1)); + is_constrained(k,2) = logical(regimesx(1).regime2(1)); + is_constrained_in_expectation(k,1) = any(regimesx(1).regime1); + is_constrained_in_expectation(k,2) = any(regimesx(1).regime1); + end + end + + if ~(options_.occbin.filter.particle.likelihood_only && ~options_.occbin.filter.particle.empirical_data_density.status) + % % store particle + [~, tmp, tmp_str]=occbin.backward_map_regime(out.regime_history(1)); + if M_.occbin.constraint_nbr==1 + tmp_str = tmp_str(2,:); + else + tmp_str = [tmp_str(2,:) tmp_str(4,:)]; + end + if number_of_simulated_regimes>0 + this_simulated_regime = find(ismember(all_simulated_regimes,{tmp_str})); + end + if number_of_simulated_regimes==0 || isempty(this_simulated_regime) + number_of_simulated_regimes = number_of_simulated_regimes+1; + simulated_regimes(number_of_simulated_regimes).index = k; + if isequal(regimesy(1),out.regime_history(1)) + simulated_regimes(number_of_simulated_regimes).is_pkf_regime = true; + else + simulated_regimes(number_of_simulated_regimes).is_pkf_regime = false; + end + simulated_regimes(number_of_simulated_regimes).obsvar = ZZ*(ss.T(my_order_var,my_order_var)*P0*ss.T(my_order_var,my_order_var)'+ss.R(my_order_var,:)*QQQ(:,:,2)*ss.R(my_order_var,:)')*ZZ' + H(di,di); + simulated_regimes(number_of_simulated_regimes).obsmean = ZZ*(ss.T(my_order_var,my_order_var)*a0+ss.C(my_order_var)); + simulated_regimes(number_of_simulated_regimes).regime = tmp_str; + simulated_regimes(number_of_simulated_regimes).ss.C = ss.C(my_order_var); + simulated_regimes(number_of_simulated_regimes).ss.R = ss.R(my_order_var,:); + simulated_regimes(number_of_simulated_regimes).ss.T = ss.T(my_order_var,my_order_var); + all_simulated_regimes = {simulated_regimes.regime}; + else + simulated_regimes(this_simulated_regime).index = [simulated_regimes(this_simulated_regime).index k]; + end + simul_regimes{k} = out.regime_history; + y10(:,k) = out.piecewise(1,my_order_var)-out.ys(my_order_var)'; + y10lin(:,k) = out.linear(1,my_order_var)-out.ys(my_order_var)'; + if M_.occbin.constraint_nbr==1 + simul_regime_exit(k,1) = max(out.regime_history.regimestart); + is_simul_constrained(k,1) = logical(out.regime_history.regime(1)); + is_simul_constrained_in_expectation(k,1) = any(out.regime_history.regime); + else + simul_regime_exit(k,1) = max(out.regime_history.regimestart1); + simul_regime_exit(k,2) = max(out.regime_history.regimestart2); + is_simul_constrained(k,1) = logical(out.regime_history.regime1(1)); + is_simul_constrained(k,2) = logical(out.regime_history.regime2(1)); + is_simul_constrained_in_expectation(k,1) = any(out.regime_history.regime1); + is_simul_constrained_in_expectation(k,2) = any(out.regime_history.regime2); + end + Cx1(:,k) = ss.C(my_order_var); + Rx1(:,:,k) = ss.R(my_order_var,:,1); + Tx1(:,:,k) = ss.T(my_order_var,my_order_var,1); + % the distribution of ZZ*y10 needs to be confronted with ZZ*a1y, ZZ*P1y*ZZ', + % i.e. the normal approximation using the regime consistent with the observable! + + if number_of_shocks_per_particle>1 + for kshock = 1:number_of_shocks_per_particle + opts_simul.SHOCKS(1,:) = ShockVectorsPerParticle(:,kshock)'; + options_.occbin.simul=opts_simul; + options_.occbin.simul.piecewise_only = false; + [~, tmp_out, tmp_ss] = occbin.solver(M_,options_,dr, endo_steady_state, exo_steady_state, exo_det_steady_state); + if tmp_out.error_flag==0 + shock_simul.regimes{k,kshock} = tmp_out.regime_history; + shock_simul.z10{k}(:,kshock) = ZZ*tmp_out.piecewise(1,my_order_var)'; + % shock_simul.y10{k}(:,kshock) = tmp_out.piecewise(1,my_order_var); + % shock_simul.y10lin{k}(:,kshock) = tmp_out.linear(1,my_order_var); + if M_.occbin.constraint_nbr==1 + shock_simul.regime_exit{k}(kshock,1) = max(tmp_out.regime_history.regimestart); + shock_simul.is_constrained{k}(kshock,1) = logical(tmp_out.regime_history.regime(1)); + shock_simul.is_constrained_in_expectation{k}(kshock,1) = any(tmp_out.regime_history.regime); + else + shock_simul.regime_exit{k}(kshock,1) = max(tmp_out.regime_history.regimestart1); + shock_simul.regime_exit{k}(kshock,2) = max(tmp_out.regime_history.regimestart2); + shock_simul.is_constrained{k}(kshock,1) = logical(tmp_out.regime_history.regime1(1)); + shock_simul.is_constrained{k}(kshock,2) = logical(tmp_out.regime_history.regime2(1)); + shock_simul.is_constrained_in_expectation{k}(kshock,1) = any(tmp_out.regime_history.regime1); + shock_simul.is_constrained_in_expectation{k}(kshock,2) = any(tmp_out.regime_history.regime2); + end + % shock_simul.C{k}(:,kshock) = tmp_ss.C(my_order_var); + % shock_simul.R{k}(:,:,kshock) = tmp_ss.R(my_order_var,:,1); + % shock_simul.T{k}(:,:,kshock) = tmp_ss.T(my_order_var,my_order_var,1); + % the distribution of ZZ*y10 needs to be confronted with ZZ*a1y, ZZ*P1y*ZZ', + % i.e. the normal approximation using the regime consistent with the observable! + end + end + % shock_simul.z10{k} = ZZ*shock_simul.y10{k}; + shock_simul.is_simul_shock_overlap(k,1)=false; + for kz=1:size(ZZ,1) + ztmp = shock_simul.z10{k}(kz,:); + [zsort, izsort] = sort(ztmp); + rtmp = shock_simul.regime_exit{k}(izsort); + if max(sign(diff(rtmp)))-min(sign(diff(rtmp)))>1 + shock_simul.is_simul_shock_overlap(k,1)=true; + % plot(zsort,rtmp), pause, + end + + end + output.filtered.shock_simul = shock_simul; + end + end + else + if ~(options_.occbin.filter.particle.likelihood_only && ~options_.occbin.filter.particle.empirical_data_density.status) + do_nothing=false; + y10(:,k) = nan(length(my_order_var),1); + end + end + +end + +nobs = size(ZZ,1); +nrow=floor(sqrt(nobs)); +ncol=ceil(nobs/nrow); + +number_of_successful_particles = length(find(success)); +options_.occbin.filter.particle.empirical_conditional_data_density.status = false; +if number_of_successful_particles==0 && info + y11=[]; + number_of_successful_particles = numel([simulated_regimes.index]); + success([simulated_regimes.index])=true; + options_.occbin.filter.particle.empirical_conditional_data_density.status = true; +end +number_of_constrained_particles = length(find(is_constrained(success,1))); +if M_.occbin.constraint_nbr==2 + number_of_constrained_particles(1,2) = length(find(is_constrained(success,2))); +end + +%% estimate empirical density of observables +is_tobit=false; +if ~(options_.occbin.filter.particle.likelihood_only && ~options_.occbin.filter.particle.empirical_data_density.status) + y100 = y10; + y10 = y10(:,success); + + % kernel density + z10 = ZZ*y10; + if nobs>2 + bw = nan(nobs,1); + for jd=1:nobs + ss=std(transpose(z10(jd,:))); + bw(jd) = ss*(4/(nobs+2)/number_of_successful_particles)^(1/(nobs+4)); + end + fobs_kernel = mvksdensity(z10',Y(di,2)','bandwidth',bw,'Function','pdf'); + else + fobs_kernel = ksdensity(z10',Y(di,2)'); + end + + % full non-parametric density + ydist = sqrt(sum((Y(di,2)-z10).^2,1)); + [s, is] = sort(ydist); + %bin population + nbins = ceil(sqrt(size(z10,2))); + + binpop = ceil(number_of_successful_particles/nbins/2)*2; + + % volume of n-dimensional sphere + binvolume = pi^(nobs/2)/gamma(nobs/2+1)*(0.5*(s(binpop)+s(binpop+1)))^nobs; + + + % samplevolume = [min(z10,[],2) max(z10,[],2)]; + % samplevolume = prod(diff(samplevolume')*1.02); + fobs = binpop/number_of_successful_particles/binvolume; + + isux = find(success); + isux = isux(is); % sorted successful deviations from data + % check for regime of Y + is_data_regime = false(number_of_simulated_regimes,1); + for kr=1:number_of_simulated_regimes + is_data_regime(kr) = any(ismember(simulated_regimes(kr).index,isux(1:binpop))); + end + + if options_.occbin.filter.particle.empirical_conditional_data_density.status || ... + (any(is_simul_constrained_in_expectation(isux(1:binpop))) && options_.occbin.filter.particle.tobit) + % Y may be constrained + is_tobit=true; +% ShockVectorsPerParticle = bsxfun(@plus,US(:,ishock)*ShockVarianceSquareRoot*transpose(norminv(qmc_scrambled(shock_variance_rank,number_of_particles,1))),zeros(shock_variance_rank,1)); + ShockVectorsInfo.US = US(:,ishock); + ShockVectorsInfo.VarianceSquareRoot = ShockVarianceSquareRoot; + ShockVectorsInfo.VarianceRank = shock_variance_rank; + [llik, llik_ppf, llik_EnKF, simulated_conditional_regimes, simulated_sample, hp_conditional_regimes] = occbin.ppf_simulated_density(number_of_particles, number_of_particles,yhat,ShockVectorsInfo, di, H, my_order_var, QQQ, Y, ZZ, base_regime, regimesy, M_, dr, endo_steady_state, exo_steady_state, exo_det_steady_state, options_, opts_simul); + y11_ppf = simulated_sample.y11; + fill_y11 = false; + if isempty(y11) + y11 = y11_ppf; + fill_y11= true; + end + is_observed = false(size(y11_ppf,1),1); + for kobs = 1:nobs + if H(kobs,kobs)==0 + is_observed(logical(ZZ(kobs,:))) = true; + if fill_y11 + y11(logical(ZZ(kobs,:)),:) = Y(di(kobs),2); + end + end + end + y11(not(is_observed),:) = y11_ppf(not(is_observed),:); + likxx=llik_ppf'; + number_of_successful_particles = length(likxx); + success = true(number_of_successful_particles,1); + else + do_nothing = true; + end + +end +% compute weights +% weights = ones(1,number_of_successful_particles)/number_of_successful_particles ; +% lnw = -likj(success); % use the joint distribution (<=> posterior kernel!) +% dfac = max(lnw); +% wtilde = weights.*exp(lnw-dfac); +% % wyy = weights(1)*exp(-(likyy-lik00)-dfac); +% wyy = weights(1)*exp(-likyy-lik0-dfac); % use joint distribution +% likka = log(sum(wtilde))+dfac; +% weights = wtilde/sum(wtilde); +% wyy = wyy/sum(wtilde); + +weights = zeros(1,number_of_particles); +weights0 = ones(1,number_of_successful_particles)/number_of_successful_particles ; +if number_of_successful_particles==0 + info = 340; + StateVectors=nan(size(yhat)); + output=struct(); + return +end +lnw0 = -likxx(success)/2; % use conditional likelihood to weight prior draws! +% we need to normalize weights, even if they are already proper likelihood values! +% to avoid that exponential gives ZEROS all over the place! +lnw0 = lnw0-max(lnw0); +wtilde0 = weights0.*exp(lnw0); +weights0 = wtilde0/sum(wtilde0); +weights(success) = weights0; + +if all(weights==0) + do_nothing = true; +end + + +ParticleOptions.resampling.method.kitagawa=true; +indx = resample(0,weights',ParticleOptions); +if options_.debug + SS2 = resample(y11',weights',ParticleOptions)'; +end +StateVectors2 = y11(:,indx); + +% updated state mean and covariance +[U,X] = svd(0.5*(Py(:,:,1)+Py(:,:,1)')); +% P= U*X*U'; +iss = find(diag(X)>1.e-12); + +% if any(iss) || ~options_.occbin.filter.particle.draw_states_from_empirical_density + +StateVectorMean = alphahaty(:,2); +if any(iss) + StateVectorVarianceSquareRoot = chol(X(iss,iss))';%reduced_rank_cholesky(ReducedForm.StateVectorVariance)'; + % Get the rank of StateVectorVarianceSquareRoot + state_variance_rank = size(StateVectorVarianceSquareRoot,2); + StateVectors = bsxfun(@plus,U(:,iss)*StateVectorVarianceSquareRoot*transpose(norminv(qmc_scrambled(state_variance_rank,number_of_particles,1))),StateVectorMean); +else + state_variance_rank = numel(StateVectorMean ); + StateVectors = bsxfun(@plus,0*transpose(norminv(qmc_scrambled(state_variance_rank,number_of_particles,1))),StateVectorMean); +end + + +%% PPF density +% % if number_of_updated_regimes==1 && not(is_tobit) +% % datalik = exp(-liky/2); +% % % datalik = exp(-updated_regimes(1).lik/2); +% % else + datalik = mean(exp(-likxx(success)/2)); +% % end + +if ~options_.occbin.filter.particle.likelihood_only + norm_datalik = sum(exp(-likxx(success))); + is_resampled_regime = false(1,number_of_updated_regimes); + pregime_updated0 = zeros(1,number_of_updated_regimes); + pregime_updated = zeros(1,number_of_updated_regimes); + datalik_regime = zeros(1,number_of_updated_regimes); + for kr=1:number_of_updated_regimes + % regime specific likelihood weighted by regime probability GIVEN the + % observable + pregime_updated0(kr) = sum(exp(-likxx(updated_regimes(kr).index)))/norm_datalik; + is_resampled_regime(kr) = any(ismember(indx,updated_regimes(kr).index)); + if is_resampled_regime(kr) + pregime_updated(kr) = sum(ismember(indx,updated_regimes(kr).index))/number_of_particles; + datalik_regime(kr) = sum(exp(-likxx(updated_regimes(kr).index)/2))/sum(success); + end + end +end + + +if ~options_.occbin.filter.particle.nograph + figure(1) + clf(1,'reset') + % subplot(nrow,ncol,kobs) + tlo = tiledlayout(nrow,ncol); + title(tlo,[' (t=' int2str(t) ')']); + tlo.TileSpacing = 'tight'; + + % set color order + all_regimes = [all_simulated_regimes all_updated_regimes(~ismember(all_updated_regimes, all_simulated_regimes))]; + number_of_regimes = length(all_regimes); + + ireg1 = find([simulated_regimes.is_pkf_regime]); + + [~, ~, tmp_str]=occbin.backward_map_regime(regimes0(1)); + if M_.occbin.constraint_nbr==1 + tmp_str = tmp_str(2,:); + else + tmp_str = [tmp_str(2,:) tmp_str(4,:)]; + end + ireg0 = find(ismember(all_simulated_regimes,tmp_str)); + + func = @(x) colorspace('RGB->Lab',x); + my_colororder= distinguishable_colors(number_of_regimes+4,'w',func); + my_colororder(ismember(my_colororder,[0 0 1],'row'),:)=[]; % remove BLUE + previous_and_current_period_regimes_are_equal = false; + if isempty(ireg0) + % t-1 regime is not present in predictive density (should never + % happen, but ...) + my_colororder(ismember(my_colororder,[1 0 0],'row'),:)=[]; % remove RED + else + if ireg0~=ireg1 + icolor0 = find(ismember(my_colororder,[1 0 0],'row')); % get the RED + my_colororder = my_colororder([1:icolor0-1,icolor0+1:number_of_simulated_regimes,icolor0,number_of_simulated_regimes+1:number_of_regimes],:); + else + ireg0 = []; + previous_and_current_period_regimes_are_equal = true; + my_colororder = my_colororder(1:number_of_regimes,:); + end + end + my_simulated_regime_order = (1:number_of_simulated_regimes); + imiddle = not(ismember(my_simulated_regime_order,[ireg0 ireg1])); + my_simulated_regime_order = [ireg1 my_simulated_regime_order(imiddle) ireg0]; + + all_regimes_order = [all_simulated_regimes(my_simulated_regime_order) all_updated_regimes(~ismember(all_updated_regimes, all_simulated_regimes))]; + updated_colororder = zeros(number_of_updated_regimes,3); + for kr=1:number_of_updated_regimes + updated_colororder(kr,:) = my_colororder(ismember(all_regimes_order,all_updated_regimes(kr)),:); + end + + % sort updated regimes like simluated for figure(1) + for kr = 1:number_of_regimes + ismember(all_regimes_order,all_updated_regimes); + end + +end + +% data density! +if ~options_.occbin.filter.particle.likelihood_only + for kobs = 1:size(ZZ,1) + obs_simul = transpose(ZZ(kobs,:)*y100); %simulated varobs + sobs_simul = sort(obs_simul(success)); % sorted values of simulated varobs + mc=nan(number_of_simulated_regimes,1); + Mc=mc; + for kr=1:number_of_simulated_regimes + mc(kr,1) = min(transpose(ZZ(kobs,:)*y100(:,simulated_regimes(kr).index))); + Mc(kr,1) = max(transpose(ZZ(kobs,:)*y100(:,simulated_regimes(kr).index))); + end + is_regime_uncertain = number_of_simulated_regimes>1; + if ~options_.occbin.filter.particle.nograph + ax = nexttile(kobs); + hold off, + end + zrange = [min(transpose(ZZ(kobs,:)*y10)) max(transpose(ZZ(kobs,:)*y10))]; + zlin = linspace(zrange(1),zrange(2),50); + + fobsk = ksdensity(transpose(ZZ(kobs,:)*y10),[zlin Y(kobs,2)]'); + fobsklin = fobsk(1:end-1); + fobsk= fobsk(end); + density.(options_.varobs{di(kobs)}).kernel = [zlin' fobsklin]; + if number_of_simulated_regimes>1 + % expected regime is uncertain + % classify support + is_overlap = false; + [mcs, imcs] = sort(mc); + Mcs = Mc(imcs); % sort max with the same ordering as min + is_sorted_pkf_regime = [simulated_regimes(imcs).is_pkf_regime]; + edge = []; + subedge=[]; + num_subedge = 0; +% % % for kr=1:number_of_simulated_regimes-1 +% % % if Mcs(kr)<mcs(kr+1) +% % % % no overlap +% % % is_overlap(kr, kr+1) = false; +% % % if Y(kobs,2)>Mcs(kr) && Y(kobs,2)<mcs(kr+1) +% % % if is_sorted_pkf_regime(kr+1) +% % % edge(kr) = 0.5*(Y(kobs,2)+Mcs(kr)); +% % % else +% % % edge(kr) = 0.5*(Y(kobs,2)+mcs(kr+1)); +% % % end +% % % else +% % % edge(kr) = 0.5*(Mcs(kr)+mcs(kr+1)); +% % % end +% % % else +% % % % overlap! +% % % edge(kr) = 0.5*(sobs_simul(find(sobs_simul<mcs(kr+1),1,'last'))+mcs(kr+1)); +% % % is_overlap(kr, kr+1) = true; +% % % for krr=kr+2:number_of_simulated_regimes +% % % if Mcs(kr)>mcs(krr) +% % % is_overlap(kr, krr) = true; +% % % end +% % % num_subedge = num_subedge+1; +% % % subedge(num_subedge) = 0.5*(sobs_simul(find(sobs_simul<mcs(krr),1,'last'))+mcs(krr)); +% % % end +% % % if Mcs(kr)<max(Mcs) +% % % num_subedge = num_subedge+1; +% % % subedge(num_subedge) = 0.5*(sobs_simul(find(sobs_simul>Mcs(kr),1,'first'))+Mcs(kr)); +% % % end +% % % +% % % end +% % % end +% % % mid = sort(unique([edge subedge])); + % m=min(transpose(ZZ(kobs,:)*y10)); + % M=max(transpose(ZZ(kobs,:)*y10)); + % bwidth = (M-m)/nbins; + % kedge=0; + % EDGES=[]; + % for ke=1:length(mid) + % kedge=kedge+1; + % EDGES(kedge) = mid(ke); + % if ke<length(mid) + % ewidth = floor((mid(ke+1)-mid(ke))/bwidth); + % for kew=1:ewidth + % kedge=kedge+1; + % EDGES(kedge) = mid(ke)+kew/(ewidth+1)*(mid(ke+1)-mid(ke)); + % end + % end + % end + % EDGES = [EDGES max(EDGES)+bwidth*(1:2*nbins)]; + % EDGES = [min(EDGES)+bwidth*(-2*nbins:-1) EDGES]; + % EDGES = EDGES(find(EDGES<m,1,'last'):find(EDGES>M,1)); + end + m=min(transpose(ZZ(kobs,:)*y10)); + M=max(transpose(ZZ(kobs,:)*y10)); + bwidth = (M-m)/nbins; + m = min(m,Y(di(kobs),2)-bwidth/2); + M = max(M,Y(di(kobs),2)+bwidth/2); + bwidth = (M-m)/nbins; + EDGES = Y(di(kobs),2)-bwidth/2+bwidth*[-2*nbins:2*nbins]; % we center data point in one bin + EDGES = EDGES(find(EDGES<m,1,'last'):find(EDGES>M,1)); + if options_.occbin.filter.particle.nograph + [pdf_sim,EDGES] = histcounts(transpose(ZZ(kobs,:)*y10), EDGES,'Normalization', 'pdf'); + % [pdf_sim,EDGES] = histcounts(transpose(ZZ(kobs,:)*y10), nbins+1, 'Normalization', 'pdf'); + else + % hh = histogram(transpose(ZZ(kobs,:)*y10), nbins+1, 'Normalization','pdf','EdgeColor','b'); + % EDGES = hh.BinEdges; + hh = histogram(transpose(ZZ(kobs,:)*y10),EDGES,'Normalization','pdf','EdgeColor','b'); + pdf_sim = hh.Values; + hold all, + end + xedges = 0.5*(EDGES(1:end-1)+EDGES(2:end)); + density.(options_.varobs{di(kobs)}).non_parametric = [xedges' pdf_sim']; + % PPF density! regime specific likelihood weighted by probability + % WITHIN the BIN + pregime = zeros(length(xedges),number_of_simulated_regimes); + pdf_ppf = nan(1,length(xedges)); + pdf_ppf_regime = zeros(length(xedges),number_of_simulated_regimes); + fobsk1 = ksdensity(transpose(ZZ(kobs,:)*y10),[xedges Y(kobs,2)]'); + for kx=1:length(xedges) + ikx = obs_simul(success)>EDGES(kx) & obs_simul(success)<EDGES(kx+1); + nkx = length(find(ikx)); + pdf_ppf(kx) = 0; + for kr=1:number_of_simulated_regimes + ikr = obs_simul(simulated_regimes(kr).index)>EDGES(kx) & obs_simul(simulated_regimes(kr).index)<EDGES(kx+1); + pregime(kx,kr) = length(find(ikr))/nkx; + if pregime(kx,kr)>0 + pdf_ppf_regime(kx,kr) = pregime(kx,kr) ... + *1./sqrt(2*pi*simulated_regimes(kr).obsvar(kobs,kobs)) ... + .*exp(-(xedges(kx)-simulated_regimes(kr).obsmean(kobs) ).^2./2./simulated_regimes(kr).obsvar(kobs,kobs)); + pdf_ppf(kx) = pdf_ppf(kx) + pdf_ppf_regime(kx,kr); + end + end + if length(find(pregime(kx,:)))>1 + pdf_ppf_regime(kx,:) = pregime(kx,:).*fobsk1(kx); + pdf_ppf(kx) = fobsk1(kx); + end + end + density.(options_.varobs{di(kobs)}).ppf = [xedges' pdf_ppf']; + density.(options_.varobs{di(kobs)}).ppf_regime = [xedges' pdf_ppf_regime]; + % while min(EDGES)>min(ObsVectors) + % end + % marginal density of observable + likk = 1/sqrt(2*pi*PZ(kobs,kobs))*exp(-(Y(di(kobs),2)-ObsVectorMean(kobs) ).^2./2./PZ(kobs,kobs)); + density_data.(options_.varobs{di(kobs)}).kernel = [Y(di(kobs),2) fobsk]; + density_data.(options_.varobs{di(kobs)}).non_parametric = [Y(di(kobs),2) pdf_sim(find(EDGES(1:end-1)<Y(di(kobs),2),1,'last'))]; + density_data.(options_.varobs{di(kobs)}).pkf = [Y(di(kobs),2) likk]; + pdf_pkf = 1./sqrt(2*pi*PZ(kobs,kobs)).*exp(-(xedges-ObsVectorMean(kobs) ).^2./2./PZ(kobs,kobs)); + density.(options_.varobs{di(kobs)}).pkf = [xedges' pdf_pkf']; + % density.(options_.varobs{di(kobs)}) = [xedges' pdf_pkf0']; + % [Nobs,EDGESobs] = histcounts(ObsVectors', EDGES,'Normalization', 'pdf'); + % [Nobs0,EDGESobs0] = histcounts(ObsVectors0', EDGES,'Normalization', 'pdf'); + + if ~options_.occbin.filter.particle.nograph + pdf_pkf0 = 1/sqrt(2*pi*PZ0(kobs,kobs))*exp(-(xedges-ObsVectorMean0(kobs) ).^2./2./PZ0(kobs,kobs)); + % hold all, plot(zlin',fobsklin,'b','linewidth',2) % kernel approx + hold all, hd(1) = plot(xedges',pdf_ppf,'b','linewidth',2); + % hh=histogram(ObsVectors',EDGES,'Normalization','pdf','DisplayStyle','stairs','linewidth',2,'EdgeColor','g'); + % Nobs = hh.Values; + hold all, hd(2) = plot(xedges',pdf_pkf,'g','linewidth',2); + + % hh=histogram(ObsVectors0',EDGES,'Normalization','pdf','DisplayStyle','stairs','linewidth',2,'linestyle','--','EdgeColor','r'); + % Nobs0 = hh.Values; + hold all, hd(3) = plot(xedges',pdf_pkf0,'r--.','linewidth',2); + if previous_and_current_period_regimes_are_equal + hd(3).Color = hd(2).Color; + end + + if length(di)>1 + [m, im] = min(abs(xedges-Y(di(kobs),2))); + hold all, hd(4) = fill(Y(di(kobs),2)+bwidth*[-0.5 -0.5 0.5 0.5],[0 pdf_ppf(im) pdf_ppf(im) 0],'b','edgecolor','b','linewidth',2,'facealpha',0.3); + else + % hold all, hd(4) = plot([Y(di(kobs),2) Y(di(kobs),2)],sort([0 datalik]),'b--*','linewidth',2); + hold all, hd(4) = fill(Y(di(kobs),2)+bwidth*[-0.5 -0.5 0.5 0.5],[0 datalik datalik 0],'b','edgecolor','b','linewidth',2,'facealpha',0.3); + % hold all, hd(4) = plot([Y(di(kobs),2) Y(di(kobs),2) Y(di(kobs),2)],sort([0 likk datalik]),'b--*','linewidth',2); + end + % hs = stairs(EDGES, [cpdf_ppf_regime(:,end:-1:1); zeros(1,number_of_simulated_regimes)]); + % func = @(x) colorspace('RGB->Lab',x); + % my_colororder= distinguishable_colors(number_of_simulated_regimes+4,'w',func); + % my_colororder(ismember(my_colororder,hd(1).Color,'row'),:)=[]; + % my_colororder(ismember(my_colororder,hd(4).EdgeColor,'row'),:)=[]; + % if isempty(ireg0) + % % t-1 regime is not present in predictive density (should never + % % happen, but ...) + % my_colororder(ismember(my_colororder,hd(3).Color,'row'),:)=[]; + % else + % if ireg0~=ireg1 + % icolor0 = find(ismember(my_colororder,hd(3).Color,'row')); + % my_colororder = my_colororder([1:icolor0-1,icolor0+1:number_of_simulated_regimes,icolor0],:); + % else + % ireg0=[]; + % my_colororder = my_colororder(1:number_of_simulated_regimes,:); + % end + % end + % my_regime_order = (1:number_of_simulated_regimes); + % imiddle = not(ismember(my_regime_order,[ireg0 ireg1])); + % my_regime_order = [ireg1 my_regime_order(imiddle) ireg0]; + cpdf_ppf_regime = cumsum(pdf_ppf_regime(:,my_simulated_regime_order),2); + cpdf_ppf_regime(pdf_ppf_regime(:,my_simulated_regime_order)==0)=NaN; + hs = stem(xedges, cpdf_ppf_regime(:,end:-1:1),':.','linewidth',1.5,'markersize',15); + tmp_colororder = my_colororder(number_of_simulated_regimes:-1:1,:); + for kr = 1:number_of_simulated_regimes + hs(kr).Color = tmp_colororder(kr,:); + end + + if kobs ==size(ZZ,1) + mytxt = {'simulated distribution','weighted distribution density', ... + 'approximated distribution','approximated distribution | t-1', ... + 'data point ppf density'}; + mytxt = [mytxt all_simulated_regimes(my_simulated_regime_order(end:-1:1))]; + % legend('simulated distribution','simulated distribution density','approximated distribution','approximated distribution | t-1','data point','Location','southoutside') + lg = legend(ax,mytxt,'Orientation','Horizontal','NumColumns',2); + lg.Layout.Tile = 'South'; % <-- Legend placement with tiled layout + end + title(options_.varobs{di(kobs)}) + + end + + + end +end + +if ~options_.occbin.filter.particle.nograph + figure(5) + clf(5,'reset') + ncomb = nchoosek(nobs,2); + nrow2=floor(sqrt(ncomb)); + ncol2=ceil(ncomb/nrow2); + + tlo2=tiledlayout(nrow2,ncol2); + title(tlo2,['predictive density (t=' int2str(t) ')']); + tlo2.TileSpacing = 'tight'; + + for ko=1:length(di)-1 + for koo=ko+1:length(di) + axo = nexttile; + for kp=1:number_of_simulated_regimes + plot(z10(ko,simulated_regimes(my_simulated_regime_order(kp)).index),z10(koo,simulated_regimes(my_simulated_regime_order(kp)).index),'.','MarkerEdgeColor',my_colororder(kp,:)) + hold all, + end + plot(Y(di(ko),2),Y(di(koo),2),'o','MarkerEdgeColor','k') + xlabel(options_.varobs{di(ko)}) + ylabel(options_.varobs{di(koo)}) + end + end + mytxt = [all_simulated_regimes(my_simulated_regime_order) 'data point']; + lgo = legend(axo,mytxt,'Orientation','Horizontal','NumColumns',2); + lgo.Layout.Tile = 'South'; % <-- Legend placement with tiled layout + + figure(4), + clf(4,'reset'), + pp = pie(datalik_regime(is_resampled_regime)/sum(datalik_regime(is_resampled_regime)), all_updated_regimes(is_resampled_regime)); + ip=0; + for kr=1:number_of_updated_regimes + if is_resampled_regime(kr) + ip=ip+1; + pp(ip).FaceColor=updated_colororder(kr,:); + ip=ip+1; + end + end + title(['data density t=' int2str(t) ', regimes contribution']) + + % iss = [1:size(Py(:,:,1),1)]; +% iss = find(diag(cov(StateVectors2'))>1.e-12); + iss = find(diag(cov(y10'))>1.e-12); + + % iss = find(sqrt(diag(Py(:,:,1)))>1.e-8); + nstate = length(iss); + nrow=floor(sqrt(nstate)); + ncol=ceil(nstate/nrow); + % func = @(x) colorspace('RGB->Lab',x); + % my_colororder= distinguishable_colors(number_of_updated_regimes,'w',func); + + figure(2) + clf(2,'reset') + set(2,'name',['t = ' int2str(t) ', updated states t|t']) + if nstate + tlo = tiledlayout(length(iss),length(iss)); + title(tlo,['t = ' int2str(t) ', updated states t|t']); + tlo.TileSpacing = 'none'; + end + for k=1:length(iss) + % h(k) = subplot(length(iss),length(iss),k+(k-1)*length(iss)); + h(k) = nexttile(k+(k-1)*length(iss)); + hold off, + % histogram(y11(k,:)','Normalization','pdf' ) + % histogram(yhat(k,:)','Normalization','pdf' ) + histogram(StateVectors(iss(k),:)','Normalization','pdf' ,'BinLimits',[min(StateVectors(iss(k),:)')-0.1*abs(min(StateVectors(iss(k),:)')),max(StateVectors(iss(k),:)')+0.1*abs(max(StateVectors(iss(k),:)'))]) + hold all, hf0=histogram(StateVectors2(iss(k),:)','Normalization','pdf' ,'BinLimits',[min(StateVectors2(iss(k),:)')-0.1*abs(min(StateVectors2(iss(k),:)')),max(StateVectors2(iss(k),:)')+0.1*abs(max(StateVectors2(iss(k),:)'))]); + % title(['updated ' M_.endo_names{dr.order_var(dr.restrict_var_list(iss(k)))} ' t|t']) + title(M_.endo_names{dr.order_var(dr.restrict_var_list(iss(k)))}) +% myax = gca; +% myax.TitleHorizontalAlignment = 'left'; + % ylabel(M_.endo_names{dr.order_var(dr.restrict_var_list(iss(k)))}) + if nstate>3 + set(gca,'Yticklabel',[]) + set(gca,'Xticklabel',[]) + end + for kj=k+1:length(iss) + for iz=1:2 + if iz==1 + ax = nexttile(k+(kj-1)*length(iss)); + plot(y10(iss(kj),:),y10(iss(k),:),'.','MarkerEdgeColor',[0.5 0.5 0.5]), + hold all, + else + ax = nexttile(kj+(k-1)*length(iss)); + end + plot(StateVectors(iss(kj),:),StateVectors(iss(k),:),'.','MarkerEdgeColor',[0.7 0.7 0.7]), + hold all, + % plot(StateVectors2(iss(kj),(ismember(indx,updated_regimes(1).index))),StateVectors2(iss(k),(ismember(indx,updated_regimes(1).index))),'.','MarkerEdgeColor',my_colororder(1,:)) + for kr=1:number_of_updated_regimes + if is_resampled_regime(kr) + plot(StateVectors2(iss(kj),(ismember(indx,updated_regimes(kr).index))),StateVectors2(iss(k),(ismember(indx,updated_regimes(kr).index))),'.','MarkerEdgeColor',updated_colororder(kr,:)) + end + end + plot(alphahaty(iss(kj),2),alphahaty(iss(k),2),'ok') + if nstate>3 + set(gca,'Yticklabel',[]) + set(gca,'Xticklabel',[]) + end + end + end + end + if M_.occbin.constraint_nbr ==1 + my_txt = mat2str(regimesy(1).regimestart); + else + my_txt = [mat2str(regimesy(1).regimestart1) '-' mat2str(regimesy(1).regimestart2)]; + end + + if nstate>1 + lg = legend(ax,[{[my_txt ' PKF']} all_updated_regimes(is_resampled_regime) {'PKF'}],'location','westoutside'); + lg.Position(1) = 0; + lg.Position(2) = 0; + % % % dim = [.1 .45 0.33 .03]; + % % % annotation('textbox',dim,'String',['. ' my_txt ' PKF'],'FitBoxToText','off','Color',[0.7 0.7 0.7],'horizontalalignment','left','verticalalignment','middle','margin',1,'EdgeColor',[0.95 0.95 0.95]); + % % % for kr = 1:number_of_updated_regimes + % % % dim = [.1 .45-kr*0.05 0.33 .03]; + % % % annotation('textbox',dim,'String',['. ' all_updated_regimes{kr}],'FitBoxToText','off','Color',my_colororder(kr,:),'horizontalalignment','left','verticalalignment','middle','margin',1,'EdgeColor',[0.95 0.95 0.95]); + % % % end + % % % dim = [.1 .45-0.02-(number_of_updated_regimes+1)*0.05 0.33 (number_of_updated_regimes+2)*0.05+0.02]; + % % % annotation('textbox',dim,'String','o PKF','FitBoxToText','off','Color','k','horizontalalignment','left','verticalalignment','bottom','margin',1); + end + % % % + % % % dim = [.05 .02 .15 .03]; + % % % annotation('textbox',dim,'String',['t = ' int2str(t)],'FitBoxToText','on','Color','k','horizontalalignment','center','verticalalignment','middle','margin',1); + % dim = [.25 .02 .3 .03]; + % annotation('textbox',dim,'String',['regime ' my_txt],'FitBoxToText','on','Color','k','horizontalalignment','center','verticalalignment','middle','margin',1); + % dim = [.55 .02 .15 .03]; + % annotation('textbox',dim,'String','PKF','FitBoxToText','on','Color','b','horizontalalignment','center','verticalalignment','middle','margin',1); + % dim = [.75 .02 .15 .03]; + % annotation('textbox',dim,'String','PPF','FitBoxToText','on','Color','r','horizontalalignment','center','verticalalignment','middle','margin',1); + figure(3) + clf(3,'reset') + iss = find(sqrt(diag(QQQ(:,:,2)))); + nshock = length(iss); + if nshock + set(3,'name',['t = ' int2str(t) ', shocks t|t']) + tlo = tiledlayout(nshock,nshock); + title(tlo,['t = ' int2str(t) ', shocks t|t']); + tlo.TileSpacing = 'none'; + end + % nrow=floor(sqrt(nshock)); + % ncol=ceil(nshock/nrow); + for k=1:nshock + % subplot(nrow,ncol,k) + % subplot(nshock,nshock,k+(k-1)*nshock); + nexttile(k+(k-1)*nshock); + hf1=histogram(epsilon(iss(k),indx),'BinLimits',[min(epsilon(iss(k),indx))-0.1*abs(min(epsilon(iss(k),indx))),max(epsilon(iss(k),indx))+0.1*abs(max(epsilon(iss(k),indx)))] ); + hf1.FaceColor=[0.8500, 0.3250, 0.0980] ; + % hf1=histogram(epsi2 ); + hold all, + hf = fill(etahaty(iss(k))+mean(diff(hf1.BinEdges))*[-0.5 0.5 0.5 -0.5 ],max(hf1.Values)*[0 0 1 1] ,'r'); + set(hf,'FaceAlpha',0.6) + hf.FaceColor=[0, 0.4470, 0.7410]; + % title([M_.exo_names{iss(k)}]) + title([M_.exo_names{iss(k)}]) +% myax = gca; +% myax.TitleHorizontalAlignment = 'left'; + % ylabel([M_.exo_names{iss(k)}]) + if nshock>3 + set(gca,'Yticklabel',[]) + set(gca,'Xticklabel',[]) + end + for kj=k+1:nshock + for iz=1:2 + if iz==1 + ax = nexttile(k+(kj-1)*nshock); + plot(ShockVectors(kj,:),ShockVectors(k,:),'.','color',[0.5 0.5 0.5]) + hold all + else + ax = nexttile(kj+(k-1)*nshock); + end + % plot(epsilon(iss(kj),indx),epsilon(iss(k),indx),'.'), + % plot(epsilon(iss(kj),indx(ismember(indx,updated_regimes(1).index))),epsilon(iss(k),indx(ismember(indx,updated_regimes(1).index))),'.','MarkerEdgeColor',my_colororder(1,:)) + % hold all, + for kr=1:number_of_updated_regimes + if is_resampled_regime(kr) + plot(epsilon(iss(kj),indx(ismember(indx,updated_regimes(kr).index))),epsilon(iss(k),indx(ismember(indx,updated_regimes(kr).index))),'.','MarkerEdgeColor',updated_colororder(kr,:)) + hold all, + end + end + plot(etahaty(iss(kj)),etahaty(iss(k)),'ok') + if nshock>3 + set(gca,'Yticklabel',[]) + set(gca,'Xticklabel',[]) + end + end + end + end + if nshock>1 + lg = legend(ax,[all_updated_regimes(is_resampled_regime) {'PKF'}],'location','eastoutside'); + lg.Position(1) = 0; + lg.Position(2) = 0; + % % for kr = 1:number_of_updated_regimes + % % dim = [.1 .45-(kr-1)*0.05 0.33 .03]; + % % annotation('textbox',dim,'String',['. ' all_updated_regimes{kr}],'FitBoxToText','off','Color',my_colororder(kr,:),'horizontalalignment','left','verticalalignment','middle','margin',1,'EdgeColor',[0.95 0.95 0.95]); + % % end + % % dim = [.1 .45-0.02-number_of_updated_regimes*0.05 0.33 (number_of_updated_regimes+1)*0.05+0.02]; + % % annotation('textbox',dim,'String','o PKF','FitBoxToText','off','Color','k','horizontalalignment','left','verticalalignment','bottom','margin',1); + end +end +% end + +use_pkf_distribution=false; +if options_.occbin.filter.particle.draw_states_from_empirical_density + if not(isequal(regimesy(1),base_regime) && any([updated_regimes.is_pkf_regime]) && (sum(ismember(indx,updated_regimes([updated_regimes.is_pkf_regime]).index))/number_of_particles)>=options_.occbin.filter.particle.use_pkf_updated_state_threshold ) +% if max(abs(ay(:,1)-alphahaty(:,2)))>1.e-10 || (not(isequal(regimesy(1),base_regime) && any([updated_regimes.is_pkf_regime]) && (sum(ismember(indx,updated_regimes([updated_regimes.is_pkf_regime]).index))/number_of_particles)>options_.occbin.filter.particle.use_pkf_updated_state_threshold )) + StateVectors = StateVectors2; + % re-sample + PS = cov(StateVectors'); + [U,X] = svd(0.5*(PS+PS')); + % P= U*X*U'; + iss = find(diag(X)>1.e-12); + + StateVectorMean = mean(StateVectors,2); + if any(iss) + StateVectorVarianceSquareRoot = chol(X(iss,iss))';%reduced_rank_cholesky(ReducedForm.StateVectorVariance)'; + % Get the rank of StateVectorVarianceSquareRoot + state_variance_rank = size(StateVectorVarianceSquareRoot,2); + StateVectors = bsxfun(@plus,U(:,iss)*StateVectorVarianceSquareRoot*transpose(norminv(qmc_scrambled(state_variance_rank,number_of_particles,1))),StateVectorMean); + else + state_variance_rank = numel(StateVectorMean ); + StateVectors = bsxfun(@plus,0*transpose(norminv(qmc_scrambled(state_variance_rank,number_of_particles,1))),StateVectorMean); + end + % updated PKF t|t according to PPF!!! +% ay(:,1) = mean(StateVectors,2); +% ay(:,2) = Ty(:,:,2)*ay(:,1)+Cy(:,2); +% Py(:,:,1) = cov(StateVectors'); +% Py(:,:,2) = Ty(:,:,2)*Py(:,:,1)*Ty(:,:,2)'+Ry(:,:,2)*QQQ(:,:,3)*Ry(:,:,2)'; +% a1y(:,2) = mean(y100(:,success),2); +% P1y(:,:,2) = cov(y100(:,success)'); + % this creates inconsistency in alphahaty in the next period! we + % need to fix it!!! + else + use_pkf_distribution=true; + end +end + +output=[]; +if ~options_.occbin.filter.particle.likelihood_only + for jk=1:size(y10,1) + [output.filtered.variables.mean(jk,1), ... + output.filtered.variables.median(jk,1), ... + output.filtered.variables.var(jk,1), ... + output.filtered.variables.hpd_interval(jk,:), ... + output.filtered.variables.deciles(jk,:), ... + output.filtered.variables.density(jk,:,:)] = posterior_moments(y10(jk,:)', options_.mh_conf_sig); + end + output.filtered.variables.particles = y100; + output.filtered.regimes.sample = simul_regimes; + output.filtered.regimes.is_constrained = is_simul_constrained; + output.filtered.regimes.is_constrained_in_expectation = is_simul_constrained_in_expectation; + [aa,bb]=histcounts(simul_regime_exit,'normalization','pdf','binmethod','integers'); + output.filtered.regimes.exit(bb(1:end-1)+0.5,1) = aa'; + isc = is_simul_constrained; + isce = is_simul_constrained_in_expectation; + rr=simul_regime_exit; + output.filtered.regimes.exit_constrained_share(1,1)=0; + for ka=2:length(output.filtered.regimes.exit) + iden = length(find(isce(rr==ka))); + if iden + output.filtered.regimes.exit_constrained_share(ka,1) = length(find(isc(rr==ka)))/iden; + else + output.filtered.regimes.exit_constrained_share(ka,1) = 0; + end + end + + [aa, bb]=histcounts(is_simul_constrained,'normalization','pdf','binmethod','integers'); + if length(aa)==1 + if bb(1)+0.5==0 + aa=[1 0]; + else + aa=[0 1]; + end + bb=-0.5:1:1.5; + end + output.filtered.regimes.is_constrained = isc; + output.filtered.regimes.prob.is_constrained = aa(2); + [aa, bb]=histcounts(is_simul_constrained_in_expectation,'normalization','pdf','binmethod','integers'); + if length(aa)==1 + if bb(1)+0.5==0 + aa=[1 0]; + else + aa=[0 1]; + end + bb=-0.5:1:1.5; + end + output.filtered.regimes.is_constrained_in_expectation = isce; + output.filtered.regimes.prob.is_constrained_in_expectation = aa(2); + + output.data.marginal_probability_distribution = density; + output.data.marginal_probability = density_data; + output.data.likelihood.empirical_kernel = fobs_kernel; + output.data.likelihood.empirical_non_parametric = fobs; + output.data.likelihood.ppf = datalik; + output.data.likelihood.pkf = exp(-liky/2); + + for jk=1:size(StateVectors,1) + [output.updated.variables.mean(jk,1), ... + output.updated.variables.median(jk,1), ... + output.updated.variables.var(jk,1), ... + output.updated.variables.hpd_interval(jk,:), ... + output.updated.variables.deciles(jk,:), ... + output.updated.variables.density(jk,:,:)] = posterior_moments(StateVectors(jk,:)', options_.mh_conf_sig); + end + output.updated.variables.particles = StateVectors; + output.updated.variables.pkf = alphahaty(:,2); + output.updated.regimes.sample = regimes(indx); + [aa,bb]=histcounts(regime_exit(indx),'normalization','pdf','binmethod','integers'); + output.updated.regimes.exit(bb(1:end-1)+0.5,1) = aa'; + isc = is_constrained(indx); + isce = is_constrained_in_expectation(indx); + rr=regime_exit(indx); + output.updated.regimes.exit_constrained_share(1,1)=0; + for ka=2:length(output.updated.regimes.exit) + iden = length(find(isce(rr==ka))); + if iden + output.updated.regimes.exit_constrained_share(ka,1) = length(find(isc(rr==ka)))/iden; + else + output.updated.regimes.exit_constrained_share(ka,1) = 0; + end + end + [aa, bb]=histcounts(is_constrained(indx),'normalization','pdf','binmethod','integers'); + if length(aa)==1 + if bb(1)+0.5==0 + aa=[1 0]; + else + aa=[0 1]; + end + bb=-0.5:1:1.5; + end + output.updated.regimes.is_constrained = isc; + output.updated.regimes.prob.is_constrained = aa(2); + + [aa, bb]=histcounts(is_constrained_in_expectation(indx),'normalization','pdf','binmethod','integers'); + if length(aa)==1 + if bb(1)+0.5==0 + aa=[1 0]; + else + aa=[0 1]; + end + bb=-0.5:1:1.5; + end + output.updated.regimes.is_constrained_in_expectation = isce; + output.updated.regimes.prob.is_constrained_in_expectation = aa(2); +end + +if options_.occbin.filter.particle.empirical_data_density.status + if options_.occbin.filter.particle.empirical_data_density.kernel_density + liky = -log(fobs_kernel)*2; + else + liky = -log(fobs)*2; + end +else + liky = -log(datalik)*2; +end + +if any(not(success)) + % keyboard +end + +end + +function [mx, Vx, lik00, log_dF00, iF00] = get_the_smoothed_density(a0, ax, Y, U, W, X, is, QQQ, Cyy, Ryy, Tyy, ZZ) + +% % % optimal smoothed state in 0 is distributed as alphaty(:,1), Vy(:,:,1), hence dy0=0 +% % dy0 = alphahatyy(:,1)-alphahaty(:,1); +% % F00 = W(:,is)'*Vy(:,:,1)*W(:,is); +% % sig00=sqrt(diag(F00)); +% % log_dF00 = log(det(F00./(sig00*sig00')))+2*sum(log(sig00)); +% % iF00 = inv(F00./(sig00*sig00'))./(sig00*sig00'); +% % lik00 = log_dF00 + transpose(dy0)*iF00*dy0 + length(is)*log(2*pi); +% % + + +% distribution of generic state in 0 depends on the regime hence dy0=0 +% mean of state in 0 +% mapping from state to observable: NOTE it depends on the smoothed regime +% via Cyy, Ryy and Tyy! +isx = find(~ismember(1:length(X),is)); +iU = inv(U); +if ~isempty(isx) + % eleminate zero variance component of state + % Y = Y-ZZ*Tyy(:,:,1)*U(:,isx)*U(:,isx)'*a0; + Y = Y-ZZ*Tyy(:,:,1)*U(:,isx)*iU(isx,:)*a0; +end +Y = Y-ZZ*Cyy(:,1); % remove constant from observable +% s0 = U(:,is)'*a0; +s0 = iU(is,:)*a0; +% s0 = U(:,is)'*(a0-Cyy(:,1)); +QQx = ZZ*Ryy(:,:,1)*QQQ(:,:,2)*transpose(Ryy(:,:,1))*ZZ'; +M = ZZ*Tyy(:,:,1)*U(:,is); +Vs = inv(inv(X(is,is)) + (M'/QQx*M)); % variance of state 0|1 +ms = Vs*(X(is,is)\s0 + (M'/QQx*Y)); % mean of state 0|1 +Vx = U(:,is)*Vs*iU(is,:); % map back to original states +mx = U(:,is)*ms+U(:,isx)*iU(isx,:)*a0; %%+Cyy(:,1); % map back to original states, also adding zero variance part! +% Vx = U(:,is)*Vs*U(:,is)'; % map back to original states +% mx = U(:,is)*ms+U(:,isx)*U(:,isx)'*a0; %%+Cyy(:,1); % map back to original states, also adding zero variance part! + +dy0 = iU(is,:)*(ax-mx); +F00 = iU(is,:)*Vx*U(:,is); +% dy0 = U(:,is)'*(ax-mx); +% F00 = U(:,is)'*Vx*U(:,is); +sig00=sqrt(diag(F00)); +log_dF00 = log(det(F00./(sig00*sig00')))+2*sum(log(sig00)); +iF00 = inv(F00./(sig00*sig00'))./(sig00*sig00'); +lik00 = log_dF00 + transpose(dy0)*iF00*dy0 + length(is)*log(2*pi); + +end diff --git a/matlab/+occbin/missing_observations_piecewise_particle_filter.m b/matlab/+occbin/missing_observations_piecewise_particle_filter.m new file mode 100644 index 0000000000000000000000000000000000000000..b789ad4dd32901631257f1dae191efe34454f145 --- /dev/null +++ b/matlab/+occbin/missing_observations_piecewise_particle_filter.m @@ -0,0 +1,433 @@ +function [LIK, lik, a, P, pfilter] = missing_observations_piecewise_particle_filter(data_index,number_of_observations,no_more_missing_observations,Y,start,last,a,P,kalman_tol,riccati_tol,rescale_prediction_error_covariance,presample,T,Q,R,H,Z,mm,pp,rr,Zflag,diffuse_periods,occbin_) +% Computes the likelihood of a state space model in the case with missing observations. +% +% INPUTS +% data_index [cell] 1*smpl cell of column vectors of indices. +% number_of_observations [integer] scalar. +% no_more_missing_observations [integer] scalar. +% Y [double] pp*smpl matrix of data. +% start [integer] scalar, index of the first observation. +% last [integer] scalar, index of the last observation. +% a [double] pp*1 vector, levels of the predicted initial state variables (E_{0}(alpha_1)). +% P [double] pp*pp matrix, covariance matrix of the initial state vector. +% kalman_tol [double] scalar, tolerance parameter (rcond). +% riccati_tol [double] scalar, tolerance parameter (riccati iteration). +% presample [integer] scalar, presampling if strictly positive. +% T [double] mm*mm transition matrix of the state equation. +% Q [double] rr*rr covariance matrix of the structural innovations. +% R [double] mm*rr matrix, mapping structural innovations to state variables. +% H [double] pp*pp (or 1*1 =0 if no measurement error) covariance matrix of the measurement errors. +% Z [integer] pp*1 vector of indices for the observed variables. +% mm [integer] scalar, dimension of the state vector. +% pp [integer] scalar, number of observed variables. +% rr [integer] scalar, number of structural innovations. +% +% OUTPUTS +% LIK [double] scalar, MINUS loglikelihood +% lik [double] vector, density of observations in each period. +% a [double] mm*1 vector, current estimate of the state vector tomorrow (E_{T}(alpha_{T+1})). +% P [double] mm*mm matrix, covariance matrix of the states. +% +% +% NOTES +% The vector "lik" is used to evaluate the jacobian of the likelihood. + +% Copyright © 2004-2021 Dynare Team +% +% This file is part of Dynare. +% +% Dynare is free software: you can redistribute it and/or modify +% it under the terms of the GNU General Public License as published by +% the Free Software Foundation, either version 3 of the License, or +% (at your option) any later version. +% +% Dynare is distributed in the hope that it will be useful, +% but WITHOUT ANY WARRANTY; without even the implied warranty of +% MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +% GNU General Public License for more details. +% +% You should have received a copy of the GNU General Public License +% along with Dynare. If not, see <https://www.gnu.org/licenses/>. + +% Set defaults +if nargin<20 + Zflag = 0; + diffuse_periods = 0; +end + +if nargin<21 + diffuse_periods = 0; +end + +if isempty(Zflag) + Zflag = 0; +end + +if isempty(diffuse_periods) + diffuse_periods = 0; +end + +if isequal(H,0) + H = zeros(pp,pp); +end + +% Get sample size. +smpl = last-start+1; + +% Initialize some variables. +dF = 1; +isqvec = false; +if ndim(Q)>2 + Qvec = Q; + Q=Q(:,:,1); + isqvec = true; +end +QQ = R*Q*transpose(R); % Variance of R times the vector of structural innovations. +t = start; % Initialization of the time index. +lik = zeros(smpl,1); % Initialization of the vector gathering the densities. +LIK = Inf; % Default value of the log likelihood. +oldK = Inf; +F_singular = true; +s = 0; +rescale_prediction_error_covariance0=rescale_prediction_error_covariance; +if occbin_.status + Qt = repmat(Q,[1 1 3]); + a0 = zeros(mm,last); + a1 = zeros(mm,last); + P0 = zeros(mm,mm,last); + P1 = zeros(mm,mm,last); + vv = zeros(pp,last); + + options_=occbin_.info{1}; + dr=occbin_.info{2}; + endo_steady_state=occbin_.info{3}; + exo_steady_state=occbin_.info{4}; + exo_det_steady_state=occbin_.info{5}; + M_=occbin_.info{6}; + occbin_options=occbin_.info{7}; + occbin_options.opts_simul.SHOCKS = []; + opts_regime.regime_history = occbin_options.opts_simul.init_regime; + opts_regime.binding_indicator = occbin_options.opts_simul.init_binding_indicator; + if isempty(opts_regime.binding_indicator) && isempty(opts_regime.regime_history) + opts_regime.binding_indicator=zeros(last+2,M_.occbin.constraint_nbr); + end + [~, ~, ~, regimes_] = occbin.check_regimes([], [], [], opts_regime, M_, options_, dr, endo_steady_state, exo_steady_state, exo_det_steady_state); + + base_regime = struct(); + if M_.occbin.constraint_nbr==1 + base_regime.regime = 0; + base_regime.regimestart = 1; + else + base_regime.regime1 = 0; + base_regime.regimestart1 = 1; + base_regime.regime2 = 0; + base_regime.regimestart2 = 1; + end + + if length(occbin_.info)>7 + TT=occbin_.info{8}; + RR=occbin_.info{9}; + CC=occbin_.info{10}; + T0=occbin_.info{11}; + R0=occbin_.info{12}; + TT = cat(3,TT,T); + RR = cat(3,RR,R); + CC = cat(2,CC,zeros(mm,1)); + if size(TT,3)<(last+1) + TT=repmat(T,1,1,last+1); + RR=repmat(R,1,1,last+1); + CC=repmat(zeros(mm,1),1,last+1); + end + + end +else + first_period_occbin_update = inf; + C=0; +end + +P=0.5*(P+P'); +[LastSeeds.Unifor, LastSeeds.Normal] = get_dynare_random_generator_state(); +% Set seed for randn(). +set_dynare_seed('default'); + +%% initialize particles +if options_.occbin.filter.particle.draw_states_from_empirical_density && any(options_.occbin.filter.particle.state_draws) + % take user provided initial states (e.g. simulated states from ergodic + % non-linear distribution + StateVectors = options_.occbin.filter.particle.state_draws; +else + %use unconditional mean and variance from baseline regime + % StateVectorMean = 0; + [U,X] = eig(P); + % P= U*X*U'; + is = find(diag(X)>1.e-12); + StateVectorVarianceSquareRoot = chol(X(is,is))';%reduced_rank_cholesky(ReducedForm.StateVectorVariance)'; + + % Get the rank of StateVectorVarianceSquareRoot + state_variance_rank = size(StateVectorVarianceSquareRoot,2); + + % % Factorize the covariance matrix of the structural innovations + % Q_lower_triangular_cholesky = chol(Q)'; + + % Initialization of the weights across particles. + options_.occbin.filter.state_covariance = true; + number_of_particles = options_.occbin.filter.particle.number_of_particles; + weights = ones(1,number_of_particles)/number_of_particles ; + if state_variance_rank + StateVectors = U(:,is)*StateVectorVarianceSquareRoot*transpose(norminv(qmc_scrambled(state_variance_rank,number_of_particles,1))); + else + StateVectors = repmat(a,[1 number_of_particles]); + end +end + +pfilter=[]; +if options_.occbin.filter.particle.ensemble_kalman_filter + % improve ergodic ... + if state_variance_rank && options_.occbin.filter.particle.initial_state_ergodic_simul + + if isqvec + Qt = cat(3,Q,Qvec(:,:,t:t+1)); + end + + [US,XS] = eig(Qt(:,:,2)); + % P= U*X*U'; + ishock = find(sqrt(abs(diag(XS)))>1.e-10); + ShockVarianceSquareRoot = chol(XS(ishock,ishock))';%reduced_rank_cholesky(ReducedForm.StateVectorVariance)'; + % Get the rank of ShockVarianceSquareRoot + shock_variance_rank = size(ShockVarianceSquareRoot,2); + opts_simul = occbin_options.opts_simul; + opts_simul.waitbar=0; + if opts_simul.restrict_state_space + my_order_var = dr.order_var(dr.restrict_var_list); + else + my_order_var = dr.order_var; + end + di=data_index{1}; + ZZ = Z(di,:); + for jk=1:10 + ShockVectors = bsxfun(@plus,US(:,ishock)*ShockVarianceSquareRoot*transpose(norminv(qmc_scrambled(shock_variance_rank,number_of_particles,1))),zeros(shock_variance_rank,1)); + [~, ~, ~, ~, sim_sample] = ... + occbin.ppf_simulated_density(number_of_particles, 1,StateVectors,ShockVectors, di, H, my_order_var, Qt, Y, ZZ, base_regime, regimes_(t:t+1), M_, dr, endo_steady_state, exo_steady_state, exo_det_steady_state, options_, opts_simul); + while not(isequal(size(StateVectors),size(sim_sample.y10))) + ShockVectors = bsxfun(@plus,US(:,ishock)*ShockVarianceSquareRoot*transpose(norminv(qmc_scrambled(shock_variance_rank,number_of_particles,1))),zeros(shock_variance_rank,1)); + [~, ~, ~, ~, sim_sample] = ... + occbin.ppf_simulated_density(number_of_particles, 1,StateVectors,ShockVectors, di, H, my_order_var, Qt, Y, ZZ, base_regime, regimes_(t:t+1), M_, dr, endo_steady_state, exo_steady_state, exo_det_steady_state, options_, opts_simul); + end + StateVectors = sim_sample.y10; + end + end + F_singular = false; + occbin_options.opts_simul.waitbar=0; + while t<=last + if t==1 + if isqvec + Qt = cat(3,Q,Qvec(:,:,t:t+1)); + end + else + if isqvec + Qt = Qvec(:,:,t-1:t+1); + end + end + [liky, StateVectors, out] = local_occbin_EnKF_iteration(StateVectors,data_index(t),Z,Y(:,t),H,Qt,t,M_,dr, endo_steady_state, exo_steady_state, exo_det_steady_state,options_,occbin_options); + s = t-start+1; + lik(s) = liky; + t=t+1; + end +else + while t<=last + a1(:,t) = a; + P1(:,:,t) = P; + C = CC(:,t+1); + R = RR(:,:,t+1); + T = TT(:,:,t+1); + if ~(isqvec) + QQ = R*Q*transpose(R); % Variance of R times the vector of structural innovations. + end + if t==1 + Pinit = P1(:,:,1); + ainit = a1(:,1); + end + s = t-start+1; + d_index = data_index{t}; + if isqvec + QQ = R*Qvec(:,:,t+1)*transpose(R); + end + if isempty(d_index) + a = T*a; + P = T*P*transpose(T)+QQ; + else + % Compute the prediction error and its variance + if Zflag + z = Z(d_index,:); + v = Y(d_index,t)-z*a; + F = z*P*z' + H(d_index,d_index); + else + z = Z(d_index); + v = Y(d_index,t) - a(z); + F = P(z,z) + H(d_index,d_index); + end + badly_conditioned_F = false; + if rescale_prediction_error_covariance + sig=sqrt(diag(F)); + if any(diag(F)<kalman_tol) || rcond(F./(sig*sig'))<kalman_tol + badly_conditioned_F = true; + end + else + if rcond(F)<kalman_tol + sig=sqrt(diag(F)); + if any(diag(F)<kalman_tol) || rcond(F./(sig*sig'))<kalman_tol + badly_conditioned_F = true; + else + rescale_prediction_error_covariance=1; + end + % badly_conditioned_F = true; + end + end + F_singular = false; + end + + occbin_options.opts_simul.waitbar=0; + if t==1 + if isqvec + Qt = cat(3,Q,Qvec(:,:,t:t+1)); + end + a00 = ainit; + a10 = [a00 a(:,1)]; + P00 = Pinit; + P10 = P1(:,:,[1 1]); + data_index0{1}=[]; + data_index0(2)=data_index(1); + v0(:,2)=vv(:,1); + Y0(:,2)=Y(:,1); + Y0(:,1)=nan; + TT01 = cat(3,T,TT(:,:,1)); + RR01 = cat(3,R,RR(:,:,1)); + CC01 = zeros(size(CC,1),2); + CC01(:,2) = CC(:,1); + + [US,XS] = eig(Qt(:,:,2)); + % P= U*X*U'; + ishock = find(sqrt(abs(diag(XS)))>1.e-10); + ShockVarianceSquareRoot = chol(XS(ishock,ishock))';%reduced_rank_cholesky(ReducedForm.StateVectorVariance)'; + % Get the rank of ShockVarianceSquareRoot + shock_variance_rank = size(ShockVarianceSquareRoot,2); + opts_simul = occbin_options.opts_simul; + if opts_simul.restrict_state_space + my_order_var = dr.order_var(dr.restrict_var_list); + else + my_order_var = dr.order_var; + end + di=data_index0{2}; + ZZ = Z(di,:); + + % improve ergodic ... + if state_variance_rank + sim_sample.y10=[]; + + for jk=1:0 %10 + while ~isequal(size(sim_sample.y10),size(StateVectors)) + ShockVectors = bsxfun(@plus,US(:,ishock)*ShockVarianceSquareRoot*transpose(norminv(qmc_scrambled(shock_variance_rank,number_of_particles,1))),zeros(shock_variance_rank,1)); + [sim_llik, sim_llik_ppf, sim_llik_EnKF, sim_regimes, sim_sample, hp_sim_regimes] = ... + occbin.ppf_simulated_density(number_of_particles, 1,StateVectors,ShockVectors, di, H, my_order_var, Qt, Y, ZZ, base_regime, regimes_(t:t+1), M_, dr, endo_steady_state, exo_steady_state, exo_det_steady_state, options_, opts_simul); + end + StateVectors = sim_sample.y10; + end + end + [ax, a1x, Px, P1x, vx, Tx, Rx, Cx, regimesx, info, M_, liky, tmpStateVectors, out] = occbin.local_pkf_iteration(StateVectors, a00, a10, P00, P10, data_index0, Z, v0, Y0, H, Qt, T0, R0, TT01, RR01, CC01, regimes_(t:t+1), isqvec, t, M_, dr, endo_steady_state, exo_steady_state, exo_det_steady_state, options_, occbin_options); + if ~options_.occbin.filter.particle.likelihood_only + pfilter = out; + end + else + if isqvec + Qt = Qvec(:,:,t-1:t+1); + end + [ax, a1x, Px, P1x, vx, Tx, Rx, Cx, regimesx, info, M_, liky, tmpStateVectors, out] = occbin.local_pkf_iteration(StateVectors, a0(:,t-1),a1(:,t-1:t),P0(:,:,t-1),P1(:,:,t-1:t),data_index(t-1:t),Z,vv(:,t-1:t),Y(:,t-1:t),H,Qt,T0,R0,TT(:,:,t-1:t),RR(:,:,t-1:t),CC(:,t-1:t),regimes_(t:t+1),isqvec,t,M_,dr, endo_steady_state, exo_steady_state, exo_det_steady_state,options_,occbin_options); + if ~options_.occbin.filter.particle.likelihood_only + pfilter.filtered.variables.mean(:,t) = out.filtered.variables.mean; + pfilter.filtered.variables.median(:,t) = out.filtered.variables.median; + pfilter.filtered.variables.var(:,t) = out.filtered.variables.var; + pfilter.filtered.variables.hpd_interval(:,:,t) = out.filtered.variables.hpd_interval; + pfilter.filtered.variables.deciles(:,:,t) = out.filtered.variables.deciles; + pfilter.filtered.variables.density(:,:,:,t) = out.filtered.variables.density; + pfilter.filtered.variables.particles(:,:,t) = out.filtered.variables.particles; + pfilter.filtered.regimes.sample(:,t) = out.filtered.regimes.sample; + pfilter.filtered.regimes.exit(1:size(out.filtered.regimes.exit,1),t) = out.filtered.regimes.exit; + pfilter.filtered.regimes.exit_constrained_share(1:size(out.filtered.regimes.exit,1),t) = out.filtered.regimes.exit_constrained_share; + pfilter.filtered.regimes.is_constrained(:,t) = out.filtered.regimes.is_constrained; + pfilter.filtered.regimes.is_constrained_in_expectation(:,t) = out.filtered.regimes.is_constrained_in_expectation; + pfilter.filtered.regimes.prob.is_constrained(t,1) = out.filtered.regimes.prob.is_constrained; + pfilter.filtered.regimes.prob.is_constrained_in_expectation(t,1) = out.filtered.regimes.prob.is_constrained_in_expectation; + if options_.occbin.filter.particle.number_of_shocks_per_particle>1 + pfilter.filtered.shock_simul(t) = out.filtered.shock_simul; + end + for jo = d_index(:)' + pfilter.data.marginal_probability_distribution.(options_.varobs{jo}).kernel(:,:,t) = out.data.marginal_probability_distribution.(options_.varobs{jo}).kernel; + pfilter.data.marginal_probability_distribution.(options_.varobs{jo}).non_parametric(:,:,t) = out.data.marginal_probability_distribution.(options_.varobs{jo}).non_parametric; + pfilter.data.marginal_probability_distribution.(options_.varobs{jo}).pkf(:,:,t) = out.data.marginal_probability_distribution.(options_.varobs{jo}).pkf; + pfilter.data.marginal_probability.(options_.varobs{jo}).kernel(:,:,t) = out.data.marginal_probability.(options_.varobs{jo}).kernel; + pfilter.data.marginal_probability.(options_.varobs{jo}).non_parametric(:,:,t) = out.data.marginal_probability.(options_.varobs{jo}).non_parametric; + pfilter.data.marginal_probability.(options_.varobs{jo}).pkf(:,:,t) = out.data.marginal_probability.(options_.varobs{jo}).pkf; + end + pfilter.data.likelihood.empirical_kernel(t,1) = out.data.likelihood.empirical_kernel; + pfilter.data.likelihood.empirical_non_parametric(t,1) = out.data.likelihood.empirical_non_parametric; + pfilter.data.likelihood.ppf(t,1) = out.data.likelihood.ppf; + pfilter.data.likelihood.pkf(t,1) = out.data.likelihood.pkf; + pfilter.updated.regimes.sample(:,t) = out.updated.regimes.sample; + pfilter.updated.regimes.exit(1:size(out.updated.regimes.exit,1),t) = out.updated.regimes.exit; + pfilter.updated.regimes.exit_constrained_share(1:size(out.updated.regimes.exit,1),t) = out.updated.regimes.exit_constrained_share; + pfilter.updated.regimes.is_constrained(:,t) = out.updated.regimes.is_constrained; + pfilter.updated.regimes.is_constrained_in_expectation(:,t) = out.updated.regimes.is_constrained_in_expectation; + pfilter.updated.regimes.prob.is_constrained(t,1) = out.updated.regimes.prob.is_constrained; + pfilter.updated.regimes.prob.is_constrained_in_expectation(t,1) = out.updated.regimes.prob.is_constrained_in_expectation; + pfilter.updated.variables.mean(:,t) = out.updated.variables.mean; + pfilter.updated.variables.median(:,t) = out.updated.variables.median; + pfilter.updated.variables.var(:,t) = out.updated.variables.var; + pfilter.updated.variables.hpd_interval(:,:,t) = out.updated.variables.hpd_interval; + pfilter.updated.variables.deciles(:,:,t) = out.updated.variables.deciles; + pfilter.updated.variables.density(:,:,:,t) = out.updated.variables.density; + pfilter.updated.variables.particles(:,:,t) = out.updated.variables.particles; + pfilter.updated.variables.pkf(:,t) = out.updated.variables.pkf; + end + end + if info + set_dynare_random_generator_state(LastSeeds.Unifor, LastSeeds.Normal); + return + end + StateVectors = tmpStateVectors; + lik(s) = liky; + + regimes_(t:t+2) = regimesx; + a0(:,t) = ax(:,1); + a1(:,t) = a1x(:,2); + a = ax(:,2); + vv(d_index,t) = vx(d_index,2); + TT(:,:,t:t+1) = Tx; + RR(:,:,t:t+1) = Rx; + CC(:,t:t+1) = Cx; + P0(:,:,t) = Px(:,:,1); + P1(:,:,t) = P1x(:,:,2); + P = Px(:,:,2); + + t = t+1; + if ~options_.occbin.filter.particle.nograph + pause(1), + end + end +end + +set_dynare_random_generator_state(LastSeeds.Unifor, LastSeeds.Normal); + +if F_singular + error('The variance of the forecast error remains singular until the end of the sample') +end + +% Divide by two. +lik(1:s) = .5*lik(1:s); + +% Compute minus the log-likelihood. +if presample>=diffuse_periods + LIK = sum(lik(1+presample-diffuse_periods:end)); +else + LIK = sum(lik); +end + diff --git a/matlab/+occbin/pkf_conditional_density.m b/matlab/+occbin/pkf_conditional_density.m new file mode 100644 index 0000000000000000000000000000000000000000..327839fbdf17d34d20a8be519cc1ca76459b63c4 --- /dev/null +++ b/matlab/+occbin/pkf_conditional_density.m @@ -0,0 +1,264 @@ +function [StateVector, StateVectorsPKF, StateVectorsPPF, liky, updated_regimes, updated_sample, updated_mode, use_pkf_distribution, error_flag] = pkf_conditional_density(StateVector0, a, a1, P, P1, Py, alphahaty, t, data_index,Z,v,Y,H,QQQ,T0,R0,TT,RR,CC,info,regimes0,base_regime,regimesy,M_, ... + dr, endo_steady_state,exo_steady_state,exo_det_steady_state,options_,occbin_options) + +if t==201 %202 %187 %202 %203 %207 %54 %41 %56 %58 %75 + do_nothing = true; +end + +yhat = StateVector0.Draws; +error_flag = 0; + +% store input values +P0=P; +try + chol(P0); +catch + P0 = 0.5*(P0+P0'); +end +a0=a; +P=P*0; +P1(:,:,1)=P; +QQ = RR(:,:,2)*QQQ(:,:,2)*transpose(RR(:,:,2)); +di=data_index{2}; +ZZ = Z(di,:); + +P1(:,:,2) = QQ; +options_.occbin.filter.state_covariance=false; +my_data_index = data_index; +my_data_index{1} = []; +my_data = Y; +my_data(:,1) = nan; + +number_of_particles = size(yhat,2); +number_of_successful_particles = 0; +updated_regimes = []; +updated_sample = []; +number_of_updated_regimes = 0; +updated_sample.success = false(number_of_particles,1); +likxmode = inf; +for k=1:number_of_particles + % parfor k=1:number_of_particles + a(:,1) = yhat(:,k); + a1(:,1) = yhat(:,k); + a1(:,2) = TT(:,:,2)*yhat(:,k); + + % conditional likelihood + options_.occbin.filter.state_covariance=true; + use_the_engine = true; + if info==0 + % start with PKF regime to speed-up search! + guess_regime = regimesy(1:2); + options_.occbin.filter.guess_regime = true; + [ax, a1x, Px, P1x, vx, Tx, Rx, Cx, regimesx, infox, ~, likxc, etahatx,alphahatx, Vxc] = ... + occbin.kalman_update_algo_1(a,a1,P,P1,my_data_index,Z,v,my_data,H,QQQ,T0,R0,TT,RR,CC,guess_regime,M_, ... + dr, endo_steady_state,exo_steady_state,exo_det_steady_state,options_,occbin_options); + options_.occbin.filter.guess_regime = false; + if infox == 0 + use_the_engine = false; + end + end + if use_the_engine + [ax, a1x, Px, P1x, vx, Tx, Rx, Cx, regimesx, infox, ~, likxc, etahatx,alphahatx, Vxc] = ... + occbin.kalman_update_engine(a,a1,P,P1,t,my_data_index,Z,v,my_data,H,QQQ,T0,R0,TT,RR,CC,regimes0,base_regime,my_data_index{2},M_, ... + dr, endo_steady_state,exo_steady_state,exo_det_steady_state,options_,occbin_options); + end + + if infox ==0 + + % handle conditional likelihood and updated regime + [~, tmp, tmp_str]=occbin.backward_map_regime(regimesx(1)); + if M_.occbin.constraint_nbr==1 + tmp_str = tmp_str(2,:); + else + tmp_str = [tmp_str(2,:) tmp_str(4,:)]; + end + if number_of_updated_regimes>0 + this_updated_regime = find(ismember(all_updated_regimes,{tmp_str})); + end + if number_of_updated_regimes==0 || isempty(this_updated_regime) + number_of_updated_regimes = number_of_updated_regimes+1; + this_updated_regime = number_of_updated_regimes; + updated_regimes(number_of_updated_regimes).index = k; + if isequal(regimesy(1),regimesx(1)) + updated_regimes(number_of_updated_regimes).is_pkf_regime = true; + else + updated_regimes(number_of_updated_regimes).is_pkf_regime = false; + end + updated_regimes(number_of_updated_regimes).regime = tmp_str; + all_updated_regimes = {updated_regimes.regime}; + if ~options_.occbin.filter.particle.likelihood_only + updated_regimes(number_of_updated_regimes).obsvar = ZZ*(Tx(:,:,1)*P0*Tx(:,:,1)'+Rx(:,:,1)*QQQ(:,:,2)*Rx(:,:,1)')*ZZ' + H(di,di); + updated_regimes(number_of_updated_regimes).obsmean = ZZ*(Tx(:,:,1)*a0+Cx(:,1)); + updated_regimes(number_of_updated_regimes).ss.C = Cx(:,1); + updated_regimes(number_of_updated_regimes).ss.R = Rx(:,:,1); + updated_regimes(number_of_updated_regimes).ss.T = Tx(:,:,1); + uF = updated_regimes(number_of_updated_regimes).obsvar; + sig=sqrt(diag(uF)); + vv = Y(di,2) - updated_regimes(number_of_updated_regimes).obsmean; + if options_.rescale_prediction_error_covariance + log_duF = log(det(uF./(sig*sig')))+2*sum(log(sig)); + iuF = inv(uF./(sig*sig'))./(sig*sig'); + else + log_duF = log(det(uF)); + iuF = inv(uF); + end + updated_regimes(number_of_updated_regimes).lik = ... + log_duF + transpose(vv)*iuF*vv + length(di)*log(2*pi); + end + else + updated_regimes(this_updated_regime).index = [updated_regimes(this_updated_regime).index k]; + end + + number_of_successful_particles = number_of_successful_particles + 1; + + likxx(k) = likxc; + + if likxc<likxmode + likxmode = likxc; + updated_mode.likxc = likxc; + updated_mode.a = ax; + updated_mode.a1 = a1x; + updated_mode.P = Px; + updated_mode.P1 = P1x; + updated_mode.v = vx; + updated_mode.T = Tx; + updated_mode.R = Rx; + updated_mode.C = Cx; + updated_mode.regimes = regimesx; + if ~options_.occbin.filter.particle.likelihood_only + updated_mode.lik = updated_regimes(this_updated_regime).lik; + end + updated_mode.etahat = etahatx; + updated_mode.alphahat = alphahatx; + updated_mode.V = Vxc; + end + + if options_.debug + + % % conditional update info + dy = W(:,is)'*(a0 - a(:,1)); + likx0(k,1) = log_dF0 + transpose(dy)*iF*dy + length(is)*log(2*pi); + [mx(:,k), Vmx(:,:,k), likmx00(k,1)] = get_the_smoothed_density(a0, a(:,1), Y(di,2), U, W, X, is, QQQ, Cx, Rx, Tx, ZZ); + dy0 = iU(is,:)*( alphahaty(:,1)- a(:,1)); + % dy0 = U(:,is)'*( alphahaty(:,1)- a(:,1)); + likx00(k,1) = log_dF00 + transpose(dy0)*iF00*dy0 + length(is)*log(2*pi); + + likx(k) = (likxx(k)+likx0(k)-likx00(k)); + likmx(k) = (likxx(k)+likx0(k)-likmx00(k)); + lik(k)=(likxx(k)-likx00(k)); + likj(k)=(likxx(k)+likx0(k)); + end + + updated_sample.epsilon(:,k)=etahatx; + updated_sample.regimes{k} = regimesx; + updated_sample.success(k)=true; + updated_sample.y01(:,k) = alphahatx(:,1); %smoothed state 0|1! + updated_sample.y11(:,k) = ax(:,1); %updated state! + if M_.occbin.constraint_nbr==1 + if logical(regimesx(1).regime(1)) + do_nothing=true; + end + updated_sample.regime_exit(k,1) = max(regimesx(1).regimestart); + updated_sample.is_constrained(k,1) = logical(regimesx(1).regime(1)); + updated_sample.is_constrained_in_expectation(k,1) = any(regimesx(1).regime); + else + updated_sample.regime_exit(k,1) = max(regimesx(1).regimestart1); + updated_sample.regime_exit(k,2) = max(regimesx(1).regimestart2); + updated_sample.is_constrained(k,1) = logical(regimesx(1).regime1(1)); + updated_sample.is_constrained(k,2) = logical(regimesx(1).regime2(1)); + updated_sample.is_constrained_in_expectation(k,1) = any(regimesx(1).regime1); + updated_sample.is_constrained_in_expectation(k,2) = any(regimesx(1).regime1); + end + + end + +end + +if number_of_successful_particles==0 + error_flag = 340; + StateVector.Draws=nan(size(yhat)); + return +end + +lnw0 = -likxx(updated_sample.success)/2; % use conditional likelihood to weight prior draws! +% we need to normalize weights, even if they are already proper likelihood values! +% to avoid that exponential gives ZEROS all over the place! +lnw0 = lnw0-max(lnw0); +weights = zeros(1,number_of_particles); +weights0 = ones(1,number_of_successful_particles)/number_of_successful_particles ; +wtilde0 = weights0.*exp(lnw0); +weights0 = wtilde0/sum(wtilde0); +weights(updated_sample.success) = weights0; +updated_sample.ESS = 1/sum(weights.^2); + +if updated_sample.ESS<0.1*number_of_particles + do_nothing = true; +end + +if all(weights==0) + do_nothing = true; +end + + +ParticleOptions.resampling.method.kitagawa=true; +indx = resample(0,weights',ParticleOptions); +StateVectorsPPF = updated_sample.y11(:,indx); +updated_sample.indx = indx; + +% updated state mean and covariance +StateVector.Variance = Py(:,:,1); +[U,X] = svd(0.5*(Py(:,:,1)+Py(:,:,1)')); +% P= U*X*U'; +iss = find(diag(X)>1.e-12); + +% if any(iss) || ~options_.occbin.filter.particle.draw_states_from_empirical_density + +StateVector.Mean = alphahaty(:,2); +if any(iss) + StateVectorVarianceSquareRoot = chol(X(iss,iss))';%reduced_rank_cholesky(ReducedForm.StateVectorVariance)'; + % Get the rank of StateVectorVarianceSquareRoot + state_variance_rank = size(StateVectorVarianceSquareRoot,2); + StateVectorsPKF = bsxfun(@plus,U(:,iss)*StateVectorVarianceSquareRoot*transpose(norminv(qmc_scrambled(state_variance_rank,number_of_particles,1))),StateVector.Mean); +else + state_variance_rank = numel(StateVector.Mean ); + StateVectorsPKF = bsxfun(@plus,0*transpose(norminv(qmc_scrambled(state_variance_rank,number_of_particles,1))),StateVector.Mean); +end + + +%% PPF density +% if number_of_updated_regimes==1 +% datalik = exp(-liky/2); +% % datalik = exp(-updated_regimes(1).lik/2); +% else + datalik = mean(exp(-likxx(updated_sample.success)/2)); +% end + +liky = -log(datalik)*2; +updated_sample.likxx = likxx; + +if options_.occbin.filter.particle.draw_states_from_empirical_density + if not(isequal(regimesy(1),base_regime) && any([updated_regimes.is_pkf_regime]) && (sum(ismember(indx,updated_regimes([updated_regimes.is_pkf_regime]).index))/number_of_particles)>=options_.occbin.filter.particle.use_pkf_updated_state_threshold ) + StateVector.Draws = StateVectorsPPF; + PS = cov(StateVector.Draws'); + [U,X] = svd(0.5*(PS+PS')); + % P= U*X*U'; + iss = find(diag(X)>1.e-12); + + StateVector.Mean = mean(StateVector.Draws,2); + if any(iss) + StateVectorVarianceSquareRoot = chol(X(iss,iss))';%reduced_rank_cholesky(ReducedForm.StateVectorVariance)'; + % Get the rank of StateVectorVarianceSquareRoot + state_variance_rank = size(StateVectorVarianceSquareRoot,2); + StateVector.Draws = bsxfun(@plus,U(:,iss)*StateVectorVarianceSquareRoot*transpose(norminv(qmc_scrambled(state_variance_rank,number_of_particles,1))),StateVector.Mean); + else + state_variance_rank = numel(StateVector.Mean ); + StateVector.Draws = bsxfun(@plus,0*transpose(norminv(qmc_scrambled(state_variance_rank,number_of_particles,1))),StateVector.Mean); + end + use_pkf_distribution=false; + else + StateVector.Draws = StateVectorsPKF; + use_pkf_distribution=true; + end +end + + diff --git a/matlab/+occbin/posterior_importance_sampling.m b/matlab/+occbin/posterior_importance_sampling.m new file mode 100644 index 0000000000000000000000000000000000000000..ab59218b8c34a137c4d44479bb25bb99310c5174 --- /dev/null +++ b/matlab/+occbin/posterior_importance_sampling.m @@ -0,0 +1,368 @@ +function oo_ = posterior_importance_sampling(M_, estim_params_, oo_, options_, bayestopt_) +% uses draws from linear MCMC to re-draw parameters using PKF + +% retrieve MCMC from linear model +DirectoryName = CheckPath('metropolis',M_.dname); +OutputDirectoryName = CheckPath('Output',M_.dname); +a=dir([DirectoryName '/' M_.fname '_mh_history*']); +load([DirectoryName '/' M_.fname '_mh_history_' int2str(length(a)-1)]); +TotalNumberOfMhFiles = sum(record.MhDraws(:,2)); +TotalNumberOfMhDraws = sum(record.MhDraws(:,1)); +npar = length(bayestopt_.name); + +KeptNumberOfMhDraws = TotalNumberOfMhDraws-floor(options_.mh_drop*TotalNumberOfMhDraws); + +B = options_.sub_draws; + +if B==KeptNumberOfMhDraws*options_.mh_nblck + % we load all retained MH runs ! + %lpost=GetAllPosteriorDraws(0, FirstMhFile, FirstLine, TotalNumberOfMhFiles, KeptNumberOfMhDraws); + %lpost=GetAllPosteriorDraws(0,record.KeepedDraws.FirstMhFile,record.KeepedDraws.FirstLine,TotalNumberOfMhFiles,KeptNumberOfMhDraws); + lpost = GetAllPosteriorDraws(options_, M_.dname, M_.fname, 0, record.KeepedDraws.FirstMhFile,record.KeepedDraws.FirstLine, TotalNumberOfMhFiles, KeptNumberOfMhDraws, options_.mh_nblck); + for j=1:npar + params0(:,j) = GetAllPosteriorDraws(options_, M_.dname, M_.fname, j, record.KeepedDraws.FirstMhFile,record.KeepedDraws.FirstLine, TotalNumberOfMhFiles, KeptNumberOfMhDraws, options_.mh_nblck); + end +else + lpost=NaN(B,1); + [error_flag,~,options_]= metropolis_draw(1,options_,estim_params_,M_); + for b=1:B + %[x(b,:), lpost(b)] = GetOneDraw(type,M_,estim_params_,oo_,options_,bayestopt_); + [params0(b,:), lpost(b)] = GetOneDraw('posterior',M_,estim_params_,oo_,options_,bayestopt_); + %params0(:,j) = GetAllPosteriorDraws(j,record.KeepedDraws.FirstMhFile,record.KeepedDraws.FirstLine,TotalNumberOfMhFiles,KeptNumberOfMhDraws); + end +end + +% make sure priordens is initialized +priordens(params0(1,:)',bayestopt_.pshape,bayestopt_.p6,bayestopt_.p7, ... + bayestopt_.p3,bayestopt_.p4,1); + +% compute posterior kernel with occbin and derive weights +lpk = nan(size(lpost)); + +hh = dyn_waitbar(0,'Computing posterior kernel'); +set(hh,'Name','Computing posterior kernel.'); + +for k=1:length(lpk) + + lpk(k) = evaluate_posterior_kernel(params0(k,:)', M_, estim_params_, oo_, options_, bayestopt_); + + dyn_waitbar(k/length(lpk), hh, sprintf('Step %u of %u', k,length(lpk))); +end + +weights = exp(lpk-lpost); +ESS = length(weights)/mean(weights.^2); +weights = weights./sum(weights); + +oo_.occbin.ESS = ESS; + +fprintf(' The effective sample size (ESS) is %.2f for %.0f draws\n',ESS,B) + +% options_.particle.resampling.method.kitagawa = true; +% options_.particle.resampling.method.stratified = false; +% options_.particle.resampling.method.smooth = false; + +% weighted posterior bootrap sample with occbin posterior kernel +resampled_params = resample(params0,weights,options_.particle); +save([DirectoryName '/' M_.fname '_occbin_importance_sampling'],'params0','resampled_params','lpost','lpk','ESS') + +% get mh mode of occbin posterior kernel +[m, im] = max(lpk); +xparam1 = params0(im,:)'; +fval = lpk(im); +hh = inv(cov(resampled_params)); +parameter_names = bayestopt_.name; +save([OutputDirectoryName '/' M_.fname '_mhocc_mode'],'fval','hh','parameter_names','xparam1') + +% print results and store them +nvx = estim_params_.nvx; +nvn = estim_params_.nvn; +ncx = estim_params_.ncx; +ncn = estim_params_.ncn; +np = estim_params_.np ; +nxs = nvx+nvn+ncx+ncn; + +oo1 = oo_; + +skipline(2) +disp('ESTIMATION RESULTS: occbin importance sampling from linear MCMC') +skipline() + + +header_width = row_header_width(M_, estim_params_, bayestopt_); +my_header = ' linear'; +header_width = max(header_width, length(my_header)); + +hpd_interval=[num2str(options_.mh_conf_sig*100), '% HPD interval']; +tit2 = sprintf('%-*s %12s %12s %23s %8s %12s\n',header_width,' ','prior mean','post. mean',hpd_interval,'prior','pstdev'); +pformat = '%-*s %12.3f % 12.4f %11.4f %11.4f %8s %12.4f'; +pformat1 = '%-*s % 12.4f %11.4f %11.4f'; +header_width1 = header_width+13; +pnames = prior_dist_names; + +if np + type = 'parameters'; + skipline() + disp(type) + disp(tit2) + + for i=1:np + ip = i+nxs; + Draws = resampled_params(:,ip); + [post_mean, post_median, post_var, hpd_interval, post_deciles, density] = posterior_moments(Draws, options_.mh_conf_sig); + name = bayestopt_.name{ip}; + oo1 = Filloo(oo1, name, type, post_mean, hpd_interval, post_median, post_var, post_deciles, density); + disp(sprintf(pformat, header_width, name, bayestopt_.p1(ip),... + post_mean, ... + hpd_interval, ... + pnames{bayestopt_.pshape(ip)+1}, ... + bayestopt_.p2(ip))); + Draws = params0(:,ip); + [post_mean, post_median, post_var, hpd_interval, post_deciles, density] = posterior_moments(Draws, options_.mh_conf_sig); + oo_ = Filloo(oo_, name, type, post_mean, hpd_interval, post_median, post_var, post_deciles, density); + disp(sprintf(pformat1, header_width1, my_header,... + post_mean, ... + hpd_interval)); + end +end + +if nvx + type = 'shocks_std'; + skipline() + disp('standard deviation of shocks') + disp(tit2) + for ip=1:nvx + Draws = resampled_params(:,ip); + [post_mean, post_median, post_var, hpd_interval, post_deciles, density] = posterior_moments(Draws, options_.mh_conf_sig); + name = bayestopt_.name{ip}; + oo1 = Filloo(oo1, name, type, post_mean, hpd_interval, post_median, post_var, post_deciles, density); + disp(sprintf(pformat, header_width, name, bayestopt_.p1(ip),... + post_mean, ... + hpd_interval, ... + pnames{bayestopt_.pshape(ip)+1}, ... + bayestopt_.p2(ip))); + Draws = params0(:,ip); + [post_mean, post_median, post_var, hpd_interval, post_deciles, density] = posterior_moments(Draws, options_.mh_conf_sig); + oo_ = Filloo(oo_, name, type, post_mean, hpd_interval, post_median, post_var, post_deciles, density); + disp(sprintf(pformat1, header_width1, my_header,... + post_mean, ... + hpd_interval)); + end +end + +if nvn + type = 'measurement_errors_std'; + skipline() + disp('standard deviation of measurement errors') + disp(tit2) + for i=1:nvn + ip=i+nvx; + Draws = resampled_params(:,ip); + [post_mean, post_median, post_var, hpd_interval, post_deciles, density] = posterior_moments(Draws, options_.mh_conf_sig); + name = bayestopt_.name{ip}; + oo1 = Filloo(oo1, name, type, post_mean, hpd_interval, post_median, post_var, post_deciles, density); + disp(sprintf(pformat, header_width, name, bayestopt_.p1(ip),... + post_mean, ... + hpd_interval, ... + pnames{bayestopt_.pshape(ip)+1}, ... + bayestopt_.p2(ip))); + Draws = params0(:,ip); + [post_mean, post_median, post_var, hpd_interval, post_deciles, density] = posterior_moments(Draws, options_.mh_conf_sig); + oo_ = Filloo(oo_, name, type, post_mean, hpd_interval, post_median, post_var, post_deciles, density); + disp(sprintf(pformat1, header_width1, my_header,... + post_mean, ... + hpd_interval)); + end +end + +if ncx + type = 'shocks_corr'; + skipline() + disp('correlation of shocks') + disp(tit2) + for i=1:ncx + ip=i+nvx+nvn; + Draws = resampled_params(:,ip); + [post_mean, post_median, post_var, hpd_interval, post_deciles, density] = posterior_moments(Draws, options_.mh_conf_sig); + name = bayestopt_.name{ip}; + name = strrep(name,'corr ',''); + NAME = strrep(name,', ','_'); + name = strrep(name,', ',','); + oo1 = Filloo(oo1, NAME, type, post_mean, hpd_interval, post_median, post_var, post_deciles, density); + disp(sprintf(pformat, header_width, name, bayestopt_.p1(ip),... + post_mean, ... + hpd_interval, ... + pnames{bayestopt_.pshape(ip)+1}, ... + bayestopt_.p2(ip))); + Draws = params0(:,ip); + [post_mean, post_median, post_var, hpd_interval, post_deciles, density] = posterior_moments(Draws, options_.mh_conf_sig); + oo_ = Filloo(oo_, NAME, type, post_mean, hpd_interval, post_median, post_var, post_deciles, density); + disp(sprintf(pformat1, header_width1, my_header,... + post_mean, ... + hpd_interval)); + end +end + +if ncn + type = 'measurement_errors_corr'; + skipline() + disp('correlation of measurement errors') + disp(tit2) + for i=1:ncn + ip=i+nvx+nvn+ncx; + Draws = resampled_params(:,ip); + [post_mean, post_median, post_var, hpd_interval, post_deciles, density] = posterior_moments(Draws, options_.mh_conf_sig); + name = bayestopt_.name{ip}; + name = strrep(name,'corr ',''); + NAME = strrep(name,', ','_'); + name = strrep(name,', ',','); + oo1 = Filloo(oo1, NAME, type, post_mean, hpd_interval, post_median, post_var, post_deciles, density); + disp(sprintf(pformat, header_width, name, bayestopt_.p1(ip),... + post_mean, ... + hpd_interval, ... + pnames{bayestopt_.pshape(ip)+1}, ... + bayestopt_.p2(ip))); + Draws = params0(:,ip); + [post_mean, post_median, post_var, hpd_interval, post_deciles, density] = posterior_moments(Draws, options_.mh_conf_sig); + oo_ = Filloo(oo_, NAME, type, post_mean, hpd_interval, post_median, post_var, post_deciles, density); + disp(sprintf(pformat1, header_width1, my_header,... + post_mean, ... + hpd_interval)); + end +end + +% this is where we store posterior statistics after importance sampling +field_list = {'posterior_mean', 'posterior_hpdinf', 'posterior_hpdsup', 'posterior_median', 'posterior_variance', 'posterior_std', 'posterior_deciles', 'posterior_density'}; +for k=1:length(field_list) + oo_.occbin.(field_list{k}) = oo1.(field_list{k}); +end + +if not(options_.nograph) + OutputDirectoryName = CheckPath('graphs',M_.dname); + + TeX = false; %options_.TeX; + + MaxNumberOfPlotPerFigure = 9;% The square root must be an integer! + nn = sqrt(MaxNumberOfPlotPerFigure); + + figurename = 'PKF and linear posteriors'; + figunumber = 0; + subplotnum = 0; + for i=1:npar + subplotnum = subplotnum+1; + if subplotnum == 1 + figunumber = figunumber+1; + hfig=dyn_figure(options_.nodisplay, 'Name', figurename); + end + [nam,texnam] = get_the_name(i, TeX, M_, estim_params_, options_); + if i <= nvx + name = M_.exo_names{estim_params_.var_exo(i,1)}; + x1 = oo_.occbin.posterior_density.shocks_std.(name)(:,1); + f1 = oo_.occbin.posterior_density.shocks_std.(name)(:,2); + x2 = oo_.posterior_density.shocks_std.(name)(:,1); + f2 = oo_.posterior_density.shocks_std.(name)(:,2); + if ~options_.mh_posterior_mode_estimation + pmod = oo_.posterior_mode.shocks_std.(name); + end + elseif i <= nvx+nvn + name = options_.varobs{estim_params_.nvn_observable_correspondence(i-nvx,1)}; + x1 = oo_.occbin.posterior_density.measurement_errors_std.(name)(:,1); + f1 = oo_.occbin.posterior_density.measurement_errors_std.(name)(:,2); + x2 = oo_.posterior_density.measurement_errors_std.(name)(:,1); + f2 = oo_.posterior_density.measurement_errors_std.(name)(:,2); + if ~options_.mh_posterior_mode_estimation + pmod = oo_.posterior_mode.measurement_errors_std.(name); + end + elseif i <= nvx+nvn+ncx + j = i - (nvx+nvn); + k1 = estim_params_.corrx(j,1); + k2 = estim_params_.corrx(j,2); + name = sprintf('%s_%s', M_.exo_names{k1}, M_.exo_names{k2}); + x1 = oo_.occbin.posterior_density.shocks_corr.(name)(:,1); + f1 = oo_.occbin.posterior_density.shocks_corr.(name)(:,2); + x2 = oo_.posterior_density.shocks_corr.(name)(:,1); + f2 = oo_.posterior_density.shocks_corr.(name)(:,2); + if ~options_.mh_posterior_mode_estimation + pmod = oo_.posterior_mode.shocks_corr.(name); + end + elseif i <= nvx+nvn+ncx+ncn + j = i - (nvx+nvn+ncx); + k1 = estim_params_.corrn(j,1); + k2 = estim_params_.corrn(j,2); + name = sprintf('%s_%s', M_.endo_names{k1}, M_.endo_names{k2}); + x1 = oo_.occbin.posterior_density.measurement_errors_corr.(name)(:,1); + f1 = oo_.occbin.posterior_density.measurement_errors_corr.(name)(:,2); + x2 = oo_.posterior_density.measurement_errors_corr.(name)(:,1); + f2 = oo_.posterior_density.measurement_errors_corr.(name)(:,2); + if ~options_.mh_posterior_mode_estimation + pmod = oo_.posterior_mode.measurement_errors_corr.(name); + end + else + j = i - (nvx+nvn+ncx+ncn); + name = M_.param_names{estim_params_.param_vals(j,1)}; + x1 = oo_.occbin.posterior_density.parameters.(name)(:,1); + f1 = oo_.occbin.posterior_density.parameters.(name)(:,2); + x2 = oo_.posterior_density.parameters.(name)(:,1); + f2 = oo_.posterior_density.parameters.(name)(:,2); + if ~options_.mh_posterior_mode_estimation + pmod = oo_.posterior_mode.parameters.(name); + end + end + top1 = max(f1); + top2 = max(f2); + top0 = max([top1; top2]); + binf1 = x1(1); + bsup1 = x1(end); + binf2 = x2(1); + bsup2 = x2(end); + borneinf = min(binf1, binf2); + bornesup = max(bsup1, bsup2); + subplot(nn, nn, subplotnum) + plot(x2, f2, '-b', 'linewidth', 2); + hold on; + plot(x1, f1, '-r', 'linewidth', 2); + % if ~options_.mh_posterior_mode_estimation + % plot([pmod pmod], [0.0 1.1*top0], '--g', 'linewidth', 2); + % end + box on + axis([borneinf bornesup 0 1.1*top0]) + if TeX + title(texnam, 'Interpreter', 'latex') + else + title(nam, 'Interpreter', 'none') + end + hold off + drawnow + if subplotnum == MaxNumberOfPlotPerFigure || i == npar + dyn_saveas(hfig,[OutputDirectoryName '/' M_.fname '_PosteriorsOccbinVsLinear' int2str(figunumber)], options_.nodisplay, options_.graph_format); + if TeX && any(strcmp('eps', cellstr(options_.graph_format))) + fprintf(fidTeX, '\\begin{figure}[H]\n'); + fprintf(fidTeX, '\\centering\n'); + fprintf(fidTeX, '\\includegraphics[width=%2.2f\\textwidth]{%s/%s_PosteriorsOccbinVsLinear%s}\n', ... + options_.figures.textwidth*min(subplotnum/nn,1), OutputDirectoryName, M_.fname, int2str(figunumber)); + fprintf(fidTeX,'\\caption{PKF and linear posteriors.}'); + fprintf(fidTeX,'\\label{Fig:PosteriorsOccbinVsLinear:%s}\n', int2str(figunumber)); + fprintf(fidTeX,'\\end{figure}\n'); + fprintf(fidTeX,' \n'); + if i == npar + fprintf(fidTeX,'%% End of TeX file.\n'); + fclose(fidTeX); + end + end + subplotnum = 0; + end + end +end + +end + +function oo = Filloo(oo, name, type, postmean, hpdinterval, postmedian, postvar, postdecile, density) +oo.posterior_mean.(type).(name) = postmean; +oo.posterior_hpdinf.(type).(name) = hpdinterval(1); +oo.posterior_hpdsup.(type).(name) = hpdinterval(2); +oo.posterior_median.(type).(name) = postmedian; +oo.posterior_variance.(type).(name) = postvar; +oo.posterior_std.(type).(name) = sqrt(postvar); +oo.posterior_deciles.(type).(name) = postdecile; +oo.posterior_density.(type).(name) = density; + +end \ No newline at end of file diff --git a/matlab/+occbin/ppf_conditional_density.m b/matlab/+occbin/ppf_conditional_density.m new file mode 100644 index 0000000000000000000000000000000000000000..2ca3dba139384a5804f5199960d47af823b86878 --- /dev/null +++ b/matlab/+occbin/ppf_conditional_density.m @@ -0,0 +1,192 @@ +function [lik, lik_ppf, lik_EnKF, y11, simulated_regimes, simul_sample, hp_simulated_regime] = ppf_conditional_density(number_of_shocks_per_particle,a0,ShockVectorsPerParticle, di, H, my_order_var, QQQ, Y, ZZ, base_regime, regimesy, M_, oo_, options_, opts_simul) +% computes empirical conditionl density for piecewise linear particle +% filter + +% [llik, shock_simul] = ppf_conditional_density(number_of_shocks_per_particle,ShockVectorsPerParticle, Y, ZZ, base_regime, regimesy, M_, oo_, options_) +% INPUTS +% a0: initial state (for conditioning) +% sample of shocks + +llik = inf; +success = false(number_of_shocks_per_particle,1); +opts_simul.SHOCKS=[]; +number_of_simulated_regimes = 0; +for k = 1:number_of_shocks_per_particle + opts_simul.SHOCKS(1,:) = ShockVectorsPerParticle(:,k)'; + options_.occbin.simul=opts_simul; + options_.occbin.simul.piecewise_only = false; + [~, out, ss] = occbin.solver(M_,oo_,options_); + if out.error_flag==0 + success(k) = true; + % % store particle + [~, tmp, tmp_str]=occbin.backward_map_regime(out.regime_history(1)); + if M_.occbin.constraint_nbr==1 + tmp_str = tmp_str(2,:); + else + tmp_str = [tmp_str(2,:) tmp_str(4,:)]; + end + if number_of_simulated_regimes>0 + this_simulated_regime = find(ismember(all_simulated_regimes,{tmp_str})); + end + if number_of_simulated_regimes==0 || isempty(this_simulated_regime) + number_of_simulated_regimes = number_of_simulated_regimes+1; + this_simulated_regime = number_of_simulated_regimes; + simulated_regimes(number_of_simulated_regimes).index = k; + if isequal(regimesy(1),out.regime_history(1)) + simulated_regimes(number_of_simulated_regimes).is_pkf_regime = true; + else + simulated_regimes(number_of_simulated_regimes).is_pkf_regime = false; + end + if isequal(base_regime,out.regime_history(1)) + simulated_regimes(number_of_simulated_regimes).is_base_regime = true; + else + simulated_regimes(number_of_simulated_regimes).is_base_regime = false; + end + simulated_regimes(number_of_simulated_regimes).obsvar = ZZ*(ss.R(my_order_var,:)*QQQ(:,:,2)*ss.R(my_order_var,:)')*ZZ' + H(di,di); + simulated_regimes(number_of_simulated_regimes).obsmean = ZZ*(ss.T(my_order_var,my_order_var)*a0+ss.C(my_order_var)); + simulated_regimes(number_of_simulated_regimes).regime = tmp_str; + simulated_regimes(number_of_simulated_regimes).ss.C = ss.C(my_order_var); + simulated_regimes(number_of_simulated_regimes).ss.R = ss.R(my_order_var,:); + simulated_regimes(number_of_simulated_regimes).ss.T = ss.T(my_order_var,my_order_var); + all_simulated_regimes = {simulated_regimes.regime}; + uF = simulated_regimes(number_of_simulated_regimes).obsvar; + sig=sqrt(diag(uF)); + vv = Y(di,2) - simulated_regimes(number_of_simulated_regimes).obsmean; + if options_.rescale_prediction_error_covariance + log_duF = log(det(uF./(sig*sig')))+2*sum(log(sig)); + iuF = inv(uF./(sig*sig'))./(sig*sig'); + else + log_duF = log(det(uF)); + iuF = inv(uF); + end + simulated_regimes(number_of_simulated_regimes).lik = ... + log_duF + transpose(vv)*iuF*vv + length(di)*log(2*pi); + simulated_regimes(number_of_simulated_regimes).Kalman_gain = (ss.R(my_order_var,:)*QQQ(:,:,2)*ss.R(my_order_var,:)')*ZZ'*iuF; + simulated_regimes(number_of_simulated_regimes).update = (ss.T(my_order_var,my_order_var)*a0+ss.C(my_order_var)) + simulated_regimes(number_of_simulated_regimes).Kalman_gain*vv; + + else + simulated_regimes(this_simulated_regime).index = [simulated_regimes(this_simulated_regime).index k]; + end + simul_sample.regime{k} = out.regime_history; + simul_sample.y10(:,k) = out.piecewise(1,my_order_var)-out.ys(my_order_var)'; + simul_sample.y10lin(:,k) = out.linear(1,my_order_var)-out.ys(my_order_var)'; + simul_sample.y11(:,k) = simulated_regimes(this_simulated_regime).update; + if M_.occbin.constraint_nbr==1 + simul_sample.exit(k,1) = max(out.regime_history.regimestart); + simul_sample.is_constrained(k,1) = logical(out.regime_history.regime(1)); + simul_sample.is_constrained_in_expectation(k,1) = any(out.regime_history.regime); + else + simul_sample.exit(k,1) = max(out.regime_history.regimestart1); + simul_sample.exit(k,2) = max(out.regime_history.regimestart2); + simul_sample.is_constrained(k,1) = logical(out.regime_history.regime1(1)); + simul_sample.is_constrained(k,2) = logical(out.regime_history.regime2(1)); + simul_sample.is_constrained_in_expectation(k,1) = any(out.regime_history.regime1); + simul_sample.is_constrained_in_expectation(k,2) = any(out.regime_history.regime2); + end + % the distribution of ZZ*y10 needs to be confronted with ZZ*a1y, ZZ*P1y*ZZ', + % i.e. the normal approximation using the regime consistent with the observable! + end +end + +number_of_successful_particles = sum(success); +simul_sample.success = success; + +if number_of_successful_particles + + % z10 = ZZ*y10; + nobs = size(ZZ,1); + z10 = ZZ*simul_sample.y10(:,success); + + % ensemble KF + mu=mean(simul_sample.y10(:,success),2); + C=cov(simul_sample.y10(:,success)'); + uF = ZZ*C*ZZ'; + sig=sqrt(diag(uF)); + vv = Y(di,2) - ZZ*mu; + if options_.rescale_prediction_error_covariance + log_duF = log(det(uF./(sig*sig')))+2*sum(log(sig)); + iuF = inv(uF./(sig*sig'))./(sig*sig'); + else + log_duF = log(det(uF)); + iuF = inv(uF); + end + lik_EnKF = ... + log_duF + transpose(vv)*iuF*vv + length(di)*log(2*pi); + + + % full non parametric density estimate +% dlik = sqrt(diag(transpose(Y(di,2)-z10)*iuF*(Y(di,2)-z10))); + dlik = log_duF+diag(transpose(Y(di,2)-z10)*iuF*(Y(di,2)-z10))+ length(di)*log(2*pi); + [sd, iss] = sort(dlik); + issux = find(success); + issux = issux(iss); + + ydist = sqrt(sum((Y(di,2)-z10).^2,1)); + [s, is] = sort(ydist); + isux = find(success); + isux = isux(is); + %bin population + nbins = ceil(sqrt(size(z10,2))); + + binpop = ceil(number_of_successful_particles/nbins/2)*2; % I center the bin on the oberved point + + % volume of n-dimensional sphere + binvolume = pi^(nobs/2)/gamma(nobs/2+1)*(0.5*(s(binpop)+s(binpop+1)))^nobs; + + % states t|t + y11 = simul_sample.y10(:,isux(randi(binpop,1))); + % use updated KF + y11 = simul_sample.y11(:,isux(randi(binpop,1))); + + % check for regime of Y + is_data_regime = false(number_of_simulated_regimes,1); + iss_data_regime = false(number_of_simulated_regimes,1); + proba = ones(number_of_simulated_regimes,1); + probaX = zeros(number_of_simulated_regimes,1); + probaY = zeros(number_of_simulated_regimes,1); + for kr=1:number_of_simulated_regimes + if options_.occbin.filter.particle.tobit && ~simulated_regimes(kr).is_base_regime + % base regime is not Tobit + proba(kr) = length(simulated_regimes(kr).index)/number_of_successful_particles; + end + % here we check if the data may be spanned for each regime + % (given we have a bunch of TRUNCATED normals, Y may be out of + % truncation ...) + is_data_regime(kr) = any(ismember(simulated_regimes(kr).index,isux(1:binpop))); + iss_data_regime(kr) = any(ismember(simulated_regimes(kr).index,issux(1:binpop))); + if is_data_regime(kr) + probaY(kr) = length(find(ismember(simulated_regimes(kr).index,isux(1:binpop))))/binpop; + probaY(kr) = length(find(ismember(simulated_regimes(kr).index,isux(1:binpop))))^2/binpop/length(simulated_regimes(kr).index); + end + if iss_data_regime(kr) + thisbin = find(ismember(issux(1:binpop),simulated_regimes(kr).index)); +% probaX(kr) = length(thisbin)/length(simulated_regimes(kr).index)/binpop; +% probaX(kr) = mean(exp(-sd(thisbin)/2))*length(thisbin)^2/length(simulated_regimes(kr).index)/binpop; + probaX(kr) = mean(exp(-sd(thisbin)/2))*length(thisbin)/length(simulated_regimes(kr).index); +% thisbin = find(ismember(issux(1:binpop),simulated_regimes(kr).index),1); +% probaX(kr) = exp(-sd(thisbin)/2); + end + end + probaX = probaX./sum(probaX); + probaY = probaY./sum(probaY); + if ~any(simul_sample.is_constrained_in_expectation(success)) + % Y is unconstrained + end + + % use most probable regime to update states + [m,im]=max((exp(-([simulated_regimes.lik]-min([simulated_regimes.lik]))./2)'.*(proba.*probaY))); + if not(simulated_regimes(im).is_pkf_regime) + stop_here = true; + end + maxbin = find(ismember(isux(1:binpop),simulated_regimes(im).index)); + y11 = mean(simul_sample.y11(:,issux(maxbin)),2); + hp_simulated_regime = simulated_regimes(im); + % likelihood is the sum of gaussians, where constrained regimes are + % weighted by their probabilities + lik_ppf = -2*(log(exp(-([simulated_regimes.lik]-min([simulated_regimes.lik]))./2)*(proba.*probaY))-min([simulated_regimes.lik])); + % samplevolume = [min(z10,[],2) max(z10,[],2)]; + % samplevolume = prod(diff(samplevolume')*1.02); + lik = -2*log(binpop/number_of_successful_particles/binvolume); + + +end diff --git a/matlab/+occbin/ppf_engine.m b/matlab/+occbin/ppf_engine.m new file mode 100644 index 0000000000000000000000000000000000000000..516e8fdbc39af7181d6ea8a236f2fdaba0de94c6 --- /dev/null +++ b/matlab/+occbin/ppf_engine.m @@ -0,0 +1,189 @@ +function [StateVectors,liky, updated_regimes, updated_sample, updated_mode, use_pkf_distribution, error_flag] = ppf_engine(StateVector0, likpkf, a, a1, P, P1, ay, a1y, Py, P1y, alphahaty, etahaty, V, t, data_index,Z,v,Y,H,QQQ,T0,R0,TT,RR,CC,info,regimes0,base_regime,regimesy,isqvec,M_, ... + dr, endo_steady_state,exo_steady_state,exo_det_steady_state,options_,occbin_options) + +if t==201 %202 %187 %202 %203 %207 %54 %41 %56 %58 %75 + do_nothing = true; +end + +info = 0; +number_of_particles = options_.occbin.filter.particle.number_of_particles; +if isempty(StateVector0) + [U,X] = svd(0.5*(P+P')); + % P= U*X*U'; % symmetric matrix! + is = find(diag(X)>1.e-12); + StateVectorVarianceSquareRoot = chol(X(is,is))';%reduced_rank_cholesky(ReducedForm.StateVectorVariance)'; + + % Get the rank of StateVectorVarianceSquareRoot + state_variance_rank = size(StateVectorVarianceSquareRoot,2); + + % % Factorize the covariance matrix of the structural innovations + % Q_lower_triangular_cholesky = chol(Q)'; + + % Initialization of the weights across particles. + if not(isempty(is)) + yhat = U(:,is)*StateVectorVarianceSquareRoot*transpose(norminv(qmc_scrambled(state_variance_rank,number_of_particles,1)))+repmat(a,[1 number_of_particles]); + else + yhat = repmat(a,[1 number_of_particles]); + end + StateVector0.Draws = yhat; + StateVector0.Mean = a; + StateVector0.Variance = P; +else + yhat = StateVector0.Draws; +end + +if ~(options_.occbin.filter.particle.likelihood_only && ~options_.occbin.filter.particle.empirical_data_density.status) + % % random shocks + [US,XS] = svd(QQQ(:,:,2)); + % P= U*X*U'; + ishock = find(sqrt(abs(diag(XS)))>1.e-10); + ShockVarianceSquareRoot = chol(XS(ishock,ishock))';%reduced_rank_cholesky(ReducedForm.StateVectorVariance)'; + % Get the rank of ShockVarianceSquareRoot + shock_variance_rank = size(ShockVarianceSquareRoot,2); + ShockVectors = bsxfun(@plus,US(:,ishock)*ShockVarianceSquareRoot*transpose(norminv(qmc_scrambled(shock_variance_rank,number_of_particles,1))),zeros(length(US),1)); +end + +if options_.occbin.filter.particle.importance_sampling_using_pkf + [StateVectors, StateVectorsPKF, StateVectorsPPF, liky, updated_regimes, updated_sample, updated_mode, use_pkf_distribution, error_flag] = occbin.ppf_state_importance_sampling(StateVector0, a, a1, P, P1, Py, alphahaty, V, t, data_index,Z,v,Y,H,QQQ,T0,R0,TT,RR,CC,info,regimes0,base_regime,regimesy,M_, ... + dr, endo_steady_state,exo_steady_state,exo_det_steady_state,options_,occbin_options); + +else + [StateVectors, StateVectorsPKF, StateVectorsPPF, liky, updated_regimes, updated_sample, updated_mode, use_pkf_distribution, error_flag] = occbin.pkf_conditional_density(StateVector0, a, a1, P, P1, Py, alphahaty, t, data_index,Z,v,Y,H,QQQ,T0,R0,TT,RR,CC,info,regimes0,base_regime,regimesy,M_, ... + dr, endo_steady_state,exo_steady_state,exo_det_steady_state,options_,occbin_options); +end + +if ~(options_.occbin.filter.particle.likelihood_only && ~options_.occbin.filter.particle.empirical_data_density.status) + di=data_index{2}; + ZZ = Z(di,:); + opts_simul = occbin_options.opts_simul; + if opts_simul.restrict_state_space + my_order_var = dr.order_var(dr.restrict_var_list); + else + my_order_var = dr.order_var; + end + number_of_shocks_per_particle = 1; + [llik, simulated_regimes, simulated_sample, hp_simulated_regimes] = ... + occbin.ppf_simulated_density(number_of_particles, number_of_shocks_per_particle,StateVector0.Draws,ShockVectors, ... + di, H, my_order_var, QQQ, Y, ZZ, base_regime, regimesy, ... + M_, dr, endo_steady_state, exo_steady_state, exo_det_steady_state, options_, opts_simul); + + +% if ~options_.occbin.filter.particle.nograph + [density, density_data] = occbin.ppf_graphs(t, updated_sample.indx, ShockVectors, StateVectorsPKF, StateVectorsPPF, updated_sample.likxx, regimes0, regimesy, ... + simulated_regimes, simulated_sample, updated_regimes, updated_sample, a1, P1, a1y, P1y, alphahaty, etahaty, ... + di, H, QQQ, Y, ZZ, dr, M_, options_); +% end + + % collect results + success = simulated_sample.success; + y100 = simulated_sample.y10; + y10 = y100(:,success); + for jk=1:size(y10,1) + [output.filtered.variables.mean(jk,1), ... + output.filtered.variables.median(jk,1), ... + output.filtered.variables.var(jk,1), ... + output.filtered.variables.hpd_interval(jk,:), ... + output.filtered.variables.deciles(jk,:), ... + output.filtered.variables.density(jk,:,:)] = posterior_moments(y10(jk,:)', options_.mh_conf_sig); + end + output.filtered.variables.particles = y100; + output.filtered.regimes.sample = simulated_sample.regime; + output.filtered.regimes.is_constrained = simulated_sample.is_constrained; + output.filtered.regimes.is_constrained_in_expectation = simulated_sample.is_constrained_in_expectation; + [aa,bb]=histcounts(simulated_sample.exit,'normalization','pdf','binmethod','integers'); + output.filtered.regimes.exit(bb(1:end-1)+0.5,1) = aa'; + isc = simulated_sample.is_constrained; + isce = simulated_sample.is_constrained_in_expectation; + rr=simulated_sample.exit; + output.filtered.regimes.exit_constrained_share(1,1)=0; + for ka=2:length(output.filtered.regimes.exit) + iden = length(find(isce(rr==ka))); + if iden + output.filtered.regimes.exit_constrained_share(ka,1) = length(find(isc(rr==ka)))/iden; + else + output.filtered.regimes.exit_constrained_share(ka,1) = 0; + end + end + + [aa, bb]=histcounts(simulated_sample.is_constrained,'normalization','pdf','binmethod','integers'); + if length(aa)==1 + if bb(1)+0.5==0 + aa=[1 0]; + else + aa=[0 1]; + end + bb=-0.5:1:1.5; + end + output.filtered.regimes.is_constrained = isc; + output.filtered.regimes.prob.is_constrained = aa(2); + [aa, bb]=histcounts(simulated_sample.is_constrained_in_expectation,'normalization','pdf','binmethod','integers'); + if length(aa)==1 + if bb(1)+0.5==0 + aa=[1 0]; + else + aa=[0 1]; + end + bb=-0.5:1:1.5; + end + output.filtered.regimes.is_constrained_in_expectation = isce; + output.filtered.regimes.prob.is_constrained_in_expectation = aa(2); + + output.data.marginal_probability_distribution = density; + output.data.marginal_probability = density_data; + output.data.likelihood.empirical_kernel = llik.kernel; + output.data.likelihood.empirical_non_parametric = llik.non_parametric; + output.data.likelihood.empirical_weighted_average = llik.ppf; + output.data.likelihood.ppf = liky; + output.data.likelihood.pkf = likpkf; + output.data.likelihood.enkf = llik.enkf; + + for jk=1:size(StateVectors.Draws,1) + [output.updated.variables.mean(jk,1), ... + output.updated.variables.median(jk,1), ... + output.updated.variables.var(jk,1), ... + output.updated.variables.hpd_interval(jk,:), ... + output.updated.variables.deciles(jk,:), ... + output.updated.variables.density(jk,:,:)] = posterior_moments(StateVectors.Draws(jk,:)', options_.mh_conf_sig); + end + output.updated.variables.particles = StateVectors.Draws; + output.updated.variables.pkf = alphahaty(:,2); + output.updated.regimes.sample = updated_sample.regimes(updated_sample.indx); + [aa,bb]=histcounts(updated_sample.regime_exit(updated_sample.indx),'normalization','pdf','binmethod','integers'); + output.updated.regimes.exit(bb(1:end-1)+0.5,1) = aa'; + isc = updated_sample.is_constrained(updated_sample.indx); + isce = updated_sample.is_constrained_in_expectation(updated_sample.indx); + rr=updated_sample.regime_exit(updated_sample.indx); + output.updated.regimes.exit_constrained_share(1,1)=0; + for ka=2:length(output.updated.regimes.exit) + iden = length(find(isce(rr==ka))); + if iden + output.updated.regimes.exit_constrained_share(ka,1) = length(find(isc(rr==ka)))/iden; + else + output.updated.regimes.exit_constrained_share(ka,1) = 0; + end + end + [aa, bb]=histcounts(updated_sample.is_constrained(updated_sample.indx),'normalization','pdf','binmethod','integers'); + if length(aa)==1 + if bb(1)+0.5==0 + aa=[1 0]; + else + aa=[0 1]; + end + bb=-0.5:1:1.5; + end + output.updated.regimes.is_constrained = isc; + output.updated.regimes.prob.is_constrained = aa(2); + + [aa, bb]=histcounts(updated_sample.is_constrained_in_expectation(updated_sample.indx),'normalization','pdf','binmethod','integers'); + if length(aa)==1 + if bb(1)+0.5==0 + aa=[1 0]; + else + aa=[0 1]; + end + bb=-0.5:1:1.5; + end + output.updated.regimes.is_constrained_in_expectation = isce; + output.updated.regimes.prob.is_constrained_in_expectation = aa(2); + +end diff --git a/matlab/+occbin/ppf_graphs.m b/matlab/+occbin/ppf_graphs.m new file mode 100644 index 0000000000000000000000000000000000000000..129d80a71698837f9ee9f1458065419cabc4bd51 --- /dev/null +++ b/matlab/+occbin/ppf_graphs.m @@ -0,0 +1,513 @@ +function [density, density_data] = ppf_graphs(t, indx, ShockVectors, StateVectorsPKF, StateVectorsPPF, likxx, regimes0, regimesy, ... + simulated_regimes, simulated_sample, updated_regimes, updated_sample, a10, P10, a1y, P1y, alphahaty, etahaty, ... + di, H, QQQ, Y, ZZ, dr, M_, options_) + +% OUTPUT +if ~options_.occbin.filter.particle.nograph + + % observed Y mean and covariance given0 + PZ0 = ZZ*P10(:,:,2)*ZZ'+H(di,di); + ObsVectorMean0 = ZZ*a10(:,2); + if isempty(reduced_rank_cholesky(PZ0)') + obs_variance_rank0=0; + else + ObsVectorVarianceSquareRoot0 = chol(PZ0)';%reduced_rank_cholesky(ReducedForm.StateVectorVariance)'; + + % Get the rank of ObsVectorVarianceSquareRoot + obs_variance_rank0 = size(ObsVectorVarianceSquareRoot0,2); + end + % ObsVectors0 = zeros(size(ObsVectorMean0,1),number_of_particles); + % % % ObsVectors0 = bsxfun(@plus,ObsVectorVarianceSquareRoot0*transpose(norminv(qmc_scrambled(obs_variance_rank,number_of_particles,1))),ObsVectorMean0); +end +% observed Y mean and covariance +PZ = ZZ*P1y(:,:,2)*ZZ'+H(di,di); +ObsVectorMean = ZZ*a1y(:,2); +ObsVectorVarianceSquareRoot = chol(PZ)';%reduced_rank_cholesky(ReducedForm.StateVectorVariance)'; + +% Get the rank of ObsVectorVarianceSquareRoot +obs_variance_rank = size(ObsVectorVarianceSquareRoot,2); + +% ObsVectors = zeros(size(ObsVectorMean,1),number_of_particles); +% % % ObsVectors = bsxfun(@plus,ObsVectorVarianceSquareRoot*transpose(norminv(qmc_scrambled(obs_variance_rank,number_of_particles,1))),ObsVectorMean); + +success = simulated_sample.success; +number_of_particles = options_.occbin.filter.particle.number_of_particles; +number_of_successful_particles = length(find(success)); +number_of_simulated_regimes = length(simulated_regimes); +number_of_updated_regimes = length(updated_regimes); + +epsilon = updated_sample.epsilon; +y100 = simulated_sample.y10; +y10 = y100(:,success); +% kernel density +z10 = ZZ*y10; +nobs = size(ZZ,1); + +if nobs>2 + bw = nan(nobs,1); + for jd=1:nobs + ss=std(transpose(z10(jd,:))); + bw(jd) = ss*(4/(nobs+2)/number_of_successful_particles)^(1/(nobs+4)); + end + fobs_kernel = mvksdensity(z10',Y(di,2)','bandwidth',bw,'Function','pdf'); +else + fobs_kernel = ksdensity(z10',Y(di,2)'); +end + +% full non-parametric density +ydist = sqrt(sum((Y(di,2)-z10).^2,1)); +[s, is] = sort(ydist); +%bin population +nbins = ceil(sqrt(size(z10,2))); + +binpop = ceil(number_of_successful_particles/nbins/2)*2; + +% volume of n-dimensional sphere +binvolume = pi^(nobs/2)/gamma(nobs/2+1)*(0.5*(s(binpop)+s(binpop+1)))^nobs; + + +% samplevolume = [min(z10,[],2) max(z10,[],2)]; +% samplevolume = prod(diff(samplevolume')*1.02); +fobs = binpop/number_of_successful_particles/binvolume; + +isux = find(success); +isux = isux(is); % sorted successful deviations from data +% check for regime of Y +is_data_regime = false(number_of_simulated_regimes,1); +for kr=1:number_of_simulated_regimes + is_data_regime(kr) = any(ismember(simulated_regimes(kr).index,isux(1:binpop))); +end + + +% PPF density +nrow=floor(sqrt(nobs)); +ncol=ceil(nobs/nrow); + +norm_datalik = sum(exp(-likxx(success))); +is_resampled_regime = false(1,number_of_updated_regimes); +pregime_updated0 = zeros(1,number_of_updated_regimes); +pregime_updated = zeros(1,number_of_updated_regimes); +datalik_regime = zeros(1,number_of_updated_regimes); +for kr=1:number_of_updated_regimes + % regime specific likelihood weighted by regime probability GIVEN the + % observable + pregime_updated0(kr) = sum(exp(-likxx(updated_regimes(kr).index)))/norm_datalik; + is_resampled_regime(kr) = any(ismember(indx,updated_regimes(kr).index)); + if is_resampled_regime(kr) + pregime_updated(kr) = sum(ismember(indx,updated_regimes(kr).index))/number_of_particles; + datalik_regime(kr) = sum(exp(-likxx(updated_regimes(kr).index)/2))/sum(success); + end +end + + +if ~options_.occbin.filter.particle.nograph +figure(1) +clf(1,'reset') +% subplot(nrow,ncol,kobs) +tlo = tiledlayout(nrow,ncol); +title(tlo,[' (t=' int2str(t) ')']); +tlo.TileSpacing = 'tight'; + +% set color order +all_simulated_regimes = {simulated_regimes.regime}; +all_updated_regimes = {updated_regimes.regime}; +all_regimes = [all_simulated_regimes all_updated_regimes(~ismember(all_updated_regimes, all_simulated_regimes))]; +number_of_regimes = length(all_regimes); + +ireg1 = find([simulated_regimes.is_pkf_regime]); + +[~, ~, tmp_str]=occbin.backward_map_regime(regimes0(1)); +if M_.occbin.constraint_nbr==1 + tmp_str = tmp_str(2,:); +else + tmp_str = [tmp_str(2,:) tmp_str(4,:)]; +end +ireg0 = find(ismember(all_simulated_regimes,tmp_str)); + +func = @(x) colorspace('RGB->Lab',x); +my_colororder= distinguishable_colors(number_of_regimes+4,'w',func); +my_colororder(ismember(my_colororder,[0 0 1],'row'),:)=[]; % remove BLUE +previous_and_current_period_regimes_are_equal = false; +if isempty(ireg0) + % t-1 regime is not present in predictive density (should never + % happen, but ...) + my_colororder(ismember(my_colororder,[1 0 0],'row'),:)=[]; % remove RED +else + if ireg0~=ireg1 + icolor0 = find(ismember(my_colororder,[1 0 0],'row')); % get the RED + my_colororder = my_colororder([1:icolor0-1,icolor0+1:number_of_simulated_regimes,icolor0,number_of_simulated_regimes+1:number_of_regimes],:); + else + ireg0 = []; + previous_and_current_period_regimes_are_equal = true; + my_colororder = my_colororder(1:number_of_regimes,:); + end +end + +my_simulated_regime_order = (1:number_of_simulated_regimes); +imiddle = not(ismember(my_simulated_regime_order,[ireg0 ireg1])); +my_simulated_regime_order = [ireg1 my_simulated_regime_order(imiddle) ireg0]; + +all_regimes_order = [all_simulated_regimes(my_simulated_regime_order) all_updated_regimes(~ismember(all_updated_regimes, all_simulated_regimes))]; +updated_colororder = zeros(number_of_updated_regimes,3); +for kr=1:number_of_updated_regimes + updated_colororder(kr,:) = my_colororder(ismember(all_regimes_order,all_updated_regimes(kr)),:); +end + +% sort updated regimes like simluated for figure(1) +for kr = 1:number_of_regimes + ismember(all_regimes_order,all_updated_regimes); +end +end + +% data density! +for kobs = 1:size(ZZ,1) + obs_simul = transpose(ZZ(kobs,:)*y100); %simulated varobs + sobs_simul = sort(obs_simul(success)); % sorted values of simulated varobs + mc=nan(number_of_simulated_regimes,1); + Mc=mc; + for kr=1:number_of_simulated_regimes + mc(kr,1) = min(transpose(ZZ(kobs,:)*y100(:,simulated_regimes(kr).index))); + Mc(kr,1) = max(transpose(ZZ(kobs,:)*y100(:,simulated_regimes(kr).index))); + end + is_regime_uncertain = number_of_simulated_regimes>1; + if ~options_.occbin.filter.particle.nograph + ax = nexttile(kobs); + hold off, + end + zrange = [min(transpose(ZZ(kobs,:)*y10)) max(transpose(ZZ(kobs,:)*y10))]; + zlin = linspace(zrange(1),zrange(2),50); + + fobsk = ksdensity(transpose(ZZ(kobs,:)*y10),[zlin Y(kobs,2)]'); + fobsklin = fobsk(1:end-1); + fobsk= fobsk(end); + density.(options_.varobs{di(kobs)}).kernel = [zlin' fobsklin]; + if number_of_simulated_regimes>1 + % expected regime is uncertain + % classify support + is_overlap = false; + [mcs, imcs] = sort(mc); + Mcs = Mc(imcs); % sort max with the same ordering as min + is_sorted_pkf_regime = [simulated_regimes(imcs).is_pkf_regime]; + edge = []; + subedge=[]; + num_subedge = 0; + end + m=min(transpose(ZZ(kobs,:)*y10)); + M=max(transpose(ZZ(kobs,:)*y10)); + bwidth = (M-m)/nbins; + m = min(m,Y(di(kobs),2)-bwidth/2); + M = max(M,Y(di(kobs),2)+bwidth/2); + bwidth = (M-m)/nbins; + EDGES = Y(di(kobs),2)-bwidth/2+bwidth*[-2*nbins:2*nbins]; % we center data point in one bin + EDGES = EDGES(find(EDGES<m,1,'last'):find(EDGES>M,1)); + if options_.occbin.filter.particle.nograph + [pdf_sim,EDGES] = histcounts(transpose(ZZ(kobs,:)*y10), EDGES,'Normalization', 'pdf'); + % [pdf_sim,EDGES] = histcounts(transpose(ZZ(kobs,:)*y10), nbins+1, 'Normalization', 'pdf'); + else + % hh = histogram(transpose(ZZ(kobs,:)*y10), nbins+1, 'Normalization','pdf','EdgeColor','b'); + % EDGES = hh.BinEdges; + hh = histogram(transpose(ZZ(kobs,:)*y10),EDGES,'Normalization','pdf','EdgeColor','b'); + pdf_sim = hh.Values; + hold all, + end + xedges = 0.5*(EDGES(1:end-1)+EDGES(2:end)); + density.(options_.varobs{di(kobs)}).non_parametric = [xedges' pdf_sim']; + % PPF density! regime specific likelihood weighted by probability + % WITHIN the BIN + pregime = zeros(length(xedges),number_of_simulated_regimes); + pdf_ppf = nan(1,length(xedges)); + pdf_ppf_regime = zeros(length(xedges),number_of_simulated_regimes); + fobsk1 = ksdensity(transpose(ZZ(kobs,:)*y10),[xedges Y(kobs,2)]'); + for kx=1:length(xedges) + ikx = obs_simul(success)>EDGES(kx) & obs_simul(success)<EDGES(kx+1); + nkx = length(find(ikx)); + pdf_ppf(kx) = 0; + for kr=1:number_of_simulated_regimes + ikr = obs_simul(simulated_regimes(kr).index)>EDGES(kx) & obs_simul(simulated_regimes(kr).index)<EDGES(kx+1); + pregime(kx,kr) = length(find(ikr))/nkx; + if pregime(kx,kr)>0 + pdf_ppf_regime(kx,kr) = pregime(kx,kr) ... + *1./sqrt(2*pi*simulated_regimes(kr).obsvar(kobs,kobs)) ... + .*exp(-(xedges(kx)-simulated_regimes(kr).obsmean(kobs) ).^2./2./simulated_regimes(kr).obsvar(kobs,kobs)); + pdf_ppf(kx) = pdf_ppf(kx) + pdf_ppf_regime(kx,kr); + end + end + if length(find(pregime(kx,:)))>1 + pdf_ppf_regime(kx,:) = pregime(kx,:).*fobsk1(kx); + pdf_ppf(kx) = fobsk1(kx); + end + end + density.(options_.varobs{di(kobs)}).ppf = [xedges' pdf_ppf']; + density.(options_.varobs{di(kobs)}).ppf_regime = [xedges' pdf_ppf_regime]; + % while min(EDGES)>min(ObsVectors) + % end + % marginal density of observable + likk = 1/sqrt(2*pi*PZ(kobs,kobs))*exp(-(Y(di(kobs),2)-ObsVectorMean(kobs) ).^2./2./PZ(kobs,kobs)); + density_data.(options_.varobs{di(kobs)}).kernel = [Y(di(kobs),2) fobsk]; + density_data.(options_.varobs{di(kobs)}).non_parametric = [Y(di(kobs),2) pdf_sim(find(EDGES(1:end-1)<Y(di(kobs),2),1,'last'))]; + density_data.(options_.varobs{di(kobs)}).pkf = [Y(di(kobs),2) likk]; + pdf_pkf = 1./sqrt(2*pi*PZ(kobs,kobs)).*exp(-(xedges-ObsVectorMean(kobs) ).^2./2./PZ(kobs,kobs)); + density.(options_.varobs{di(kobs)}).pkf = [xedges' pdf_pkf']; + % density.(options_.varobs{di(kobs)}) = [xedges' pdf_pkf0']; + % [Nobs,EDGESobs] = histcounts(ObsVectors', EDGES,'Normalization', 'pdf'); + % [Nobs0,EDGESobs0] = histcounts(ObsVectors0', EDGES,'Normalization', 'pdf'); + + if ~options_.occbin.filter.particle.nograph + pdf_pkf0 = 1/sqrt(2*pi*PZ0(kobs,kobs))*exp(-(xedges-ObsVectorMean0(kobs) ).^2./2./PZ0(kobs,kobs)); + % hold all, plot(zlin',fobsklin,'b','linewidth',2) % kernel approx + hold all, hd(1) = plot(xedges',pdf_ppf,'b','linewidth',2); + % hh=histogram(ObsVectors',EDGES,'Normalization','pdf','DisplayStyle','stairs','linewidth',2,'EdgeColor','g'); + % Nobs = hh.Values; + hold all, hd(2) = plot(xedges',pdf_pkf,'g','linewidth',2); + + % hh=histogram(ObsVectors0',EDGES,'Normalization','pdf','DisplayStyle','stairs','linewidth',2,'linestyle','--','EdgeColor','r'); + % Nobs0 = hh.Values; + hold all, hd(3) = plot(xedges',pdf_pkf0,'r--.','linewidth',2); + if previous_and_current_period_regimes_are_equal + hd(3).Color = hd(2).Color; + end + + if length(di)>1 + [m, im] = min(abs(xedges-Y(di(kobs),2))); + hold all, hd(4) = fill(Y(di(kobs),2)+bwidth*[-0.5 -0.5 0.5 0.5],[0 pdf_ppf(im) pdf_ppf(im) 0],'b','edgecolor','b','linewidth',2,'facealpha',0.3); + else + % hold all, hd(4) = plot([Y(di(kobs),2) Y(di(kobs),2)],sort([0 datalik]),'b--*','linewidth',2); + hold all, hd(4) = fill(Y(di(kobs),2)+bwidth*[-0.5 -0.5 0.5 0.5],[0 datalik datalik 0],'b','edgecolor','b','linewidth',2,'facealpha',0.3); + % hold all, hd(4) = plot([Y(di(kobs),2) Y(di(kobs),2) Y(di(kobs),2)],sort([0 likk datalik]),'b--*','linewidth',2); + end + cpdf_ppf_regime = cumsum(pdf_ppf_regime(:,my_simulated_regime_order),2); + cpdf_ppf_regime(pdf_ppf_regime(:,my_simulated_regime_order)==0)=NaN; + hs = stem(xedges, cpdf_ppf_regime(:,end:-1:1),':.','linewidth',1.5,'markersize',15); + tmp_colororder = my_colororder(number_of_simulated_regimes:-1:1,:); + for kr = 1:number_of_simulated_regimes + hs(kr).Color = tmp_colororder(kr,:); + end + + if kobs ==size(ZZ,1) + mytxt = {'simulated distribution','weighted distribution density', ... + 'approximated distribution','approximated distribution | t-1', ... + 'data point ppf density'}; + mytxt = [mytxt all_simulated_regimes(my_simulated_regime_order(end:-1:1))]; + % legend('simulated distribution','simulated distribution density','approximated distribution','approximated distribution | t-1','data point','Location','southoutside') + lg = legend(ax,mytxt,'Orientation','Horizontal','NumColumns',2); + lg.Layout.Tile = 'South'; % <-- Legend placement with tiled layout + end + title(options_.varobs{di(kobs)}) + + end + + +end + +if ~options_.occbin.filter.particle.nograph + figure(5) + clf(5,'reset') + ncomb = nchoosek(nobs,2); + nrow2=floor(sqrt(ncomb)); + ncol2=ceil(ncomb/nrow2); + + tlo2=tiledlayout(nrow2,ncol2); + title(tlo2,['predictive density (t=' int2str(t) ')']); + tlo2.TileSpacing = 'tight'; + + for ko=1:length(di)-1 + for koo=ko+1:length(di) + axo = nexttile; + for kp=1:number_of_simulated_regimes + plot(z10(ko,simulated_regimes(my_simulated_regime_order(kp)).index),z10(koo,simulated_regimes(my_simulated_regime_order(kp)).index),'.','MarkerEdgeColor',my_colororder(kp,:)) + hold all, + end + plot(Y(di(ko),2),Y(di(koo),2),'o','MarkerEdgeColor','k') + xlabel(options_.varobs{di(ko)}) + ylabel(options_.varobs{di(koo)}) + end + end + mytxt = [all_simulated_regimes(my_simulated_regime_order) 'data point']; + lgo = legend(axo,mytxt,'Orientation','Horizontal','NumColumns',2); + lgo.Layout.Tile = 'South'; % <-- Legend placement with tiled layout + + figure(4), + clf(4,'reset'), + pp = pie(datalik_regime(is_resampled_regime)/sum(datalik_regime(is_resampled_regime)), all_updated_regimes(is_resampled_regime)); + ip=0; + for kr=1:number_of_updated_regimes + if is_resampled_regime(kr) + ip=ip+1; + pp(ip).FaceColor=updated_colororder(kr,:); + ip=ip+1; + end + end + title(['data density t=' int2str(t) ', regimes contribution']) + + % iss = [1:size(Py(:,:,1),1)]; + % iss = find(diag(cov(StateVectorsPPF'))>1.e-12); + iss = find(diag(cov(y10'))>1.e-12); + + % iss = find(sqrt(diag(Py(:,:,1)))>1.e-8); + nstate = length(iss); + nrow=floor(sqrt(nstate)); + ncol=ceil(nstate/nrow); + % func = @(x) colorspace('RGB->Lab',x); + % my_colororder= distinguishable_colors(number_of_updated_regimes,'w',func); + + figure(2) + clf(2,'reset') + set(2,'name',['t = ' int2str(t) ', updated states t|t']) + if nstate + tlo = tiledlayout(length(iss),length(iss)); + title(tlo,['t = ' int2str(t) ', updated states t|t']); + tlo.TileSpacing = 'none'; + end + for k=1:length(iss) + % h(k) = subplot(length(iss),length(iss),k+(k-1)*length(iss)); + h(k) = nexttile(k+(k-1)*length(iss)); + hold off, + % histogram(y11(k,:)','Normalization','pdf' ) + % histogram(yhat(k,:)','Normalization','pdf' ) + histogram(StateVectorsPKF(iss(k),:)','Normalization','pdf' ,'BinLimits',[min(StateVectorsPKF(iss(k),:)')-0.1*abs(min(StateVectorsPKF(iss(k),:)')),max(StateVectorsPKF(iss(k),:)')+0.1*abs(max(StateVectorsPKF(iss(k),:)'))]) + hold all, hf0=histogram(StateVectorsPPF(iss(k),:)','Normalization','pdf' ,'BinLimits',[min(StateVectorsPPF(iss(k),:)')-0.1*abs(min(StateVectorsPPF(iss(k),:)')),max(StateVectorsPPF(iss(k),:)')+0.1*abs(max(StateVectorsPPF(iss(k),:)'))]); + % title(['updated ' M_.endo_names{dr.order_var(dr.restrict_var_list(iss(k)))} ' t|t']) + title(M_.endo_names{dr.order_var(dr.restrict_var_list(iss(k)))}) + % myax = gca; + % myax.TitleHorizontalAlignment = 'left'; + % ylabel(M_.endo_names{dr.order_var(dr.restrict_var_list(iss(k)))}) + if nstate>3 + set(gca,'Yticklabel',[]) + set(gca,'Xticklabel',[]) + end + for kj=k+1:length(iss) + for iz=1:2 + if iz==1 + ax = nexttile(k+(kj-1)*length(iss)); + plot(y10(iss(kj),:),y10(iss(k),:),'.','MarkerEdgeColor',[0.5 0.5 0.5]), + hold all, + else + ax = nexttile(kj+(k-1)*length(iss)); + end + plot(StateVectorsPKF(iss(kj),:),StateVectorsPKF(iss(k),:),'.','MarkerEdgeColor',[0.7 0.7 0.7]), + hold all, + % plot(StateVectorsPPF(iss(kj),(ismember(indx,updated_regimes(1).index))),StateVectorsPPF(iss(k),(ismember(indx,updated_regimes(1).index))),'.','MarkerEdgeColor',my_colororder(1,:)) + for kr=1:number_of_updated_regimes + if is_resampled_regime(kr) + plot(StateVectorsPPF(iss(kj),(ismember(indx,updated_regimes(kr).index))),StateVectorsPPF(iss(k),(ismember(indx,updated_regimes(kr).index))),'.','MarkerEdgeColor',updated_colororder(kr,:)) + end + end + plot(alphahaty(iss(kj),2),alphahaty(iss(k),2),'ok') + if abs(diff(ax.XLim))<0.02*abs(alphahaty(iss(kj),2)) + ax.XLim = [alphahaty(iss(kj),2)-abs(alphahaty(iss(kj),2))*0.01 ... + alphahaty(iss(kj),2)+abs(alphahaty(iss(kj),2))*0.01]; + end + if abs(diff(ax.YLim))<0.02*abs(alphahaty(iss(k),2)) + ax.YLim = [alphahaty(iss(k),2)-abs(alphahaty(iss(k),2))*0.01 ... + alphahaty(iss(k),2)+abs(alphahaty(iss(k),2))*0.01]; + end + if nstate>3 + set(gca,'Yticklabel',[]) + set(gca,'Xticklabel',[]) + end + end + end + end + if M_.occbin.constraint_nbr ==1 + my_txt = mat2str(regimesy(1).regimestart); + else + my_txt = [mat2str(regimesy(1).regimestart1) '-' mat2str(regimesy(1).regimestart2)]; + end + + if nstate>1 + lg = legend(ax,[{[my_txt ' PKF']} all_updated_regimes(is_resampled_regime) {'PKF'}],'location','westoutside'); + lg.Position(1) = 0; + lg.Position(2) = 0; + % % % dim = [.1 .45 0.33 .03]; + % % % annotation('textbox',dim,'String',['. ' my_txt ' PKF'],'FitBoxToText','off','Color',[0.7 0.7 0.7],'horizontalalignment','left','verticalalignment','middle','margin',1,'EdgeColor',[0.95 0.95 0.95]); + % % % for kr = 1:number_of_updated_regimes + % % % dim = [.1 .45-kr*0.05 0.33 .03]; + % % % annotation('textbox',dim,'String',['. ' all_updated_regimes{kr}],'FitBoxToText','off','Color',my_colororder(kr,:),'horizontalalignment','left','verticalalignment','middle','margin',1,'EdgeColor',[0.95 0.95 0.95]); + % % % end + % % % dim = [.1 .45-0.02-(number_of_updated_regimes+1)*0.05 0.33 (number_of_updated_regimes+2)*0.05+0.02]; + % % % annotation('textbox',dim,'String','o PKF','FitBoxToText','off','Color','k','horizontalalignment','left','verticalalignment','bottom','margin',1); + end + % % % + % % % dim = [.05 .02 .15 .03]; + % % % annotation('textbox',dim,'String',['t = ' int2str(t)],'FitBoxToText','on','Color','k','horizontalalignment','center','verticalalignment','middle','margin',1); + % dim = [.25 .02 .3 .03]; + % annotation('textbox',dim,'String',['regime ' my_txt],'FitBoxToText','on','Color','k','horizontalalignment','center','verticalalignment','middle','margin',1); + % dim = [.55 .02 .15 .03]; + % annotation('textbox',dim,'String','PKF','FitBoxToText','on','Color','b','horizontalalignment','center','verticalalignment','middle','margin',1); + % dim = [.75 .02 .15 .03]; + % annotation('textbox',dim,'String','PPF','FitBoxToText','on','Color','r','horizontalalignment','center','verticalalignment','middle','margin',1); + figure(3) + clf(3,'reset') + iss = find(sqrt(diag(QQQ(:,:,2)))); + nshock = length(iss); + if nshock + set(3,'name',['t = ' int2str(t) ', shocks t|t']) + tlo = tiledlayout(nshock,nshock); + title(tlo,['t = ' int2str(t) ', shocks t|t']); + tlo.TileSpacing = 'none'; + end + % nrow=floor(sqrt(nshock)); + % ncol=ceil(nshock/nrow); + for k=1:nshock + % subplot(nrow,ncol,k) + % subplot(nshock,nshock,k+(k-1)*nshock); + nexttile(k+(k-1)*nshock); + hf1=histogram(epsilon(iss(k),indx),'BinLimits',[min(epsilon(iss(k),indx))-0.1*abs(min(epsilon(iss(k),indx))),max(epsilon(iss(k),indx))+0.1*abs(max(epsilon(iss(k),indx)))] ); + hf1.FaceColor=[0.8500, 0.3250, 0.0980] ; + % hf1=histogram(epsi2 ); + hold all, + hf = fill(etahaty(iss(k))+mean(diff(hf1.BinEdges))*[-0.5 0.5 0.5 -0.5 ],max(hf1.Values)*[0 0 1 1] ,'r'); + set(hf,'FaceAlpha',0.6) + hf.FaceColor=[0, 0.4470, 0.7410]; + % title([M_.exo_names{iss(k)}]) + title([M_.exo_names{iss(k)}]) + % myax = gca; + % myax.TitleHorizontalAlignment = 'left'; + % ylabel([M_.exo_names{iss(k)}]) + if nshock>3 + set(gca,'Yticklabel',[]) + set(gca,'Xticklabel',[]) + end + for kj=k+1:nshock + for iz=1:2 + if iz==1 + ax = nexttile(k+(kj-1)*nshock); + plot(ShockVectors(kj,:),ShockVectors(k,:),'.','color',[0.5 0.5 0.5]) + hold all + else + ax = nexttile(kj+(k-1)*nshock); + end + % plot(epsilon(iss(kj),indx),epsilon(iss(k),indx),'.'), + % plot(epsilon(iss(kj),indx(ismember(indx,updated_regimes(1).index))),epsilon(iss(k),indx(ismember(indx,updated_regimes(1).index))),'.','MarkerEdgeColor',my_colororder(1,:)) + % hold all, + for kr=1:number_of_updated_regimes + if is_resampled_regime(kr) + plot(epsilon(iss(kj),indx(ismember(indx,updated_regimes(kr).index))),epsilon(iss(k),indx(ismember(indx,updated_regimes(kr).index))),'.','MarkerEdgeColor',updated_colororder(kr,:)) + hold all, + end + end + plot(etahaty(iss(kj)),etahaty(iss(k)),'ok') + if nshock>3 + set(gca,'Yticklabel',[]) + set(gca,'Xticklabel',[]) + end + end + end + end + if nshock>1 + lg = legend(ax,[all_updated_regimes(is_resampled_regime) {'PKF'}],'location','eastoutside'); + lg.Position(1) = 0; + lg.Position(2) = 0; + % % for kr = 1:number_of_updated_regimes + % % dim = [.1 .45-(kr-1)*0.05 0.33 .03]; + % % annotation('textbox',dim,'String',['. ' all_updated_regimes{kr}],'FitBoxToText','off','Color',my_colororder(kr,:),'horizontalalignment','left','verticalalignment','middle','margin',1,'EdgeColor',[0.95 0.95 0.95]); + % % end + % % dim = [.1 .45-0.02-number_of_updated_regimes*0.05 0.33 (number_of_updated_regimes+1)*0.05+0.02]; + % % annotation('textbox',dim,'String','o PKF','FitBoxToText','off','Color','k','horizontalalignment','left','verticalalignment','bottom','margin',1); + end +end +% end + + +end + diff --git a/matlab/+occbin/ppf_simulated_density.m b/matlab/+occbin/ppf_simulated_density.m new file mode 100644 index 0000000000000000000000000000000000000000..820c04e76c321e693cd615ea4732d662547221be --- /dev/null +++ b/matlab/+occbin/ppf_simulated_density.m @@ -0,0 +1,243 @@ +function [llik, simulated_regimes, simul_sample, hp_simul_regimes] = ppf_simulated_density(number_of_particles, number_of_shocks_per_particle,StateVector,ShockVectorsInfo, di, H, my_order_var, QQQ, Y, ZZ, base_regime, regimesy, M_, dr, endo_steady_state, exo_steady_state, exo_det_steady_state, options_, opts_simul) +% computes empirical conditionl density for piecewise linear particle +% filter + +% [llik, shock_simul] = ppf_conditional_density(number_of_shocks_per_particle,ShockVectorsPerParticle, Y, ZZ, base_regime, regimesy, M_, dr, endo_steady_state, exo_steady_state, exo_det_steady_state, options_) +% INPUTS +% a0: initial state (for conditioning) +% sample of shocks + +if number_of_shocks_per_particle==1 + hp_simul_regimes = []; + ShockVectors = ShockVectorsInfo; + simul_sample.success = false(number_of_particles,1); +end +llik.kernel = zeros(number_of_particles,1); +llik.non_parametric = zeros(number_of_particles,1); +llik.ppf = zeros(number_of_particles,1); +llik.enkf = zeros(number_of_particles,1); +y11 = zeros(size(StateVector)); +y11_EnKF = zeros(size(StateVector)); +opts_simul.SHOCKS=[]; +number_of_simulated_regimes = 0; +for k = 1:number_of_particles + if number_of_shocks_per_particle>1 + ShockVectorsPerParticle = bsxfun(@plus,ShockVectorsInfo.US*ShockVectorsInfo.VarianceSquareRoot*transpose(norminv(qmc_scrambled(ShockVectorsInfo.VarianceRank,number_of_shocks_per_particle,1))),zeros(ShockVectorsInfo.VarianceRank,1)); + end + if opts_simul.restrict_state_space + tmp=zeros(M_.endo_nbr,1); + tmp(dr.restrict_var_list,1)=StateVector(:,k); + opts_simul.endo_init = tmp(dr.inv_order_var,1); + else + opts_simul.endo_init = StateVector(dr.inv_order_var,k); + end + options_.noprint = true; + if number_of_shocks_per_particle==1 + a0 = StateVector(:,k); + opts_simul.SHOCKS(1,:) = ShockVectors(:,k)'; + options_.occbin.simul=opts_simul; + options_.occbin.simul.piecewise_only = false; + [~, out, ss] = occbin.solver(M_,options_,dr, endo_steady_state, exo_steady_state, exo_det_steady_state); + if out.error_flag==0 + simul_sample.success(k) = true; + % % store particle + [~, tmp, tmp_str]=occbin.backward_map_regime(out.regime_history(1)); + if M_.occbin.constraint_nbr==1 + tmp_str = tmp_str(2,:); + else + tmp_str = [tmp_str(2,:) tmp_str(4,:)]; + end + if number_of_simulated_regimes>0 + this_simulated_regime = find(ismember(all_simulated_regimes,{tmp_str})); + end + if number_of_simulated_regimes==0 || isempty(this_simulated_regime) + number_of_simulated_regimes = number_of_simulated_regimes+1; + this_simulated_regime = number_of_simulated_regimes; + simulated_regimes(number_of_simulated_regimes).index = k; + if isequal(regimesy(1),out.regime_history(1)) + simulated_regimes(number_of_simulated_regimes).is_pkf_regime = true; + else + simulated_regimes(number_of_simulated_regimes).is_pkf_regime = false; + end + if isequal(base_regime,out.regime_history(1)) + simulated_regimes(number_of_simulated_regimes).is_base_regime = true; + else + simulated_regimes(number_of_simulated_regimes).is_base_regime = false; + end + simulated_regimes(number_of_simulated_regimes).obsvar = ZZ*(ss.R(my_order_var,:)*QQQ(:,:,2)*ss.R(my_order_var,:)')*ZZ' + H(di,di); + simulated_regimes(number_of_simulated_regimes).obsmean = ZZ*(ss.T(my_order_var,my_order_var)*a0+ss.C(my_order_var)); + simulated_regimes(number_of_simulated_regimes).regime = tmp_str; + simulated_regimes(number_of_simulated_regimes).ss.C = ss.C(my_order_var); + simulated_regimes(number_of_simulated_regimes).ss.R = ss.R(my_order_var,:); + simulated_regimes(number_of_simulated_regimes).ss.T = ss.T(my_order_var,my_order_var); + all_simulated_regimes = {simulated_regimes.regime}; + uF = simulated_regimes(number_of_simulated_regimes).obsvar; + sig=sqrt(diag(uF)); + vv = Y(di,2) - simulated_regimes(number_of_simulated_regimes).obsmean; + if options_.rescale_prediction_error_covariance + log_duF = log(det(uF./(sig*sig')))+2*sum(log(sig)); + iuF = inv(uF./(sig*sig'))./(sig*sig'); + else + log_duF = log(det(uF)); + iuF = inv(uF); + end + simulated_regimes(number_of_simulated_regimes).lik = ... + log_duF + transpose(vv)*iuF*vv + length(di)*log(2*pi); + simulated_regimes(number_of_simulated_regimes).Kalman_gain = (ss.R(my_order_var,:)*QQQ(:,:,2)*ss.R(my_order_var,:)')*ZZ'*iuF; + + else + simulated_regimes(this_simulated_regime).index = [simulated_regimes(this_simulated_regime).index k]; + end + simul_sample.regime{k} = out.regime_history; + simul_sample.y10(:,k) = out.piecewise(1,my_order_var)-out.ys(my_order_var)'; + simul_sample.y10lin(:,k) = out.linear(1,my_order_var)-out.ys(my_order_var)'; + % this is in the spirit of EnKF, but with regime specific + % Kalman gain +% simul_sample.y11(:,k) = (out.piecewise(1,my_order_var)'-out.ys(my_order_var)) + simulated_regimes(this_simulated_regime).Kalman_gain*(Y(di,2) - ZZ*simul_sample.y10(:,k)); +% simul_sample.y11_regime(:,k) = simulated_regimes(this_simulated_regime).update; + omean = (simulated_regimes(this_simulated_regime).ss.T*a0+simulated_regimes(this_simulated_regime).ss.C); + simul_sample.y11(:,k) = omean + simulated_regimes(this_simulated_regime).Kalman_gain*(Y(di,2) - ZZ*omean); + + if M_.occbin.constraint_nbr==1 + simul_sample.exit(k,1) = max(out.regime_history.regimestart); + simul_sample.is_constrained(k,1) = logical(out.regime_history.regime(1)); + simul_sample.is_constrained_in_expectation(k,1) = any(out.regime_history.regime); + else + simul_sample.exit(k,1) = max(out.regime_history.regimestart1); + simul_sample.exit(k,2) = max(out.regime_history.regimestart2); + simul_sample.is_constrained(k,1) = logical(out.regime_history.regime1(1)); + simul_sample.is_constrained(k,2) = logical(out.regime_history.regime2(1)); + simul_sample.is_constrained_in_expectation(k,1) = any(out.regime_history.regime1); + simul_sample.is_constrained_in_expectation(k,2) = any(out.regime_history.regime2); + end + % the distribution of ZZ*y10 needs to be confronted with ZZ*a1y, ZZ*P1y*ZZ', + % i.e. the normal approximation using the regime consistent with the observable! + end + else + [llik.non_parametric(k), llik.ppf(k), llik.enkf(k), y11(:,k),simulated_regimes{k},simul_sample{k},hp_simul_regimes(k)] = occbin.ppf_conditional_density(number_of_shocks_per_particle,StateVector(:,k),ShockVectorsPerParticle, di, H, my_order_var, QQQ, Y, ZZ, base_regime, regimesy, M_, oo_, options_, opts_simul); +% hp_simul_regimes{k} = hp_simulated_regime.regime; + end +end + +if number_of_shocks_per_particle==1 + success = simul_sample.success; + number_of_successful_particles = sum(success); + + if number_of_successful_particles + + % z10 = ZZ*y10; + nobs = size(ZZ,1); + z10 = ZZ*simul_sample.y10(:,success); + + % ensemble KF + mu=mean(simul_sample.y10(:,success),2); + C=cov(simul_sample.y10(:,success)'); + uF = ZZ*C*ZZ' + H(di,di); + sig=sqrt(diag(uF)); + vv = Y(di,2) - ZZ*mu; + if options_.rescale_prediction_error_covariance + log_duF = log(det(uF./(sig*sig')))+2*sum(log(sig)); + iuF = inv(uF./(sig*sig'))./(sig*sig'); + else + log_duF = log(det(uF)); + iuF = inv(uF); + end + llik.enkf = log_duF + transpose(vv)*iuF*vv + length(di)*log(2*pi); + Kalman_gain = C*ZZ'*iuF; + y11_EnKF = simul_sample.y11*nan; + y11_EnKF(:,success) = simul_sample.y10(:,success)+Kalman_gain*(Y(di,2)-ZZ*simul_sample.y10(:,success)); + + % full non parametric density estimate + % dlik = sqrt(diag(transpose(Y(di,2)-z10)*iuF*(Y(di,2)-z10))); + dlik = log_duF+diag(transpose(Y(di,2)-z10)*iuF*(Y(di,2)-z10))+ length(di)*log(2*pi); + [sd, iss] = sort(dlik); + issux = find(success); + issux = issux(iss); + + ydist = sqrt(sum((Y(di,2)-z10).^2,1)); + [s, is] = sort(ydist); + isux = find(success); + isux = isux(is); + %bin population + nbins = ceil(sqrt(size(z10,2))); + + binpop = ceil(number_of_successful_particles/nbins/2)*2; % I center the bin on the oberved point + + % volume of n-dimensional sphere + binvolume = pi^(nobs/2)/gamma(nobs/2+1)*(0.5*(s(binpop)+s(binpop+1)))^nobs; + llik.non_parametric = -2*log(binpop/number_of_successful_particles/binvolume); + + % kernel density + if nobs>2 + bw = nan(nobs,1); + for jd=1:nobs + ss=std(transpose(z10(jd,:))); + bw(jd) = ss*(4/(nobs+2)/number_of_successful_particles)^(1/(nobs+4)); + end + fobs_kernel = mvksdensity(z10',Y(di,2)','bandwidth',bw,'Function','pdf'); + else + fobs_kernel = ksdensity(z10',Y(di,2)'); + end + llik.kernel = -2*log(fobs_kernel); + + % states t|t + y11 = simul_sample.y10(:,isux(randi(binpop,1))); + % use updated KF + y11 = simul_sample.y11(:,isux(randi(binpop,1))); + + % check for regime of Y + is_data_regime = false(number_of_simulated_regimes,1); + iss_data_regime = false(number_of_simulated_regimes,1); + proba = ones(number_of_simulated_regimes,1); + probaX = zeros(number_of_simulated_regimes,1); + probaY = zeros(number_of_simulated_regimes,1); + for kr=1:number_of_simulated_regimes + if options_.occbin.filter.particle.tobit && ~simulated_regimes(kr).is_base_regime + % base regime is not Tobit + proba(kr) = length(simulated_regimes(kr).index)/number_of_successful_particles; + end + % here we check if the data may be spanned for each regime + % (given we have a bunch of TRUNCATED normals, Y may be out of + % truncation ...) + is_data_regime(kr) = any(ismember(simulated_regimes(kr).index,isux(1:binpop))); + iss_data_regime(kr) = any(ismember(simulated_regimes(kr).index,issux(1:binpop))); + if is_data_regime(kr) + probaY(kr) = length(find(ismember(simulated_regimes(kr).index,isux(1:binpop))))/binpop; + probaY(kr) = length(find(ismember(simulated_regimes(kr).index,isux(1:binpop))))^2/binpop/length(simulated_regimes(kr).index); + end + if iss_data_regime(kr) + thisbin = find(ismember(issux(1:binpop),simulated_regimes(kr).index)); + % probaX(kr) = length(thisbin)/length(simulated_regimes(kr).index)/binpop; + % probaX(kr) = mean(exp(-sd(thisbin)/2))*length(thisbin)^2/length(simulated_regimes(kr).index)/binpop; + probaX(kr) = mean(exp(-sd(thisbin)/2))*length(thisbin)/length(simulated_regimes(kr).index); + % thisbin = find(ismember(issux(1:binpop),simulated_regimes(kr).index),1); + % probaX(kr) = exp(-sd(thisbin)/2); + end + end + probaX = probaX./sum(probaX); + probaY = probaY./sum(probaY); + if ~any(simul_sample.is_constrained_in_expectation(success)) + % Y is unconstrained + end + + % use most probable regime to update states + [m,im]=max((exp(-[simulated_regimes.lik]./2)'.*(proba.*probaY))); + maxbin = find(ismember(isux(1:binpop),simulated_regimes(im).index)); + simul_sample.y11(:,success) = StateVector(:,success) + simulated_regimes(im).Kalman_gain*(Y(di,2) - ZZ*StateVector(:,success)); + y11 = simul_sample.y11; % use particle updated state ... + + ay = mean(simul_sample.y11(:,issux(maxbin)),2); % updated state ... + hp_simul_regimes = simulated_regimes(im); + % likelihood is the sum of gaussians, where constrained regimes are + % weighted by their probabilities + llik.ppf = -2*log(exp(-[simulated_regimes.lik]./2)*(proba.*probaY)); + % samplevolume = [min(z10,[],2) max(z10,[],2)]; + % samplevolume = prod(diff(samplevolume')*1.02); + + + end + + +end + +simul_sample.y11_EnKF = y11_EnKF; \ No newline at end of file diff --git a/matlab/+occbin/ppf_state_importance_sampling.m b/matlab/+occbin/ppf_state_importance_sampling.m new file mode 100644 index 0000000000000000000000000000000000000000..ee738104176169cba8fb6677f875a28349fa0bb4 --- /dev/null +++ b/matlab/+occbin/ppf_state_importance_sampling.m @@ -0,0 +1,385 @@ +function [StateVector, StateVectorsPKF, StateVectorsPPF, liky, updated_regimes, updated_sample, updated_mode, use_pkf_distribution, error_flag] = ppf_state_importance_sampling(StateVector0, a, a1, P, P1, Py, alphahaty, V, t, data_index,Z,v,Y,H,QQQ,T0,R0,TT,RR,CC,info,regimes0,base_regime,regimesy,M_, ... + dr, endo_steady_state,exo_steady_state,exo_det_steady_state,options_,occbin_options) + +if t==203 %173 %201 %202 %187 %202 %203 %207 %54 %41 %56 %58 %75 + do_nothing = true; +end + +% prior distribution in t-1 +% [UP,XP,WP] = svd(0.5*(P+P')); +[UP,XP,WP] = svd(0.5*(StateVector0.Variance+StateVector0.Variance')); +isp = find(diag(XP)>options_.kalman_tol); +ns = length(isp); +if ns==0 + disp(['ppf: time ' int2str(t) ' no state uncertainty!']); +end +UP = UP(:,isp); +WP = WP(:,isp); +S = XP(isp,isp); %UP(:,isp)'*(0.5*(P+P'))*WP(:,isp); +log_dS = log(det(S)); +iS = inv(S); + +% yhat = StateVector0.Draws; + +error_flag = 0; + +% draws states in t-1 from PKF +number_of_particles = options_.occbin.filter.particle.number_of_particles; +[U,X] = svd(0.5*(V(:,:,1)+V(:,:,1)')); +% P= U*X*U'; % symmetric matrix! +is = find(diag(X)>options_.kalman_tol); +StateVectorVarianceSquareRoot = chol(X(is,is))';%reduced_rank_cholesky(ReducedForm.StateVectorVariance)'; + +% Get the rank of StateVectorVarianceSquareRoot +state_variance_rank = size(StateVectorVarianceSquareRoot,2); +U = U(:,is); +log_dS0 = log(det(X(is,is))); +iS0 = 1./diag(X(is,is)); + +% % Factorize the covariance matrix of the structural innovations +% Q_lower_triangular_cholesky = chol(Q)'; + +% Initialization of the weights across particles. +qmc_base = norminv(qmc_scrambled(state_variance_rank,number_of_particles,1)); +if not(isempty(is)) + yhat = bsxfun(@plus,U(:,is)*StateVectorVarianceSquareRoot*transpose(qmc_base),alphahaty(:,1)); +else + yhat = repmat(alphahaty(:,1),[1 number_of_particles]); +end +vv0 = U'*bsxfun(@plus,yhat,-alphahaty(:,1)); +lpost = log_dS0 + iS0'*vv0.^2 + ns*log(2*pi); + + +% store input values +P0=P; +try + chol(P0); +catch + P0 = 0.5*(P0+P0'); +end +a0=a; +P=P*0; +P1(:,:,1)=P; +QQ = RR(:,:,2)*QQQ(:,:,2)*transpose(RR(:,:,2)); +di=data_index{2}; +ZZ = Z(di,:); + +P1(:,:,2) = QQ; +options_.occbin.filter.state_covariance=false; +my_data_index = data_index; +my_data_index{1} = []; +my_data = Y; +my_data(:,1) = nan; + +number_of_particles = size(yhat,2); +niter = 0; +ESS = 0; +smc_scale = 1; +lprior = nan(size(lpost)); +while ESS<0.5*number_of_particles || smc_scale<1 + niter = niter+1; + number_of_successful_particles = 0; + updated_regimes = []; + updated_sample = []; + number_of_updated_regimes = 0; + updated_sample.success = false(number_of_particles,1); + likxmode = inf; + for k=1:number_of_particles + % parfor k=1:number_of_particles + a(:,1) = yhat(:,k); + a1(:,1) = yhat(:,k); + a1(:,2) = TT(:,:,2)*yhat(:,k); + + % sampling distribution of states in t-1 is the 'posterior' t-1|t from PKF, so we need to compute posterior kernel + % where the prior is the distribution of states t-1|t-1 + % vv = UP'*(a(:,1)-a0); + vv = UP'*(a(:,1)-StateVector0.Mean); + lprior(k) = log_dS + transpose(vv)*iS*vv + ns*log(2*pi); + % vv0 = U'*(a(:,1)-alphahaty(:,1)); + % lpost(k) = log_dS0 + transpose(vv0)*iS0*vv0 + ns*log(2*pi); + + % conditional likelihood + options_.occbin.filter.state_covariance=true; + use_the_engine = true; + if info==0 + % start with PKF regime to speed-up search! + guess_regime = regimesy(1:2); + options_.occbin.filter.guess_regime = true; + [ax, a1x, Px, P1x, vx, Tx, Rx, Cx, regimesx, infox, ~, likxc, etahatx,alphahatx, Vxc] = ... + occbin.kalman_update_algo_1(a,a1,P,P1,my_data_index,Z,v,my_data,H,QQQ,T0,R0,TT,RR,CC,guess_regime,M_, ... + dr, endo_steady_state,exo_steady_state,exo_det_steady_state,options_,occbin_options); + options_.occbin.filter.guess_regime = false; + if infox == 0 + use_the_engine = false; + end + end + if use_the_engine + [ax, a1x, Px, P1x, vx, Tx, Rx, Cx, regimesx, infox, ~, likxc, etahatx,alphahatx, Vxc] = ... + occbin.kalman_update_engine(a,a1,P,P1,t,my_data_index,Z,v,my_data,H,QQQ,T0,R0,TT,RR,CC,regimes0,base_regime,my_data_index{2},M_, ... + dr, endo_steady_state,exo_steady_state,exo_det_steady_state,options_,occbin_options); + end + + if infox ==0 + + % handle conditional likelihood and updated regime + [~, tmp, tmp_str]=occbin.backward_map_regime(regimesx(1)); + if M_.occbin.constraint_nbr==1 + tmp_str = tmp_str(2,:); + else + tmp_str = [tmp_str(2,:) tmp_str(4,:)]; + end + if number_of_updated_regimes>0 + this_updated_regime = find(ismember(all_updated_regimes,{tmp_str})); + end + if number_of_updated_regimes==0 || isempty(this_updated_regime) + number_of_updated_regimes = number_of_updated_regimes+1; + this_updated_regime = number_of_updated_regimes; + updated_regimes(number_of_updated_regimes).index = k; + if isequal(regimesy(1),regimesx(1)) + updated_regimes(number_of_updated_regimes).is_pkf_regime = true; + else + updated_regimes(number_of_updated_regimes).is_pkf_regime = false; + end + updated_regimes(number_of_updated_regimes).regime = tmp_str; + all_updated_regimes = {updated_regimes.regime}; + if ~options_.occbin.filter.particle.likelihood_only + updated_regimes(number_of_updated_regimes).obsvar = ZZ*(Tx(:,:,1)*P0*Tx(:,:,1)'+Rx(:,:,1)*QQQ(:,:,2)*Rx(:,:,1)')*ZZ' + H(di,di); + updated_regimes(number_of_updated_regimes).obsmean = ZZ*(Tx(:,:,1)*a0+Cx(:,1)); + updated_regimes(number_of_updated_regimes).ss.C = Cx(:,1); + updated_regimes(number_of_updated_regimes).ss.R = Rx(:,:,1); + updated_regimes(number_of_updated_regimes).ss.T = Tx(:,:,1); + uF = updated_regimes(number_of_updated_regimes).obsvar; + sig=sqrt(diag(uF)); + vv = Y(di,2) - updated_regimes(number_of_updated_regimes).obsmean; + if options_.rescale_prediction_error_covariance + log_duF = log(det(uF./(sig*sig')))+2*sum(log(sig)); + iuF = inv(uF./(sig*sig'))./(sig*sig'); + else + log_duF = log(det(uF)); + iuF = inv(uF); + end + updated_regimes(number_of_updated_regimes).lik = ... + log_duF + transpose(vv)*iuF*vv + length(di)*log(2*pi); + end + else + updated_regimes(this_updated_regime).index = [updated_regimes(this_updated_regime).index k]; + end + + number_of_successful_particles = number_of_successful_particles + 1; + + likxx(k) = likxc; + + if likxc<likxmode + likxmode = likxc; + updated_mode.likxc = likxc; + updated_mode.a = ax; + updated_mode.a1 = a1x; + updated_mode.P = Px; + updated_mode.P1 = P1x; + updated_mode.v = vx; + updated_mode.T = Tx; + updated_mode.R = Rx; + updated_mode.C = Cx; + updated_mode.regimes = regimesx; + if ~options_.occbin.filter.particle.likelihood_only + updated_mode.lik = updated_regimes(this_updated_regime).lik; + end + updated_mode.etahat = etahatx; + updated_mode.alphahat = alphahatx; + updated_mode.V = Vxc; + end + + updated_sample.epsilon(:,k)=etahatx; + updated_sample.regimes{k} = regimesx; + updated_sample.success(k)=true; + updated_sample.y01(:,k) = alphahatx(:,1); %smoothed state 0|1! + updated_sample.y11(:,k) = ax(:,1); %updated state! + if M_.occbin.constraint_nbr==1 + if logical(regimesx(1).regime(1)) + do_nothing=true; + end + updated_sample.regime_exit(k,1) = max(regimesx(1).regimestart); + updated_sample.is_constrained(k,1) = logical(regimesx(1).regime(1)); + updated_sample.is_constrained_in_expectation(k,1) = any(regimesx(1).regime); + else + updated_sample.regime_exit(k,1) = max(regimesx(1).regimestart1); + updated_sample.regime_exit(k,2) = max(regimesx(1).regimestart2); + updated_sample.is_constrained(k,1) = logical(regimesx(1).regime1(1)); + updated_sample.is_constrained(k,2) = logical(regimesx(1).regime2(1)); + updated_sample.is_constrained_in_expectation(k,1) = any(regimesx(1).regime1); + updated_sample.is_constrained_in_expectation(k,2) = any(regimesx(1).regime1); + end + + end + + end + + if number_of_successful_particles==0 + error_flag = 340; + StateVector.Draws=nan(size(yhat)); + return + end + + + % lnw0 = -likxx(updated_sample.success)/2-lprior(updated_sample.success)/2+lpost(updated_sample.success)/2; % use posterior kernel to weight t-1 draws based on PKF! + ESS=0; + do_resample = false; + while ESS<0.5*number_of_particles + lnw0 = -likxx(updated_sample.success)/2-lprior(updated_sample.success)/2+lpost(updated_sample.success)/2; % use posterior kernel to weight t-1 draws based on PKF! + lnw0 = lnw0*smc_scale; + % we need to normalize weights, even if they are already proper likelihood values! + % to avoid that exponential gives ZEROS all over the place! + lnw0 = lnw0-max(lnw0); + weights = zeros(1,number_of_particles); + weights0 = ones(1,number_of_successful_particles)/number_of_successful_particles ; + wtilde0 = weights0.*exp(lnw0); + weights0 = wtilde0/sum(wtilde0); + weights(updated_sample.success) = weights0; + ESS = 1/sum(weights.^2); + + if ESS<0.5*number_of_particles + do_resample = true; + if niter==1 + break + else + smc_scale = smc_scale*0.98; + end + end + end + + if not(do_resample) && smc_scale<1 + while ESS>0.5*number_of_particles && smc_scale<1 + smc_scale = min(1,smc_scale*1.1); + lnw0 = -likxx(updated_sample.success)/2-lprior(updated_sample.success)/2+lpost(updated_sample.success)/2; % use posterior kernel to weight t-1 draws based on PKF! + lnw0 = lnw0*smc_scale; + % we need to normalize weights, even if they are already proper likelihood values! + % to avoid that exponential gives ZEROS all over the place! + lnw0 = lnw0-max(lnw0); + weights = zeros(1,number_of_particles); + weights0 = ones(1,number_of_successful_particles)/number_of_successful_particles ; + wtilde0 = weights0.*exp(lnw0); + weights0 = wtilde0/sum(wtilde0); + weights(updated_sample.success) = weights0; + ESS = 1/sum(weights.^2); + end + while ESS<0.5*number_of_particles + smc_scale = smc_scale*0.98; + lnw0 = -likxx(updated_sample.success)/2-lprior(updated_sample.success)/2+lpost(updated_sample.success)/2; % use posterior kernel to weight t-1 draws based on PKF! + lnw0 = lnw0*smc_scale; + % we need to normalize weights, even if they are already proper likelihood values! + % to avoid that exponential gives ZEROS all over the place! + lnw0 = lnw0-max(lnw0); + weights = zeros(1,number_of_particles); + weights0 = ones(1,number_of_successful_particles)/number_of_successful_particles ; + wtilde0 = weights0.*exp(lnw0); + weights0 = wtilde0/sum(wtilde0); + weights(updated_sample.success) = weights0; + ESS = 1/sum(weights.^2); + end + do_resample=true; + end + + ParticleOptions.resampling.method.kitagawa=true; + indx = resample(0,weights',ParticleOptions); + if all(weights==0) + do_nothing = true; + end + + StateVectorsPPF = updated_sample.y11(:,indx); + if do_resample + if niter==1 + % start from prior + StateVectorsPPFMean = StateVector0.Mean; + StateVectorsPPFVariance = StateVector0.Variance; + else + StateVectorsPPFVariance = cov(yhat(:,indx)'); + StateVectorsPPFMean = mean(yhat(:,indx),2); + end + % now do svd and sample from gaussian approx thereof ... + [U,X] = svd(0.5*(StateVectorsPPFVariance+StateVectorsPPFVariance')); + % P= U*X*U'; % symmetric matrix! + is = find(diag(X)>options_.kalman_tol); + if length(is)~=ns + is=isp; + end + StateVectorVarianceSquareRoot = chol(X(is,is))';%reduced_rank_cholesky(ReducedForm.StateVectorVariance)'; + + % Get the rank of StateVectorVarianceSquareRoot + state_variance_rank = size(StateVectorVarianceSquareRoot,2); + U = U(:,is); + log_dS0 = log(det(X(is,is))); + iS0 = 1./diag(X(is,is)); + + % % Factorize the covariance matrix of the structural innovations + % Q_lower_triangular_cholesky = chol(Q)'; + + % Initialization of the weights across particles. + if not(isempty(is)) + yhat = bsxfun(@plus,U(:,is)*StateVectorVarianceSquareRoot*transpose(qmc_base),StateVectorsPPFMean); + else + yhat = repmat(StateVectorsPPFMean,[1 number_of_particles]); + end + vv0 = U'*bsxfun(@plus,yhat,-StateVectorsPPFMean); + lpost = log_dS0 + iS0'*vv0.^2 + ns*log(2*pi); + end +end + +updated_sample.indx = indx; +updated_sample.ESS = ESS; + +% updated state mean and covariance +StateVector.Variance = Py(:,:,1); +[U,X] = svd(0.5*(Py(:,:,1)+Py(:,:,1)')); +% P= U*X*U'; +iss = find(diag(X)>1.e-12); + +% if any(iss) || ~options_.occbin.filter.particle.draw_states_from_empirical_density + +StateVector.Mean = alphahaty(:,2); +if any(iss) + StateVectorVarianceSquareRoot = chol(X(iss,iss))';%reduced_rank_cholesky(ReducedForm.StateVectorVariance)'; + % Get the rank of StateVectorVarianceSquareRoot + state_variance_rank = size(StateVectorVarianceSquareRoot,2); + StateVectorsPKF = bsxfun(@plus,U(:,iss)*StateVectorVarianceSquareRoot*transpose(norminv(qmc_scrambled(state_variance_rank,number_of_particles,1))),StateVector.Mean); +else + state_variance_rank = numel(StateVector.Mean ); + StateVectorsPKF = bsxfun(@plus,0*transpose(norminv(qmc_scrambled(state_variance_rank,number_of_particles,1))),StateVector.Mean); +end + +%% PPF density +% if number_of_updated_regimes==1 +% datalik = exp(-liky/2); +% % datalik = exp(-updated_regimes(1).lik/2); +% else +datalik = mean(exp(-likxx(updated_sample.success)/2-lprior(updated_sample.success)/2+lpost(updated_sample.success)/2)); +% end + +liky = -log(datalik)*2; +updated_sample.likxx = likxx; + +if options_.occbin.filter.particle.draw_states_from_empirical_density + if not(isequal(regimesy(1),base_regime) && any([updated_regimes.is_pkf_regime]) && (sum(ismember(indx,updated_regimes([updated_regimes.is_pkf_regime]).index))/number_of_particles)>=options_.occbin.filter.particle.use_pkf_updated_state_threshold ) + StateVector.Draws = StateVectorsPPF; + StateVector.Variance = cov(StateVector.Draws'); + + StateVector.Mean = mean(StateVector.Draws,2); + % [U,X] = svd(0.5*(StateVector.Variance+StateVector.Variance')); + % % P= U*X*U'; + % iss = find(diag(X)>1.e-12); + % if any(iss) + % StateVectorVarianceSquareRoot = chol(X(iss,iss))';%reduced_rank_cholesky(ReducedForm.StateVectorVariance)'; + % % Get the rank of StateVectorVarianceSquareRoot + % state_variance_rank = size(StateVectorVarianceSquareRoot,2); + % StateVectors = bsxfun(@plus,U(:,iss)*StateVectorVarianceSquareRoot*transpose(norminv(qmc_scrambled(state_variance_rank,number_of_particles,1))),StateVector.Mean); + % else + % state_variance_rank = numel(StateVectorMean ); + % StateVectors = bsxfun(@plus,0*transpose(norminv(qmc_scrambled(state_variance_rank,number_of_particles,1))),StateVector.Mean); + % end + use_pkf_distribution=false; + else + StateVector.Draws = StateVectorsPKF; + use_pkf_distribution=true; + end +end + + diff --git a/matlab/+occbin/set_default_options.m b/matlab/+occbin/set_default_options.m index 04e13d54df14d98f8a7b783848481936f19cddeb..63d9ab387a8c6fa877081cc0110b27de8f908d9d 100644 --- a/matlab/+occbin/set_default_options.m +++ b/matlab/+occbin/set_default_options.m @@ -42,6 +42,22 @@ if ismember(flag,{'all'}) end if ismember(flag,{'filter','all'}) + options_occbin_.filter.particle.draw_states_from_empirical_density = true; + options_occbin_.filter.particle.empirical_conditional_data_density.status = false; + options_occbin_.filter.particle.empirical_data_density.kernel_density = true; + options_occbin_.filter.particle.empirical_data_density.status = false; + options_occbin_.filter.particle.ensemble_kalman_filter = false; + options_occbin_.filter.particle.importance_sampling_using_pkf = true; + options_occbin_.filter.particle.initial_state_ergodic_simul = false; + options_occbin_.filter.particle.likelihood_only = true; + options_occbin_.filter.particle.number_of_particles = 127; + options_occbin_.filter.particle.number_of_shocks_per_particle = 1; + options_occbin_.filter.particle.nograph = true; + options_occbin_.filter.particle.state_draws = []; + options_occbin_.filter.particle.status = false; + options_occbin_.filter.particle.tobit = false; + options_occbin_.filter.particle.use_pkf_updated_state_threshold = 1; + options_occbin_.filter.init_periods_using_particles = false; options_occbin_.filter.state_covariance = false; options_occbin_.filter.guess_regime = false; options_occbin_.filter.periodic_solution = true; @@ -72,7 +88,9 @@ end if ismember(flag,{'likelihood','all'}) options_occbin_.likelihood.brute_force_regime_guess = true; + options_occbin_.likelihood.brute_force_extra_regime_guess = true; options_occbin_.likelihood.curb_retrench = false; + options_occbin_.likelihood.first_period_binding_regime_allowed = 1; options_occbin_.likelihood.first_period_occbin_update = 1; options_occbin_.likelihood.full_output = false; options_occbin_.likelihood.IF_likelihood = false; @@ -175,6 +193,8 @@ if ismember(flag,{'shock_decomp','all'}) options_occbin_.shock_decomp.shocks_only=false; options_occbin_.shock_decomp.total_effect=false; options_occbin_.shock_decomp.conditional_only=true; + options_occbin_.shock_decomp.use_shock_groups=''; + options_occbin_.shock_decomp.TINIT = dates(); % date to initialize states for shock decomp % options_occbin_.shock_decomp.use_shock_groups=options_.plot_shock_decomp.use_shock_groups; end @@ -223,6 +243,7 @@ if ismember(flag,{'smoother','all'}) options_occbin_.smoother.piecewise_only = true; options_occbin_.smoother.plot = true; options_occbin_.smoother.status=true; + options_occbin_.smoother.store_results=true; options_occbin_.smoother.waitbar=true; % options.occbin.smoother.restrict_state_space = 1; end diff --git a/matlab/+occbin/shock_decomposition.m b/matlab/+occbin/shock_decomposition.m index f050772fc1706cac59a553562eaf9ce04aa56258..d2a5a051b2110d384d4fe7000ce177161dfea66f 100644 --- a/matlab/+occbin/shock_decomposition.m +++ b/matlab/+occbin/shock_decomposition.m @@ -159,6 +159,11 @@ as_lin=zeros(M_.endo_nbr,length(tinit:gend)); % linear smoother reconstructed att=zeros(M_.endo_nbr,length(tinit:gend)); % linear initial condition effect inn=zeros(M_.endo_nbr,length(tinit:gend)); % linear aggreage shocks effect without att, i.e. as_lin = inn+att; deco=zeros(M_.endo_nbr,M_.exo_nbr,length(tinit:gend)); % full decomposition into individual shocks +deco_init = zeros(M_.endo_nbr,M_.endo_nbr+2,length(tinit:gend)); % full decomposition into individual initval +deco_init(:,end,:) = oo_.occbin.linear_smoother.alphahat(:,tinit:gend); +for i=1:M_.endo_nbr + deco_init(i,i,1) = oo_.occbin.linear_smoother.alphahat(i,tinit); +end att(:,1)=oo_.occbin.linear_smoother.alphahat(:,tinit); as_lin(:,1)=oo_.occbin.linear_smoother.alphahat(:,tinit); @@ -178,6 +183,11 @@ for j=2:length(tinit:gend) end end + if j>1 + deco_init(:,1:M_.endo_nbr,j) = TM*deco_init(:,1:M_.endo_nbr,j-1); + end + deco_init(:,M_.endo_nbr+1,j) = deco_init(:,M_.endo_nbr+2,j) - sum(deco_init(:,1:M_.endo_nbr,j),2); + end as_lin=as_lin(oo_.dr.inv_order_var,:); % linear smoother reconstructed att=att(oo_.dr.inv_order_var,:); % linear initial condition effect @@ -186,15 +196,22 @@ deco=deco(oo_.dr.inv_order_var,:,:); % full decomposition into individual shocks deco(:,M_.exo_nbr+1,:)=att; deco(:,M_.exo_nbr+2,:)=as_lin; oo_.occbin.linear_smoother.decomp=deco; +deco_init=deco_init(oo_.dr.inv_order_var,oo_.dr.inv_order_var,:); % full decomposition into individual initval +oo_.occbin.linear_smoother.init_decomp=deco_init; %%%%%%%%%%%%%%%%%%%% END LINEAR shock decomp %% -%%%%%%%%%%%%%%%%%%%% piecewise COONDITIONAL shock decomp +%%%%%%%%%%%%%%%%%%%% piecewise CONDITIONAL shock decomp as_p=zeros(M_.endo_nbr,length(tinit:gend)); % smoother reconstructed att_p=zeros(M_.endo_nbr,length(tinit:gend)); % initial condition effect inn_p=zeros(M_.endo_nbr,length(tinit:gend)); % aggreage shocks effect without att, i.e. as_lin = inn+att; deco_p=zeros(M_.endo_nbr,M_.exo_nbr,length(tinit:gend)); % full decomposition into individual shocks +deco_p_init = zeros(M_.endo_nbr,M_.endo_nbr+2,length(tinit:gend)); % full decomposition into individual initval +deco_p_init(:,end,:) = oo_.occbin.smoother.alphahat(:,tinit:gend); +for i=1:M_.endo_nbr + deco_p_init(i,i,1) = oo_.occbin.smoother.alphahat(i,tinit); +end reg_p=zeros(M_.endo_nbr,length(tinit:gend)); % pure regime effect (CONST 'shocks') att_p(:,1)=oo_.occbin.smoother.alphahat(:,tinit); @@ -221,6 +238,10 @@ for j=2:length(tinit:gend) end end + if j>1 + deco_p_init(:,1:M_.endo_nbr,j) = TM*deco_p_init(:,1:M_.endo_nbr,j-1); +end + deco_p_init(:,M_.endo_nbr+1,j) = deco_p_init(:,M_.endo_nbr+2,j) - sum(deco_p_init(:,1:M_.endo_nbr,j),2); end as_p=as_p(oo_.dr.inv_order_var,:); % occbin smoother reconstructed att_p=att_p(oo_.dr.inv_order_var,:); % occbin initial condition effect @@ -230,11 +251,19 @@ reg_p=reg_p(oo_.dr.inv_order_var,:); % occbin pure regime effect (CONST 'shocks' i_reg=strmatch('EPS_REGIME',M_.exo_names,'exact'); if ~isempty(i_reg) deco_p(:,i_reg,:)=reg_p; +else +att_p = att_p + reg_p; end deco_p(:,M_.exo_nbr+1,:)=att_p; deco_p(:,M_.exo_nbr+2,:)=as_p; %deco_p(:,M_.exo_nbr+3,:)=as_p-reg_p; oo_.occbin.smoother.decomp=deco_p; +if isempty(i_reg) + deco_p(:,M_.exo_nbr+1,:)=att_p-reg_p; +end +deco_p_init=deco_p_init(oo_.dr.inv_order_var,oo_.dr.inv_order_var,:); % full decomposition into individual initval +oo_.occbin.smoother.init_decomp=deco_p_init; + rr=abs(deco_p(:,1:end-1,:)); if isequal(use_shock_groups,'ALL') && ~isempty(i_reg) rr(:,i_reg,:)=0; diff --git a/matlab/+occbin/solve_one_constraint.m b/matlab/+occbin/solve_one_constraint.m index b8a16fb4a11b17714cf949e7b5e397da3b756f4a..9cfd6377648dedb38f39bba10ea23b11828fa052 100644 --- a/matlab/+occbin/solve_one_constraint.m +++ b/matlab/+occbin/solve_one_constraint.m @@ -123,6 +123,9 @@ for shock_period = 1:n_shocks_periods dyn_waitbar(shock_period/n_shocks_periods, hh_fig, sprintf('Period %u of %u', shock_period,n_shocks_periods)); end + if shock_period == 171 + do_nothing=true; + end regime_change_this_iteration=true; iter = 0; nperiods_endogenously_increased = false; diff --git a/matlab/+occbin/squeeze_shock_decomposition.m b/matlab/+occbin/squeeze_shock_decomposition.m index bbd54f048d3e601b792be5a4bd5a4e91c9ff4180..3d2a705f3ef7d81d483c9e9af283cc3a17ce4407 100644 --- a/matlab/+occbin/squeeze_shock_decomposition.m +++ b/matlab/+occbin/squeeze_shock_decomposition.m @@ -22,7 +22,18 @@ options_.occbin.shock_decomp.i_var = i_var; if isfield (oo_.occbin.smoother,'decomp') oo_.occbin.smoother.decomp = oo_.occbin.smoother.decomp(i_var,:,:); oo_.occbin.smoother.wdecomp = oo_.occbin.smoother.wdecomp(i_var,:,:); + if isfield (oo_.occbin,'linear_smoother') + oo_.occbin.linear_smoother.decomp = oo_.occbin.linear_smoother.decomp(i_var,:,:); end +end + +if isfield (oo_.occbin.smoother,'init_decomp') + oo_.occbin.smoother.init_decomp = oo_.occbin.smoother.init_decomp(i_var,:,:); + if isfield (oo_.occbin,'linear_smoother') + oo_.occbin.linear_smoother.init_decomp = oo_.occbin.linear_smoother.init_decomp(i_var,:,:); + end +end + if isfield(oo_.occbin,'shock_decomp') fnames = fieldnames(oo_.occbin.shock_decomp); diff --git a/matlab/default_option_values.m b/matlab/default_option_values.m index 9e6b685ac83f528558a918dc5f387b7d959fa18b..9b47162cfb121f7406c756291c1f33e593732c75 100644 --- a/matlab/default_option_values.m +++ b/matlab/default_option_values.m @@ -486,7 +486,11 @@ options_.posterior_sampler_options.slice.WR=[]; options_.posterior_sampler_options.slice.mode_files=[]; options_.posterior_sampler_options.slice.mode=[]; options_.posterior_sampler_options.slice.initial_step_size=0.8; +options_.posterior_sampler_options.slice.use_prior_draws=false; options_.posterior_sampler_options.slice.save_tmp_file=1; +options_.posterior_sampler_options.slice.draw_init_state_from_smoother=false; +options_.posterior_sampler_options.slice.draw_init_state_with_rotated_slice=false; +options_.posterior_sampler_options.slice.fast_likelihood_evaluation_for_rejection=true; % Independent Metropolis-Hastings options_.posterior_sampler_options.imh.proposal_distribution = 'rand_multivariate_normal'; options_.posterior_sampler_options.imh.use_mh_covariance_matrix=0; @@ -758,6 +762,8 @@ options_.endogenous_prior = false; options_.endogenous_prior_restrictions.irf={}; options_.endogenous_prior_restrictions.moment={}; +options_.init_state_endogenous_prior = false; + % OSR Optimal Simple Rules options_.osr.opt_algo=4; diff --git a/matlab/estimation/draw_init_state_from_smoother.m b/matlab/estimation/draw_init_state_from_smoother.m new file mode 100644 index 0000000000000000000000000000000000000000..044cc6f8e842d8993c05af05d8f797ffcbf04c30 --- /dev/null +++ b/matlab/estimation/draw_init_state_from_smoother.m @@ -0,0 +1,360 @@ +function [xparam1, logpost0, mh_bounds, M_] = draw_init_state_from_smoother(init,sampler_options,xparam1,logpost0,mh_bounds, ... + dataset_,dataset_info,options_,M_,estim_params_,bayestopt_,BoundsInfo,dr, endo_steady_state, exo_steady_state, exo_det_steady_state,derivatives_info) +% Computes the endogenous log prior addition to the initial prior +% +% INPUTS +% - xparam1 [double] current values for the estimated parameters. +% - mh_bounds [structure] containing mh_bounds +% - dataset_ [structure] dataset after transformations +% - dataset_info [structure] storing informations about the +% sample; not used but required for interface +% - options_ [structure] Matlab's structure describing the current options +% - M_ [structure] Matlab's structure describing the model +% - estim_params_ [structure] characterizing parameters to be estimated +% - bayestopt_ [structure] describing the priors +% - BoundsInfo [structure] containing prior bounds +% - dr [structure] Reduced form model. +% - endo_steady_state [vector] steady state value for endogenous variables +% - exo_steady_state [vector] steady state value for exogenous variables +% - exo_det_steady_state [vector] steady state value for exogenous deterministic variables +% - derivatives_info [structure] derivative info for identification +% +% OUTPUTS +% - xparam1 [double] current values for the estimated parameters. +% - mh_bounds [structure] containing mh_bounds +% - M_ [structure] Matlab's structure describing the model + +% Copyright © 2024 Dynare Team +% +% This file is part of Dynare. +% +% Dynare is free software: you can redistribute it and/or modify +% it under the terms of the GNU General Public License as published by +% the Free Software Foundation, either version 3 of the License, or +% (at your option) any later version. +% +% Dynare is distributed in the hope that it will be useful, +% but WITHOUT ANY WARRANTY; without even the implied warranty of +% MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +% GNU General Public License for more details. +% +% You should have received a copy of the GNU General Public License +% along with Dynare. If not, see <https://www.gnu.org/licenses/>. + +% varargin +% 1 2 3 4 5 6 7 8 9 10 11 12 +% dataset_,dataset_info,options_,M_,estim_params_,bayestopt_,BoundsInfo,dr, endo_steady_state, exo_steady_state, exo_det_steady_state,derivatives_info + +if nargin<17 + derivatives_info=[]; +end + +target_accepted = 5; +if not(islogical(init)) + target_accepted = init(2); + init=logical(init(1)); +end + +if not(init) +% logpost0=-dsge_likelihood(xparam1,dataset_,dataset_info,options_,M_,estim_params_,bayestopt_,BoundsInfo,dr, endo_steady_state, exo_steady_state, exo_det_steady_state,derivatives_info); + xparam10 = xparam1; +end + +filter_initial_state=M_.filter_initial_state; +M_.filter_initial_state = []; +error_flag=0; +options_.lik_init=1; +if init + [Pstar, info]=get_pstar(xparam1,options_,M_,estim_params_,bayestopt_,BoundsInfo,dr, endo_steady_state, exo_steady_state, exo_det_steady_state); + if info(1) + return + end + + state_uncertainty0=zeros(M_.endo_nbr,M_.endo_nbr); + state_uncertainty0(dr.restrict_var_list,dr.restrict_var_list)=Pstar; + alphahat0=zeros(M_.endo_nbr,1); + + +end + data = dataset_.data; + data_index = dataset_info.missing.aindex; + missing_value = dataset_info.missing.state; + + % Set number of observations + gend = dataset_.nobs; + options_.smoother_redux=true; + options_.smoothed_state_uncertainty = true; + options_.occbin.smoother.debug = false; + options_.occbin.smoother.plot = false; + options_.occbin.smoother.store_results = false; + options_.occbin.smoother.waitbar = false; + oo_.dr = dr; + oo_.steady_state= endo_steady_state; + oo_.exo_steady_state = exo_steady_state; + oo_.exo_det_steady_state = exo_det_steady_state; + if options_.occbin.smoother.status + if not(init) + % check first that PKF with latent states provides sensible + % likelihood + options_.init_state_endogenous_prior=false; + theta_struct.fval=logpost0-10; + theta_struct.params=xparam1; + logpost2 = -dsge_likelihood(theta_struct,dataset_,dataset_info,options_,M_,estim_params_,bayestopt_,BoundsInfo,dr, endo_steady_state, exo_steady_state, exo_det_steady_state,derivatives_info); + options_.init_state_endogenous_prior=true; + if (logpost0-logpost2)<1.e3 + [atT,innov,measurement_error,updated_variables,ys,trend_coeff,aK,~,~,P,PK,decomp,Trend,state_uncertainty,oo_,bayestopt_,alphahat0,state_uncertainty0] = occbin.DSGE_smoother(xparam1,gend,transpose(data),data_index,missing_value,M_,oo_,options_,bayestopt_,estim_params_,dataset_,dataset_info); + % error_flag = oo_.occbin.smoother.error_flag; + else + oo_.occbin.smoother.error_flag = 313; + alphahat0 = []; + end + end + if init || (oo_.occbin.smoother.error_flag && isempty(alphahat0)) + options_.occbin.smoother.status=false; + [atT,innov,measurement_error,updated_variables,ys,trend_coeff,aK,~,~,P,PK,decomp,Trend,state_uncertainty,oo_,bayestopt_,alphahat0,state_uncertainty0] = DsgeSmoother(xparam1,gend,transpose(data),data_index,missing_value,M_,oo_,options_,bayestopt_,estim_params_); + options_.occbin.smoother.status=true; + end + else + [atT,innov,measurement_error,updated_variables,ys,trend_coeff,aK,~,~,P,PK,decomp,Trend,state_uncertainty,oo_,bayestopt_,alphahat0,state_uncertainty0] = DsgeSmoother(xparam1,gend,transpose(data),data_index,missing_value,M_,oo_,options_,bayestopt_,estim_params_); + end +% end +M_.filter_initial_state = filter_initial_state; +options_.lik_init=2; +if error_flag==0 + % smoother converged! + params0 = M_.params; + % draw initial state from smoothed distribution + [U,X] = svd(0.5*(state_uncertainty0(dr.restrict_var_list(bayestopt_.mf0),dr.restrict_var_list(bayestopt_.mf0))+state_uncertainty0(dr.restrict_var_list(bayestopt_.mf0),dr.restrict_var_list(bayestopt_.mf0))')); + % [U,X] = svd(0.5*(Pstar+Pstar')); +% [U,X] = svd(0.5*(P(dr.restrict_var_list(bayestopt_.mf0),dr.restrict_var_list(bayestopt_.mf0),1)+P(dr.restrict_var_list(bayestopt_.mf0),dr.restrict_var_list(bayestopt_.mf0),1)')); + % P= U*X*U'; % symmetric matrix! + is = find(diag(X)>options_.kalman_tol); + StateVectorVarianceSquareRoot = chol(X(is,is))';%reduced_rank_cholesky(ReducedForm.StateVectorVariance)'; + + % Get the rank of StateVectorVarianceSquareRoot + state_variance_rank = size(StateVectorVarianceSquareRoot,2); + U = U(:,is); + + % + M0=M_; + pvec=[]; + for ii=1:size(M_.filter_initial_state,1) + if ~isempty(M_.filter_initial_state{ii,1}) + tmp1 = strrep(M_.filter_initial_state{ii,2},');',''); + tmp1 = strrep(tmp1,'M_.params(',''); + pvec = [pvec eval(tmp1)]; + end + end + [~,~,IB] = intersect(M_.param_names(pvec),bayestopt_.name,'stable'); + M_.params(pvec) = xparam1(IB); + + % + fast_likelihood_evaluation_for_rejection = false; + if isfield(sampler_options,'fast_likelihood_evaluation_for_rejection') && sampler_options.fast_likelihood_evaluation_for_rejection + fast_likelihood_evaluation_for_rejection = true; + end + + alphahat00 = alphahat0; + if not(init) + if options_.init_state_endogenous_prior + % independent MH + % check probability of smoothed in t=0 (which is the mode for + % linear case, but how about occbin?) + yhat = alphahat0(dr.restrict_var_list(bayestopt_.mf0)); + alphahat01=alphahat00; + alphahat01(dr.restrict_var_list(bayestopt_.mf0))=yhat; + alphahat01 = alphahat01(dr.inv_order_var); + + for ii=1:size(M_.filter_initial_state,1) + if ~isempty(M_.filter_initial_state{ii,1}) + if options_.loglinear && ~options_.logged_steady_state + eval([strrep(M_.filter_initial_state{ii,2},';','') ' = exp(log(dr.ys(ii))+alphahat01(ii));']); + elseif ~options_.loglinear && ~options_.logged_steady_state + eval([strrep(M_.filter_initial_state{ii,2},';','') '= dr.ys(ii)+alphahat01(ii);']) + else + error('The steady state is logged. This should not happen. Please contact the developers') + end + end + end + xparam11=xparam1; + xparam11(IB) = M_.params(pvec); + if not(all(xparam11(:)>=sampler_options.bounds.lb) && all(xparam11(:)<=sampler_options.bounds.ub)) + xparam11(xparam11(:)<sampler_options.bounds.lb)=sampler_options.bounds.lb(xparam11(:)<sampler_options.bounds.lb)+sqrt(eps); + xparam11(xparam11(:)>sampler_options.bounds.ub)=sampler_options.bounds.ub(xparam11(:)>sampler_options.bounds.ub)-sqrt(eps); + end + + is_smoothed_state_optimal=true; + [xcheck, icheck]=set_init_state(xparam11,options_,M_,estim_params_,bayestopt_,BoundsInfo,dr, endo_steady_state, exo_steady_state, exo_det_steady_state); + if icheck + xparam11=xcheck; + end + logpost1 = -dsge_likelihood(xparam11,dataset_,dataset_info,options_,M_,estim_params_,bayestopt_,BoundsInfo,dr, endo_steady_state, exo_steady_state, exo_det_steady_state,derivatives_info); + if logpost1<logpost0 + is_smoothed_state_optimal=false; + end + logpostSMO = logpost1; + xparamSMO = xparam11; + end + if not(options_.init_state_endogenous_prior) || not(is_smoothed_state_optimal) + + % use previous draw RW MH + + for ii=1:size(M_.filter_initial_state,1) + if ~isempty(M_.filter_initial_state{ii,1}) + if options_.loglinear && ~options_.logged_steady_state + eval(['alphahat0(ii) = log(' strrep(M_.filter_initial_state{ii,2},';','') ') - log(dr.ys(ii));']); + elseif ~options_.loglinear && ~options_.logged_steady_state + eval(['alphahat0(ii) = ' strrep(M_.filter_initial_state{ii,2},';','') '- dr.ys(ii);']) + else + error('The steady state is logged. This should not happen. Please contact the developers') + end + end + end + alphahat0=alphahat0(dr.order_var); + end + end + + naccepted=0; + if not(init) + logpost00=logpost0; + end + nattempts=0; + while naccepted<target_accepted && nattempts<10 + niter = 0; + nattempts = nattempts+1; + while naccepted<target_accepted && niter<20 + niter = niter+1; + new_draw_out_of_bounds= true; + icount = 0; + while new_draw_out_of_bounds && icount<10 + icount = icount+1; + draw_base = randn(state_variance_rank,1); + if not(isempty(is)) + yhat = U(:,is)*StateVectorVarianceSquareRoot*draw_base+alphahat0(dr.restrict_var_list(bayestopt_.mf0)); + else + yhat = alphahat0(dr.restrict_var_list(bayestopt_.mf0)); + end + alphahat01=alphahat00; + alphahat01(dr.restrict_var_list(bayestopt_.mf0))=yhat; + alphahat01 = alphahat01(dr.inv_order_var); + + for ii=1:size(M_.filter_initial_state,1) + if ~isempty(M_.filter_initial_state{ii,1}) + if options_.loglinear && ~options_.logged_steady_state + eval([strrep(M_.filter_initial_state{ii,2},';','') ' = exp(log(dr.ys(ii))+alphahat01(ii));']); + elseif ~options_.loglinear && ~options_.logged_steady_state + eval([strrep(M_.filter_initial_state{ii,2},';','') '= dr.ys(ii)+alphahat01(ii);']) + else + error('The steady state is logged. This should not happen. Please contact the developers') + end + end + end + xparam11=xparam1; + xparam11(IB) = M_.params(pvec); + if all(xparam11(:)>=sampler_options.bounds.lb) && all(xparam11(:)<=sampler_options.bounds.ub) + new_draw_out_of_bounds = false; + end + end + if new_draw_out_of_bounds + xparam11(xparam11(:)<sampler_options.bounds.lb)=sampler_options.bounds.lb(xparam11(:)<sampler_options.bounds.lb)+sqrt(eps); + xparam11(xparam11(:)>sampler_options.bounds.ub)=sampler_options.bounds.ub(xparam11(:)>sampler_options.bounds.ub)-sqrt(eps); + end + [xcheck, icheck]=set_init_state(xparam11,options_,M_,estim_params_,bayestopt_,BoundsInfo,dr, endo_steady_state, exo_steady_state, exo_det_steady_state); + if icheck + do_nothing=true; + xparam11=xcheck; + end + if init + xparam1=xparam11; + break + end + lnrand = log(rand); + if fast_likelihood_evaluation_for_rejection + theta_struct.fval=lnrand+logpost0-10; + theta_struct.params=xparam11; + logpost1 = -dsge_likelihood(theta_struct,dataset_,dataset_info,options_,M_,estim_params_,bayestopt_,BoundsInfo,dr, endo_steady_state, exo_steady_state, exo_det_steady_state,derivatives_info); + if (logpost1 >= theta_struct.fval) + logcheck = -dsge_likelihood(xparam11,dataset_,dataset_info,options_,M_,estim_params_,bayestopt_,BoundsInfo,dr, endo_steady_state, exo_steady_state, exo_det_steady_state,derivatives_info); + if logcheck~=logpost1 + do_nothing = true; + end + logpost1 = logcheck; + end + else + logpost1 = -dsge_likelihood(xparam11,dataset_,dataset_info,options_,M_,estim_params_,bayestopt_,BoundsInfo,dr, endo_steady_state, exo_steady_state, exo_det_steady_state,derivatives_info); + end + if logpost1<logpost0 + r = logpost1-logpost0; + if (logpost1 > -inf) && (lnrand < r) + accepted = 1; + xparam1 = xparam11; + else + accepted = 0; + M_.params = params0; + logpost1 = logpost0; + end + else + xparam1 = xparam11; + accepted = 1; + end + if accepted + logpost0 = logpost1; + naccepted = naccepted+1; + M_.params(pvec) = xparam1(IB); + if options_.init_state_endogenous_prior && logpostSMO<logpost0 + is_smoothed_state_optimal=false; + end + + if not(options_.init_state_endogenous_prior && is_smoothed_state_optimal) %nattempts==1) + for ii=1:size(M_.filter_initial_state,1) + if ~isempty(M_.filter_initial_state{ii,1}) + if options_.loglinear && ~options_.logged_steady_state + eval(['alphahat0(ii) = log(' strrep(M_.filter_initial_state{ii,2},';','') ') - log(dr.ys(ii));']); + elseif ~options_.loglinear && ~options_.logged_steady_state + eval(['alphahat0(ii) = ' strrep(M_.filter_initial_state{ii,2},';','') '- dr.ys(ii);']) + else + error('The steady state is logged. This should not happen. Please contact the developers') + end + end + end + alphahat0=alphahat0(dr.order_var); + end + end + end + if init + break + end + if naccepted==0 && options_.init_state_endogenous_prior + if not(is_smoothed_state_optimal) + M_.params(pvec) = xparam1(IB); + alphahat00 = alphahat0; + + for ii=1:size(M_.filter_initial_state,1) + if ~isempty(M_.filter_initial_state{ii,1}) + if options_.loglinear && ~options_.logged_steady_state + eval(['alphahat0(ii) = log(' strrep(M_.filter_initial_state{ii,2},';','') ') - log(dr.ys(ii));']); + elseif ~options_.loglinear && ~options_.logged_steady_state + eval(['alphahat0(ii) = ' strrep(M_.filter_initial_state{ii,2},';','') '- dr.ys(ii);']) + else + error('The steady state is logged. This should not happen. Please contact the developers') + end + end + end + alphahat0=alphahat0(dr.order_var); + else + StateVectorVarianceSquareRoot = StateVectorVarianceSquareRoot*0.66; + end + end + end +end +if naccepted==0 || (options_.init_state_endogenous_prior && not(is_smoothed_state_optimal)) + do_nothing = true; +end +if not(init) + if isstruct(mh_bounds) + mh_bounds.lb(IB)= xparam1(IB); + mh_bounds.ub(IB)= xparam1(IB); + end + disp(['naccepted=' int2str(naccepted) '| niter=' int2str(niter) '| nattempts=' int2str(nattempts)]) +end + diff --git a/matlab/estimation/dsge_likelihood.m b/matlab/estimation/dsge_likelihood.m index 6a72e9e38a4b97c9e3325e36d49582a01e56713a..24e952f67c47d703eb7e10f48618bbd5e764b749 100644 --- a/matlab/estimation/dsge_likelihood.m +++ b/matlab/estimation/dsge_likelihood.m @@ -73,7 +73,14 @@ Hess = []; % (Don't do the transformation if xparam1 is empty, otherwise it would become a % 0×1 matrix, which create issues with older MATLABs when comparing with [] in % check_bounds_and_definiteness_estimation) +test_rejection = false; if ~isempty(xparam1) + if isstruct(xparam1) + lnprior = priordens(xparam1.params(:),bayestopt_.pshape,bayestopt_.p6,bayestopt_.p7,bayestopt_.p3,bayestopt_.p4); + options_.likelihood_base_value = (xparam1.fval-lnprior); + xparam1 = xparam1.params; + test_rejection = true; + end xparam1 = xparam1(:); end @@ -266,7 +273,15 @@ switch options_.lik_init Pinf = []; a = zeros(mm,1); a = set_Kalman_starting_values(a,M_,dr,options_,bayestopt_); - a_0_given_tm1 = T*a; %set state prediction for first Kalman step; + a_0_given_tm0 = a; + if not(options_.occbin.likelihood.status && options_.occbin.likelihood.first_period_occbin_update==1) + a_0_given_tm1 = T*a; %set state prediction for first Kalman step; + if options_.Harvey_scale_factor==0 + Pstar = T*Pstar*T' + R*Q*R'; + end + else + a_0_given_tm1 = a; + end if options_.occbin.likelihood.status Z =zeros(length(bayestopt_.mf),size(T,1)); for i = 1:length(bayestopt_.mf) @@ -594,6 +609,19 @@ end %------------------------------------------------------------------------------ % 4. Likelihood evaluation %------------------------------------------------------------------------------ +if test_rejection && options_.init_state_endogenous_prior + if not(options_.lik_init==2 && options_.Harvey_scale_factor==0) + error('Init state endogenous prior not supported without conditional likelihood') + else + [lnpriorendoinitstate, lnpriorinitstate] = init_state_endogenous_prior(a_0_given_tm0,T,R,Q,xparam1,bayestopt_,M_,options_); + end + % note lnprior has been already removed! + options_.likelihood_base_value = options_.likelihood_base_value-lnpriorendoinitstate +lnpriorinitstate; + if occbin_.status + occbin_.info{1} = options_; + end +end + if options_.heteroskedastic_filter Q=Qvec; end @@ -636,12 +664,25 @@ if ((kalman_algo==1) || (kalman_algo==3))% Multivariate Kalman Filter end end else - [LIK,lik] = missing_observations_kalman_filter(dataset_info.missing.aindex,dataset_info.missing.number_of_observations,dataset_info.missing.no_more_missing_observations,Y,diffuse_periods+1,size(Y,2), ... +% if occbin_.status && options_.occbin.filter.particle.status +% [LIK,lik,aT,PT, pfilter] = occbin.missing_observations_piecewise_particle_filter(dataset_info.missing.aindex,dataset_info.missing.number_of_observations,dataset_info.missing.no_more_missing_observations,Y,diffuse_periods+1,size(Y,2), ... +% a_0_given_tm1, Pstar, ... +% kalman_tol, options_.riccati_tol, ... +% options_.rescale_prediction_error_covariance, ... +% options_.presample, ... +% T,Q,R,H,Z,mm,pp,rr,Zflag,diffuse_periods, occbin_); +% if ~isinf(LIK) +% filter_output.occbin.particle_filter = pfilter; +% end +% else + + [LIK,lik,aT, PT] = missing_observations_kalman_filter(dataset_info.missing.aindex,dataset_info.missing.number_of_observations,dataset_info.missing.no_more_missing_observations,Y,diffuse_periods+1,size(Y,2), ... a_0_given_tm1, Pstar, ... kalman_tol, options_.riccati_tol, ... options_.rescale_prediction_error_covariance, ... options_.presample, ... T,Q,R,H,Z,mm,pp,rr,Zflag,diffuse_periods, occbin_); +% end if occbin_.status && isinf(LIK) fval = Inf; info(1) = 320; @@ -828,6 +869,14 @@ if options_.endogenous_prior==1 [lnpriormom] = endogenous_prior(Y,dataset_info,Pstar,bayestopt_,H); fval = (likelihood-lnprior-lnpriormom); end +elseif options_.init_state_endogenous_prior + if not(options_.lik_init==2 && options_.Harvey_scale_factor==0) + error('Init state endogenous prior not supported without conditional likelihood') + elseif not(test_rejection) + [lnpriorendoinitstate, lnpriorinitstate] = init_state_endogenous_prior(a_0_given_tm0,T,R,Q,xparam1,bayestopt_,M_,options_); + end + fval = likelihood-lnpriorendoinitstate - (lnprior-lnpriorinitstate); + else fval = (likelihood-lnprior); end diff --git a/matlab/estimation/dynare_estimation_1.m b/matlab/estimation/dynare_estimation_1.m index a1a01039280d585d8009bea5f34d4244804ef580..94d03c868aafe04973b859ffdc1d1b83f4251137 100644 --- a/matlab/estimation/dynare_estimation_1.m +++ b/matlab/estimation/dynare_estimation_1.m @@ -205,15 +205,15 @@ if isequal(options_.mode_compute,0) && isempty(options_.mode_file) && ~options_. end else if options_.occbin.smoother.status - [atT,innov,measurement_error,updated_variables,ys,trend_coeff,aK,~,~,P,PK,decomp,Trend,state_uncertainty,oo_,bayestopt_] = occbin.DSGE_smoother(xparam1,gend,transpose(data),data_index,missing_value,M_,oo_,options_,bayestopt_,estim_params_,dataset_,dataset_info); + [atT,innov,measurement_error,updated_variables,ys,trend_coeff,aK,~,~,P,PK,decomp,Trend,state_uncertainty,oo_,bayestopt_,a0T,state_uncertainty0] = occbin.DSGE_smoother(xparam1,gend,transpose(data),data_index,missing_value,M_,oo_,options_,bayestopt_,estim_params_,dataset_,dataset_info); if oo_.occbin.smoother.error_flag(1)==0 - [oo_]=store_smoother_results(M_,oo_,options_,bayestopt_,dataset_,dataset_info,atT,innov,measurement_error,updated_variables,ys,trend_coeff,aK,P,PK,decomp,Trend,state_uncertainty); + [oo_]=store_smoother_results(M_,oo_,options_,bayestopt_,dataset_,dataset_info,atT,innov,measurement_error,updated_variables,ys,trend_coeff,aK,P,PK,decomp,Trend,state_uncertainty,a0T,state_uncertainty0); else fprintf('\nOccbin: smoother did not succeed. No results will be written to oo_.\n') end else - [atT,innov,measurement_error,updated_variables,ys,trend_coeff,aK,~,~,P,PK,decomp,Trend,state_uncertainty,oo_,bayestopt_] = DsgeSmoother(xparam1,gend,transpose(data),data_index,missing_value,M_,oo_,options_,bayestopt_,estim_params_); - [oo_]=store_smoother_results(M_,oo_,options_,bayestopt_,dataset_,dataset_info,atT,innov,measurement_error,updated_variables,ys,trend_coeff,aK,P,PK,decomp,Trend,state_uncertainty); + [atT,innov,measurement_error,updated_variables,ys,trend_coeff,aK,~,~,P,PK,decomp,Trend,state_uncertainty,oo_,bayestopt_,a0T,state_uncertainty0] = DsgeSmoother(xparam1,gend,transpose(data),data_index,missing_value,M_,oo_,options_,bayestopt_,estim_params_); + [oo_]=store_smoother_results(M_,oo_,options_,bayestopt_,dataset_,dataset_info,atT,innov,measurement_error,updated_variables,ys,trend_coeff,aK,P,PK,decomp,Trend,state_uncertainty,a0T,state_uncertainty0); end end if options_.forecast > 0 @@ -432,6 +432,7 @@ if issmc(options_) || (any(bayestopt_.pshape>0) && options_.mh_replic) || (any( posterior_sampler_options = options_.posterior_sampler_options.current_options; posterior_sampler_options.invhess = invhess; [posterior_sampler_options, options_, bayestopt_] = check_posterior_sampler_options(posterior_sampler_options, M_.fname, M_.dname, options_, bounds, bayestopt_); + bounds=posterior_sampler_options.bounds; % store current options in global options_.posterior_sampler_options.current_options = posterior_sampler_options; if options_.mh_replic diff --git a/matlab/estimation/get_init_state.m b/matlab/estimation/get_init_state.m new file mode 100644 index 0000000000000000000000000000000000000000..4a0380282c066e0af538fda76dc1132e1b0982fd --- /dev/null +++ b/matlab/estimation/get_init_state.m @@ -0,0 +1,54 @@ +function a = get_init_state(a,xparam1,bayestopt_,estim_params_,dr,M_,options_) +% Computes the endogenous log prior addition to the initial prior +% +% INPUTS +% xparam1 [double] n vector of estimated params +% bayestopt_ [structure] +% dr [structure] +% M_ [structure] +% +% OUTPUTS +% a [double] k vector of initial states + +% Copyright © 2024 Dynare Team +% +% This file is part of Dynare. +% +% Dynare is free software: you can redistribute it and/or modify +% it under the terms of the GNU General Public License as published by +% the Free Software Foundation, either version 3 of the License, or +% (at your option) any later version. +% +% Dynare is distributed in the hope that it will be useful, +% but WITHOUT ANY WARRANTY; without even the implied warranty of +% MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +% GNU General Public License for more details. +% +% You should have received a copy of the GNU General Public License +% along with Dynare. If not, see <https://www.gnu.org/licenses/>. + +M_ = set_all_parameters(xparam1,estim_params_,M_); + +pvec=[]; +for ii=1:size(M_.filter_initial_state,1) + if ~isempty(M_.filter_initial_state{ii,1}) + tmp1 = strrep(M_.filter_initial_state{ii,2},');',''); + tmp1 = strrep(tmp1,'M_.params(',''); + pvec = [pvec eval(tmp1)]; + end +end +% [~,~,IB] = intersect(M_.param_names(pvec),bayestopt_.name,'stable'); + +for ii=1:size(M_.filter_initial_state,1) + if ~isempty(M_.filter_initial_state{ii,1}) + if options_.loglinear && ~options_.logged_steady_state + eval(['a(ii) = log(' strrep(M_.filter_initial_state{ii,2},';','') ') - log(dr.ys(ii));']); + elseif ~options_.loglinear && ~options_.logged_steady_state + eval(['a(ii) = ' strrep(M_.filter_initial_state{ii,2},';','') '- dr.ys(ii);']) + else + error('The steady state is logged. This should not happen. Please contact the developers') + end + end +end + +a=a(dr.order_var); diff --git a/matlab/estimation/get_init_state_estim_params.m b/matlab/estimation/get_init_state_estim_params.m new file mode 100644 index 0000000000000000000000000000000000000000..8b9de46e7f58bd65ae2c1d0747f1e28736be1b33 --- /dev/null +++ b/matlab/estimation/get_init_state_estim_params.m @@ -0,0 +1,51 @@ +function [IB, IS, IP] = get_init_state_estim_params(M_, bayestopt_, dr) +% Computes the endogenous log prior addition to the initial prior +% +% INPUTS +% - M_ [structure] Matlab's structure describing the model +% - bayestopt_ [structure] describing the priors +% +% OUTPUTS +% - IB [integer] array contain indices associated to initial states in estimated +% param vector + +% Copyright © 2024 Dynare Team +% +% This file is part of Dynare. +% +% Dynare is free software: you can redistribute it and/or modify +% it under the terms of the GNU General Public License as published by +% the Free Software Foundation, either version 3 of the License, or +% (at your option) any later version. +% +% Dynare is distributed in the hope that it will be useful, +% but WITHOUT ANY WARRANTY; without even the implied warranty of +% MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +% GNU General Public License for more details. +% +% You should have received a copy of the GNU General Public License +% along with Dynare. If not, see <https://www.gnu.org/licenses/>. + +% varargin +% 1 2 3 4 5 6 7 8 9 10 11 12 +% dataset_,dataset_info,options_,M_,estim_params_,bayestopt_,BoundsInfo,dr, endo_steady_state, exo_steady_state, exo_det_steady_state,derivatives_info + +pvec=[]; +for ii=1:size(M_.filter_initial_state,1) + if ~isempty(M_.filter_initial_state{ii,1}) + tmp1 = strrep(M_.filter_initial_state{ii,2},');',''); + tmp1 = strrep(tmp1,'M_.params(',''); + pvec = [pvec eval(tmp1)]; + end +end +[~,~,IB] = intersect(M_.param_names(pvec),bayestopt_.name,'stable'); +IB = sort(IB); +IP = 1:length(bayestopt_.name); +IP = IP(not(ismember(bayestopt_.name,bayestopt_.name(IB)))); +% IP containts indices in estim params vector that are NTO init states + +nam=M_.endo_names(dr.order_var); +nam=nam(dr.restrict_var_list(bayestopt_.mf0)); +for k=1:length(nam), nam{k} = [nam{k} 'init']; end +[C,IA,IS] = intersect(bayestopt_.name(IB),nam,'stable'); +% IS contains state indices corresponding estim params \ No newline at end of file diff --git a/matlab/estimation/get_init_state_prior.m b/matlab/estimation/get_init_state_prior.m new file mode 100644 index 0000000000000000000000000000000000000000..6402e6f13cd9281236f479d113ace9844fce0e60 --- /dev/null +++ b/matlab/estimation/get_init_state_prior.m @@ -0,0 +1,50 @@ +function [UP, XP] = get_init_state_prior(xparam1, options_,M_,estim_params_,bayestopt_,BoundsInfo,dr, endo_steady_state, exo_steady_state, exo_det_steady_state) + +% Computes the endogenous log prior addition to the initial prior +% +% INPUTS +% xparam1 [double] n vector of estimated params +% bayestopt_ [structure] +% dr [structure] +% M_ [structure] +% +% OUTPUTS +% xparam1 [double] n vector of estimated params +% icheck [logical] flag for the need to update xparam1 + +% Copyright © 2024 Dynare Team +% +% This file is part of Dynare. +% +% Dynare is free software: you can redistribute it and/or modify +% it under the terms of the GNU General Public License as published by +% the Free Software Foundation, either version 3 of the License, or +% (at your option) any later version. +% +% Dynare is distributed in the hope that it will be useful, +% but WITHOUT ANY WARRANTY; without even the implied warranty of +% MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +% GNU General Public License for more details. +% +% You should have received a copy of the GNU General Public License +% along with Dynare. If not, see <https://www.gnu.org/licenses/>. + +% [Pstar, info]=get_pstar(xparam1,options_,varargin{4:end}); +icheck=false; + +filter_initial_state=M_.filter_initial_state; +M_.filter_initial_state=[]; +options_.lik_init=1; +options_.init_state_endogenous_prior=false; +[Pstar, info] = get_pstar(xparam1,options_,M_,estim_params_,bayestopt_,BoundsInfo,dr, endo_steady_state, exo_steady_state, exo_det_steady_state); +if info(1) + return +end +M_.filter_initial_state=filter_initial_state; + +[UP,XP] = svd(0.5*(Pstar(bayestopt_.mf0,bayestopt_.mf0)+Pstar(bayestopt_.mf0,bayestopt_.mf0)')); +isp = find(diag(XP)>options_.kalman_tol); +% isn = find(diag(XP)<=options_.kalman_tol); +% UPN = UP(:,isn); +UP = UP(:,isp); +XP = XP(isp,isp); diff --git a/matlab/estimation/get_pstar.m b/matlab/estimation/get_pstar.m new file mode 100644 index 0000000000000000000000000000000000000000..e2a43831bc6f841a810f87c03bf0942fbcd1efad --- /dev/null +++ b/matlab/estimation/get_pstar.m @@ -0,0 +1,199 @@ +function [Pstar, info] = get_pstar(xparam1,options_,M_,estim_params_,bayestopt_,BoundsInfo,dr, endo_steady_state, exo_steady_state, exo_det_steady_state) +% [Pstar, info] = get_pstar(xparam1,options_,M_,estim_params_,bayestopt_,BoundsInfo,dr, endo_steady_state, exo_steady_state, exo_det_steady_state) +% Evaluates the unconditional variance of states +% +% INPUTS +% - xparam1 [double] current values for the estimated parameters. +% - options_ [structure] Matlab's structure describing the current options +% - M_ [structure] Matlab's structure describing the model +% - estim_params_ [structure] characterizing parameters to be estimated +% - bayestopt_ [structure] describing the priors +% - BoundsInfo [structure] containing prior bounds +% - dr [structure] Reduced form model. +% - endo_steady_state [vector] steady state value for endogenous variables +% - exo_steady_state [vector] steady state value for exogenous variables +% - exo_det_steady_state [vector] steady state value for exogenous deterministic variables +% - derivatives_info [structure] derivative info for identification +% +% OUTPUTS +% - Pstar [double] state matrix +% +% This function is called by: posterior_sampler_iteration + +% Copyright © 2024 Dynare Team +% +% This file is part of Dynare. +% +% Dynare is free software: you can redistribute it and/or modify +% it under the terms of the GNU General Public License as published by +% the Free Software Foundation, either version 3 of the License, or +% (at your option) any later version. +% +% Dynare is distributed in the hope that it will be useful, +% but WITHOUT ANY WARRANTY; without even the implied warranty of +% MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +% GNU General Public License for more details. +% +% You should have received a copy of the GNU General Public License +% along with Dynare. If not, see <https://www.gnu.org/licenses/>. + +% Initial author: stephane DOT adjemian AT univ DASH lemans DOT FR + +% Initialization of the returned variables and others... +%------------------------------------------------------------------------------ +% 1. Get the structural parameters & define penalties +%------------------------------------------------------------------------------ +M_ = set_all_parameters(xparam1,estim_params_,M_); +Pstar=[]; +[fval,info,exit_flag,Q,H]=check_bounds_and_definiteness_estimation(xparam1, M_, estim_params_, BoundsInfo); +if info(1) + return +end + +%------------------------------------------------------------------------------ +% 2. call model setup & reduction program +%------------------------------------------------------------------------------ +is_restrict_state_space = true; + % Linearize the model around the deterministic steady state and extract the matrices of the state equation (T and R). + if options_.gabaix + [T,R,SteadyState,info,dr, M_.params] = gabaix_dynare_resolve(M_,options_,dr, endo_steady_state, exo_steady_state, exo_det_steady_state,'restrict'); + else + [T,R,SteadyState,info,dr, M_.params] = dynare_resolve(M_,options_,dr, endo_steady_state, exo_steady_state, exo_det_steady_state,'restrict'); + end + occbin_.status = false; + +% Return, with endogenous penalty when possible, if dynare_resolve issues an error code (defined in resol). +if info(1) + if info(1) == 3 || info(1) == 4 || info(1) == 5 || info(1)==6 ||info(1) == 19 ||... + info(1) == 20 || info(1) == 21 || info(1) == 23 || info(1) == 26 || ... + info(1) == 81 || info(1) == 84 || info(1) == 85 || info(1) == 86 || ... + info(1) == 401 || info(1) == 402 || info(1) == 403 || ... %cycle reduction + info(1) == 411 || info(1) == 412 || info(1) == 413 % logarithmic reduction + %meaningful second entry of output that can be used + if isnan(info(2)) + info(4) = 0.1; + else + info(4) = info(2); + end + return + else + fval = Inf; + info(4) = 0.1; + return + end +end + +% check endogenous prior restrictions + +if is_restrict_state_space +%% Define a vector of indices for the observed variables. Is this really usefull?... + bayestopt_.mf = bayestopt_.mf1; +else +%get location of observed variables and requested smoothed variables in +%decision rules + bayestopt_.mf = bayestopt_.smoother_var_list(bayestopt_.smoother_mf); +end + + +switch options_.lik_init + case 0% start from states in first_period-1 + [a_0_given_tm1,Pstar]=set_Kalman_starting_state(M_); + if options_.occbin.likelihood.status + Z =zeros(length(bayestopt_.mf),size(T,1)); + for i = 1:length(bayestopt_.mf) + Z(i,bayestopt_.mf(i))=1; + end + Zflag = 1; + else + Zflag = 0; + end + + case 1% Standard initialization with the steady state of the state equation. + Pstar=lyapunov_solver(T,R,Q,options_); + + case 2% Initialization with large numbers on the diagonal of the covariance matrix if the states (for non stationary models). + Pstar = options_.Harvey_scale_factor*eye(mm); + + case 3% Diffuse Kalman filter (Durbin and Koopman) + % Use standard kalman filter except if the univariate filter is explicitely choosen. + if kalman_algo == 0 + kalman_algo = 3; + elseif ~((kalman_algo == 3) || (kalman_algo == 4)) + error(['The model requires Diffuse filter, but you specified a different Kalman filter. You must set options_.kalman_algo ' ... + 'to 0 (default), 3 or 4']) + end + [Pstar,Pinf] = compute_Pinf_Pstar(Z,T,R,Q,options_.qz_criterium); + Z =zeros(length(bayestopt_.mf),size(T,1)); + for i = 1:length(bayestopt_.mf) + Z(i,bayestopt_.mf(i))=1; + end + Zflag = 1; + if options_.heteroskedastic_filter + QQ=Qvec; + else + QQ=Q; + end + + case 4% Start from the solution of the Riccati equation. + if kalman_algo ~= 2 + kalman_algo = 1; + end + try + if isequal(H,0) + Pstar = kalman_steady_state(transpose(T),R*Q*transpose(R),transpose(build_selection_matrix(Z,mm,length(Z)))); + else + Pstar = kalman_steady_state(transpose(T),R*Q*transpose(R),transpose(build_selection_matrix(Z,mm,length(Z))),H); + end + catch ME + disp(ME.message) + disp('dsge_likelihood:: I am not able to solve the Riccati equation, so I switch to lik_init=1!'); + options_.lik_init = 1; + Pstar=lyapunov_solver(T,R,Q,options_); + end + Pinf = []; + a = zeros(mm,1); + a = set_Kalman_starting_values(a,M_,dr,options_,bayestopt_); + a_0_given_tm1 = T*a; + if options_.occbin.likelihood.status + Z =zeros(length(bayestopt_.mf),size(T,1)); + for i = 1:length(bayestopt_.mf) + Z(i,bayestopt_.mf(i))=1; + end + Zflag = 1; + else + Zflag = 0; + end + case 5 % Old diffuse Kalman filter only for the non stationary variables + [eigenvect, eigenv] = eig(T); + eigenv = diag(eigenv); + nstable = length(find(abs(abs(eigenv)-1) > 1e-7)); + unstable = find(abs(abs(eigenv)-1) < 1e-7); + V = eigenvect(:,unstable); + indx_unstable = find(sum(abs(V),2)>1e-5); + stable = find(sum(abs(V),2)<1e-5); + nunit = length(eigenv) - nstable; + Pstar = options_.Harvey_scale_factor*eye(nunit); + if kalman_algo ~= 2 + kalman_algo = 1; + end + R_tmp = R(stable, :); + T_tmp = T(stable,stable); + Pstar_tmp=lyapunov_solver(T_tmp,R_tmp,Q,options_); + Pstar(stable, stable) = Pstar_tmp; + Pinf = []; + a = zeros(mm,1); + a = set_Kalman_starting_values(a,M_,dr,options_,bayestopt_); + a_0_given_tm1 = T*a; + if options_.occbin.likelihood.status + Z =zeros(length(bayestopt_.mf),size(T,1)); + for i = 1:length(bayestopt_.mf) + Z(i,bayestopt_.mf(i))=1; + end + Zflag = 1; + else + Zflag = 0; + end + otherwise + error('dsge_likelihood:: Unknown initialization approach for the Kalman filter!') +end + diff --git a/matlab/estimation/init_state_endogenous_prior.m b/matlab/estimation/init_state_endogenous_prior.m new file mode 100644 index 0000000000000000000000000000000000000000..713afaefa088066bf34845bda87780c86f087907 --- /dev/null +++ b/matlab/estimation/init_state_endogenous_prior.m @@ -0,0 +1,69 @@ +function [lnpriorendoinitstate, lnpriorinitstate] = init_state_endogenous_prior(a_0_given_tm0,T,R,Q,xparam1,bayestopt_,M_,options_) +% Computes the endogenous log prior addition to the initial prior +% +% INPUTS +% a_0_given_tm0 [double] k vector of initial states +% T [double] k*k state matrix +% R [double] k*n impact shock matric +% Q [double] k*n impact shock matric +% options_ [structure] +% +% OUTPUTS +% lnpriorinitstate [double] scalar of log init state endogenous prior value + +% Copyright © 2024 Dynare Team +% +% This file is part of Dynare. +% +% Dynare is free software: you can redistribute it and/or modify +% it under the terms of the GNU General Public License as published by +% the Free Software Foundation, either version 3 of the License, or +% (at your option) any later version. +% +% Dynare is distributed in the hope that it will be useful, +% but WITHOUT ANY WARRANTY; without even the implied warranty of +% MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +% GNU General Public License for more details. +% +% You should have received a copy of the GNU General Public License +% along with Dynare. If not, see <https://www.gnu.org/licenses/>. + +Pstar=lyapunov_solver(T,R,Q,options_); + +[UP,XP,WP] = svd(0.5*(Pstar(bayestopt_.mf0,bayestopt_.mf0)+Pstar(bayestopt_.mf0,bayestopt_.mf0)')); +isp = find(diag(XP)>options_.kalman_tol); +isn = find(diag(XP)<=options_.kalman_tol); +ns = size(XP,1); +UPN = UP(:,isn); +UP = UP(:,isp); +S = XP(isp,isp); %UP(:,isp)'*(0.5*(P+P'))*WP(:,isp); +log_dS = log(det(S)); +iS = inv(S); + +vv = UP'*(a_0_given_tm0(bayestopt_.mf0)); +lnpriorendoinitstate = -(log_dS + transpose(vv)*iS*vv + ns*log(2*pi))/2; + +mycheck =max(abs(UPN'*a_0_given_tm0(bayestopt_.mf0))); +if mycheck>options_.kalman_tol*10 + do_nothing=true; +end +% now I remove original state prior declared +pvec=[]; +for ii=1:size(M_.filter_initial_state,1) + if ~isempty(M_.filter_initial_state{ii,1}) + tmp1 = strrep(M_.filter_initial_state{ii,2},');',''); + tmp1 = strrep(tmp1,'M_.params(',''); + pvec = [pvec eval(tmp1)]; + end +end +[~,~,IB] = intersect(M_.param_names(pvec),bayestopt_.name,'stable'); + +lnpriorinitstate = priordens(xparam1(IB),bayestopt_.pshape(IB),bayestopt_.p6(IB),bayestopt_.p7(IB),bayestopt_.p3(IB),bayestopt_.p4(IB),true); +% lnpriorendoinitstate = lnpriorendoinitstate-lnpriorstates; + +% re-set persistent variables in prior dens! +lnprior0 = priordens(xparam1,bayestopt_.pshape,bayestopt_.p6,bayestopt_.p7,bayestopt_.p3,bayestopt_.p4,true); + + + + diff --git a/matlab/estimation/pm3.m b/matlab/estimation/pm3.m index 820c83ca4935b1e001c015f6469dea50146c94c8..5f4411bb9d38678320071f6fe5bcae30d48987f6 100644 --- a/matlab/estimation/pm3.m +++ b/matlab/estimation/pm3.m @@ -1,4 +1,4 @@ -function oo_=pm3(M_,options_,oo_,n1,n2,ifil,B,tit1,tit2,tit_tex,names1,names2,name3,DirectoryName,var_type,dispString) +function [oo_, B1]=pm3(M_,options_,oo_,n1,n2,ifil,B,tit1,tit2,tit_tex,names1,names2,name3,DirectoryName,var_type,dispString) % Computes, stores and plots the posterior moment statistics. % @@ -82,12 +82,19 @@ fprintf(['%s: ' tit1 '\n'],dispString); k = 0; filter_step_ahead_indicator=0; filter_covar_indicator=0; +init_state_uncert_indicator=0; state_uncert_indicator=0; for file = 1:ifil loaded_file=load([DirectoryName '/' M_.fname var_type int2str(file)]); stock=loaded_file.stock; - if strcmp(var_type,'_filter_step_ahead') + if strcmp(var_type,'_param') + if file==1 %on first run, initialize variable for storing filter_step_ahead + stock1 = zeros(n1,B); + end + k = k(end)+(1:size(stock,1)); + stock1(:,k) = stock'; + elseif strcmp(var_type,'_filter_step_ahead') if file==1 %on first run, initialize variable for storing filter_step_ahead stock1_filter_step_ahead=NaN(n1,n2,B,length(options_.filter_step_ahead)); stock1 = zeros(n1,n2,B); @@ -109,9 +116,20 @@ for file = 1:ifil filter_covar_indicator=1; k = k(end)+(1:size(stock,4)); stock1_filter_covar(:,:,:,k) = stock; + elseif strcmp(var_type,'_init_state') + if file==1 %on first run, initialize variable for storing filter_step_ahead + stock1 = zeros(n1,B); + end + k = k(end)+(1:size(stock,2)); + stock1(:,k) = stock; + elseif strcmp(var_type,'_occbin_regime') + k = k(end)+(1:size(stock,2)); + stock1(:,k) = stock; + elseif strcmp(var_type,'_occbin_realtime_regime') + k = k(end)+(1:size(stock,2)); + stock1(:,k) = stock; elseif strcmp(var_type,'_trend_coeff') if file==1 %on first run, initialize variable for storing filter_step_ahead - stock1_filter_step_ahead=NaN(n1,n2,B,length(options_.filter_step_ahead)); stock1 = zeros(n1,B); end k = k(end)+(1:size(stock,2)); @@ -123,6 +141,13 @@ for file = 1:ifil state_uncert_indicator=1; k = k(end)+(1:size(stock,4)); stock1_state_uncert(:,:,:,k) = stock; + elseif strcmp(var_type,'_init_state_uncert') + if file==1 %on first run, initialize variable for storing filter_step_ahead + stock1_init_state_uncert=NaN(n1,size(stock,2),B); + end + init_state_uncert_indicator=1; + k = k(end)+(1:size(stock,3)); + stock1_init_state_uncert(:,:,k) = stock; else if file==1 %on first run, initialize variable for storing filter_step_ahead stock1 = zeros(n1,n2,B); @@ -131,6 +156,10 @@ for file = 1:ifil stock1(:,:,k) = stock; end end +if nargout==2 + B1=max(k); + return +end clear stock if filter_step_ahead_indicator clear stock_filter_step_ahead @@ -165,8 +194,33 @@ elseif filter_covar_indicator oo_.FilterCovariance.HPDsup=squeeze(hpd_interval(:,:,:,2)); fprintf(['%s: ' tit1 ', done!\n'],dispString); return +elseif init_state_uncert_indicator + draw_dimension=3; + oo_.Smoother.Init_State_uncertainty=struct(); + oo_.Smoother.Init_State_uncertainty.Mean = squeeze(mean(stock1_init_state_uncert,draw_dimension)); + oo_.Smoother.Init_State_uncertainty.Median = squeeze(median(stock1_init_state_uncert,draw_dimension)); + oo_.Smoother.Init_State_uncertainty.var = squeeze(var(stock1_init_state_uncert,0,draw_dimension)); + if size(stock1_init_state_uncert,draw_dimension)>2 + hpd_interval = quantile(stock1_init_state_uncert,[(1-options_.mh_conf_sig)/2 (1-options_.mh_conf_sig)/2+options_.mh_conf_sig],draw_dimension); + else + size_matrix=size(stock1_init_state_uncert); + hpd_interval=NaN([size_matrix(1:2),2]); + end + if size(stock1_init_state_uncert,draw_dimension)>9 + post_deciles =quantile(stock1_init_state_uncert,0.1:0.1:0.9,draw_dimension); + else + size_matrix=size(stock1_init_state_uncert); + post_deciles=NaN([size_matrix(1:2),9]); + end + oo_.Smoother.Init_State_uncertainty.post_deciles=post_deciles; + oo_.Smoother.Init_State_uncertainty.HPDinf=squeeze(hpd_interval(:,:,1)); + oo_.Smoother.Init_State_uncertainty.HPDsup=squeeze(hpd_interval(:,:,2)); + fprintf(['%s: ' tit1 ', done!\n'],dispString); + return + elseif state_uncert_indicator draw_dimension=4; + oo_.Smoother.State_uncertainty=struct(); oo_.Smoother.State_uncertainty.Mean = squeeze(mean(stock1_state_uncert,draw_dimension)); oo_.Smoother.State_uncertainty.Median = squeeze(median(stock1_state_uncert,draw_dimension)); oo_.Smoother.State_uncertainty.var = squeeze(var(stock1_state_uncert,0,draw_dimension)); @@ -189,7 +243,7 @@ elseif state_uncert_indicator return end -if strcmp(var_type,'_trend_coeff') %two dimensional arrays +if strcmp(var_type,'_trend_coeff') || strcmp(var_type,'_init_state') %two dimensional arrays for i = 1:nvar if options_.estimation.moments_posterior_density.indicator [Mean(1,i),Median(1,i),Var(1,i),HPD(:,1,i),Distrib(:,1,i),Density(:,:,1,i)] = ... @@ -199,6 +253,25 @@ if strcmp(var_type,'_trend_coeff') %two dimensional arrays posterior_moments(squeeze(stock1(SelecVariables(i),:)),options_.mh_conf_sig); end end +elseif strcmp(var_type,'_occbin_regime') || strcmp(var_type,'_occbin_realtime_regime') %two structure arrays + for j=1:n1 + for i = 1:n2 + for k=1:B + if j==1 + tmp_regime_info(1,k) = stock1(i,k).regimestart(end)-1; + else + if not(isempty(find(stock1(i,k).regime==1,1))) + tmp_regime_info(1,k) = stock1(i,k).regimestart(find(stock1(i,k).regime==1,1)); % first period kick in + else + tmp_regime_info(1,k)=0; + end + end + end + + [Mean(i,j),Median(i,j),Var(i,j),HPD(:,i,j),Distrib(:,i,j)] = ... + posterior_moments(tmp_regime_info,options_.mh_conf_sig); + end + end else %three dimensional arrays for i = 1:nvar for j = 1:n2 @@ -239,7 +312,7 @@ if filter_step_ahead_indicator %write matrices corresponding to ML oo_.FilteredVariablesKStepAheadVariances=FilteredVariablesKStepAheadVariances; end -if strcmp(var_type,'_trend_coeff') || strcmp(var_type,'_smoothed_trend') || strcmp(var_type,'_smoothed_trend') +if strcmp(var_type,'_trend_coeff') || strcmp(var_type,'_init_state') || strcmp(var_type,'_smoothed_trend') || strcmp(var_type,'_occbin_regime') || strcmp(var_type,'_occbin_realtime_regime') for i = 1:nvar name = deblank(names1{SelecVariables(i)}); oo_.Smoother.(name3).Mean.(name) = Mean(:,i); @@ -248,7 +321,7 @@ if strcmp(var_type,'_trend_coeff') || strcmp(var_type,'_smoothed_trend') || strc oo_.Smoother.(name3).deciles.(name) = Distrib(:,:,i); oo_.Smoother.(name3).HPDinf.(name) = HPD(1,:,i)'; oo_.Smoother.(name3).HPDsup.(name) = HPD(2,:,i)'; - if options_.estimation.moments_posterior_density.indicator + if not(strcmp(var_type,'_occbin_regime') || strcmp(var_type,'_occbin_realtime_regime')) && options_.estimation.moments_posterior_density.indicator oo_.Smoother.(name3).density.(name) = Density(:,:,:,i); end end @@ -281,7 +354,8 @@ else end end -if strcmp(var_type,'_trend_coeff') || max(max(abs(Mean(:,:))))<=10^(-6) || all(all(isnan(Mean))) +if strcmp(var_type,'_trend_coeff') || strcmp(var_type,'_init_state') || strcmp(var_type,'_occbin_regime') || strcmp(var_type,'_occbin_realtime_regime') ... + || max(max(abs(Mean(:,:))))<=10^(-6) || all(all(isnan(Mean))) fprintf(['%s: ' tit1 ', done!\n'],dispString); return %not do plots end diff --git a/matlab/estimation/posterior_sampler_initialization.m b/matlab/estimation/posterior_sampler_initialization.m index 194f4c3c30f1401043e3157d734ecae1a143fd33..981771483c4c336a59259801608fa704649b8a6d 100644 --- a/matlab/estimation/posterior_sampler_initialization.m +++ b/matlab/estimation/posterior_sampler_initialization.m @@ -195,6 +195,9 @@ if ~options_.load_mh_file && ~options_.mh_recover while ~validate && trial <= 10 if isempty(d) candidate = Prior.draw(); + if options_.occbin.likelihood.status + options_.occbin.likelihood.brute_force_extra_regime_guess = false; + end else if isfield(options_,'mh_init_scale') if trial==1 @@ -206,6 +209,14 @@ if ~options_.load_mh_file && ~options_.mh_recover end end if all(candidate(:) >= mh_bounds.lb) && all(candidate(:) <= mh_bounds.ub) +% if isfield(options_.posterior_sampler_options.current_options,'draw_init_state_from_smoother') && options_.posterior_sampler_options.current_options.draw_init_state_from_smoother + if strcmp(func2str(TargetFun),'dsge_likelihood') && options_.init_state_endogenous_prior + init = true; + [candidate, ~, ~, M_] = draw_init_state_from_smoother(init,options_.posterior_sampler_options.current_options,candidate,NaN,mh_bounds, ... + dataset_,dataset_info,options_,M_,estim_params_,bayestopt_,mh_bounds,oo_.dr, oo_.steady_state, oo_.exo_steady_state, oo_.exo_det_steady_state); + assignin('caller','params',M_.params) + evalin('caller','M_.params=params;') + end ix2(j,new_estimated_parameters) = candidate(new_estimated_parameters); ilogpo2(j) = - feval(TargetFun,ix2(j,:)',dataset_,dataset_info,options_,M_,estim_params_,bayestopt_,mh_bounds,oo_.dr, oo_.steady_state,oo_.exo_steady_state,oo_.exo_det_steady_state); if isfinite(ilogpo2(j)) % log density is finite @@ -242,6 +253,7 @@ if ~options_.load_mh_file && ~options_.mh_recover end end trial = trial+1; + init_iter = 0; end end if ~validate && trial>10 @@ -297,6 +309,8 @@ if ~options_.load_mh_file && ~options_.mh_recover for j=1:NumberOfBlocks % we set a different seed for the random generator for each block then we record the corresponding random generator state (vector) options_=set_dynare_seed_local_options(options_,options_.DynareRandomStreams.seed+j); + % use the following (instead) to recover old initialization of seeds + % set_dynare_seed_local_options(options_,options_.DynareRandomStreams.seed+j); % record.Seeds keeps a vector of the random generator state and not the scalar seed despite its name [record.InitialSeeds(j).Unifor,record.InitialSeeds(j).Normal] = get_dynare_random_generator_state(); end diff --git a/matlab/estimation/posterior_sampler_iteration.m b/matlab/estimation/posterior_sampler_iteration.m index efb6e147054788e837ac2b383f5ae49abeb14eae..8126a348576abf83c986348aff6a230aac53b5ff 100644 --- a/matlab/estimation/posterior_sampler_iteration.m +++ b/matlab/estimation/posterior_sampler_iteration.m @@ -50,6 +50,12 @@ mh_bounds = sampler_options.bounds; switch posterior_sampling_method case 'slice' + if isfield(sampler_options,'draw_init_state_from_smoother') && sampler_options.draw_init_state_from_smoother + if not(strcmp(func2str(TargetFun),'dsge_likelihood')) + error('draw_init_state_from_smoother:: I cannot determine the input arguments for DsgeSmoother') + end + end + [par, logpost, neval] = slice_sampler(TargetFun,last_draw, [mh_bounds.lb mh_bounds.ub], sampler_options,varargin{:}); accepted = 1; case 'random_walk_metropolis_hastings' diff --git a/matlab/estimation/prior_posterior_statistics.m b/matlab/estimation/prior_posterior_statistics.m index 6971f9465660c4f7e6b624b54786b382638b83bd..20dfe9ad9a035b542786a86595cb4f9623775e8b 100644 --- a/matlab/estimation/prior_posterior_statistics.m +++ b/matlab/estimation/prior_posterior_statistics.m @@ -314,6 +314,14 @@ if ~isnumeric(options_.parallel) end end +% to check for possible non converged smoothers +[~, B1]=pm3(M_,options_,oo_,length(bayestopt_.name),1,ifil(5),B,'Params',... + [],[],bayestopt_.name,... + bayestopt_.name,'Params',DirectoryName,'_param',dispString); +if B1<B + B=B1; +end + if options_.smoother oo_=pm3(M_,options_,oo_,endo_nbr,gend,ifil(1),B,'Smoothed variables',... varlist, M_.endo_names_tex,M_.endo_names,... @@ -321,9 +329,12 @@ if options_.smoother oo_=pm3(M_,options_,oo_,exo_nbr,gend,ifil(2),B,'Smoothed shocks',... M_.exo_names,M_.exo_names_tex,M_.exo_names,... M_.exo_names,'SmoothedShocks',DirectoryName,'_inno',dispString); - oo_=pm3(M_,options_,oo_,endo_nbr,1,ifil(9),B,'Trend_coefficients',... + oo_=pm3(M_,options_,oo_,endo_nbr,1,ifil(9),B,'Trend coefficients',... varlist,M_.endo_names_tex,M_.endo_names,... varlist,'TrendCoeff',DirectoryName,'_trend_coeff',dispString); + oo_=pm3(M_,options_,oo_,endo_nbr,1,ifil(9),B,'Init State',... + varlist,M_.endo_names_tex,M_.endo_names,... + varlist,'Init_State',DirectoryName,'_init_state',dispString); oo_=pm3(M_,options_,oo_,endo_nbr,gend,ifil(10),B,'Smoothed constant',... varlist,M_.endo_names_tex,M_.endo_names,... varlist,'Constant',DirectoryName,'_smoothed_constant',dispString); @@ -338,6 +349,9 @@ if options_.smoother oo_=pm3(M_,options_,oo_,endo_nbr,endo_nbr,ifil(13),B,'State Uncertainty',... varlist,M_.endo_names_tex,M_.endo_names,... varlist,'StateUncertainty',DirectoryName,'_state_uncert',dispString); + oo_=pm3(M_,options_,oo_,endo_nbr,1,ifil(13),B,'Initial State Uncertainty',... + varlist,M_.endo_names_tex,M_.endo_names,... + varlist,'InitStateUncertainty',DirectoryName,'_init_state_uncert',dispString); end if nvn @@ -349,6 +363,16 @@ if options_.smoother meas_error_names,texnames,meas_error_names,... meas_error_names,'SmoothedMeasurementErrors',DirectoryName,'_error',dispString); end + + if options_.occbin.smoother.status + oo_=pm3(M_,options_,oo_,2,gend,ifil(5),B,'Occbin regime',... + [],[],{'duration','start'},... + {'duration','start'},'Occbin_Regime',DirectoryName,'_occbin_regime',dispString); + oo_=pm3(M_,options_,oo_,2,gend,ifil(5),B,'Occbin realtime regime',... + [],[],{'duration','start'},... + {'duration','start'},'Occbin_Realtime_Regime',DirectoryName,'_occbin_realtime_regime',dispString); + end + end if options_.filtered_vars diff --git a/matlab/estimation/prior_posterior_statistics_core.m b/matlab/estimation/prior_posterior_statistics_core.m index 65660dabb59324575ac2bbcee33e3d90c0aff31b..0fc52d2b59fe6391a508baf1b3dc6c3f5e8dd296 100644 --- a/matlab/estimation/prior_posterior_statistics_core.m +++ b/matlab/estimation/prior_posterior_statistics_core.m @@ -19,9 +19,11 @@ function myoutput=prior_posterior_statistics_core(myinputs,fpar,B,whoiam, ThisMa % _forc_point_ME; % _filter_covar; % _trend_coeff; +% _init_state; % _smoothed_trend; % _smoothed_constant; % _state_uncert; +% _init_state_uncert; % % ALGORITHM % Portion of prior_posterior.m function. @@ -161,12 +163,17 @@ end %initialize arrays if run_smoother + stock_init_smooth=NaN(endo_nbr,MAX_n_trend_coeff); stock_smooth=NaN(endo_nbr,gend,MAX_nsmoo); stock_update=NaN(endo_nbr,gend,MAX_nsmoo); stock_innov=NaN(M_.exo_nbr,gend,MAX_ninno); stock_smoothed_constant=NaN(endo_nbr,gend,MAX_n_smoothed_constant); stock_smoothed_trend=NaN(endo_nbr,gend,MAX_n_smoothed_trend); stock_trend_coeff = zeros(endo_nbr,MAX_n_trend_coeff); + if options_.occbin.smoother.status +% stock_occbin_regime = struct(); +% stock_occbin_realtime_regime = struct(); + end if horizon stock_forcst_mean= NaN(endo_nbr,horizon,MAX_nforc1); stock_forcst_point = NaN(endo_nbr,horizon,MAX_nforc2); @@ -189,6 +196,7 @@ if filter_covariance end if smoothed_state_uncertainty stock_smoothed_uncert = zeros(endo_nbr,endo_nbr,gend,MAX_n_smoothed_state_uncertainty); + stock_init_smoothed_uncert = zeros(endo_nbr,endo_nbr,MAX_n_smoothed_state_uncertainty); end opts_local = options_; @@ -226,17 +234,24 @@ for b=fpar:B fprintf('prior_posterior_statistics: This should not happen. Please contact the developers.\n',message) end if options_.occbin.smoother.status + opts_local.occbin.smoother.debug = false; + opts_local.occbin.smoother.plot = false; + opts_local.occbin.smoother.store_results = false; opts_local.occbin.simul.waitbar=0; - opts_local.occbin.smoother.waitbar = 0; + opts_local.occbin.smoother.waitbar = false; opts_local.occbin.smoother.linear_smoother=false; % speed-up - [alphahat,etahat,epsilonhat,alphatilde,SteadyState,trend_coeff,aK,~,~,P,~,~,trend_addition,state_uncertainty,oo_,bayestopt_] = ... + [alphahat,etahat,epsilonhat,alphatilde,SteadyState,trend_coeff,aK,~,~,P,~,~,trend_addition,state_uncertainty,oo_,bayestopt_,a0T,state_uncertainty0] = ... occbin.DSGE_smoother(deep,gend,Y,data_index,missing_value,M_,oo_,opts_local,bayestopt_,estim_params_); if oo_.occbin.smoother.error_flag(1) message=get_error_message(oo_.occbin.smoother.error_flag,opts_local); fprintf('\nprior_posterior_statistics: One of the draws failed with the error:\n%s\n',message) + continue + else + stock_occbin_regime(:,irun(5))=oo_.occbin.smoother.regime_history; + stock_occbin_realtime_regime(:,irun(5))=oo_.occbin.smoother.realtime_regime_history; end else - [alphahat,etahat,epsilonhat,alphatilde,SteadyState,trend_coeff,aK,~,~,P,~,~,trend_addition,state_uncertainty,oo_,bayestopt_] = ... + [alphahat,etahat,epsilonhat,alphatilde,SteadyState,trend_coeff,aK,~,~,P,~,~,trend_addition,state_uncertainty,oo_,bayestopt_,a0T,state_uncertainty0] = ... DsgeSmoother(deep,gend,Y,data_index,missing_value,M_,oo_,opts_local,bayestopt_,estim_params_); end @@ -255,6 +270,7 @@ for b=fpar:B stock_update(dr.order_var,:,irun(1)) = alphatilde(1:endo_nbr,:)+ ... constant_part; end + stock_init_smooth(dr.order_var,irun(9)) = a0T(1:endo_nbr)+constant_part(:,1); stock_smoothed_constant(dr.order_var,:,irun(10))=constant_part; %% Compute constant for observables if options_.prefilter == 1 %as mean is taken after log transformation, no distinction is needed here @@ -275,9 +291,11 @@ for b=fpar:B stock_smoothed_constant(IdObs,:,irun(10))=stock_smoothed_constant(IdObs,:,irun(10))+mean_correction; %smoothed variables are E_T(y_t) so no trend shift is required stock_smooth(IdObs,:,irun(1))=stock_smooth(IdObs,:,irun(1))+trend_addition+mean_correction; + stock_init_smooth(IdObs,irun(9))=stock_init_smooth(IdObs,irun(9))+trend_addition(:,1)+mean_correction(:,1); %updated variables are E_t(y_t) so no trend shift is required stock_update(IdObs,:,irun(1))=stock_update(IdObs,:,irun(1))+trend_addition+mean_correction; else + stock_init_smooth(IdObs,irun(9))=stock_init_smooth(IdObs,irun(9))+trend_addition(:,1); stock_smooth(IdObs,:,irun(1))=stock_smooth(IdObs,:,irun(1))+trend_addition; stock_update(IdObs,:,irun(1))=stock_update(IdObs,:,irun(1))+trend_addition; end @@ -366,6 +384,7 @@ for b=fpar:B end if smoothed_state_uncertainty stock_smoothed_uncert(dr.order_var,dr.order_var,:,irun(13)) = state_uncertainty; + stock_init_smoothed_uncert(dr.order_var,dr.order_var,irun(13)) = state_uncertainty0; end else [~,~,SteadyState] = dynare_resolve(M_,options_,oo_.dr,oo_.steady_state,oo_.exo_steady_state,oo_.exo_det_steady_state); @@ -431,6 +450,16 @@ for b=fpar:B if RemoteFlag==1 OutputFileName_param = [OutputFileName_param; {[DirectoryName filesep], [M_.fname '_param' int2str(ifil(5)) '.mat']}]; end + if options_.occbin.smoother.status + stock = stock_occbin_regime(:,1:irun(5)-1); + save([DirectoryName '/' M_.fname '_occbin_regime' int2str(ifil(5)) '.mat'],'stock'); + stock = stock_occbin_realtime_regime(:,1:irun(5)-1); + save([DirectoryName '/' M_.fname '_occbin_realtime_regime' int2str(ifil(5)) '.mat'],'stock'); + if RemoteFlag==1 + OutputFileName_param = [OutputFileName_param; {[DirectoryName filesep], [M_.fname '_occbin_regime' int2str(ifil(5)) '.mat']}]; + OutputFileName_param = [OutputFileName_param; {[DirectoryName filesep], [M_.fname '_occbin_realtime_regime' int2str(ifil(5)) '.mat']}]; + end + end irun(5) = 1; end @@ -466,11 +495,14 @@ for b=fpar:B irun_index=9; if run_smoother && (irun(irun_index) > MAX_n_trend_coeff || b == B) - stock = stock_trend_coeff(:,1:irun(irun_index)-1); ifil(irun_index) = ifil(irun_index) + 1; + stock = stock_trend_coeff(:,1:irun(irun_index)-1); save([DirectoryName '/' M_.fname '_trend_coeff' int2str(ifil(irun_index)) '.mat'],'stock'); + stock = stock_init_smooth(:,1:irun(irun_index)-1); + save([DirectoryName '/' M_.fname '_init_state' int2str(ifil(irun_index)) '.mat'],'stock'); if RemoteFlag==1 OutputFileName_trend_coeff = [OutputFileName_trend_coeff; {[DirectoryName filesep], [M_.fname '_trend_coeff' int2str(ifil(irun_index)) '.mat']}]; + OutputFileName_init_state = [OutputFileName_init_state; {[DirectoryName filesep], [M_.fname '_init_state' int2str(ifil(irun_index)) '.mat']}]; end irun(irun_index) = 1; end @@ -510,11 +542,14 @@ for b=fpar:B irun_index=13; if run_smoother && smoothed_state_uncertainty && (irun(irun_index) > MAX_n_smoothed_state_uncertainty || b == B) - stock = stock_smoothed_uncert(:,:,:,1:irun(irun_index)-1); ifil(irun_index) = ifil(irun_index) + 1; + stock = stock_smoothed_uncert(:,:,:,1:irun(irun_index)-1); save([DirectoryName '/' M_.fname '_state_uncert' int2str(ifil(irun_index)) '.mat'],'stock'); + stock = stock_init_smoothed_uncert(:,:,1:irun(irun_index)-1); + save([DirectoryName '/' M_.fname '_init_state_uncert' int2str(ifil(irun_index)) '.mat'],'stock'); if RemoteFlag==1 OutputFileName_state_uncert = [OutputFileName_state_uncert; {[DirectoryName filesep], [M_.fname '_state_uncert' int2str(ifil(irun_index)) '.mat']}]; + OutputFileName_init_state_uncert = [OutputFileName_init_state_uncert; {[DirectoryName filesep], [M_.fname '_init_state_uncert' int2str(ifil(irun_index)) '.mat']}]; end irun(irun_index) = 1; end @@ -536,9 +571,11 @@ if RemoteFlag==1 OutputFileName_forc_point_ME; OutputFileName_filter_covar; OutputFileName_trend_coeff; + OutputFileName_init_state; OutputFileName_smoothed_trend; OutputFileName_smoothed_constant; - OutputFileName_state_uncert]; + OutputFileName_state_uncert; + OutputFileName_init_state_uncert]; end dyn_waitbar_close(h); diff --git a/matlab/estimation/rotated_slice_sampler.m b/matlab/estimation/rotated_slice_sampler.m index 45f5c9a41676cf2bdb745d0742231946cd5863ce..1b78a0196231a4891d663262d384a80c64b8b1df 100644 --- a/matlab/estimation/rotated_slice_sampler.m +++ b/matlab/estimation/rotated_slice_sampler.m @@ -48,6 +48,23 @@ W1=[]; if isfield(sampler_options,'WR') W1 = sampler_options.WR; end +rthetaprior=[]; +if isfield(sampler_options,'rthetaprior') + rthetaprior = sampler_options.rthetaprior; +end +endo_init_state = false; +if isfield(sampler_options,'endo_init_state') + endo_init_state = sampler_options.endo_init_state.status; +end +IP = 1:length(theta); +if endo_init_state + IP = sampler_options.endo_init_state.IP; +end + +fast_likelihood_evaluation_for_rejection = false; +if isfield(sampler_options,'fast_likelihood_evaluation_for_rejection') && sampler_options.fast_likelihood_evaluation_for_rejection + fast_likelihood_evaluation_for_rejection = true; +end if ~isempty(sampler_options.mode) mm = sampler_options.mode; n = length(mm); @@ -75,13 +92,20 @@ else end npar=size(V1,2); +fname = int2str(sampler_options.curr_block); + for it=1:npar theta0 = theta; neval(it) = 0; xold = 0; - tb=sort([(thetaprior(:,1)-theta)./V1(:,it) (thetaprior(:,2)-theta)./V1(:,it)],2); - XLB=max(tb(:,1)); - XUB=min(tb(:,2)); + if not(isempty(rthetaprior)) + XLB = rthetaprior(it,1); + XUB = rthetaprior(it,2); + else + tb=sort([(thetaprior(IP,1)-theta(IP))./V1(IP,it) (thetaprior(IP,2)-theta(IP))./V1(IP,it)],2); + XLB=max(tb(:,1)); + XUB=min(tb(:,2)); + end if isempty(W1) W = (XUB-XLB); %*0.8; else @@ -109,7 +133,17 @@ for it=1:npar while(L > XLB) xsim = L; theta = theta0+xsim*V1(:,it); - fxl = -feval(objective_function,theta,varargin{:}); + if endo_init_state + theta1 = theta; + [theta, icheck]=set_init_state(theta1,varargin{3:end}); + end + if fast_likelihood_evaluation_for_rejection + theta_struct.fval=Z-10; + theta_struct.params=theta; + fxl = -feval(objective_function,theta_struct,varargin{:}); + else + fxl = -feval(objective_function,theta,varargin{:}); + end neval(it) = neval(it) + 1; if (fxl <= Z) break @@ -119,7 +153,17 @@ for it=1:npar while(R < XUB) xsim = R; theta = theta0+xsim*V1(:,it); - fxr = -feval(objective_function,theta,varargin{:}); + if endo_init_state + theta1 = theta; + [theta, icheck]=set_init_state(theta1,varargin{3:end}); + end + if fast_likelihood_evaluation_for_rejection + theta_struct.fval=Z-10; + theta_struct.params=theta; + fxr = -feval(objective_function,theta_struct,varargin{:}); + else + fxr = -feval(objective_function,theta,varargin{:}); + end neval(it) = neval(it) + 1; if (fxr <= Z) break @@ -134,7 +178,17 @@ for it=1:npar u = rand(1,1); xsim = L + u*(R - L); theta = theta0+xsim*V1(:,it); - fxsim = -feval(objective_function,theta,varargin{:}); + if endo_init_state + theta1 = theta; + [theta, icheck]=set_init_state(theta1,varargin{3:end}); + end + if fast_likelihood_evaluation_for_rejection + theta_struct.fval=Z-10; + theta_struct.params=theta; + fxsim = -feval(objective_function,theta_struct,varargin{:}); + else + fxsim = -feval(objective_function,theta,varargin{:}); + end neval(it) = neval(it) + 1; if (xsim > xold) R = xsim; @@ -142,5 +196,10 @@ for it=1:npar L = xsim; end end + if endo_init_state && icheck %(icheck || islow) + do_nothing=true; + [theta, fxsim, ~, M_] = draw_init_state_from_smoother([false 1],sampler_options,theta,fxsim,thetaprior,varargin{:}); + end + save([varargin{4}.dname filesep 'metropolis/slice_iter_info_' fname],'neval','it','theta','fxsim') end end diff --git a/matlab/estimation/set_init_state.m b/matlab/estimation/set_init_state.m new file mode 100644 index 0000000000000000000000000000000000000000..4b1512c74ab5562954fd3bf4e84c166dfb5e7cb7 --- /dev/null +++ b/matlab/estimation/set_init_state.m @@ -0,0 +1,116 @@ +function [xparam1, icheck] = set_init_state(xparam1, options_,M_,estim_params_,bayestopt_,BoundsInfo,dr, endo_steady_state, exo_steady_state, exo_det_steady_state) + +% Computes the endogenous log prior addition to the initial prior +% +% INPUTS +% xparam1 [double] n vector of estimated params +% bayestopt_ [structure] +% dr [structure] +% M_ [structure] +% +% OUTPUTS +% xparam1 [double] n vector of estimated params +% icheck [logical] flag for the need to update xparam1 + +% Copyright © 2024 Dynare Team +% +% This file is part of Dynare. +% +% Dynare is free software: you can redistribute it and/or modify +% it under the terms of the GNU General Public License as published by +% the Free Software Foundation, either version 3 of the License, or +% (at your option) any later version. +% +% Dynare is distributed in the hope that it will be useful, +% but WITHOUT ANY WARRANTY; without even the implied warranty of +% MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +% GNU General Public License for more details. +% +% You should have received a copy of the GNU General Public License +% along with Dynare. If not, see <https://www.gnu.org/licenses/>. + +% [Pstar, info]=get_pstar(xparam1,options_,varargin{4:end}); +icheck=false; + +filter_initial_state=M_.filter_initial_state; +M_.filter_initial_state=[]; +options_.lik_init=1; +options_.init_state_endogenous_prior=false; +[Pstar, info] = get_pstar(xparam1,options_,M_,estim_params_,bayestopt_,BoundsInfo,dr, endo_steady_state, exo_steady_state, exo_det_steady_state); +if info(1) + return +end +M_.filter_initial_state=filter_initial_state; + +[UP,XP,WP] = svd(0.5*(Pstar(bayestopt_.mf0,bayestopt_.mf0)+Pstar(bayestopt_.mf0,bayestopt_.mf0)')); +isp = find(diag(XP)>options_.kalman_tol); +isn = find(diag(XP)<=options_.kalman_tol); +UPN = UP(:,isn); +[~,im]=max(abs(UPN)); +if length(unique(im))<length(im) + in=im*nan; + for k=1:length(isn) + [~, tmp] = max(abs(UPN(:,k))); + if not(ismember(tmp,in)) + in(k) = tmp; + else + [~, tmp] = sort(abs(UPN(:,k))); + not_found = true; + offset=0; + while not_found + offset=offset+1; + if not(ismember(tmp(end-offset),in)) + in(k) = tmp(end-offset); + not_found = false; + end + end + end + end + im=in; +else + do_nothing=true; +end + +a = get_init_state(zeros(size(Pstar,1)),xparam1,bayestopt_,estim_params_,dr,M_,options_); + +mycheck =max(abs(UPN'*a(dr.restrict_var_list(bayestopt_.mf0)))); +if mycheck>options_.kalman_tol + icheck=true; + + nstates = length(M_.state_var); + + % check that state draws are in the null space of prior states 0|0 + in=setdiff(1:nstates,im); + + a1=a; + % a1(bayestopt_.mf0(im))=-UPN(im,:)'\(UPN(in,:)'*a(bayestopt_.mf0(in))); + a1(dr.restrict_var_list(bayestopt_.mf0(im)))=-UPN(im,:)'\(UPN(in,:)'*a(dr.restrict_var_list(bayestopt_.mf0(in)))); + % alphahat01 = zeros(M_.endo_nbr,1); + % alphahat01(dr.restrict_var_list) = a1; + alphahat01 = a1(dr.inv_order_var); %declaration order + + % map params associated to init states + pvec=[]; + for ii=1:size(M_.filter_initial_state,1) + if ~isempty(M_.filter_initial_state{ii,1}) + tmp1 = strrep(M_.filter_initial_state{ii,2},');',''); + tmp1 = strrep(tmp1,'M_.params(',''); + pvec = [pvec eval(tmp1)]; + end + end + [~,~,IB] = intersect(M_.param_names(pvec),bayestopt_.name,'stable'); + + for ii=1:size(M_.filter_initial_state,1) + if ~isempty(M_.filter_initial_state{ii,1}) + if options_.loglinear && ~options_.logged_steady_state + eval([strrep(M_.filter_initial_state{ii,2},';','') ' = exp(log(dr.ys(ii))+alphahat01(ii));']); + elseif ~options_.loglinear && ~options_.logged_steady_state + eval([strrep(M_.filter_initial_state{ii,2},';','') '= dr.ys(ii)+alphahat01(ii);']) + else + error('The steady state is logged. This should not happen. Please contact the developers') + end + end + end + % xparam0=xparam1; + xparam1(IB) = M_.params(pvec); +end diff --git a/matlab/estimation/slice_sampler.m b/matlab/estimation/slice_sampler.m index 326df5ff0d227c687df005c0876a6186a528e540..7cf0db32c6b23a90084de0d7ac79eac428e6d91a 100644 --- a/matlab/estimation/slice_sampler.m +++ b/matlab/estimation/slice_sampler.m @@ -41,8 +41,76 @@ function [theta, fxsim, neval] = slice_sampler(objective_function,theta,thetapri % You should have received a copy of the GNU General Public License % along with Dynare. If not, see <https://www.gnu.org/licenses/>. +persistent try_prior_draws sto_block + +% % % fname0=fname; +fname = int2str(sampler_options.curr_block); + +endo_init_state = false; +draw_endo_init_state_from_smoother=false; +draw_endo_init_state_with_rotated_slice = false; +if isfield(sampler_options,'draw_init_state_with_rotated_slice') && sampler_options.draw_init_state_with_rotated_slice + endo_init_state = true; + draw_endo_init_state_with_rotated_slice = true; +end +if isfield(sampler_options,'draw_init_state_from_smoother') && sampler_options.draw_init_state_from_smoother + endo_init_state = true; + draw_endo_init_state_from_smoother=true; +end +thetaprior0=thetaprior; +if endo_init_state + [IB, IS, IP] = get_init_state_estim_params(varargin{4}, varargin{6}, varargin{8}); + thetaprior(IB,1) = theta(IB); + thetaprior(IB,2) = theta(IB); +end + if sampler_options.rotated %&& ~isempty(sampler_options.V1), + sampler_options.endo_init_state.status = endo_init_state; + if endo_init_state + do_nothing = true; + % draw only params first! + [V1, D]=eig(sampler_options.invhess(IP,IP)); + sampler_options.WR=sqrt(diag(D))*3; + sampler_options.V1(:,:)=0; + sampler_options.V1(:,IB)=[]; + sampler_options.V1(IP,IP)=V1; + sampler_options.endo_init_state.IB = IB; + sampler_options.endo_init_state.IP = IP; + end [theta, fxsim, neval] = rotated_slice_sampler(objective_function,theta,thetaprior,sampler_options,varargin{:}); + if endo_init_state + % draw initial states + do_nothing=true; + if draw_endo_init_state_from_smoother + [theta, fxsim, ~, M_] = draw_init_state_from_smoother([false 5],sampler_options,theta,fxsim,thetaprior,varargin{:}); + elseif draw_endo_init_state_with_rotated_slice + [V, D]=get_init_state_prior(theta,varargin{3:end}); + % take eigenvectors of state priors and set zero wieghts for other + % params + nslice = size(V,2); + V=V(IS,:); + V1 = zeros(length(theta),size(V,2)); + V1(IB,:) = V; + sampler_options.V1=V1; + stderr = sqrt(diag(D)); + sampler_options.WR=stderr*3; + for k=1:nslice + bounds.lb(k) = norminv(1e-10, 0, stderr(k)); + bounds.ub(k) = norminv(1-1e-10, 0, stderr(k)); + end + sampler_options.rthetaprior=[bounds.lb(:) bounds.ub(:)]; + % here params are fixed, so no need to account for change in + % state prior! + sampler_options.endo_init_state.status = false; + % sampler_options.WR=sampler_options.initial_step_size*(bounds.ub-bounds.lb); + [theta, fxsim, neval1] = rotated_slice_sampler(objective_function,theta,thetaprior,sampler_options,varargin{:}); + [~, icheck]=set_init_state(theta,varargin{3:end}); + if icheck + do_nothing = true; + end + neval(IB(1:nslice)) = neval1(1:nslice); + end + end if isempty(sampler_options.mode) % jumping return else @@ -50,26 +118,90 @@ if sampler_options.rotated %&& ~isempty(sampler_options.V1), end end +fast_likelihood_evaluation_for_rejection = false; +if isfield(sampler_options,'fast_likelihood_evaluation_for_rejection') && sampler_options.fast_likelihood_evaluation_for_rejection + fast_likelihood_evaluation_for_rejection = true; +end + +use_prior_draws = false; +if isfield(sampler_options,'use_prior_draws') && sampler_options.use_prior_draws + use_prior_draws = true; + if isempty(try_prior_draws) || isempty(sto_block) || sto_block~=sampler_options.curr_block + try_prior_draws=true; + end +end + +if isempty(sto_block) || sto_block~=sampler_options.curr_block + sto_block = sampler_options.curr_block; +end + theta=theta(:); npar = length(theta); W1 = sampler_options.W1; neval = zeros(npar,1); -% % % fname0=fname; -fname = [ int2str(sampler_options.curr_block)]; - Prior = dprior(varargin{6},varargin{3}.prior_trunc); +if use_prior_draws && try_prior_draws + fxsim = evalin('caller','last_posterior'); + Z1 = fxsim + log(rand(1,1)); + ilogpo2=-inf; + nattempts=0; + while ilogpo2<Z1 && nattempts<10 + nattempts=nattempts+1; + + + validate=false; + while not(validate) + candidate = Prior.draw(); + + if all(candidate >= thetaprior0(:,1)) && all(candidate <= thetaprior0(:,2)) + if endo_init_state + init = true; + [candidate, ~, ~, M_] = draw_init_state_from_smoother(init,sampler_options,candidate,nan,thetaprior,varargin{:}); + end + if fast_likelihood_evaluation_for_rejection + theta_struct.fval=Z1-10; + theta_struct.params=candidate; + itest = -feval(objective_function,theta_struct,varargin{:}); + else + itest = -feval(objective_function,candidate,varargin{:}); + end + if isfinite(itest) + validate=true; + end + + end + end + if itest>ilogpo2 + ilogpo2= itest; + best_candidate = candidate; + end + end + if ilogpo2>=Z1 + theta= best_candidate; + fxsim = ilogpo2; + else + try_prior_draws = false; + end +end + + it=0; +islow=false(npar,1); while it<npar it=it+1; neval(it) = 0; W = W1(it); xold = theta(it); + theta0=theta; % XLB = thetaprior(3); % XUB = thetaprior(4); XLB = thetaprior(it,1); XUB = thetaprior(it,2); + if XLB==XUB + continue + end % ------------------------------------------------------- @@ -102,13 +234,32 @@ while it<npar % STEPPING-OUT PROCEDURE % ------------------------------------------------------------- u = rand(1,1); + %L = max(W*XLB+(1-W)*xold,xold-W*u); + %R = min(W*XUB+(1-W)*xold,L+W); L = max(XLB,xold-W*u); R = min(XUB,L+W); mytxt{it,1} = ''; while(L > XLB) xsim = L; theta(it) = xsim; - fxl = -feval(objective_function,theta,varargin{:}); + if endo_init_state + theta0(it) = xsim; + [theta, icheck]=set_init_state(theta0,varargin{3:end}); + end + if fast_likelihood_evaluation_for_rejection + theta_struct.fval=Z-10; + theta_struct.params=theta; + fxl = -feval(objective_function,theta_struct,varargin{:}); + % if (fxl >= Z) + % fxl1 = -feval(objective_function,theta,varargin{:}); + % if fxl1~=fxl + % do_nothing = true; + % end + % fxl = fxl1; + % end + else + fxl = -feval(objective_function,theta,varargin{:}); + end neval(it) = neval(it) + 1; if (fxl <= Z) break @@ -118,6 +269,10 @@ while it<npar L=XLB; xsim = L; theta(it) = xsim; + if endo_init_state + theta0(it) = xsim; + [theta, icheck]=set_init_state(theta0,varargin{3:end}); + end fxl = -feval(objective_function,theta,varargin{:}); icount = 0; while (isinf(fxl) || isnan(fxl)) && icount<300 @@ -125,6 +280,10 @@ while it<npar L=L+sqrt(eps); xsim = L; theta(it) = xsim; + if endo_init_state + theta0(it) = xsim; + [theta, icheck]=set_init_state(theta0,varargin{3:end}); + end fxl = -feval(objective_function,theta,varargin{:}); end mytxt{it,1} = sprintf('Getting L for [%s] is taking too long.', varargin{6}.name{it}); @@ -137,7 +296,24 @@ while it<npar while(R < XUB) xsim = R; theta(it) = xsim; - fxr = -feval(objective_function,theta,varargin{:}); + if endo_init_state + theta0(it) = xsim; + [theta, icheck]=set_init_state(theta0,varargin{3:end}); + end + if fast_likelihood_evaluation_for_rejection + theta_struct.fval=Z-10; + theta_struct.params=theta; + fxr = -feval(objective_function,theta_struct,varargin{:}); + % if (fxr >= Z) + % fxr1 = -feval(objective_function,theta,varargin{:}); + % if fxr1~=fxr + % do_nothing = true; + % end + % fxr = fxr1; + % end + else + fxr = -feval(objective_function,theta,varargin{:}); + end neval(it) = neval(it) + 1; if (fxr <= Z) break @@ -147,6 +323,10 @@ while it<npar R=XUB; xsim = R; theta(it) = xsim; + if endo_init_state + theta0(it) = xsim; + [theta, icheck]=set_init_state(theta0,varargin{3:end}); + end fxr = -feval(objective_function,theta,varargin{:}); icount = 0; while (isinf(fxr) || isnan(fxr)) && icount<300 @@ -154,6 +334,10 @@ while it<npar R=R-sqrt(eps); xsim = R; theta(it) = xsim; + if endo_init_state + theta0(it) = xsim; + [theta, icheck]=set_init_state(theta0,varargin{3:end}); + end fxr = -feval(objective_function,theta,varargin{:}); end mytxt{it,2} = sprintf('Getting R for [%s] is taking too long.', varargin{6}.name{it}); @@ -170,7 +354,25 @@ while it<npar u = rand(1,1); xsim = L + u*(R - L); theta(it) = xsim; - fxsim = -feval(objective_function,theta,varargin{:}); + if endo_init_state + theta0(it) = xsim; + [theta, icheck]=set_init_state(theta0,varargin{3:end}); + end + if fast_likelihood_evaluation_for_rejection + theta_struct.fval=Z-10; + theta_struct.params=theta; + fxsim = -feval(objective_function,theta_struct,varargin{:}); + % % % cross check + % % if (fxsim >= Z) + % % fxsim1 = -feval(objective_function,theta,varargin{:}); + % % if fxsim1~=fxsim + % % do_nothing = true; + % % end + % % fxsim = fxsim1; + % % end + else + fxsim = -feval(objective_function,theta,varargin{:}); + end neval(it) = neval(it) + 1; if (xsim > xold) R = xsim; @@ -180,17 +382,96 @@ while it<npar if (R-L)<1.e-6 %neval(it)>(30+neval2) fprintf('The sampling for parameter [%s] is taking too long as the sampling set is too tight. Check the prior.\n', varargin{6}.name{it}) mytxt{it,3} = sprintf('Sampling [%s] is taking too long.', varargin{6}.name{it}); - save([varargin{4}.dname filesep 'metropolis/slice_iter_info_' fname],'mytxt','neval','it') + % save([varargin{4}.dname filesep 'metropolis/slice_iter_info_' fname],'mytxt','neval','it') + islow(it)=true; + theta(it) = xold; + fxsim = fxold; break end end - save([varargin{4}.dname filesep 'metropolis/slice_iter_info_' fname],'mytxt','neval','it','theta','fxsim') if isinf(fxsim) || isnan(fxsim) theta(it) = xold; fxsim = fxold; disp('SLICE: posterior density is infinite. Reset values at initial ones.') end + if endo_init_state && icheck %(icheck || islow) + do_nothing=true; + [theta, fxsim, ~, M_] = draw_init_state_from_smoother([false 1],sampler_options,theta,fxsim,thetaprior,varargin{:}); + end + if isinf(fxsim) || isnan(fxsim) + theta(it) = xold; + fxsim = fxold; + disp('SLICE: posterior density is infinite. Reset values at initial ones.') + end + save([varargin{4}.dname filesep 'metropolis/slice_iter_info_' fname],'mytxt','neval','it','theta','fxsim') +end + +if any(islow) + fxsim = -feval(objective_function,theta,varargin{:}); + Z1 = fxsim + log(rand(1,1)); + ilogpo2=-inf; + nattempts=0; + while ilogpo2<Z1 && nattempts<10 + nattempts=nattempts+1; + + + validate=false; + while not(validate) + candidate = theta; + my_candidate = Prior.draw(); + candidate(islow) = my_candidate(islow); + + if all(candidate(islow) >= thetaprior(islow,1)) && all(candidate(islow) <= thetaprior(islow,2)) + if endo_init_state + init = true; + [candidate, ~, ~, M_] = draw_init_state_from_smoother(init,sampler_options,candidate,nan,thetaprior,varargin{:}); + end + itest = -feval(objective_function,candidate,varargin{:}); + if isfinite(itest) + validate=true; + end + + end + end + if itest>ilogpo2 + ilogpo2= itest; + best_candidate = candidate; + end + end + theta= best_candidate; + fxsim = ilogpo2; +end +if endo_init_state + % draw initial states + do_nothing=true; + if draw_endo_init_state_from_smoother + [theta, fxsim, ~, M_] = draw_init_state_from_smoother([false 5],sampler_options,theta,fxsim,thetaprior,varargin{:}); + elseif draw_endo_init_state_with_rotated_slice + [V, D]=get_init_state_prior(theta,varargin{3:end}); + % take eigenvectors of state priors and set zero wieghts for other + % params + nslice = size(V,2); + V=V(IS,:); + V1 = zeros(length(theta),size(V,2)); + V1(IB,:) = V; + sampler_options.V1=V1; + stderr = sqrt(diag(D)); + sampler_options.WR=stderr*3; + for k=1:nslice + bounds.lb(k) = norminv(1e-10, 0, stderr(k)); + bounds.ub(k) = norminv(1-1e-10, 0, stderr(k)); + end + sampler_options.rthetaprior=[bounds.lb(:) bounds.ub(:)]; + % sampler_options.WR=sampler_options.initial_step_size*(bounds.ub-bounds.lb); + [theta, fxsim, neval1] = rotated_slice_sampler(objective_function,theta,thetaprior,sampler_options,varargin{:}); + [~, icheck]=set_init_state(theta,varargin{3:end}); + if icheck + do_nothing = true; + end + neval(IB(1:nslice)) = neval1(1:nslice); + end + save([varargin{4}.dname filesep 'metropolis/slice_iter_info_' fname],'mytxt','neval','it','theta','fxsim') end if sampler_options.rotated && ~isempty(sampler_options.mode) % jumping diff --git a/matlab/kalman/DsgeSmoother.m b/matlab/kalman/DsgeSmoother.m index c84b800286e301ff5a65c02db894c4c4c5f723e9..61d8e774c3497e0def95c46f9c34ab4c28265442 100644 --- a/matlab/kalman/DsgeSmoother.m +++ b/matlab/kalman/DsgeSmoother.m @@ -263,8 +263,17 @@ end if kalman_algo == 1 || kalman_algo == 3 a_initial = zeros(np,1); - a_initial=set_Kalman_smoother_starting_values(a_initial,M_,oo_,options_); - a_initial=T*a_initial; %set state prediction for first Kalman step; + if options_.smoother_redux + a_initial=set_Kalman_starting_values(a_initial,M_,oo_.dr,options_,bayestopt_); + else + a_initial=set_Kalman_smoother_starting_values(a_initial,M_,oo_,options_); + end + if options_.lik_init==2 && options_.Harvey_scale_factor==0 + a_initial=[a_initial T*a_initial]; %set state prediction for first Kalman step; + Pstar = cat(3,Pstar*0, R1*Q*R1'); + else + a_initial=T*a_initial; %set state prediction for first Kalman step; + end [alphahat,epsilonhat,etahat,ahat,P,aK,PK,decomp,state_uncertainty, aahat, eehat, d, alphahat0, aalphahat0, state_uncertainty0] = missing_DiffuseKalmanSmootherH1_Z(a_initial,ST, ... Z,R1,Q,H,Pinf,Pstar, ... data1,vobs,np,smpl,data_index, ... @@ -319,8 +328,20 @@ if kalman_algo == 2 || kalman_algo == 4 end a_initial = zeros(np,1); - a_initial=set_Kalman_smoother_starting_values(a_initial,M_,oo_,options_); - a_initial=ST*a_initial; %set state prediction for first Kalman step; + if options_.smoother_redux + a_initial=set_Kalman_starting_values(a_initial,M_,oo_.dr,options_,bayestopt_); + else + a_initial=set_Kalman_smoother_starting_values(a_initial,M_,oo_,options_); + end + if not(options_.occbin.smoother.status) % && options_.occbin.smoother.first_period_occbin_update==1) + if options_.lik_init==2 && options_.Harvey_scale_factor==0 + a_initial=[a_initial ST*a_initial]; %set state prediction for first Kalman step; + Pstar = cat(3,Pstar, R1*Q*R1'); + else + a_initial=ST*a_initial; %set state prediction for first Kalman step; + end + end + [alphahat,epsilonhat,etahat,ahat,P,aK,PK,decomp,state_uncertainty, aahat, eehat, d, alphahat0, aalphahat0, state_uncertainty0, regimes_,TT,RR,CC,TTx,RRx,CCx] = missing_DiffuseKalmanSmootherH3_Z(a_initial,ST, ... Z,R1,Q,diag(H), ... Pinf,Pstar,data1,vobs,np,smpl,data_index, ... @@ -368,6 +389,10 @@ else end if options_.occbin.smoother.status + if isempty(alphahat0) + % something went wrong + return + end % reconstruct occbin smoother if length_varargin>0 % sequence of regimes is provided in input @@ -728,3 +753,31 @@ if isfield(M_,'filter_initial_state') && ~isempty(M_.filter_initial_state) end end +function a=set_Kalman_starting_values(a,M_,dr,options_,bayestopt_) +% function a=set_Kalman_starting_values(a,M_,dr,options_,bayestopt_) +% Sets initial states guess for Kalman filter/smoother based on M_.filter_initial_state +% +% INPUTS +% o a [double] (p*1) vector of states +% o M_ [structure] decribing the model +% o dr [structure] storing the decision rules +% o options_ [structure] describing the options +% o bayestopt_ [structure] describing the priors +% +% OUTPUTS +% o a [double] (p*1) vector of set initial states + +if isfield(M_,'filter_initial_state') && ~isempty(M_.filter_initial_state) + state_indices=dr.order_var(dr.restrict_var_list(bayestopt_.mf0)); + for ii=1:size(state_indices,1) + if ~isempty(M_.filter_initial_state{state_indices(ii),1}) + if options_.loglinear && ~options_.logged_steady_state + a(bayestopt_.mf0(ii)) = log(eval(M_.filter_initial_state{state_indices(ii),2})) - log(dr.ys(state_indices(ii))); + elseif ~options_.loglinear && ~options_.logged_steady_state + a(bayestopt_.mf0(ii)) = eval(M_.filter_initial_state{state_indices(ii),2}) - dr.ys(state_indices(ii)); + else + error('The steady state is logged. This should not happen. Please contact the developers') + end + end + end +end diff --git a/matlab/kalman/evaluate_smoother.m b/matlab/kalman/evaluate_smoother.m index f093701df8e428693db326b38b37710e69bc5026..4ba6859f76bae13b59028301eb63f937935f0962 100644 --- a/matlab/kalman/evaluate_smoother.m +++ b/matlab/kalman/evaluate_smoother.m @@ -118,16 +118,16 @@ if options_.occbin.smoother.status bayestopt_.mf = bayestopt_.smoother_var_list(bayestopt_.smoother_mf); end else - [atT,innov,measurement_error,updated_variables,ys,trend_coeff,aK,~,~,P,PK,decomp,Trend,state_uncertainty,oo_,bayestopt_] = ... + [atT,innov,measurement_error,updated_variables,ys,trend_coeff,aK,~,~,P,PK,decomp,Trend,state_uncertainty,oo_,bayestopt_,alphahat0,state_uncertainty0] = ... occbin.DSGE_smoother(parameters,dataset_.nobs,transpose(dataset_.data),dataset_info.missing.aindex,dataset_info.missing.state,M_,oo_,options_,bayestopt_,estim_params_,dataset_,dataset_info); end else - [atT,innov,measurement_error,updated_variables,ys,trend_coeff,aK,~,~,P,PK,decomp,Trend,state_uncertainty,oo_,bayestopt_] = ... + [atT,innov,measurement_error,updated_variables,ys,trend_coeff,aK,~,~,P,PK,decomp,Trend,state_uncertainty,oo_,bayestopt_,alphahat0,state_uncertainty0] = ... DsgeSmoother(parameters,dataset_.nobs,transpose(dataset_.data),dataset_info.missing.aindex,dataset_info.missing.state,M_,oo_,options_,bayestopt_,estim_params_); end if ~(options_.occbin.smoother.status && options_.occbin.smoother.inversion_filter) if ~options_.occbin.smoother.status || (options_.occbin.smoother.status && oo_.occbin.smoother.error_flag==0) - [oo_]=store_smoother_results(M_,oo_,options_,bayestopt_,dataset_,dataset_info,atT,innov,measurement_error,updated_variables,ys,trend_coeff,aK,P,PK,decomp,Trend,state_uncertainty); + [oo_]=store_smoother_results(M_,oo_,options_,bayestopt_,dataset_,dataset_info,atT,innov,measurement_error,updated_variables,ys,trend_coeff,aK,P,PK,decomp,Trend,state_uncertainty,alphahat0,state_uncertainty0); end else if ~oo_.occbin.smoother.error_flag diff --git a/matlab/kalman/likelihood/kalman_filter.m b/matlab/kalman/likelihood/kalman_filter.m index acafbff267c5645e476563546b4dab6e731ab8a7..7b9b6cd2f1963e978069f55956e4639e286508af 100644 --- a/matlab/kalman/likelihood/kalman_filter.m +++ b/matlab/kalman/likelihood/kalman_filter.m @@ -167,6 +167,9 @@ end rescale_prediction_error_covariance0=rescale_prediction_error_covariance; while notsteady && t<=last s = t-start+1; + if t==last + do_noting=true; + end if isqvec QQ = R*Qvec(:,:,t+1)*transpose(R); end diff --git a/matlab/kalman/likelihood/missing_observations_kalman_filter.m b/matlab/kalman/likelihood/missing_observations_kalman_filter.m index 348daa46b35f7e39579fd5db66d05a98e4b55205..6ad0447285e4039b277cee43c6d619277ff7dd2c 100644 --- a/matlab/kalman/likelihood/missing_observations_kalman_filter.m +++ b/matlab/kalman/likelihood/missing_observations_kalman_filter.m @@ -73,6 +73,9 @@ if isequal(H,0) end P=0.5*(P+P'); +[LastSeeds.Unifor, LastSeeds.Normal] = get_dynare_random_generator_state(); +% Set seed for randn(). +set_dynare_seed('default'); % Get sample size. smpl = last-start+1; @@ -152,6 +155,18 @@ else C=0; end +check_rejection = false; +if occbin_.status && isfield(options_,'likelihood_base_value') && not(isempty(options_.likelihood_base_value)) + % compute upper bound data density + check_rejection = true; + likUB = lik; + if isqvec + likUB = max_data_density(likUB, t, start, last, occbin_, options_, RR, TT, P, Q, R, T, H, data_index, Zflag, Z, kalman_tol, rescale_prediction_error_covariance, no_more_missing_observations, riccati_tol, isqvec, Qvec); + else + likUB = max_data_density(likUB, t, start, last, occbin_, options_, RR, TT, P, Q, R, T, H, data_index, Zflag, Z, kalman_tol, rescale_prediction_error_covariance, no_more_missing_observations, riccati_tol, isqvec); + end +end + while notsteady && t<=last if occbin_.status a1(:,t) = a; @@ -173,7 +188,7 @@ while notsteady && t<=last QQ = R*Qvec(:,:,t+1)*transpose(R); end if isempty(d_index) - a = T*a; + a = T*a+C; P = T*P*transpose(T)+QQ; else % Compute the prediction error and its variance @@ -206,6 +221,7 @@ while notsteady && t<=last if badly_conditioned_F && (~occbin_.status || (occbin_.status && t<first_period_occbin_update)) % if ~all(abs(F(:))<kalman_tol), then use univariate filter, otherwise this is a % pathological case and the draw is discarded + set_dynare_random_generator_state(LastSeeds.Unifor, LastSeeds.Normal); return else F_singular = false; @@ -248,7 +264,9 @@ while notsteady && t<=last end end if occbin_.status && t>=first_period_occbin_update - + if t==39 + do_nothing= true; + end occbin_options.opts_simul.waitbar=0; if t==1 if isqvec @@ -268,20 +286,55 @@ while notsteady && t<=last CC01 = zeros(size(CC,1),2); CC01(:,2) = CC(:,1); % insert here kalman update engine - [ax, a1x, Px, P1x, vx, Tx, Rx, Cx, regx, info, M_, likx] = occbin.kalman_update_engine(a00, a10, P00, P10, t, data_index0, Z, v0, Y0, H, Qt, T0, R0, TT01, RR01, CC01, regimes_(t:t+1), base_regime, d_index, M_, dr, endo_steady_state,exo_steady_state,exo_det_steady_state, options_, occbin_options); -% [ax, a1x, Px, P1x, vx, Tx, Rx, Cx, regimes_(t:t+2), info, M_, likx] = occbin.kalman_update_algo_1(a00, a10, P00, P10, data_index0, Z, v0, Y0, H, Qt, T0, R0, TT01, RR01, CC01, regimes_(t:t+1), M_, dr, endo_steady_state,exo_steady_state,exo_det_steady_state, options_, occbin_options); + [ax, a1x, Px, P1x, vx, Tx, Rx, Cx, regx, info, M_, likx, etahaty, alphahaty, V] = occbin.kalman_update_engine(a00, a10, P00, P10, t, data_index0, Z, v0, Y0, H, Qt, T0, R0, TT01, RR01, CC01, regimes_(t:t+1), base_regime, d_index, M_, dr, endo_steady_state,exo_steady_state,exo_det_steady_state, options_, occbin_options); + if t<options_.occbin.likelihood.first_period_binding_regime_allowed && not(isequal(regx(1),base_regime)) + likx= inf; + end + if options_.occbin.filter.init_periods_using_particles || options_.occbin.filter.particle.status + [StateVector,liky, updated_regimes, updated_sample, updated_mode, use_pkf_distribution] = ... + occbin.ppf_engine([], likx, a00, a10, P00, P10,ax,a1x,Px,P1x, alphahaty, etahaty, V, t, ... + data_index0,Z,v0,Y0,H,Qt,T0,R0,TT01,RR01,CC01,info,regimes_(t:t+1),base_regime,regx,isqvec, ... + M_,dr, endo_steady_state,exo_steady_state,exo_det_steady_state,options_,occbin_options); + likx = liky; + if use_pkf_distribution + options_.occbin.filter.init_periods_using_particles = false; + end + end else if isqvec Qt = Qvec(:,:,t-1:t+1); end % insert here kalman update engine - [ax, a1x, Px, P1x, vx, Tx, Rx, Cx, regx, info, M_, likx] = occbin.kalman_update_engine(a0(:,t-1),a1(:,t-1:t),P0(:,:,t-1),P1(:,:,t-1:t),t,data_index(t-1:t),Z,vv(:,t-1:t),Y(:,t-1:t),H,Qt,T0,R0,TT(:,:,t-1:t),RR(:,:,t-1:t),CC(:,t-1:t),regimes_(t:t+1),base_regime,d_index,M_,dr, endo_steady_state,exo_steady_state,exo_det_steady_state,options_,occbin_options); -% [ax, a1x, Px, P1x, vx, Tx, Rx, Cx, regimes_(t:t+2), info, M_, likx] = occbin.kalman_update_algo_1(a0(:,t-1),a1(:,t-1:t),P0(:,:,t-1),P1(:,:,t-1:t),data_index(t-1:t),Z,vv(:,t-1:t),Y(:,t-1:t),H,Qt,T0,R0,TT(:,:,t-1:t),RR(:,:,t-1:t),CC(:,t-1:t),regimes_(t:t+1),M_,dr,endo_steady_state,exo_steady_state,exo_det_steady_state,options_,occbin_options); + [ax, a1x, Px, P1x, vx, Tx, Rx, Cx, regx, info, M_, likx, etahaty, alphahaty, V] = occbin.kalman_update_engine(a0(:,t-1),a1(:,t-1:t),P0(:,:,t-1),P1(:,:,t-1:t),t,data_index(t-1:t),Z,vv(:,t-1:t),Y(:,t-1:t),H,Qt,T0,R0,TT(:,:,t-1:t),RR(:,:,t-1:t),CC(:,t-1:t),regimes_(t:t+1),base_regime,d_index,M_,dr, endo_steady_state,exo_steady_state,exo_det_steady_state,options_,occbin_options); + if t<options_.occbin.likelihood.first_period_binding_regime_allowed && not(isequal(regx(1),base_regime)) + likx= inf; + end + if options_.occbin.filter.init_periods_using_particles || options_.occbin.filter.particle.status + [StateVector,liky, updated_regimes, updated_sample, updated_mode, use_pkf_distribution, infoy] = ... + occbin.ppf_engine(StateVector, likx, a0(:,t-1),a1(:,t-1:t),P0(:,:,t-1),P1(:,:,t-1:t),ax,a1x,Px,P1x, alphahaty, etahaty,V,t, ... + data_index(t-1:t),Z,vv(:,t-1:t),Y(:,t-1:t),H,Qt,T0,R0,TT(:,:,t-1:t),RR(:,:,t-1:t),CC(:,t-1:t), ... + info,regimes_(t:t+1),base_regime,regx,isqvec, ... + M_,dr, endo_steady_state,exo_steady_state,exo_det_steady_state,options_,occbin_options); + if infoy==0 + likx = liky; + if use_pkf_distribution + options_.occbin.filter.init_periods_using_particles = false; + end + end + end end if info if options_.debug fprintf('\nmissing_observations_kalman_filter:PKF failed in period %u with: %s\n', t, get_error_message(info,options_)); end + set_dynare_random_generator_state(LastSeeds.Unifor, LastSeeds.Normal); + return + end + if isinf(likx) % lik is inf but info = 0 ! + if options_.debug + fprintf('\nmissing_observations_kalman_filter:PKF failed in period %u with: Inf in likelihood value\n', t); + end + set_dynare_random_generator_state(LastSeeds.Unifor, LastSeeds.Normal); return end if options_.occbin.likelihood.use_updated_regime @@ -300,6 +353,21 @@ while notsteady && t<=last P = Px(:,:,2); end + if check_rejection + likUB(s) = lik(s); + if presample>=diffuse_periods + LIKUB = -sum(0.5*likUB(1+presample-diffuse_periods:end)); + else + LIKUB = -sum(0.5*likUB); + end + if LIKUB < options_.likelihood_base_value + do_nothing = true; + t=last+1; + s = t-start; + lik=likUB; + break + end + end t = t+1; end @@ -321,3 +389,80 @@ if presample>=diffuse_periods else LIK = sum(lik); end + +set_dynare_random_generator_state(LastSeeds.Unifor, LastSeeds.Normal); + +end + +function lik = max_data_density(lik, t, start, last, occbin_, options_, RR, TT, P, Q, R, T, H, data_index, Zflag, Z, kalman_tol, rescale_prediction_error_covariance, no_more_missing_observations, riccati_tol, isqvec, Qvec) + +if t==1 && options_.lik_init==2 && options_.Harvey_scale_factor==0 && (options_.occbin.likelihood.status && options_.occbin.likelihood.first_period_occbin_update==1) + P = T*P*T' + R*Q*R'; +end + +oldK = Inf; +notsteady = 1; +if ~(isqvec) + QQ = R*Q*transpose(R); % Variance of R times the vector of structural innovations. +end + +while t<=last + % compute everything with base regime +% if occbin_.status +% R = RR(:,:,t+1); +% T = TT(:,:,t+1); +% if ~(isqvec) +% QQ = R*Q*transpose(R); % Variance of R times the vector of structural innovations. +% end +% end + s = t-start+1; + d_index = data_index{t}; + if isqvec + QQ = R*Qvec(:,:,t+1)*transpose(R); + end + if notsteady==0 + lik(s) = log_dF + length(d_index)*log(2*pi); + else + if isempty(d_index) + P = T*P*transpose(T)+QQ; + else + % Compute the prediction error and its variance + if Zflag + z = Z(d_index,:); + F = z*P*z' + H(d_index,d_index); + else + z = Z(d_index); + F = P(z,z) + H(d_index,d_index); + end + if rescale_prediction_error_covariance + sig=sqrt(diag(F)); + else + if rcond(F)<kalman_tol + sig=sqrt(diag(F)); + end + end + if rescale_prediction_error_covariance + log_dF = log(det(F./(sig*sig')))+2*sum(log(sig)); + iF = inv(F./(sig*sig'))./(sig*sig'); + else + log_dF = log(det(F)); + iF = inv(F); + end + lik(s) = log_dF + length(d_index)*log(2*pi); + if Zflag + K = P*z'*iF; + P = T*(P-K*z*P)*transpose(T)+QQ; + else + K = P(:,z)*iF; + P = T*(P-K*P(z,:))*transpose(T)+QQ; + end + if t>=no_more_missing_observations && ~isqvec + notsteady = max(abs(K(:)-oldK))>riccati_tol; + oldK = K(:); + end + end + end + t = t+1; +end + +end \ No newline at end of file diff --git a/matlab/kalman/missing_DiffuseKalmanSmootherH1_Z.m b/matlab/kalman/missing_DiffuseKalmanSmootherH1_Z.m index 4b23b4cc856d56ac988fb6b771e0840c82fe9ded..e273a7d605cf7cb34b43fceb5b44a3a7e418d298 100644 --- a/matlab/kalman/missing_DiffuseKalmanSmootherH1_Z.m +++ b/matlab/kalman/missing_DiffuseKalmanSmootherH1_Z.m @@ -83,10 +83,16 @@ function [alphahat,epsilonhat,etahat,atilde,P,aK,PK,decomp,V,aalphahat,eetahat,d d = 0; decomp = []; spinf = size(Pinf1); -spstar = size(Pstar1); +spstar = size(Pstar1(:,:,1)); v = zeros(pp,smpl); a = zeros(mm,smpl+1); -a(:,1) = a_initial; +if size(a_initial,2)>1 + a0 = a_initial(:,1); + a(:,1) = a_initial(:,2); +else + a0 = a_initial; + a(:,1) = a_initial; +end atilde = zeros(mm,smpl); aK = zeros(nk,mm,smpl+nk); if filter_covariance_flag @@ -106,7 +112,13 @@ Kstar = zeros(mm,pp,smpl); Kinf = zeros(mm,pp,smpl); P = zeros(mm,mm,smpl+1); Pstar = zeros(spstar(1),spstar(2),smpl+1); -Pstar(:,:,1) = Pstar1; +if size(Pstar1,3)>1 + P0 = Pstar1(:,:,1); + Pstar(:,:,1) = Pstar1(:,:,2); +else + P0 = Pstar1; + Pstar(:,:,1) = Pstar1; +end Pinf = zeros(spinf(1),spinf(2),smpl+1); Pinf(:,:,1) = Pinf1; rr = size(Q,1); @@ -358,8 +370,8 @@ while t>d+1 end if d==0 % get smoother in period t=0 - a0 = a(:,1); - P0 = P(:,:,1); +% a0 = a(:,1); +% P0 = P(:,:,1); L0=T; r0 = L0'*r(:,1); %compute r_{t-1}, DK (2012), eq. 4.38 with Z=0 alphahat0 = a0 + P0*r0; %DK (2012), eq. 4.35 diff --git a/matlab/kalman/missing_DiffuseKalmanSmootherH3_Z.m b/matlab/kalman/missing_DiffuseKalmanSmootherH3_Z.m index 366e415f4a8a51b728427a1d363dfde2abaa0cf7..c238bed36cbebe0caa30d338a42a72c58c492b1d 100644 --- a/matlab/kalman/missing_DiffuseKalmanSmootherH3_Z.m +++ b/matlab/kalman/missing_DiffuseKalmanSmootherH3_Z.m @@ -92,12 +92,19 @@ end d = 0; decomp = []; spinf = size(Pinf1); -spstar = size(Pstar1); +spstar = size(Pstar1(:,:,1)); v = zeros(pp,smpl); a = zeros(mm,smpl); a1 = zeros(mm,smpl+1); -a(:,1) = a_initial; -a1(:,1) = a_initial; +if size(a_initial,2)>1 + a(:,1) = a_initial(:,2); + a1(:,1) = a_initial(:,2); + ainit = a_initial(:,1); +else + a(:,1) = a_initial; + a1(:,1) = a_initial; + ainit = a1(:,1); +end aK = zeros(nk,mm,smpl+nk); Fstar = zeros(pp,smpl); @@ -114,7 +121,12 @@ else PK = []; end Pstar = zeros(spstar(1),spstar(2),smpl); -Pstar(:,:,1) = Pstar1; +Pinit = Pstar1(:,:,1); +if size(Pstar1,3)>1 + Pstar(:,:,1) = Pstar1(:,:,2); +else + Pstar(:,:,1) = Pstar1; +end Pinf = zeros(spinf(1),spinf(2),smpl); Pinf(:,:,1) = Pinf1; Pstar1 = Pstar; @@ -368,12 +380,13 @@ Pinf = Pinf(:,:,1:d); Pstar1 = Pstar1(:,:,1:d); Pinf1 = Pinf1(:,:,1:d); notsteady = 1; +lik=zeros(1,smpl); while notsteady && t<smpl t = t+1; - if t==1 - Pinit = P(:,:,1); - ainit = a1(:,1); - end +% if t==1 +% Pinit = P(:,:,1); +% ainit = a1(:,1); +% end a(:,t) = a1(:,t); P1(:,:,t) = P(:,:,t); di = data_index{t}'; @@ -414,6 +427,7 @@ while notsteady && t<smpl end if ~error_flag regimes_(t:t+2)=tmp; + lik(t)=likx; else varargout{1} = []; varargout{2} = []; diff --git a/matlab/kalman/save_display_classical_smoother_results.m b/matlab/kalman/save_display_classical_smoother_results.m index f7f2f981e710722c51b9f8ff466d92773f743cac..f45c8fcb41823657d5f5b6658b115d6539bdad1f 100644 --- a/matlab/kalman/save_display_classical_smoother_results.m +++ b/matlab/kalman/save_display_classical_smoother_results.m @@ -52,17 +52,17 @@ if options_.occbin.smoother.status && options_.occbin.smoother.inversion_filter end else if options_.occbin.smoother.status - [atT,innov,measurement_error,updated_variables,ys,trend_coeff,aK,~,~,P,PK,decomp,Trend,state_uncertainty,oo_,bayestopt_] = occbin.DSGE_smoother(xparam1,dataset_.nobs,transpose(dataset_.data),dataset_info.missing.aindex,dataset_info.missing.state,M_,oo_,options_,bayestopt_,estim_params_,dataset_,dataset_info); + [atT,innov,measurement_error,updated_variables,ys,trend_coeff,aK,~,~,P,PK,decomp,Trend,state_uncertainty,oo_,bayestopt_,a0T,state_uncertainty0] = occbin.DSGE_smoother(xparam1,dataset_.nobs,transpose(dataset_.data),dataset_info.missing.aindex,dataset_info.missing.state,M_,oo_,options_,bayestopt_,estim_params_,dataset_,dataset_info); if oo_.occbin.smoother.error_flag(1)==0 - oo_ = store_smoother_results(M_,oo_,options_,bayestopt_,dataset_,dataset_info,atT,innov,measurement_error,updated_variables,ys,trend_coeff,aK,P,PK,decomp,Trend,state_uncertainty); + oo_ = store_smoother_results(M_,oo_,options_,bayestopt_,dataset_,dataset_info,atT,innov,measurement_error,updated_variables,ys,trend_coeff,aK,P,PK,decomp,Trend,state_uncertainty,a0T,state_uncertainty0); else fprintf('\nOccbin: smoother did not succeed. No results will be written to oo_.\n') end else - [atT,innov,measurement_error,updated_variables,ys,trend_coeff,aK,~,~,P,PK,decomp,Trend,state_uncertainty,oo_,bayestopt_] = DsgeSmoother(xparam1,dataset_.nobs,transpose(dataset_.data),dataset_info.missing.aindex,dataset_info.missing.state,M_,oo_,options_,bayestopt_,estim_params_); - oo_ = store_smoother_results(M_,oo_,options_,bayestopt_,dataset_,dataset_info,atT,innov,measurement_error,updated_variables,ys,trend_coeff,aK,P,PK,decomp,Trend,state_uncertainty); + [atT,innov,measurement_error,updated_variables,ys,trend_coeff,aK,~,~,P,PK,decomp,Trend,state_uncertainty,oo_,bayestopt_,a0T,state_uncertainty0] = DsgeSmoother(xparam1,dataset_.nobs,transpose(dataset_.data),dataset_info.missing.aindex,dataset_info.missing.state,M_,oo_,options_,bayestopt_,estim_params_); + oo_ = store_smoother_results(M_,oo_,options_,bayestopt_,dataset_,dataset_info,atT,innov,measurement_error,updated_variables,ys,trend_coeff,aK,P,PK,decomp,Trend,state_uncertainty,a0T,state_uncertainty0); end - [oo_,yf]=store_smoother_results(M_,oo_,options_,bayestopt_,dataset_,dataset_info,atT,innov,measurement_error,updated_variables,ys,trend_coeff,aK,P,PK,decomp,Trend,state_uncertainty); + [oo_,yf]=store_smoother_results(M_,oo_,options_,bayestopt_,dataset_,dataset_info,atT,innov,measurement_error,updated_variables,ys,trend_coeff,aK,P,PK,decomp,Trend,state_uncertainty,a0T,state_uncertainty0); end if ~options_.nograph [nbplt,nr,nc,~,~,nstar] = pltorg(M_.exo_nbr); diff --git a/matlab/kalman/store_smoother_results.m b/matlab/kalman/store_smoother_results.m index 534f7ba7288514a6d82c1cc8300fdb7e9fbe8140..d4f8cbffa508ba969b417f2f7ce5055f4799467b 100644 --- a/matlab/kalman/store_smoother_results.m +++ b/matlab/kalman/store_smoother_results.m @@ -1,4 +1,4 @@ -function [oo_, yf]=store_smoother_results(M_,oo_,options_,bayestopt_,dataset_,dataset_info,atT,innov,measurement_error,updated_variables,ys,trend_coeff,aK,P,PK,decomp,Trend,state_uncertainty) +function [oo_, yf]=store_smoother_results(M_,oo_,options_,bayestopt_,dataset_,dataset_info,atT,innov,measurement_error,updated_variables,ys,trend_coeff,aK,P,PK,decomp,Trend,state_uncertainty,a0T,state_uncertainty0) % oo_=store_smoother_results(M_,oo_,options_,bayestopt_,dataset_,atT,innov,measurement_error,updated_variables,ys,trend_coeff,aK,P,PK,decomp,Trend) % Writes the smoother results into respective fields in oo_ % @@ -26,6 +26,9 @@ function [oo_, yf]=store_smoother_results(M_,oo_,options_,bayestopt_,dataset_,da % Trend [double] [nvarobs*T] matrix of trends in observables % state_uncertainty [double] (K,K,T) array, storing the uncertainty % about the smoothed state (decision-rule order) +% a0T [double] (m*1) matrix, smoothed endogenous variables (a_{0|T}) (decision-rule order) +% state_uncertainty [double] (K,K,1) array, storing the uncertainty +% about the smoothed state in t=0 (decision-rule order) % % Outputs: % oo_ [structure] storing the results: @@ -154,9 +157,6 @@ if options_.filter_covariance oo_.Smoother.Variance = P; end -if options_.smoothed_state_uncertainty - oo_.Smoother.State_uncertainty=state_uncertainty; -end %get indices of smoothed variables i_endo_in_bayestopt_smoother_varlist = bayestopt_.smoother_saved_var_list; i_endo_in_dr_matrices=bayestopt_.smoother_var_list(i_endo_in_bayestopt_smoother_varlist); @@ -197,6 +197,9 @@ for i_endo_in_bayestopt_smoother_varlist=bayestopt_.smoother_saved_var_list' constant_current_variable=repmat((ys(i_endo_declaration_order)),gend,1); end oo_.Smoother.Constant.(M_.endo_names{i_endo_declaration_order})=constant_current_variable; + if nargin>18 + oo_.Smoother.Init_State.(M_.endo_names{i_endo_declaration_order})=a0T(i_endo_in_dr)+constant_current_variable(1); + end oo_.SmoothedVariables.(M_.endo_names{i_endo_declaration_order})=atT(i_endo_in_dr,:)'+constant_current_variable; if ~isempty(options_.nk) && options_.nk > 0 % && ~((any(bayestopt_.pshape > 0) && options_.mh_replic) || (any(bayestopt_.pshape> 0) && options_.load_mh_file)) oo_.FilteredVariables.(M_.endo_names{i_endo_declaration_order})=squeeze(aK(1,i_endo_in_dr,2:end-(options_.nk-1)))+constant_current_variable; @@ -256,6 +259,8 @@ if options_.filter_covariance end if options_.smoothed_state_uncertainty oo_.Smoother.State_uncertainty(oo_.dr.order_var,oo_.dr.order_var,:)=state_uncertainty; + oo_.Smoother.Init_State_uncertainty(oo_.dr.order_var,oo_.dr.order_var)=state_uncertainty0; + oo_.Smoother.Init_State_Vector(:,1)=a0T(oo_.dr.inv_order_var)+ys; end %% get smoothed shocks