masterParallel.m 26.2 KB
Newer Older
Marco Ratto's avatar
Marco Ratto committed
1
function [fOutVar,nBlockPerCPU, totCPU] = masterParallel(Parallel,fBlock,nBlock,NamFileInput,fname,fInputVar,fGlobalVar,Parallel_info,initialize)
2 3 4 5 6 7
% PARALLEL CONTEXT
% This is the most important function for the management of DYNARE parallel
% computing.
% It is the top-level function called on the master computer when parallelizing a task.


Marco Ratto's avatar
Marco Ratto committed
8
% This function have two main computational startegy for manage the matlab worker (slave process).
9
% 0 Simple Close/Open Stategy:
Marco Ratto's avatar
Marco Ratto committed
10 11 12
% In this case the new matlab istances (slave process) are open when
% necessary and then closed. This can happen many times during the
% simulation of a model.
13 14

% 1 Alway Open Stategy:
Marco Ratto's avatar
Marco Ratto committed
15 16 17 18 19
% In this case we have a more sophisticated management of slave processes,
% which are no longer closed at the end of each job. The slave processes
% waits for a new job (if exist). If a slave do not receives a new job after a
% fixed time it is destroyed. This solution removes the computational
% time necessary to Open/Close new matlab istances.
20 21 22 23 24 25 26

% The first (point 0) is the default Strategy
% i.e.(Parallel_info.leaveSlaveOpen=0). This value can be changed by the
% user in xxx.mod file or it is changed by the programmer if it necessary to
% reduce the overall computational time. See for example the
% prior_posterior_statistics.m.

27 28 29
% The number of parallelized threads will be equal to (nBlock-fBlock+1).
%
% INPUTS
30 31
%  o Parallel [struct vector]   copy of options_.parallel
%  o fBlock [int]               index number of the first thread
32
%                              (between 1 and nBlock)
33 34
%  o nBlock [int]               index number of the last thread
%  o NamFileInput [cell array]  containins the list of input files to be
35 36 37 38
%                              copied in the working directory of remote slaves
%                              2 columns, as many lines as there are files
%                              - first column contains directory paths
%                              - second column contains filenames
39
%  o fname [string]             name of the function to be parallelized, and
40
%                              which will be run on the slaves
41
%  o fInputVar [struct]         structure containing local variables to be used
42
%                              by fName on the slaves
43
%  o fGlobalVar [struct]        structure containing global variables to be used
44 45 46
%                              by fName on the slaves
%
% OUTPUT
47
%  o fOutVar [struct vector]    result of the parallel computation, one
48
%                              struct per thread
49
%  o nBlockPerCPU [int vector]  for each CPU used, indicates the number of
50
%                              threads run on that CPU
51
%  o totCPU [int]               total number of CPU used (can be lower than
52 53 54
%                              the number of CPU declared in "Parallel", if
%                              the number of required threads is lower)

55
% Copyright (C) 2009-2010 Dynare Team
56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71
%
% This file is part of Dynare.
%
% Dynare is free software: you can redistribute it and/or modify
% it under the terms of the GNU General Public License as published by
% the Free Software Foundation, either version 3 of the License, or
% (at your option) any later version.
%
% Dynare is distributed in the hope that it will be useful,
% but WITHOUT ANY WARRANTY; without even the implied warranty of
% MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
% GNU General Public License for more details.
%
% You should have received a copy of the GNU General Public License
% along with Dynare.  If not, see <http://www.gnu.org/licenses/>.

72 73


Marco Ratto's avatar
Marco Ratto committed
74 75 76 77 78 79 80 81 82
% If islocal==0, create a new directory for remote computation.
% This directory is named using current data and time,
% is used only one time and then deleted.

persistent PRCDir
% PRCDir= Present Remote Computational Directory!

Strategy=Parallel_info.leaveSlaveOpen;

83
islocal = 1;
Marco Ratto's avatar
Marco Ratto committed
84
for j=1:length(Parallel),
85
    islocal=islocal*Parallel(j).Local;
Marco Ratto's avatar
Marco Ratto committed
86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102
end
if nargin>8 && initialize==1
    if islocal == 0,
        PRCDir=CreateTimeString();
        assignin('base','PRCDirTmp',PRCDir),
        evalin('base','options_.parallel_info.RemoteTmpFolder=PRCDirTmp;')
        evalin('base','clear PRCDirTmp,')
    else
        % Delete the traces (if exists) of last local section computations.
        if Strategy==1,
            mydelete(['slaveParallel_input*.mat']);
        end
    end
    return
end


103
for j=1:length(Parallel),
104
    if isempty(Parallel(j).MatlabPath),
105 106 107
        Parallel(j).MatlabPath = 'matlab';
    end
end
Marco Ratto's avatar
Marco Ratto committed
108 109 110 111 112 113 114

% if Strategy==0
%     disp('User Strategy Now Is Open/Close (0)');
% else
%     disp('User Strategy Now Is Always Open (1)');
% end

115
if Strategy==1
Marco Ratto's avatar
Marco Ratto committed
116
    totCPU=0;
117
end
118

Marco Ratto's avatar
Marco Ratto committed
119

120
% Determine my hostname and my working directory.
Marco Ratto's avatar
Marco Ratto committed
121

122 123
DyMo=pwd;
fInputVar.DyMo=DyMo;
ratto's avatar
ratto committed
124 125
if isunix || (~matlab_ver_less_than('7.4') && ismac) ,
    [tempo, MasterName]=system('hostname --fqdn');
Marco Ratto's avatar
Marco Ratto committed
126
else
127 128 129 130 131
    [tempo, MasterName]=system('hostname');
end
MasterName=deblank(MasterName);
fInputVar.MasterName = MasterName;

Marco Ratto's avatar
Marco Ratto committed
132 133
% keyboard;

134 135
% Save input data for use by the slaves.
switch Strategy
Marco Ratto's avatar
Marco Ratto committed
136 137 138 139 140 141 142 143
    case 0
        if exist('fGlobalVar'),
            save([fname,'_input.mat'],'fInputVar','fGlobalVar')
        else
            save([fname,'_input.mat'],'fInputVar')
        end
        save([fname,'_input.mat'],'Parallel','-append')
        
144
    case 1
Marco Ratto's avatar
Marco Ratto committed
145 146 147 148 149 150
        if exist('fGlobalVar'),
            save(['temp_input.mat'],'fInputVar','fGlobalVar')
        else
            save(['temp_input.mat'],'fInputVar')
        end
        save(['temp_input.mat'],'Parallel','-append')
151 152
end

153

154 155 156
% Determine the total number of available CPUs, and the number of threads
% to run on each CPU.

Marco Ratto's avatar
Marco Ratto committed
157 158 159
[nCPU, totCPU, nBlockPerCPU, totSlaves] = distributeJobs(Parallel, fBlock, nBlock);
offset0 = fBlock-1;

160

161
% Clean up remnants of previous runs.
162
mydelete(['comp_status_',fname,'*.mat'])
163
mydelete(['P_',fname,'*End.txt'])
Marco Ratto's avatar
Marco Ratto committed
164

165 166 167

% Create a shell script containing the commands to launch the required tasks on the slaves
fid = fopen('ConcurrentCommand1.bat','w+');
Marco Ratto's avatar
Marco Ratto committed
168 169 170 171 172 173 174 175 176


% Creo la directory in cui effettuare i calcoli in remoto ...
if isempty(PRCDir) && ~islocal,
    error('PRCDir not initialized!')
else
    dynareParallelMkDir(PRCDir,Parallel(1:totSlaves));
end

177 178
for j=1:totCPU,
    
179 180 181 182 183
    
    if Strategy==1
        command1 = ' ';
    end
    
184 185 186 187 188 189 190 191 192
    indPC=min(find(nCPU>=j));
    
    if indPC>1
        nCPU0 = nCPU(indPC-1);
    else
        nCPU0=0;
    end
    offset = sum(nBlockPerCPU(1:j-1))+offset0;
    
Marco Ratto's avatar
Marco Ratto committed
193 194
        
    % Creo un file che mi serve per sapere se la computazione di un
195
    % blocco parallelo (una core) è terminata oppure no!
Marco Ratto's avatar
Marco Ratto committed
196
    
197 198 199
    fid1=fopen(['P_',fname,'_',int2str(j),'End.txt'],'w+');
    fclose(fid1);
    
Marco Ratto's avatar
Marco Ratto committed
200 201 202 203 204 205 206
    if Strategy==1,
        
        fblck = offset+1;
        nblck = sum(nBlockPerCPU(1:j));
        save temp_input fblck nblck fname -append;
        copyfile('temp_input.mat',['slaveJob',int2str(j),'.mat'])
        if Parallel(indPC).Local ==0,
207 208
            fid1=fopen(['stayalive',int2str(j),'.txt'],'w+');
            fclose(fid1);
Marco Ratto's avatar
Marco Ratto committed
209 210 211 212 213 214
            dynareParallelSendFiles(['stayalive',int2str(j),'.txt'],PRCDir,Parallel(indPC));
            mydelete(['stayalive',int2str(j),'.txt']);
        end
        % Wait for possibly local alive CPU to start the new job or close by
        % internal criteria.
        pause(1);
215 216
        newInstance = 0;
        
Marco Ratto's avatar
Marco Ratto committed
217 218
        % Check if j CPU is already alive.        
        if isempty(dynareParallelDir(['P_slave_',int2str(j),'End.txt'],PRCDir,Parallel(indPC)));
219 220
            fid1=fopen(['P_slave_',int2str(j),'End.txt'],'w+');
            fclose(fid1);
Marco Ratto's avatar
Marco Ratto committed
221 222 223 224 225
            if Parallel(indPC).Local==0,
                dynareParallelSendFiles(['P_slave_',int2str(j),'End.txt'],PRCDir,Parallel(indPC));
                delete(['P_slave_',int2str(j),'End.txt']);
            end
            
226 227 228 229 230
            newInstance = 1;
            storeGlobalVars( ['slaveParallel_input',int2str(j)]);
            save( ['slaveParallel_input',int2str(j)],'Parallel','-append');
            % Prepare global vars for Slave.
        end
Marco Ratto's avatar
Marco Ratto committed
231
    else
232
        
233
        % Se la computazione è remota, lo creo in locale, lo copio directory remota (creata
Marco Ratto's avatar
Marco Ratto committed
234 235 236 237 238 239 240 241 242 243
        % sopra) e lo cancello in locale. Lo stesso vale per gli altri
        % dati.
        save( ['slaveParallel_input',int2str(j)],'Parallel');
        
        if Parallel(indPC).Local==0,
            dynareParallelSendFiles(['P_',fname,'_',int2str(j),'End.txt'],PRCDir,Parallel(indPC));
            delete(['P_',fname,'_',int2str(j),'End.txt']);
            
            dynareParallelSendFiles(['slaveParallel_input',int2str(j),'.mat'],PRCDir,Parallel(indPC));
            delete(['slaveParallel_input',int2str(j),'.mat']);
244 245
        
        end
Marco Ratto's avatar
Marco Ratto committed
246 247

    end
248
    
Marco Ratto's avatar
Marco Ratto committed
249 250 251 252 253 254 255 256 257
    %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
    % The following 'switch - case' code is the core of this function!
    switch Strategy
        case 0
            
            if Parallel(indPC).Local == 1, %Run on the local machine (localhost).
                
                if isunix || (~matlab_ver_less_than('7.4') && ismac),
                    if exist('OCTAVE_VERSION')
258
                        command1=['octave --eval "addpath(''',Parallel(indPC).DynarePath,'''), dynareroot = dynare_config(); fParallel(',int2str(offset+1),',',int2str(sum(nBlockPerCPU(1:j))),',',int2str(j),',',int2str(indPC),',''',fname,''')" &'];
Marco Ratto's avatar
Marco Ratto committed
259
                    else
260
                        command1=[Parallel(indPC).MatlabPath,' -nosplash -nodesktop -minimize -r "addpath(''',Parallel(indPC).DynarePath,'''), dynareroot = dynare_config(); fParallel(',int2str(offset+1),',',int2str(sum(nBlockPerCPU(1:j))),',',int2str(j),',',int2str(indPC),',''',fname,''')" &'];
Marco Ratto's avatar
Marco Ratto committed
261
                    end
262
                else
Marco Ratto's avatar
Marco Ratto committed
263
                    if exist('OCTAVE_VERSION')
264
                        command1=['start /B psexec -W ',DyMo, ' -a ',int2str(Parallel(indPC).NumCPU(j-nCPU0)),' -low  octave --eval "addpath(''',Parallel(indPC).DynarePath,'''), dynareroot = dynare_config(); fParallel(',int2str(offset+1),',',int2str(sum(nBlockPerCPU(1:j))),',',int2str(j),',',int2str(indPC),',''',fname,''')"'];
Marco Ratto's avatar
Marco Ratto committed
265
                    else
266
                        command1=['start /B psexec -W ',DyMo, ' -a ',int2str(Parallel(indPC).NumCPU(j-nCPU0)),' -low  ',Parallel(indPC).MatlabPath,' -nosplash -nodesktop -minimize -r "addpath(''',Parallel(indPC).DynarePath,'''), dynareroot = dynare_config(); fParallel(',int2str(offset+1),',',int2str(sum(nBlockPerCPU(1:j))),',',int2str(j),',',int2str(indPC),',''',fname,''')"'];
Marco Ratto's avatar
Marco Ratto committed
267
                    end
268
                end
Marco Ratto's avatar
Marco Ratto committed
269 270 271 272
            else                            % Parallel(indPC).Local==0: Run using network on remote machine or also on local machine.
                if j==nCPU0+1,
                    dynareParallelSendFiles([fname,'_input.mat'],PRCDir,Parallel(indPC));
                    dynareParallelSendFiles(NamFileInput,PRCDir,Parallel(indPC));
273
                end
Marco Ratto's avatar
Marco Ratto committed
274
                
ratto's avatar
ratto committed
275
                if isunix || (~matlab_ver_less_than('7.4') && ismac),
Marco Ratto's avatar
Marco Ratto committed
276
                    if exist('OCTAVE_VERSION'),
277
                        command1=['ssh ',Parallel(indPC).user,'@',Parallel(indPC).PcName,' "cd ',Parallel(indPC).RemoteFolder,'/',PRCDir, '; octave --eval \"addpath(''',Parallel(indPC).DynarePath,'''), dynareroot = dynare_config(); fParallel(',int2str(offset+1),',',int2str(sum(nBlockPerCPU(1:j))),',',int2str(j),',',int2str(indPC),',''',fname,''');\" " &'];
Marco Ratto's avatar
Marco Ratto committed
278
                    else
279
                        command1=['ssh ',Parallel(indPC).user,'@',Parallel(indPC).PcName,' "cd ',Parallel(indPC).RemoteFolder,'/',PRCDir, '; ',Parallel(indPC).MatlabPath,' -nosplash -nodesktop -minimize -r \"addpath(''',Parallel(indPC).DynarePath,'''), dynareroot = dynare_config(); fParallel(',int2str(offset+1),',',int2str(sum(nBlockPerCPU(1:j))),',',int2str(j),',',int2str(indPC),',''',fname,''');\" " &'];
Marco Ratto's avatar
Marco Ratto committed
280
                    end
281
                else
Marco Ratto's avatar
Marco Ratto committed
282 283 284
                    if ~strcmp(Parallel(indPC).PcName,MasterName), % Run on a remote machine!
                        if exist('OCTAVE_VERSION'),
                            command1=['start /B psexec \\',Parallel(indPC).PcName,' -e -u ',Parallel(indPC).user,' -p ',Parallel(indPC).passwd,' -W ',Parallel(indPC).RemoteDrive,':\',Parallel(indPC).RemoteFolder,'\',PRCDir,'\ -a ',int2str(Parallel(indPC).NumCPU(j-nCPU0)), ...
285
                                ' -low  octave --eval "addpath(''',Parallel(indPC).DynarePath,'''), dynareroot = dynare_config(); fParallel(',int2str(offset+1),',',int2str(sum(nBlockPerCPU(1:j))),',',int2str(j),',',int2str(indPC),',''',fname,''')"'];
Marco Ratto's avatar
Marco Ratto committed
286 287
                        else
                            command1=['start /B psexec \\',Parallel(indPC).PcName,' -e -u ',Parallel(indPC).user,' -p ',Parallel(indPC).passwd,' -W ',Parallel(indPC).RemoteDrive,':\',Parallel(indPC).RemoteFolder,'\',PRCDir,'\ -a ',int2str(Parallel(indPC).NumCPU(j-nCPU0)), ...
288
                                ' -low  ',Parallel(indPC).MatlabPath,' -nosplash -nodesktop -minimize -r "addpath(''',Parallel(indPC).DynarePath,'''), dynareroot = dynare_config(); fParallel(',int2str(offset+1),',',int2str(sum(nBlockPerCPU(1:j))),',',int2str(j),',',int2str(indPC),',''',fname,''')"'];
Marco Ratto's avatar
Marco Ratto committed
289 290 291 292
                        end
                    else % Run on the local machine via the network
                        if exist('OCTAVE_VERSION'),
                            command1=['start /B psexec \\',Parallel(indPC).PcName,' -e -W ',Parallel(indPC).RemoteDrive,':\',Parallel(indPC).RemoteFolder,'\',PRCDir,'\ -a ',int2str(Parallel(indPC).NumCPU(j-nCPU0)), ...
293
                                ' -low  octave --eval "addpath(''',Parallel(indPC).DynarePath,'''), dynareroot = dynare_config(); fParallel(',int2str(offset+1),',',int2str(sum(nBlockPerCPU(1:j))),',',int2str(j),',',int2str(indPC),',''',fname,''')"'];
Marco Ratto's avatar
Marco Ratto committed
294 295
                        else
                            command1=['start /B psexec \\',Parallel(indPC).PcName,' -e -W ',Parallel(indPC).RemoteDrive,':\',Parallel(indPC).RemoteFolder,'\',PRCDir,'\ -a ',int2str(Parallel(indPC).NumCPU(j-nCPU0)), ...
296
                                ' -low  ',Parallel(indPC).MatlabPath,' -nosplash -nodesktop -minimize -r "addpath(''',Parallel(indPC).DynarePath,'''), dynareroot = dynare_config(); fParallel(',int2str(offset+1),',',int2str(sum(nBlockPerCPU(1:j))),',',int2str(j),',',int2str(indPC),',''',fname,''')"'];
297 298 299
                        end
                    end
                end
300 301
            end
            
Marco Ratto's avatar
Marco Ratto committed
302 303 304 305 306 307
            
        case 1
            if Parallel(indPC).Local == 1 & newInstance, % Run on the local machine.
                if isunix || (~matlab_ver_less_than('7.4') && ismac),
                    if exist('OCTAVE_VERSION')
                        %command1=['octave --eval fParallel\(',int2str(offset+1),',',int2str(sum(nBlockPerCPU(1:j))),',',int2str(j),',',int2str(indPC),',\''',fname,'\''\) &'];
308
                        command1=['octave --eval "addpath(''',Parallel(indPC).DynarePath,'''), dynareroot = dynare_config(); slaveParallel(',int2str(j),',',int2str(indPC),')" &'];
Marco Ratto's avatar
Marco Ratto committed
309 310
                    else
                        %command1=[Parallel(indPC).MatlabPath,' -nosplash -nodesktop -minimize -r fParallel\(',int2str(offset+1),',',int2str(sum(nBlockPerCPU(1:j))),',',int2str(j),',',int2str(indPC),',\''',fname,'\''\) &'];
311
                        command1=[Parallel(indPC).MatlabPath,' -nosplash -nodesktop -minimize -r "addpath(''',Parallel(indPC).DynarePath,'''), dynareroot = dynare_config(); slaveParallel(',int2str(j),',',int2str(indPC),')" &'];
312 313
                    end
                else
Marco Ratto's avatar
Marco Ratto committed
314 315
                    if exist('OCTAVE_VERSION')
                        %command1=['start /B psexec -W ',DyMo, ' -a ',int2str(Parallel(indPC).NumCPU(j-nCPU0)),' -low  octave --eval fParallel(',int2str(offset+1),',',int2str(sum(nBlockPerCPU(1:j))),',',int2str(j),',',int2str(indPC),',''',fname,''')'];
316
                        command1=['start /B psexec -W ',DyMo, ' -a ',int2str(Parallel(indPC).NumCPU(j-nCPU0)),' -low  octave --eval "addpath(''',Parallel(indPC).DynarePath,'''), dynareroot = dynare_config(); slaveParallel(',int2str(j),',',int2str(indPC),')"'];
317
                    else
Marco Ratto's avatar
Marco Ratto committed
318
                        %command1=['start /B psexec -W ',DyMo, ' -a ',int2str(Parallel(indPC).NumCPU(j-nCPU0)),' -low  ',Parallel(indPC).MatlabPath,' -nosplash -nodesktop -minimize -r fParallel(',int2str(offset+1),',',int2str(sum(nBlockPerCPU(1:j))),',',int2str(j),',',int2str(indPC),',''',fname,''')'];
319
                        command1=['start /B psexec -W ',DyMo, ' -a ',int2str(Parallel(indPC).NumCPU(j-nCPU0)),' -low  ',Parallel(indPC).MatlabPath,' -nosplash -nodesktop -minimize -r "addpath(''',Parallel(indPC).DynarePath,'''), dynareroot = dynare_config(); slaveParallel(',int2str(j),',',int2str(indPC),')"'];
320
                    end
Marco Ratto's avatar
Marco Ratto committed
321 322
                end
            elseif Parallel(indPC).Local==0, % Run using network on remote machine or also on local machine.
323 324 325
                if j==nCPU0+1,
                    dynareParallelSendFiles(NamFileInput,PRCDir,Parallel(indPC));
                end
Marco Ratto's avatar
Marco Ratto committed
326
                dynareParallelSendFiles(['P_',fname,'_',int2str(j),'End.txt'],PRCDir,Parallel(indPC));
327
                delete(['P_',fname,'_',int2str(j),'End.txt']);
Marco Ratto's avatar
Marco Ratto committed
328
                dynareParallelSendFiles(['slaveJob',int2str(j),'.mat'],PRCDir,Parallel(indPC));
329
                delete(['slaveJob',int2str(j),'.mat']);
Marco Ratto's avatar
Marco Ratto committed
330 331 332 333
                if newInstance,
                    dynareParallelSendFiles(['slaveParallel_input',int2str(j),'.mat'],PRCDir,Parallel(indPC))
                    if isunix || (~matlab_ver_less_than('7.4') && ismac),
                        if exist('OCTAVE_VERSION'),
334
                            command1=['ssh ',Parallel(indPC).user,'@',Parallel(indPC).PcName,' "cd ',Parallel(indPC).RemoteFolder,'/',PRCDir '; octave --eval \"addpath(''',Parallel(indPC).DynarePath,'''), dynareroot = dynare_config(); slaveParallel(',int2str(j),',',int2str(indPC),');\" " &'];
Marco Ratto's avatar
Marco Ratto committed
335
                        else
336
                            command1=['ssh ',Parallel(indPC).user,'@',Parallel(indPC).PcName,' "cd ',Parallel(indPC).RemoteFolder,'/',PRCDir '; ',Parallel(indPC).MatlabPath,' -nosplash -nodesktop -minimize -r \"addpath(''',Parallel(indPC).DynarePath,'''), dynareroot = dynare_config(); slaveParallel(',int2str(j),',',int2str(indPC),');\" " &'];
Marco Ratto's avatar
Marco Ratto committed
337
                        end
338
                    else
Marco Ratto's avatar
Marco Ratto committed
339 340 341
                        if ~strcmp(Parallel(indPC).PcName,MasterName), % Run on a remote machine.
                            if exist('OCTAVE_VERSION'),
                                command1=['start /B psexec \\',Parallel(indPC).PcName,' -e -u ',Parallel(indPC).user,' -p ',Parallel(indPC).passwd,' -W ',Parallel(indPC).RemoteDrive,':\',Parallel(indPC).RemoteFolder,'\',PRCDir,'\ -a ',int2str(Parallel(indPC).NumCPU(j-nCPU0)), ...
342
                                    ' -low  octave --eval "addpath(''',Parallel(indPC).DynarePath,'''), dynareroot = dynare_config(); slaveParallel(',int2str(j),',',int2str(indPC),')"'];
Marco Ratto's avatar
Marco Ratto committed
343 344
                            else
                                command1=['start /B psexec \\',Parallel(indPC).PcName,' -e -u ',Parallel(indPC).user,' -p ',Parallel(indPC).passwd,' -W ',Parallel(indPC).RemoteDrive,':\',Parallel(indPC).RemoteFolder,'\',PRCDir,'\ -a ',int2str(Parallel(indPC).NumCPU(j-nCPU0)), ...
345
                                    ' -low  ',Parallel(indPC).MatlabPath,' -nosplash -nodesktop -minimize -r "addpath(''',Parallel(indPC).DynarePath,'''), dynareroot = dynare_config(); slaveParallel(',int2str(j),',',int2str(indPC),')"'];
Marco Ratto's avatar
Marco Ratto committed
346 347 348 349
                            end
                        else % Run on the local machine via the network.
                            if exist('OCTAVE_VERSION'),
                                command1=['start /B psexec \\',Parallel(indPC).PcName,' -e -W ',Parallel(indPC).RemoteDrive,':\',Parallel(indPC).RemoteFolder,'\',PRCDir,'\ -a ',int2str(Parallel(indPC).NumCPU(j-nCPU0)), ...
350
                                    ' -low  octave --eval "addpath(''',Parallel(indPC).DynarePath,'''), dynareroot = dynare_config(); slaveParallel(',int2str(j),',',int2str(indPC),')"'];
Marco Ratto's avatar
Marco Ratto committed
351 352
                            else
                                command1=['start /B psexec \\',Parallel(indPC).PcName,' -e -W ',Parallel(indPC).RemoteDrive,':\',Parallel(indPC).RemoteFolder,'\',PRCDir,'\ -a ',int2str(Parallel(indPC).NumCPU(j-nCPU0)), ...
353
                                    ' -low  ',Parallel(indPC).MatlabPath,' -nosplash -nodesktop -minimize -r "addpath(''',Parallel(indPC).DynarePath,'''), dynareroot = dynare_config(); slaveParallel(',int2str(j),',',int2str(indPC),')"'];
Marco Ratto's avatar
Marco Ratto committed
354 355
                            end
                        end
356
                    end
357 358
                end
            end
Marco Ratto's avatar
Marco Ratto committed
359 360 361 362 363
            
    end
    
    fprintf(fid,'%s\n',command1);
    
364
end
Marco Ratto's avatar
Marco Ratto committed
365 366

fclose(fid);
367 368

% Run the slaves.
ratto's avatar
ratto committed
369
if isunix || (~matlab_ver_less_than('7.4') && ismac),
370 371 372 373 374 375
    system('sh ConcurrentCommand1.bat &');
    pause(1)
else
    system('ConcurrentCommand1.bat');
end

Marco Ratto's avatar
Marco Ratto committed
376

377 378 379 380
% Wait for the slaves to finish their job, and display some progress
% information meanwhile.


381 382
if exist('OCTAVE_VERSION'),
    diary off;
383
    printf('\n');
384 385
else
    hfigstatus = figure('name',['Parallel ',fname],...
Marco Ratto's avatar
Marco Ratto committed
386 387 388 389 390 391 392 393
        'DockControls','off', ...
        'IntegerHandle','off', ...
        'Interruptible','off', ...
        'MenuBar', 'none', ...
        'NumberTitle','off', ...
        'Renderer','Painters', ...
        'Resize','off');
    
394 395 396
    vspace = 0.1;
    ncol = ceil(totCPU/10);
    hspace = 0.9/ncol;
Marco Ratto's avatar
Marco Ratto committed
397 398 399 400 401 402 403 404 405 406
    hstatus(1) = axes('position',[0.05/ncol 0.92 0.9/ncol 0.03], ...
        'box','on','xtick',[],'ytick',[],'xlim',[0 1],'ylim',[0 1]);
    set(hstatus(1),'Units','pixels')
    hpixel = get(hstatus(1),'Position');
    hfigure = get(hfigstatus,'Position');
    hfigure(4)=hpixel(4)*10/3*min(10,totCPU);
    set(hfigstatus,'Position',hfigure)
    set(hstatus(1),'Units','normalized'),
    vspace = max(0.1,1/totCPU);
    vstart = 1-vspace+0.2*vspace;
407 408
    for j=1:totCPU,
        jrow = mod(j-1,10)+1;
Marco Ratto's avatar
Marco Ratto committed
409 410 411 412 413 414
        jcol = ceil(j/10);
        hstatus(j) = axes('position',[0.05/ncol+(jcol-1)/ncol vstart-vspace*(jrow-1) 0.9/ncol 0.3*vspace], ...
            'box','on','xtick',[],'ytick',[],'xlim',[0 1],'ylim',[0 1]);
        hpat(j) = patch([0 0 0 0],[0 1 1 0],'r','EdgeColor','r');
        htit(j) = title(['Initialize ...']);

415
    end
Marco Ratto's avatar
Marco Ratto committed
416
    
417 418 419
    cumBlockPerCPU = cumsum(nBlockPerCPU);
end
pcerdone = NaN(1,totCPU);
Marco Ratto's avatar
Marco Ratto committed
420 421 422 423 424 425 426
idCPU = NaN(1,totCPU);

delete(['comp_status_',fname,'*.mat']);

ForEver=1;

while (ForEver)
427
    
428 429
    waitbarString = '';
    statusString = '';
430 431
    
    
Marco Ratto's avatar
Marco Ratto committed
432 433 434 435
    
     pause(1)
        
    try
436
        if islocal ==0,
Marco Ratto's avatar
Marco Ratto committed
437 438 439 440 441 442 443
            dynareParallelGetFiles(['comp_status_',fname,'*.mat'],PRCDir,Parallel(1:totSlaves));
        end
    catch
    end
    
%     stax = dir(['comp_status_',fname,'*.mat']);
    for j=1:totCPU, %length(stax),
444
        try
Marco Ratto's avatar
Marco Ratto committed
445 446 447
            if ~isempty(['comp_status_',fname,int2str(j),'.mat'])
                load(['comp_status_',fname,int2str(j),'.mat']);% (stax(j).name)
            end
448
            pcerdone(j) = prtfrc;
Marco Ratto's avatar
Marco Ratto committed
449
            idCPU(j) = njob;
450
            if exist('OCTAVE_VERSION'),
451
                statusString = [statusString, int2str(j), ' %3.f%% done! '];
452
            else
Marco Ratto's avatar
Marco Ratto committed
453 454
                status_String{j} = waitbarString;
                status_Title{j} = waitbarTitle;
455
            end
Marco Ratto's avatar
Marco Ratto committed
456
%             if prtfrc==1, delete(stax(j).name), end
457
        catch
458 459
%             pcerdone(j) = NaN;
%             idCPU(j) = NaN;
Marco Ratto's avatar
Marco Ratto committed
460 461 462
%             if j>1
%                 j=j-1;
%             end
463 464 465 466 467
        end
    end
    if exist('OCTAVE_VERSION'),
        printf([statusString,'\r'], 100 .* pcerdone);
    else
Marco Ratto's avatar
Marco Ratto committed
468
        for j=1:totCPU, %length(stax)
469
            try
Marco Ratto's avatar
Marco Ratto committed
470 471 472 473
                set(hpat(j),'XData',[0 0 pcerdone(j) pcerdone(j)]);
                set(htit(j),'String',[status_Title{j},' - ',status_String{j}]);
            catch ME

474
            end
475 476
        end
    end
Marco Ratto's avatar
Marco Ratto committed
477 478 479 480 481 482 483 484 485 486 487 488 489 490 491 492 493 494 495 496 497
    
    
    
    if isempty(dynareParallelDir(['P_',fname,'_*End.txt'],PRCDir,Parallel(1:totSlaves)));
        HoTuttiGliOutput=0;
        for j=1:totCPU,
            if ~isempty(dynareParallelDir([fname,'_output_',int2str(j),'.mat'],PRCDir,Parallel(1:totSlaves)))
                HoTuttiGliOutput=HoTuttiGliOutput+1;
            end
        end
        if HoTuttiGliOutput==totCPU,
            
            mydelete(['comp_status_',fname,'*.mat'])
            if ~exist('OCTAVE_VERSION'),
                close(hfigstatus),
            else
                printf('\n');
                diary on;
            end
            
            break
498
        else
Marco Ratto's avatar
Marco Ratto committed
499
            disp('Waiting for output files from slaves ...')
500 501
        end
    end
Marco Ratto's avatar
Marco Ratto committed
502
      
503 504
end

505
% keyboard;
Marco Ratto's avatar
Marco Ratto committed
506 507


508
% Create return value.
509
iscrash = 0;
510
for j=1:totCPU,
511 512
    indPC=min(find(nCPU>=j));
    dynareParallelGetFiles([fname,'_output_',int2str(j),'.mat'],PRCDir,Parallel(indPC));
513 514
    load([fname,'_output_',int2str(j),'.mat'],'fOutputVar');
    delete([fname,'_output_',int2str(j),'.mat']);
Marco Ratto's avatar
Marco Ratto committed
515
    if isfield(fOutputVar,'OutputFileName'),
516
        dynareParallelGetFiles([fOutputVar.OutputFileName],PRCDir,Parallel(indPC));
Marco Ratto's avatar
Marco Ratto committed
517 518
    end
    if isfield(fOutputVar,'error'),
Marco Ratto's avatar
Marco Ratto committed
519
        disp(['Job number ',int2str(j),' crashed with error:']);
520
        iscrash=1;
Marco Ratto's avatar
Marco Ratto committed
521
%         keyboard;
522
        disp([fOutputVar.error.message]);
523 524 525
        for jstack=1:length(fOutputVar.error.stack)
            fOutputVar.error.stack(jstack),
        end
526 527
    else
        fOutVar(j)=fOutputVar;
Marco Ratto's avatar
Marco Ratto committed
528
    end
529
end
530 531 532 533
if iscrash,
   error('Remote jobs crashed');
end
            
Marco Ratto's avatar
Marco Ratto committed
534 535 536 537 538
pause(1), % wait for all remote diary off completed

% Cleanup. (Only if the computing is executed remotly).
dynareParallelGetFiles('*.log',PRCDir,Parallel(1:totSlaves));

539
switch Strategy
Marco Ratto's avatar
Marco Ratto committed
540 541 542 543 544 545 546 547 548 549 550 551 552 553 554 555 556 557 558 559 560 561
    case 0
        for indPC=1:length(Parallel)
            if Parallel(indPC).Local == 0
               dynareParallelRmDir(PRCDir,Parallel(indPC));
            end
            
            if isempty(dir('dynareParallelLogFiles'))
                [A B C]=rmdir('dynareParallelLogFiles');
                mkdir('dynareParallelLogFiles');
            end
            
            copyfile('*.log','dynareParallelLogFiles');
            delete([fname,'*.log']);
            
            %%%%%%%%     E' GIUSTO???    %%%%%%%%%%%%%
            mydelete(['*_core*_input*.mat']);
            if Parallel(indPC).Local == 1
                delete(['slaveParallel_input*.mat']);
            end
            %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
                
        end
562
        
Marco Ratto's avatar
Marco Ratto committed
563 564 565 566 567 568 569 570 571 572 573 574 575 576 577 578 579
        delete ConcurrentCommand1.bat
    case 1
        delete(['temp_input.mat'])
        if newInstance,
            if isempty(dir('dynareParallelLogFiles'))
                [A B C]=rmdir('dynareParallelLogFiles');
                mkdir('dynareParallelLogFiles');
            end
        end
        copyfile('*.log','dynareParallelLogFiles');
        if newInstance,
            delete ConcurrentCommand1.bat
        end
end



580