masterParallel.m 27.7 KB
Newer Older
Marco Ratto's avatar
Marco Ratto committed
1
function [fOutVar,nBlockPerCPU, totCPU] = masterParallel(Parallel,fBlock,nBlock,NamFileInput,fname,fInputVar,fGlobalVar,Parallel_info,initialize)
2 3 4 5 6
% PARALLEL CONTEXT
% This is the most important function for the management of DYNARE parallel
% computing.
% It is the top-level function called on the master computer when parallelizing a task.

Marco Ratto's avatar
Marco Ratto committed
7
% This function have two main computational startegy for manage the matlab worker (slave process).
8
% 0 Simple Close/Open Stategy:
Marco Ratto's avatar
Marco Ratto committed
9 10 11
% In this case the new matlab istances (slave process) are open when
% necessary and then closed. This can happen many times during the
% simulation of a model.
12 13

% 1 Alway Open Stategy:
Marco Ratto's avatar
Marco Ratto committed
14 15 16 17 18
% In this case we have a more sophisticated management of slave processes,
% which are no longer closed at the end of each job. The slave processes
% waits for a new job (if exist). If a slave do not receives a new job after a
% fixed time it is destroyed. This solution removes the computational
% time necessary to Open/Close new matlab istances.
19 20 21 22 23 24 25

% The first (point 0) is the default Strategy
% i.e.(Parallel_info.leaveSlaveOpen=0). This value can be changed by the
% user in xxx.mod file or it is changed by the programmer if it necessary to
% reduce the overall computational time. See for example the
% prior_posterior_statistics.m.

26 27 28
% The number of parallelized threads will be equal to (nBlock-fBlock+1).
%
% INPUTS
29 30
%  o Parallel [struct vector]   copy of options_.parallel
%  o fBlock [int]               index number of the first thread
31
%                               (between 1 and nBlock)
32 33
%  o nBlock [int]               index number of the last thread
%  o NamFileInput [cell array]  containins the list of input files to be
34 35 36 37
%                               copied in the working directory of remote slaves
%                               2 columns, as many lines as there are files
%                               - first column contains directory paths
%                               - second column contains filenames
38
%  o fname [string]             name of the function to be parallelized, and
39
%                               which will be run on the slaves
40
%  o fInputVar [struct]         structure containing local variables to be used
41
%                               by fName on the slaves
42
%  o fGlobalVar [struct]        structure containing global variables to be used
43 44 45
%                               by fName on the slaves
%  o Parallel_info              []
%  o initialize                 []
46 47
%
% OUTPUT
48
%  o fOutVar [struct vector]   result of the parallel computation, one
49
%                              struct per thread
50
%  o nBlockPerCPU [int vector] for each CPU used, indicates the number of
51
%                              threads run on that CPU
52
%  o totCPU [int]              total number of CPU used (can be lower than
53 54 55
%                              the number of CPU declared in "Parallel", if
%                              the number of required threads is lower)

56
% Copyright (C) 2009-2010 Dynare Team
57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72
%
% This file is part of Dynare.
%
% Dynare is free software: you can redistribute it and/or modify
% it under the terms of the GNU General Public License as published by
% the Free Software Foundation, either version 3 of the License, or
% (at your option) any later version.
%
% Dynare is distributed in the hope that it will be useful,
% but WITHOUT ANY WARRANTY; without even the implied warranty of
% MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
% GNU General Public License for more details.
%
% You should have received a copy of the GNU General Public License
% along with Dynare.  If not, see <http://www.gnu.org/licenses/>.

73 74


Marco Ratto's avatar
Marco Ratto committed
75 76 77 78 79
% If islocal==0, create a new directory for remote computation.
% This directory is named using current data and time,
% is used only one time and then deleted.

persistent PRCDir
80
% PRCDir = Present Remote Computational Directory!
Marco Ratto's avatar
Marco Ratto committed
81 82 83

Strategy=Parallel_info.leaveSlaveOpen;

84
islocal = 1;
Marco Ratto's avatar
Marco Ratto committed
85
for j=1:length(Parallel),
86
    islocal=islocal*Parallel(j).Local;
Marco Ratto's avatar
Marco Ratto committed
87 88 89 90 91 92 93 94
end
if nargin>8 && initialize==1
    if islocal == 0,
        PRCDir=CreateTimeString();
        assignin('base','PRCDirTmp',PRCDir),
        evalin('base','options_.parallel_info.RemoteTmpFolder=PRCDirTmp;')
        evalin('base','clear PRCDirTmp,')
    else
95
        % Delete the traces (if existing) of last local session of computations.
Marco Ratto's avatar
Marco Ratto committed
96 97 98 99 100 101 102 103
        if Strategy==1,
            mydelete(['slaveParallel_input*.mat']);
        end
    end
    return
end


104 105 106

% Only for testing!

Marco Ratto's avatar
Marco Ratto committed
107 108 109 110 111 112
% if Strategy==0
%     disp('User Strategy Now Is Open/Close (0)');
% else
%     disp('User Strategy Now Is Always Open (1)');
% end

113
if Strategy==1
Marco Ratto's avatar
Marco Ratto committed
114
    totCPU=0;
115
end
116

Marco Ratto's avatar
Marco Ratto committed
117

118
% Determine my hostname and my working directory.
Marco Ratto's avatar
Marco Ratto committed
119

120
DyMo=pwd;
121 122
% fInputVar.DyMo=DyMo;
if ~(isunix || (~matlab_ver_less_than('7.4') && ismac)) ,
123
    [tempo, MasterName]=system('hostname');
124
    MasterName=deblank(MasterName);
125
end
126
% fInputVar.MasterName = MasterName;
127

Marco Ratto's avatar
Marco Ratto committed
128

129 130
% Save input data for use by the slaves.
switch Strategy
Marco Ratto's avatar
Marco Ratto committed
131 132 133 134 135 136 137 138
    case 0
        if exist('fGlobalVar'),
            save([fname,'_input.mat'],'fInputVar','fGlobalVar')
        else
            save([fname,'_input.mat'],'fInputVar')
        end
        save([fname,'_input.mat'],'Parallel','-append')
        
139
    case 1
Marco Ratto's avatar
Marco Ratto committed
140 141 142 143 144 145
        if exist('fGlobalVar'),
            save(['temp_input.mat'],'fInputVar','fGlobalVar')
        else
            save(['temp_input.mat'],'fInputVar')
        end
        save(['temp_input.mat'],'Parallel','-append')
146 147
end

148

149 150 151
% Determine the total number of available CPUs, and the number of threads
% to run on each CPU.

Marco Ratto's avatar
Marco Ratto committed
152 153 154
[nCPU, totCPU, nBlockPerCPU, totSlaves] = distributeJobs(Parallel, fBlock, nBlock);
offset0 = fBlock-1;

155

156
% Clean up remnants of previous runs.
157 158
mydelete(['comp_status_',fname,'*.mat']);
mydelete(['P_',fname,'*End.txt']);
Marco Ratto's avatar
Marco Ratto committed
159

160

161 162
% Create a shell script containing the commands to launch the required
% tasks on the slaves.
163
fid = fopen('ConcurrentCommand1.bat','w+');
Marco Ratto's avatar
Marco Ratto committed
164 165


166
% Create the directory devoted to remote computation.
Marco Ratto's avatar
Marco Ratto committed
167 168 169 170 171 172
if isempty(PRCDir) && ~islocal,
    error('PRCDir not initialized!')
else
    dynareParallelMkDir(PRCDir,Parallel(1:totSlaves));
end

173

174 175
for j=1:totCPU,
    
176 177 178 179 180
    
    if Strategy==1
        command1 = ' ';
    end
    
181 182
    indPC=min(find(nCPU>=j));
    
183 184 185 186 187 188 189 190 191 192 193 194 195
    % According to the information contained in configuration file, compThread can limit MATLAB
    % to a single computational thread. By default, MATLAB makes use of the multithreading
    % capabilities of the computer on which it is running. Nevertheless
    % exsperimental results show as matlab native
    % multithreading limit the performaces when the parallel computing is active.
    
    
    if strcmp('true',Parallel(indPC).SingleCompThread),
        compThread = '-singleCompThread';
    else
        compThread = '';
    end
    
196 197 198 199 200 201 202
    if indPC>1
        nCPU0 = nCPU(indPC-1);
    else
        nCPU0=0;
    end
    offset = sum(nBlockPerCPU(1:j-1))+offset0;
    
203 204
    % Create a file used to monitoring if a parallel block (core)
    % computation is finished or not.
Marco Ratto's avatar
Marco Ratto committed
205
    
206 207 208
    fid1=fopen(['P_',fname,'_',int2str(j),'End.txt'],'w+');
    fclose(fid1);
    
Marco Ratto's avatar
Marco Ratto committed
209 210 211 212 213 214 215
    if Strategy==1,
        
        fblck = offset+1;
        nblck = sum(nBlockPerCPU(1:j));
        save temp_input fblck nblck fname -append;
        copyfile('temp_input.mat',['slaveJob',int2str(j),'.mat'])
        if Parallel(indPC).Local ==0,
216 217
            fid1=fopen(['stayalive',int2str(j),'.txt'],'w+');
            fclose(fid1);
Marco Ratto's avatar
Marco Ratto committed
218 219 220 221 222 223
            dynareParallelSendFiles(['stayalive',int2str(j),'.txt'],PRCDir,Parallel(indPC));
            mydelete(['stayalive',int2str(j),'.txt']);
        end
        % Wait for possibly local alive CPU to start the new job or close by
        % internal criteria.
        pause(1);
224 225
        newInstance = 0;
        
226
        % Check if j CPU is already alive.
Marco Ratto's avatar
Marco Ratto committed
227
        if isempty(dynareParallelDir(['P_slave_',int2str(j),'End.txt'],PRCDir,Parallel(indPC)));
228 229
            fid1=fopen(['P_slave_',int2str(j),'End.txt'],'w+');
            fclose(fid1);
Marco Ratto's avatar
Marco Ratto committed
230 231 232 233 234
            if Parallel(indPC).Local==0,
                dynareParallelSendFiles(['P_slave_',int2str(j),'End.txt'],PRCDir,Parallel(indPC));
                delete(['P_slave_',int2str(j),'End.txt']);
            end
            
235 236 237 238 239
            newInstance = 1;
            storeGlobalVars( ['slaveParallel_input',int2str(j)]);
            save( ['slaveParallel_input',int2str(j)],'Parallel','-append');
            % Prepare global vars for Slave.
        end
Marco Ratto's avatar
Marco Ratto committed
240
    else
241
        
242 243 244 245
        % If the computation is executed remotely all the necessary files
        % are created localy, then copied in remote directory and then
        % deleted (loacal)!
        
Marco Ratto's avatar
Marco Ratto committed
246 247 248 249 250 251 252 253
        save( ['slaveParallel_input',int2str(j)],'Parallel');
        
        if Parallel(indPC).Local==0,
            dynareParallelSendFiles(['P_',fname,'_',int2str(j),'End.txt'],PRCDir,Parallel(indPC));
            delete(['P_',fname,'_',int2str(j),'End.txt']);
            
            dynareParallelSendFiles(['slaveParallel_input',int2str(j),'.mat'],PRCDir,Parallel(indPC));
            delete(['slaveParallel_input',int2str(j),'.mat']);
254
            
255
        end
256
        
Marco Ratto's avatar
Marco Ratto committed
257
    end
258
    
Marco Ratto's avatar
Marco Ratto committed
259 260 261 262 263
    %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
    % The following 'switch - case' code is the core of this function!
    switch Strategy
        case 0
            
264
            if Parallel(indPC).Local == 1,                                  % 0.1 Run on the local machine (localhost).
Marco Ratto's avatar
Marco Ratto committed
265 266 267
                
                if isunix || (~matlab_ver_less_than('7.4') && ismac),
                    if exist('OCTAVE_VERSION')
268
                        command1=['octave --eval "addpath(''',Parallel(indPC).DynarePath,'''), dynareroot = dynare_config(); fParallel(',int2str(offset+1),',',int2str(sum(nBlockPerCPU(1:j))),',',int2str(j),',',int2str(indPC),',''',fname,''')" &'];
Marco Ratto's avatar
Marco Ratto committed
269
                    else
270
                        command1=[Parallel(indPC).MatlabOctavePath,' -nosplash -nodesktop -minimize ',compThread,' -r "addpath(''',Parallel(indPC).DynarePath,'''), dynareroot = dynare_config(); fParallel(',int2str(offset+1),',',int2str(sum(nBlockPerCPU(1:j))),',',int2str(j),',',int2str(indPC),',''',fname,''')" &'];
Marco Ratto's avatar
Marco Ratto committed
271
                    end
272
                else
Marco Ratto's avatar
Marco Ratto committed
273
                    if exist('OCTAVE_VERSION')
274
                        command1=['start /B psexec -W ',DyMo, ' -a ',int2str(Parallel(indPC).CPUnbr(j-nCPU0)),' -low  octave --eval "addpath(''',Parallel(indPC).DynarePath,'''), dynareroot = dynare_config(); fParallel(',int2str(offset+1),',',int2str(sum(nBlockPerCPU(1:j))),',',int2str(j),',',int2str(indPC),',''',fname,''')"'];
Marco Ratto's avatar
Marco Ratto committed
275
                    else
276
                        command1=['start /B psexec -W ',DyMo, ' -a ',int2str(Parallel(indPC).CPUnbr(j-nCPU0)),' -low  ',Parallel(indPC).MatlabOctavePath,' -nosplash -nodesktop -minimize ',compThread,' -r "addpath(''',Parallel(indPC).DynarePath,'''), dynareroot = dynare_config(); fParallel(',int2str(offset+1),',',int2str(sum(nBlockPerCPU(1:j))),',',int2str(j),',',int2str(indPC),',''',fname,''')"'];
Marco Ratto's avatar
Marco Ratto committed
277
                    end
278
                end
279
            else                                                            % 0.2 Parallel(indPC).Local==0: Run using network on remote machine or also on local machine.
Marco Ratto's avatar
Marco Ratto committed
280 281 282
                if j==nCPU0+1,
                    dynareParallelSendFiles([fname,'_input.mat'],PRCDir,Parallel(indPC));
                    dynareParallelSendFiles(NamFileInput,PRCDir,Parallel(indPC));
283
                end
Marco Ratto's avatar
Marco Ratto committed
284
                
ratto's avatar
ratto committed
285
                if isunix || (~matlab_ver_less_than('7.4') && ismac),
Marco Ratto's avatar
Marco Ratto committed
286
                    if exist('OCTAVE_VERSION'),
287
                        command1=['ssh ',Parallel(indPC).UserName,'@',Parallel(indPC).ComputerName,' "cd ',Parallel(indPC).RemoteDirectory,'/',PRCDir, '; octave --eval \"addpath(''',Parallel(indPC).DynarePath,'''), dynareroot = dynare_config(); fParallel(',int2str(offset+1),',',int2str(sum(nBlockPerCPU(1:j))),',',int2str(j),',',int2str(indPC),',''',fname,''');\" " &'];
Marco Ratto's avatar
Marco Ratto committed
288
                    else
289
                        command1=['ssh ',Parallel(indPC).UserName,'@',Parallel(indPC).ComputerName,' "cd ',Parallel(indPC).RemoteDirectory,'/',PRCDir, '; ',Parallel(indPC).MatlabOctavePath,' -nosplash -nodesktop -minimize ',compThread,' -r \"addpath(''',Parallel(indPC).DynarePath,'''), dynareroot = dynare_config(); fParallel(',int2str(offset+1),',',int2str(sum(nBlockPerCPU(1:j))),',',int2str(j),',',int2str(indPC),',''',fname,''');\" " &'];
Marco Ratto's avatar
Marco Ratto committed
290
                    end
291
                else
292
                    if ~strcmp(Parallel(indPC).ComputerName,MasterName),  % 0.3 Run on a remote machine!
Marco Ratto's avatar
Marco Ratto committed
293
                        if exist('OCTAVE_VERSION'),
294
                            command1=['start /B psexec \\',Parallel(indPC).ComputerName,' -e -u ',Parallel(indPC).UserName,' -p ',Parallel(indPC).Password,' -W ',Parallel(indPC).RemoteDrive,':\',Parallel(indPC).RemoteDirectory,'\',PRCDir,'\ -a ',int2str(Parallel(indPC).CPUnbr(j-nCPU0)), ...
295
                                ' -low  octave --eval "addpath(''',Parallel(indPC).DynarePath,'''), dynareroot = dynare_config(); fParallel(',int2str(offset+1),',',int2str(sum(nBlockPerCPU(1:j))),',',int2str(j),',',int2str(indPC),',''',fname,''')"'];
Marco Ratto's avatar
Marco Ratto committed
296
                        else
297
                            command1=['start /B psexec \\',Parallel(indPC).ComputerName,' -e -u ',Parallel(indPC).UserName,' -p ',Parallel(indPC).Password,' -W ',Parallel(indPC).RemoteDrive,':\',Parallel(indPC).RemoteDirectory,'\',PRCDir,'\ -a ',int2str(Parallel(indPC).CPUnbr(j-nCPU0)), ...
298
                                ' -low  ',Parallel(indPC).MatlabOctavePath,' -nosplash -nodesktop -minimize ',compThread,' -r "addpath(''',Parallel(indPC).DynarePath,'''), dynareroot = dynare_config(); fParallel(',int2str(offset+1),',',int2str(sum(nBlockPerCPU(1:j))),',',int2str(j),',',int2str(indPC),',''',fname,''')"'];
Marco Ratto's avatar
Marco Ratto committed
299
                        end
300
                    else                                                  % 0.4 Run on the local machine via the network
Marco Ratto's avatar
Marco Ratto committed
301
                        if exist('OCTAVE_VERSION'),
302
                            command1=['start /B psexec \\',Parallel(indPC).ComputerName,' -e -W ',Parallel(indPC).RemoteDrive,':\',Parallel(indPC).RemoteDirectory,'\',PRCDir,'\ -a ',int2str(Parallel(indPC).CPUnbr(j-nCPU0)), ...
303
                                ' -low  octave --eval "addpath(''',Parallel(indPC).DynarePath,'''), dynareroot = dynare_config(); fParallel(',int2str(offset+1),',',int2str(sum(nBlockPerCPU(1:j))),',',int2str(j),',',int2str(indPC),',''',fname,''')"'];
Marco Ratto's avatar
Marco Ratto committed
304
                        else
305
                            command1=['start /B psexec \\',Parallel(indPC).ComputerName,' -e -W ',Parallel(indPC).RemoteDrive,':\',Parallel(indPC).RemoteDirectory,'\',PRCDir,'\ -a ',int2str(Parallel(indPC).CPUnbr(j-nCPU0)), ...
306
                                ' -low  ',Parallel(indPC).MatlabOctavePath,' -nosplash -nodesktop -minimize ',compThread,' -r "addpath(''',Parallel(indPC).DynarePath,'''), dynareroot = dynare_config(); fParallel(',int2str(offset+1),',',int2str(sum(nBlockPerCPU(1:j))),',',int2str(j),',',int2str(indPC),',''',fname,''')"'];
307 308 309
                        end
                    end
                end
310 311
            end
            
Marco Ratto's avatar
Marco Ratto committed
312 313
            
        case 1
314
            if Parallel(indPC).Local == 1 & newInstance,                   % 1.1 Run on the local machine.
Marco Ratto's avatar
Marco Ratto committed
315 316
                if isunix || (~matlab_ver_less_than('7.4') && ismac),
                    if exist('OCTAVE_VERSION')
317
                        command1=['octave --eval "addpath(''',Parallel(indPC).DynarePath,'''), dynareroot = dynare_config(); slaveParallel(',int2str(j),',',int2str(indPC),')" &'];
Marco Ratto's avatar
Marco Ratto committed
318
                    else
319
                        command1=[Parallel(indPC).MatlabOctavePath,' -nosplash -nodesktop -minimize ',compThread,' -r "addpath(''',Parallel(indPC).DynarePath,'''), dynareroot = dynare_config(); slaveParallel(',int2str(j),',',int2str(indPC),')" &'];
320 321
                    end
                else
Marco Ratto's avatar
Marco Ratto committed
322
                    if exist('OCTAVE_VERSION')
323
                        command1=['start /B psexec -W ',DyMo, ' -a ',int2str(Parallel(indPC).CPUnbr(j-nCPU0)),' -low  octave --eval "addpath(''',Parallel(indPC).DynarePath,'''), dynareroot = dynare_config(); slaveParallel(',int2str(j),',',int2str(indPC),')"'];
324
                    else
325
                        command1=['start /B psexec -W ',DyMo, ' -a ',int2str(Parallel(indPC).CPUnbr(j-nCPU0)),' -low  ',Parallel(indPC).MatlabOctavePath,' -nosplash -nodesktop -minimize ',compThread,' -r "addpath(''',Parallel(indPC).DynarePath,'''), dynareroot = dynare_config(); slaveParallel(',int2str(j),',',int2str(indPC),')"'];
326
                    end
Marco Ratto's avatar
Marco Ratto committed
327
                end
328
            elseif Parallel(indPC).Local==0,                                % 1.2 Run using network on remote machine or also on local machine.
329 330 331
                if j==nCPU0+1,
                    dynareParallelSendFiles(NamFileInput,PRCDir,Parallel(indPC));
                end
Marco Ratto's avatar
Marco Ratto committed
332
                dynareParallelSendFiles(['P_',fname,'_',int2str(j),'End.txt'],PRCDir,Parallel(indPC));
333
                delete(['P_',fname,'_',int2str(j),'End.txt']);
Marco Ratto's avatar
Marco Ratto committed
334
                dynareParallelSendFiles(['slaveJob',int2str(j),'.mat'],PRCDir,Parallel(indPC));
335
                delete(['slaveJob',int2str(j),'.mat']);
Marco Ratto's avatar
Marco Ratto committed
336 337 338 339
                if newInstance,
                    dynareParallelSendFiles(['slaveParallel_input',int2str(j),'.mat'],PRCDir,Parallel(indPC))
                    if isunix || (~matlab_ver_less_than('7.4') && ismac),
                        if exist('OCTAVE_VERSION'),
340
                            command1=['ssh ',Parallel(indPC).UserName,'@',Parallel(indPC).ComputerName,' "cd ',Parallel(indPC).RemoteDirectory,'/',PRCDir '; octave --eval \"addpath(''',Parallel(indPC).DynarePath,'''), dynareroot = dynare_config(); slaveParallel(',int2str(j),',',int2str(indPC),');\" " &'];
Marco Ratto's avatar
Marco Ratto committed
341
                        else
342
                            command1=['ssh ',Parallel(indPC).UserName,'@',Parallel(indPC).ComputerName,' "cd ',Parallel(indPC).RemoteDirectory,'/',PRCDir '; ',Parallel(indPC).MatlabOctavePath,' -nosplash -nodesktop -minimize ',compThread,' -r \"addpath(''',Parallel(indPC).DynarePath,'''), dynareroot = dynare_config(); slaveParallel(',int2str(j),',',int2str(indPC),');\" " &'];
Marco Ratto's avatar
Marco Ratto committed
343
                        end
344
                    else
345
                        if ~strcmp(Parallel(indPC).ComputerName,MasterName), % 1.3 Run on a remote machine.
Marco Ratto's avatar
Marco Ratto committed
346
                            if exist('OCTAVE_VERSION'),
347
                                command1=['start /B psexec \\',Parallel(indPC).ComputerName,' -e -u ',Parallel(indPC).UserName,' -p ',Parallel(indPC).Password,' -W ',Parallel(indPC).RemoteDrive,':\',Parallel(indPC).RemoteDirectory,'\',PRCDir,'\ -a ',int2str(Parallel(indPC).CPUnbr(j-nCPU0)), ...
348
                                    ' -low  octave --eval "addpath(''',Parallel(indPC).DynarePath,'''), dynareroot = dynare_config(); slaveParallel(',int2str(j),',',int2str(indPC),')"'];
Marco Ratto's avatar
Marco Ratto committed
349
                            else
350
                                command1=['start /B psexec \\',Parallel(indPC).ComputerName,' -e -u ',Parallel(indPC).UserName,' -p ',Parallel(indPC).Password,' -W ',Parallel(indPC).RemoteDrive,':\',Parallel(indPC).RemoteDirectory,'\',PRCDir,'\ -a ',int2str(Parallel(indPC).CPUnbr(j-nCPU0)), ...
351
                                    ' -low  ',Parallel(indPC).MatlabOctavePath,' -nosplash -nodesktop -minimize ',compThread,' -r "addpath(''',Parallel(indPC).DynarePath,'''), dynareroot = dynare_config(); slaveParallel(',int2str(j),',',int2str(indPC),')"'];
Marco Ratto's avatar
Marco Ratto committed
352
                            end
353
                        else                                                % 1.4 Run on the local machine via the network.
Marco Ratto's avatar
Marco Ratto committed
354
                            if exist('OCTAVE_VERSION'),
355
                                command1=['start /B psexec \\',Parallel(indPC).ComputerName,' -e -W ',Parallel(indPC).RemoteDrive,':\',Parallel(indPC).RemoteDirectory,'\',PRCDir,'\ -a ',int2str(Parallel(indPC).CPUnbr(j-nCPU0)), ...
356
                                    ' -low  octave --eval "addpath(''',Parallel(indPC).DynarePath,'''), dynareroot = dynare_config(); slaveParallel(',int2str(j),',',int2str(indPC),')"'];
Marco Ratto's avatar
Marco Ratto committed
357
                            else
358
                                command1=['start /B psexec \\',Parallel(indPC).ComputerName,' -e -W ',Parallel(indPC).RemoteDrive,':\',Parallel(indPC).RemoteDirectory,'\',PRCDir,'\ -a ',int2str(Parallel(indPC).CPUnbr(j-nCPU0)), ...
359
                                    ' -low  ',Parallel(indPC).MatlabOctavePath,' -nosplash -nodesktop -minimize ',compThread,' -r "addpath(''',Parallel(indPC).DynarePath,'''), dynareroot = dynare_config(); slaveParallel(',int2str(j),',',int2str(indPC),')"'];
Marco Ratto's avatar
Marco Ratto committed
360 361
                            end
                        end
362
                    end
363 364
                end
            end
Marco Ratto's avatar
Marco Ratto committed
365 366 367 368 369
            
    end
    
    fprintf(fid,'%s\n',command1);
    
370
end
Marco Ratto's avatar
Marco Ratto committed
371

372 373 374 375 376 377 378 379
% In This way we are sure that the file 'ConcurrentCommand1.bat' is
% closed and then it can be deleted!
while (1)
    StatusOfCC1_bat = fclose(fid);
    if StatusOfCC1_bat==0
        break
    end
end
380 381

% Run the slaves.
ratto's avatar
ratto committed
382
if isunix || (~matlab_ver_less_than('7.4') && ismac),
383 384 385 386 387 388
    system('sh ConcurrentCommand1.bat &');
    pause(1)
else
    system('ConcurrentCommand1.bat');
end

Marco Ratto's avatar
Marco Ratto committed
389

390 391 392 393 394 395 396
% For matlab enviroment with options_.console_mode = 0:
% create a parallel (local/remote) specialized computational status bars!

global options_



397
% Create a parallel (local/remote) specialized computational status bars!
398

399
if exist('OCTAVE_VERSION') || (options_.console_mode == 1),
400
    diary off;
401 402 403 404 405
    if exist('OCTAVE_VERSION')
        printf('\n');
    else
        fprintf('\n');
    end
406 407
else
    hfigstatus = figure('name',['Parallel ',fname],...
Marco Ratto's avatar
Marco Ratto committed
408 409 410 411 412 413 414 415
        'DockControls','off', ...
        'IntegerHandle','off', ...
        'Interruptible','off', ...
        'MenuBar', 'none', ...
        'NumberTitle','off', ...
        'Renderer','Painters', ...
        'Resize','off');
    
416 417 418
    vspace = 0.1;
    ncol = ceil(totCPU/10);
    hspace = 0.9/ncol;
Marco Ratto's avatar
Marco Ratto committed
419 420 421 422 423 424 425 426 427 428
    hstatus(1) = axes('position',[0.05/ncol 0.92 0.9/ncol 0.03], ...
        'box','on','xtick',[],'ytick',[],'xlim',[0 1],'ylim',[0 1]);
    set(hstatus(1),'Units','pixels')
    hpixel = get(hstatus(1),'Position');
    hfigure = get(hfigstatus,'Position');
    hfigure(4)=hpixel(4)*10/3*min(10,totCPU);
    set(hfigstatus,'Position',hfigure)
    set(hstatus(1),'Units','normalized'),
    vspace = max(0.1,1/totCPU);
    vstart = 1-vspace+0.2*vspace;
429 430
    for j=1:totCPU,
        jrow = mod(j-1,10)+1;
Marco Ratto's avatar
Marco Ratto committed
431 432 433 434 435
        jcol = ceil(j/10);
        hstatus(j) = axes('position',[0.05/ncol+(jcol-1)/ncol vstart-vspace*(jrow-1) 0.9/ncol 0.3*vspace], ...
            'box','on','xtick',[],'ytick',[],'xlim',[0 1],'ylim',[0 1]);
        hpat(j) = patch([0 0 0 0],[0 1 1 0],'r','EdgeColor','r');
        htit(j) = title(['Initialize ...']);
436
        
437
    end
Marco Ratto's avatar
Marco Ratto committed
438
    
439 440 441
    cumBlockPerCPU = cumsum(nBlockPerCPU);
end
pcerdone = NaN(1,totCPU);
Marco Ratto's avatar
Marco Ratto committed
442 443 444 445
idCPU = NaN(1,totCPU);

delete(['comp_status_',fname,'*.mat']);

446 447 448 449

% Wait for the slaves to finish their job, and display some progress
% information meanwhile.

450 451 452 453 454 455 456 457 458 459 460 461 462 463 464 465 466 467 468 469 470 471 472 473 474 475 476 477 478 479
% Caption for console mode computing ...

if (options_.console_mode == 1)
    fnameTemp=fname;
    
    L=length(fnameTemp);
    
    PoCo=strfind(fnameTemp,'_core');
    
    for i=PoCo:L
        if i==PoCo
            fnameTemp(i)=' ';
        else
            fnameTemp(i)='.';
        end
    end
     
    for i=1:L
        if  fnameTemp(i)=='_';
            fnameTemp(i)=' ';
        end
    end
    
     fnameTemp(L)='';
    
    Information=['Parallel ' fnameTemp ' Computing ...'];
    fprintf([Information,'\n\n']);
    
end

480

Marco Ratto's avatar
Marco Ratto committed
481
ForEver=1;
482
statusString = '';
Marco Ratto's avatar
Marco Ratto committed
483 484

while (ForEver)
485
    
486
    waitbarString = '';
487
    statusString0 = repmat('\b',1,length(sprintf(statusString, 100 .* pcerdone)));
488
    statusString = '';
489
    
490
    pause(1)
491
    
Marco Ratto's avatar
Marco Ratto committed
492
    try
493
        if islocal ==0,
Marco Ratto's avatar
Marco Ratto committed
494 495 496 497 498
            dynareParallelGetFiles(['comp_status_',fname,'*.mat'],PRCDir,Parallel(1:totSlaves));
        end
    catch
    end
    
499
    for j=1:totCPU,
500
        try
Marco Ratto's avatar
Marco Ratto committed
501
            if ~isempty(['comp_status_',fname,int2str(j),'.mat'])
502
                load(['comp_status_',fname,int2str(j),'.mat']);
Marco Ratto's avatar
Marco Ratto committed
503
            end
504
            pcerdone(j) = prtfrc;
Marco Ratto's avatar
Marco Ratto committed
505
            idCPU(j) = njob;
506
            if exist('OCTAVE_VERSION') || (options_.console_mode == 1),
507
                statusString = [statusString, int2str(j), ' %3.f%% done! '];
508
            else
Marco Ratto's avatar
Marco Ratto committed
509 510
                status_String{j} = waitbarString;
                status_Title{j} = waitbarTitle;
511
            end
512 513
        catch % ME
            % To define!
514 515 516
            if exist('OCTAVE_VERSION') || (options_.console_mode == 1),
                statusString = [statusString, int2str(j), ' %3.f%% done! '];
            end
517 518
        end
    end
519 520 521 522 523 524 525 526 527
    if exist('OCTAVE_VERSION') || (options_.console_mode == 1),
        if exist('OCTAVE_VERSION')
            printf([statusString,'\r'], 100 .* pcerdone);
        else
            if ~isempty(statusString)
                fprintf([statusString0,statusString], 100 .* pcerdone); 
            end
        end
        
528
    else
529
        for j=1:totCPU,
530
            try
Marco Ratto's avatar
Marco Ratto committed
531 532 533
                set(hpat(j),'XData',[0 0 pcerdone(j) pcerdone(j)]);
                set(htit(j),'String',[status_Title{j},' - ',status_String{j}]);
            catch ME
534
                
535
            end
536 537
        end
    end
Marco Ratto's avatar
Marco Ratto committed
538 539 540 541 542 543 544 545
    
    if isempty(dynareParallelDir(['P_',fname,'_*End.txt'],PRCDir,Parallel(1:totSlaves)));
        HoTuttiGliOutput=0;
        for j=1:totCPU,
            if ~isempty(dynareParallelDir([fname,'_output_',int2str(j),'.mat'],PRCDir,Parallel(1:totSlaves)))
                HoTuttiGliOutput=HoTuttiGliOutput+1;
            end
        end
546
        
Marco Ratto's avatar
Marco Ratto committed
547
        if HoTuttiGliOutput==totCPU,
548 549 550 551 552 553 554 555
            mydelete(['comp_status_',fname,'*.mat']);
            if exist('OCTAVE_VERSION')|| (options_.console_mode == 1),
                if exist('OCTAVE_VERSION')
                    printf('\n');
                else
                    fprintf('\n');
                    fprintf(['End Parallel Session ....','\n\n']);
                end
Marco Ratto's avatar
Marco Ratto committed
556
                diary on;
557 558
            else
                close(hfigstatus),
Marco Ratto's avatar
Marco Ratto committed
559 560 561
            end
            
            break
562
        else
Marco Ratto's avatar
Marco Ratto committed
563
            disp('Waiting for output files from slaves ...')
564 565
        end
    end
566
    
567 568
end

569
% Create return value.
570
iscrash = 0;
571
for j=1:totCPU,
572 573
    indPC=min(find(nCPU>=j));
    dynareParallelGetFiles([fname,'_output_',int2str(j),'.mat'],PRCDir,Parallel(indPC));
574 575
    load([fname,'_output_',int2str(j),'.mat'],'fOutputVar');
    delete([fname,'_output_',int2str(j),'.mat']);
Marco Ratto's avatar
Marco Ratto committed
576
    if isfield(fOutputVar,'OutputFileName'),
577
        dynareParallelGetFiles([fOutputVar.OutputFileName],PRCDir,Parallel(indPC));
Marco Ratto's avatar
Marco Ratto committed
578 579
    end
    if isfield(fOutputVar,'error'),
Marco Ratto's avatar
Marco Ratto committed
580
        disp(['Job number ',int2str(j),' crashed with error:']);
581 582
        iscrash=1;
        disp([fOutputVar.error.message]);
583 584 585
        for jstack=1:length(fOutputVar.error.stack)
            fOutputVar.error.stack(jstack),
        end
586 587
    else
        fOutVar(j)=fOutputVar;
Marco Ratto's avatar
Marco Ratto committed
588
    end
589
end
590
if iscrash,
591
    error('Remote jobs crashed');
592
end
Marco Ratto's avatar
Marco Ratto committed
593

594 595 596
pause(1), % Wait for all remote diary off completed

% Cleanup.
Marco Ratto's avatar
Marco Ratto committed
597 598
dynareParallelGetFiles('*.log',PRCDir,Parallel(1:totSlaves));

599
switch Strategy
Marco Ratto's avatar
Marco Ratto committed
600 601 602
    case 0
        for indPC=1:length(Parallel)
            if Parallel(indPC).Local == 0
603
                dynareParallelRmDir(PRCDir,Parallel(indPC));
Marco Ratto's avatar
Marco Ratto committed
604 605 606 607 608 609 610 611 612 613 614
            end
            
            if isempty(dir('dynareParallelLogFiles'))
                [A B C]=rmdir('dynareParallelLogFiles');
                mkdir('dynareParallelLogFiles');
            end
            
            copyfile('*.log','dynareParallelLogFiles');
            delete([fname,'*.log']);
            
            mydelete(['*_core*_input*.mat']);
615 616 617 618
            %             if Parallel(indPC).Local == 1
            %                 delete(['slaveParallel_input*.mat']);
            %             end
            
Marco Ratto's avatar
Marco Ratto committed
619
        end
620
        
Marco Ratto's avatar
Marco Ratto committed
621 622 623 624 625 626 627 628 629 630 631 632 633 634 635 636 637
        delete ConcurrentCommand1.bat
    case 1
        delete(['temp_input.mat'])
        if newInstance,
            if isempty(dir('dynareParallelLogFiles'))
                [A B C]=rmdir('dynareParallelLogFiles');
                mkdir('dynareParallelLogFiles');
            end
        end
        copyfile('*.log','dynareParallelLogFiles');
        if newInstance,
            delete ConcurrentCommand1.bat
        end
end



638