Commit a02dd60d authored by ratto's avatar ratto
Browse files

Modified version of parallel routines, with more sophisticated management of...

Modified version of parallel routines, with more sophisticated management of slave processes, which are not closed after the completion of each job but can stay in stand-by until a new job is assigned to them.

git-svn-id: https://www.dynare.org/svn/dynare/trunk@3379 ac1d8469-bf42-47a9-8791-bf33cf982152
parent 0378c804
function [fOutVar,nBlockPerCPU, totCPU] = masterParallelMan(Parallel,fBlock,nBlock,NamFileInput,fname,fInputVar,fGlobalVar)
% Top-level function called on the master computer when parallelizing a task.
% This version has amore sophisticated management of slave processes,
% which are no longer closed at the end of each job
%
% The number of parallelized threads will be equal to (nBlock-fBlock+1).
%
% INPUTS
% Parallel [struct vector] copy of options_.parallel
% fBlock [int] index number of the first thread
% (between 1 and nBlock)
% nBlock [int] index number of the last thread
% NamFileInput [cell array] containins the list of input files to be
% copied in the working directory of remote slaves
% 2 columns, as many lines as there are files
% - first column contains directory paths
% - second column contains filenames
% fname [string] name of the function to be parallelized, and
% which will be run on the slaves
% fInputVar [struct] structure containing local variables to be used
% by fName on the slaves
% fGlobalVar [struct] structure containing global variables to be used
% by fName on the slaves
%
% OUTPUT
% fOutVar [struct vector] result of the parallel computation, one
% struct per thread
% nBlockPerCPU [int vector] for each CPU used, indicates the number of
% threads run on that CPU
% totCPU [int] total number of CPU used (can be lower than
% the number of CPU declared in "Parallel", if
% the number of required threads is lower)
% Copyright (C) 2009 Dynare Team
%
% This file is part of Dynare.
%
% Dynare is free software: you can redistribute it and/or modify
% it under the terms of the GNU General Public License as published by
% the Free Software Foundation, either version 3 of the License, or
% (at your option) any later version.
%
% Dynare is distributed in the hope that it will be useful,
% but WITHOUT ANY WARRANTY; without even the implied warranty of
% MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
% GNU General Public License for more details.
%
% You should have received a copy of the GNU General Public License
% along with Dynare. If not, see <http://www.gnu.org/licenses/>.
persistent initialize
if isempty(initialize),
mydelete(['P_slave_*End.txt']);
mydelete(['slaveParallel_input*.mat']);
pause(1),
end
totCPU=0;
% Determine my hostname and my working directory
DyMo=pwd;
fInputVar.DyMo=DyMo;
if isunix || (~matlab_ver_less_than('7.4') && ismac) ,
% [tempo, MasterName]=system(['ifconfig | grep ''inet addr:''| grep -v ''127.0.0.1'' | cut -d: -f2 | awk ''{ print $1}''']);
[tempo, MasterName]=system('hostname --fqdn');
else
[tempo, MasterName]=system('hostname');
end
MasterName=deblank(MasterName);
fInputVar.MasterName = MasterName;
% Save input data for use by the slaves
if exist('fGlobalVar'),
save(['temp_input.mat'],'fInputVar','fGlobalVar')
else
save(['temp_input.mat'],'fInputVar')
end
save(['temp_input.mat'],'Parallel','-append')
% Determine the total number of available CPUs, and the number of threads to run on each CPU
for j=1:length(Parallel),
nCPU(j)=length(Parallel(j).NumCPU);
totCPU=totCPU+nCPU(j);
end
nCPU=cumsum(nCPU);
offset0 = fBlock-1;
if (nBlock-offset0)>totCPU,
diff = mod((nBlock-offset0),totCPU);
nBlockPerCPU(1:diff) = ceil((nBlock-offset0)/totCPU);
nBlockPerCPU(diff+1:totCPU) = floor((nBlock-offset0)/totCPU);
else
nBlockPerCPU(1:nBlock-offset0)=1;
totCPU = nBlock-offset0;
end
% Clean up remnants of previous runs
mydelete(['comp_status_',fname,'*.mat'])
mydelete(['P_',fname,'*End.txt']);
% Create a shell script containing the commands to launch the required tasks on the slaves
fid = fopen('ConcurrentCommand1.bat','w+');
for j=1:totCPU,
indPC=min(find(nCPU>=j));
if indPC>1
nCPU0 = nCPU(indPC-1);
else
nCPU0=0;
end
offset = sum(nBlockPerCPU(1:j-1))+offset0;
fid1=fopen(['P_',fname,'_',int2str(j),'End.txt'],'w+');
fclose(fid1);
fblck = offset+1;
nblck = sum(nBlockPerCPU(1:j));
save temp_input fblck nblck fname -append;
copyfile('temp_input.mat',['slaveJob',int2str(j),'.mat'])
if Parallel(indPC).Local ==0,
fid1=fopen(['stayalive',int2str(j),'.txt'],'w+');
fclose(fid1);
if isunix || (~matlab_ver_less_than('7.4') && ismac),
system(['scp stayalive',int2str(j),'.txt ',Parallel(indPC).user,'@',Parallel(indPC).PcName,':',Parallel(indPC).RemoteFolder]);
else
copyfile(['stayalive',int2str(j),'.txt'], ['\\',Parallel(indPC).PcName,'\',Parallel(indPC).RemoteDrive,'$\',Parallel(indPC).RemoteFolder]);
end
mydelete(['stayalive',int2str(j),'.txt'],'w+');
end
pause(1); % wait for possibly local alive CPU to start the new job or close by internal criteria
newInstance = 0;
if isempty( dir(['P_slave_',int2str(j),'End.txt'])); % check if j CPU is already alive
fid1=fopen(['P_slave_',int2str(j),'End.txt'],'w+');
fclose(fid1);
newInstance = 1;
storeGlobalVars( ['slaveParallel_input',int2str(j)]);
save( ['slaveParallel_input',int2str(j)],'Parallel','-append');
% prepare global vars for Slave
end
if Parallel(indPC).Local == 1 & newInstance, % run on the local machine
if isunix || (~matlab_ver_less_than('7.4') && ismac),
if exist('OCTAVE_VERSION')
% command1=['octave --eval fParallel\(',int2str(offset+1),',',int2str(sum(nBlockPerCPU(1:j))),',',int2str(j),',',int2str(indPC),',\''',fname,'\''\) &'];
command1=['octave --eval slaveParallel\(',int2str(j),',',int2str(indPC),'\) &'];
else
% command1=['matlab -nosplash -nodesktop -minimize -r fParallel\(',int2str(offset+1),',',int2str(sum(nBlockPerCPU(1:j))),',',int2str(j),',',int2str(indPC),',\''',fname,'\''\) &'];
command1=['matlab -nosplash -nodesktop -minimize -r slaveParallel\(',int2str(j),',',int2str(indPC),'\) &'];
end
else
if exist('OCTAVE_VERSION')
% command1=['start /B psexec -W ',DyMo, ' -a ',int2str(Parallel(indPC).NumCPU(j-nCPU0)),' -low octave --eval fParallel(',int2str(offset+1),',',int2str(sum(nBlockPerCPU(1:j))),',',int2str(j),',',int2str(indPC),',''',fname,''')'];
command1=['start /B psexec -W ',DyMo, ' -a ',int2str(Parallel(indPC).NumCPU(j-nCPU0)),' -low octave --eval slaveParallel(',int2str(j),',',int2str(indPC),')'];
else
% command1=['start /B psexec -W ',DyMo, ' -a ',int2str(Parallel(indPC).NumCPU(j-nCPU0)),' -low matlab -nosplash -nodesktop -minimize -r fParallel(',int2str(offset+1),',',int2str(sum(nBlockPerCPU(1:j))),',',int2str(j),',',int2str(indPC),',''',fname,''')'];
command1=['start /B psexec -W ',DyMo, ' -a ',int2str(Parallel(indPC).NumCPU(j-nCPU0)),' -low matlab -nosplash -nodesktop -minimize -r slaveParallel(',int2str(j),',',int2str(indPC),')'];
end
end
else
if isunix || (~matlab_ver_less_than('7.4') && ismac),
% [tempo, RemoteName]=system(['ssh ',Parallel(indPC).user,'@',Parallel(indPC).PcName,' "ifconfig | grep \''inet addr:\''| grep -v \''127.0.0.1\'' | cut -d: -f2 | awk \''{ print $1}\''"']);
[tempo, RemoteName]=system(['ssh ',Parallel(indPC).user,'@',Parallel(indPC).PcName,' "hostname --fqdn"']);
RemoteName=RemoteName(1:end-1);
RemoteFolder = Parallel(indPC).RemoteFolder;
else
RemoteName = Parallel(indPC).PcName;
RemoteFolder = [Parallel(indPC).RemoteDrive,':\',Parallel(indPC).RemoteFolder];
end
remoteFlag=1;
if strcmpi(RemoteName,MasterName),
if ~copyfile(['P_',fname,'_',int2str(j),'End.txt'],RemoteFolder),
remoteFlag=0;
end
end
if remoteFlag,
if j==nCPU0+1,
if newInstance, % clean remote folder
if isunix || (~matlab_ver_less_than('7.4') && ismac),
system(['ssh ',Parallel(indPC).user,'@',Parallel(indPC).PcName,' rm -fr ',Parallel(indPC).RemoteFolder,'/*']);
else
mydelete('*.*',['\\',Parallel(indPC).PcName,'\',Parallel(indPC).RemoteDrive,'$\',Parallel(indPC).RemoteFolder,'\']);
adir=dir(['\\',Parallel(indPC).PcName,'\',Parallel(indPC).RemoteDrive,'$\',Parallel(indPC).RemoteFolder,'\']);
for jdir=3:length(adir)
STATUS = rmdir(['\\',Parallel(indPC).PcName,'\',Parallel(indPC).RemoteDrive,'$\',Parallel(indPC).RemoteFolder,'\',adir(jdir).name],'s');
if STATUS == 0,
disp(['Warning!: Directory ',adir(jdir).name,' could not be removed from ',Parallel(indPC).PcName,'.'])
end
end
end
end
if isunix || (~matlab_ver_less_than('7.4') && ismac),
system(['scp ',fname,'_input.mat ',Parallel(indPC).user,'@',Parallel(indPC).PcName,':',Parallel(indPC).RemoteFolder]);
for jfil=1:size(NamFileInput,1)
if ~isempty(NamFileInput{jfil,1})
system(['ssh ',Parallel(indPC).user,'@',Parallel(indPC).PcName,' mkdir -p ',Parallel(indPC).RemoteFolder,'/',NamFileInput{jfil,1}])
end
system(['scp ',NamFileInput{jfil,1},NamFileInput{jfil,2},' ',Parallel(indPC).user,'@',Parallel(indPC).PcName,':',Parallel(indPC).RemoteFolder,'/',NamFileInput{jfil,1}]);
end
system(['scp slaveJob',int2str(j),'.mat ',Parallel(indPC).user,'@',Parallel(indPC).PcName,':',Parallel(indPC).RemoteFolder]);
if newInstance,
system(['scp slaveParallel_input',int2str(j),'.mat ',Parallel(indPC).user,'@',Parallel(indPC).PcName,':',Parallel(indPC).RemoteFolder]);
end
else
copyfile([fname,'_input.mat'], ['\\',Parallel(indPC).PcName,'\',Parallel(indPC).RemoteDrive,'$\',Parallel(indPC).RemoteFolder]);
for jfil=1:size(NamFileInput,1)
copyfile([NamFileInput{jfil,1},NamFileInput{jfil,2}],['\\',Parallel(indPC).PcName,'\',Parallel(indPC).RemoteDrive,'$\',Parallel(indPC).RemoteFolder,'\',NamFileInput{jfil,1}])
end
copyfile(['slaveJob',int2str(j),'.mat'], ['\\',Parallel(indPC).PcName,'\',Parallel(indPC).RemoteDrive,'$\',Parallel(indPC).RemoteFolder]);
if newInstance,
copyfile(['slaveParallel_input',int2str(j),'.mat'], ['\\',Parallel(indPC).PcName,'\',Parallel(indPC).RemoteDrive,'$\',Parallel(indPC).RemoteFolder]);
end
end
end
end
if newInstance,
if isunix || (~matlab_ver_less_than('7.4') && ismac),
if exist('OCTAVE_VERSION'),
command1=['ssh ',Parallel(indPC).user,'@',Parallel(indPC).PcName,' "cd ',Parallel(indPC).RemoteFolder, '; octave --eval fParallel\(',int2str(offset+1),',',int2str(sum(nBlockPerCPU(1:j))),',',int2str(j),',',int2str(indPC),',\''',fname,'\''\);" &'];
else
command1=['ssh ',Parallel(indPC).user,'@',Parallel(indPC).PcName,' "cd ',Parallel(indPC).RemoteFolder, '; matlab -nosplash -nodesktop -minimize -r fParallel\(',int2str(offset+1),',',int2str(sum(nBlockPerCPU(1:j))),',',int2str(j),',',int2str(indPC),',\''',fname,'\''\);" &'];
end
else
if ~strcmp(Parallel(indPC).PcName,MasterName), % run on a remote machine
if exist('OCTAVE_VERSION'),
command1=['start /B psexec \\',Parallel(indPC).PcName,' -e -u ',Parallel(indPC).user,' -p ',Parallel(indPC).passwd,' -W ',Parallel(indPC).RemoteDrive,':\',Parallel(indPC).RemoteFolder,'\ -a ',int2str(Parallel(indPC).NumCPU(j-nCPU0)), ...
' -low octave --eval fParallel(',int2str(offset+1),',',int2str(sum(nBlockPerCPU(1:j))),',',int2str(j),',',int2str(indPC),',''',fname,''')'];
else
command1=['start /B psexec \\',Parallel(indPC).PcName,' -e -u ',Parallel(indPC).user,' -p ',Parallel(indPC).passwd,' -W ',Parallel(indPC).RemoteDrive,':\',Parallel(indPC).RemoteFolder,'\ -a ',int2str(Parallel(indPC).NumCPU(j-nCPU0)), ...
' -low matlab -nosplash -nodesktop -minimize -r fParallel(',int2str(offset+1),',',int2str(sum(nBlockPerCPU(1:j))),',',int2str(j),',',int2str(indPC),',''',fname,''')'];
end
else % run on the local machine via the network
if exist('OCTAVE_VERSION'),
command1=['start /B psexec \\',Parallel(indPC).PcName,' -e -W ',Parallel(indPC).RemoteDrive,':\',Parallel(indPC).RemoteFolder,'\ -a ',int2str(Parallel(indPC).NumCPU(j-nCPU0)), ...
' -low octave --eval fParallel(',int2str(offset+1),',',int2str(sum(nBlockPerCPU(1:j))),',',int2str(j),',',int2str(indPC),',''',fname,''')'];
else
command1=['start /B psexec \\',Parallel(indPC).PcName,' -e -W ',Parallel(indPC).RemoteDrive,':\',Parallel(indPC).RemoteFolder,'\ -a ',int2str(Parallel(indPC).NumCPU(j-nCPU0)), ...
' -low matlab -nosplash -nodesktop -minimize -r fParallel(',int2str(offset+1),',',int2str(sum(nBlockPerCPU(1:j))),',',int2str(j),',',int2str(indPC),',''',fname,''')'];
end
end
end
end
end
fprintf(fid,'%s\n',command1);
end
fclose(fid);
% Run the slaves
if isunix || (~matlab_ver_less_than('7.4') && ismac),
system('sh ConcurrentCommand1.bat &');
pause(1)
else
system('ConcurrentCommand1.bat');
end
% Wait for the slaves to finish their job, and display some progress information meanwhile
t0=cputime;
t00=cputime;
hh=NaN(1,nBlock);
if exist('OCTAVE_VERSION'),
diary off;
else
hfigstatus = figure('name',['Parallel ',fname],...
'MenuBar', 'none', ...
'NumberTitle','off');
vspace = 0.1;
ncol = ceil(totCPU/10);
hspace = 0.9/ncol;
for j=1:totCPU,
jrow = mod(j-1,10)+1;
jcol = ceil(j/10);
hstatus(j) = axes('position',[0.05/ncol+(jcol-1)/ncol 0.92-vspace*(jrow-1) 0.9/ncol 0.03], ...
'box','on','xtick',[],'ytick',[],'xlim',[0 1],'ylim',[0 1]);
end
cumBlockPerCPU = cumsum(nBlockPerCPU);
end
pcerdone = NaN(1,totCPU);
while (1)
waitbarString = '';
statusString = '';
pause(1)
stax = dir(['comp_status_',fname,'*.mat']);
for j=1:length(stax),
try
load(stax(j).name)
pcerdone(j) = prtfrc;
if exist('OCTAVE_VERSION'),
statusString = [statusString, waitbarString, ', %3.f%% done! '];
else
status_String{j} = waitbarString;
status_Title{j} = waitbarTitle;
idCPU(j) = njob;
end
if prtfrc==1, delete(stax(j).name), end
catch
end
end
if exist('OCTAVE_VERSION'),
printf([statusString,'\r'], 100 .* pcerdone);
else
figure(hfigstatus),
for j=1:length(stax),
axes(hstatus(idCPU(j))),
hpat = findobj(hstatus(idCPU(j)),'Type','patch');
if ~isempty(hpat),
set(hpat,'XData',[0 0 pcerdone(j) pcerdone(j)])
else
patch([0 0 pcerdone(j) pcerdone(j)],[0 1 1 0],'r','EdgeColor','r')
end
title([status_Title{j},' - ',status_String{j}]);
end
end
if isempty(dir(['P_',fname,'_*End.txt']))
mydelete(['comp_status_',fname,'*.mat'])
if ~exist('OCTAVE_VERSION'),
close(hfigstatus),
else
printf('\n');
diary on;
end
break
end
end
% Create return value
for j=1:totCPU,
load([fname,'_output_',int2str(j),'.mat'],'fOutputVar');
delete([fname,'_output_',int2str(j),'.mat']);
fOutVar(j)=fOutputVar;
end
% Cleanup
delete([fname,'_input.mat'])
delete ConcurrentCommand1.bat
function slaveParallel(whoiam,ThisMatlab)
% In a parallelization context, this function is launched on slave
% machines, to initialize MATLAB and DYNARE environment and waits for
% instructions sent by the Master.
%
% INPUTS
% whoiam [int] index number of this CPU among all CPUs in the
% cluster
% ThisMatlab [int] index number of this slave machine in the cluster
% Copyright (C) 2009 Dynare Team
%
% This file is part of Dynare.
%
% Dynare is free software: you can redistribute it and/or modify
% it under the terms of the GNU General Public License as published by
% the Free Software Foundation, either version 3 of the License, or
% (at your option) any later version.
%
% Dynare is distributed in the hope that it will be useful,
% but WITHOUT ANY WARRANTY; without even the implied warranty of
% MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
% GNU General Public License for more details.
%
% You should have received a copy of the GNU General Public License
% along with Dynare. If not, see <http://www.gnu.org/licenses/>.
global funcName
warning off;
diary off;
delete( ['slaveParallel_',int2str(whoiam),'.log']);
diary( ['slaveParallel_',int2str(whoiam),'.log']);
% configure dynare environment
dynareroot = dynare_config();
% Load input data
load( ['slaveParallel_input',int2str(whoiam)])
%loads fGlobalVar Parallel
if exist('fGlobalVar'),
globalVars = fieldnames(fGlobalVar);
for j=1:length(globalVars),
eval(['global ',globalVars{j},';'])
end
struct2local(fGlobalVar);
clear fGlobalVar
end
t0=clock;
fslave = dir( ['slaveParallel_input',int2str(whoiam),'.mat']);
while (etime(clock,t0)<1200 && ~isempty(fslave)) || ~isempty(dir(['stayalive',int2str(whoiam),'.txt'])),
if ~isempty(dir(['stayalive',int2str(whoiam),'.txt'])),
t0=clock;
delete(['stayalive',int2str(whoiam),'.txt']);
end
% I wait for 20 min or while mater asks to exit (i.e. it cancels fslave file)
pause(1);
fjob = dir(['slaveJob',int2str(whoiam),'.mat']);
if ~isempty(fjob),
clear fGlobalVar fInputVar fblck nblck fname
load(['slaveJob',int2str(whoiam),'.mat']);
funcName=fname; % update global job name
% loads values for fblck nblck fname fGlobalVar fInputVar
delete(['slaveJob',int2str(whoiam),'.mat']);
if exist('fGlobalVar'),
globalVars = fieldnames(fGlobalVar);
% for j=1:length(globalVars),
% eval(['global ',globalVars{j},';'])
% end
struct2local(fGlobalVar);
end
fInputVar.Parallel = Parallel;
% Launch the routine to be run in parallel
tic,
fOutputVar = feval(fname, fInputVar ,fblck, nblck, whoiam, ThisMatlab);
toc,
if isfield(fOutputVar,'OutputFileName'),
OutputFileName = fOutputVar.OutputFileName;
else
OutputFileName = '';
end
if(whoiam)
% Save the output result
save([ fname,'_output_',int2str(whoiam),'.mat'],'fOutputVar' )
% Inform the master that the job is finished, and transfer the output data
if Parallel(ThisMatlab).Local
delete(['P_',fname,'_',int2str(whoiam),'End.txt']);
else
if isunix || (~matlab_ver_less_than('7.4') && ismac),
for j=1:size(OutputFileName,1),
system(['scp ',OutputFileName{j,1},OutputFileName{j,2},' ',Parallel(ThisMatlab).user,'@',fInputVar.MasterName,':',fInputVar.DyMo,'/',OutputFileName{j,1}]);
end
system(['scp ',fname,'_output_',int2str(whoiam),'.mat ',Parallel(ThisMatlab).user,'@',fInputVar.MasterName,':',fInputVar.DyMo]);
system(['ssh ',Parallel(ThisMatlab).user,'@',fInputVar.MasterName,' rm -f ',fInputVar.DyMo,'/P_',fname,'_',int2str(whoiam),'End.txt']);
% system(['fusermount -u ~/MasterRemoteMirror_',fname,'_',int2str(whoiam)]);
% system(['rm -r ~/MasterRemoteMirror_',fname,'_',int2str(whoiam)]);
else
for j=1:size(OutputFileName,1),
copyfile([OutputFileName{j,1},OutputFileName{j,2}],['\\',fInputVar.MasterName,'\',fInputVar.DyMo(1),'$\',fInputVar.DyMo(4:end),'\',OutputFileName{j,1}])
end
copyfile([fname,'_output_',int2str(whoiam),'.mat'],['\\',fInputVar.MasterName,'\',fInputVar.DyMo(1),'$\',fInputVar.DyMo(4:end)]);
delete(['\\',fInputVar.MasterName,'\',fInputVar.DyMo(1),'$\',fInputVar.DyMo(4:end),'\P_',fname,'_',int2str(whoiam),'End.txt']);
end
end
end
disp(['Job ',fname,' on CPU ',int2str(whoiam),' completed.'])
t0 =clock; % re-set waiting time of 20 mins
end
fslave = dir( ['slaveParallel_input',int2str(whoiam),'.mat']); % check if Master asks to exit
end
if Parallel(ThisMatlab).Local
delete(['P_slave_',int2str(whoiam),'End.txt']);
else
if isunix || (~matlab_ver_less_than('7.4') && ismac),
system(['ssh ',Parallel(ThisMatlab).user,'@',fInputVar.MasterName,' rm -f ',fInputVar.DyMo,'/P_slave_',int2str(whoiam),'End.txt']);
else
delete(['\\',fInputVar.MasterName,'\',fInputVar.DyMo(1),'$\',fInputVar.DyMo(4:end),'\P_slave_',int2str(whoiam),'End.txt']);
end
end
disp(['slaveParallel on CPU ',int2str(whoiam),' completed.'])
diary off;
exit;
function storeGlobalVars(fname,append)
% Stores all global vars in structure fGlobalVar and saves it in the file fname.mat
%
% INPUTS
% fname [str] name of the file
% Copyright (C) 2009 Dynare Team
%
% This file is part of Dynare.
%
% Dynare is free software: you can redistribute it and/or modify
% it under the terms of the GNU General Public License as published by
% the Free Software Foundation, either version 3 of the License, or
% (at your option) any later version.
%
% Dynare is distributed in the hope that it will be useful,
% but WITHOUT ANY WARRANTY; without even the implied warranty of
% MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
% GNU General Public License for more details.
%
% You should have received a copy of the GNU General Public License
% along with Dynare. If not, see <http://www.gnu.org/licenses/>.
GlobalNames = who('global');
varlist = '';
for j=1:length(GlobalNames);
eval(['global ',GlobalNames{j},';']);
eval(['fGlobalVar.',GlobalNames{j},'=',GlobalNames{j},';']);
end
if nargin<2,
save(fname,'fGlobalVar');
else
save(fname,'fGlobalVar','-append');
end
\ No newline at end of file
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment