Skip to content
Snippets Groups Projects
Commit abb57c9f authored by fangq's avatar fangq
Browse files

massively speeding up loadjson for parsing large collection of unstructured small objects 

git-svn-id: http://svn.code.sf.net/p/iso2mesh/code/trunk/jsonlab@487 786e58fb-9377-0410-9ff7-e4ac0ac0635c
parent afbedadd
Branches
Tags
No related merge requests found
Show whitespace changes
Inline Side-by-side
loadjson.m
+ 5
− 37
View file @ abb57c9f
...@@ -109,41 +109,15 @@ if(jsoncount==1 && iscell(data)) ...@@ -109,41 +109,15 @@ if(jsoncount==1 && iscell(data))
data=data{1}; data=data{1};
end end
if(~isempty(data))
if(isstruct(data)) % data can be a struct array
data=jstruct2array(data);
elseif(iscell(data))
data=jcell2array(data);
end
end
if(isfield(opt,'progressbar_')) if(isfield(opt,'progressbar_'))
close(opt.progressbar_); close(opt.progressbar_);
end end
%%
function newdata=jcell2array(data)
len=length(data);
newdata=data;
for i=1:len
if(isstruct(data{i}))
newdata{i}=jstruct2array(data{i});
elseif(iscell(data{i}))
newdata{i}=jcell2array(data{i});
end
end
%%------------------------------------------------------------------------- %%-------------------------------------------------------------------------
function newdata=jstruct2array(data) function newdata=jstruct2array(data)
fn=fieldnames(data); fn=fieldnames(data);
newdata=data; newdata=data;
len=length(data); len=length(data);
for i=1:length(fn) % depth-first
for j=1:len
if(isstruct(getfield(data(j),fn{i})))
newdata(j)=setfield(newdata(j),fn{i},jstruct2array(getfield(data(j),fn{i})));
end
end
end
if(~isempty(strmatch('x0x5F_ArrayType_',fn)) && ~isempty(strmatch('x0x5F_ArrayData_',fn))) if(~isempty(strmatch('x0x5F_ArrayType_',fn)) && ~isempty(strmatch('x0x5F_ArrayData_',fn)))
newdata=cell(len,1); newdata=cell(len,1);
for j=1:len for j=1:len
...@@ -214,6 +188,9 @@ function object = parse_object(varargin) ...@@ -214,6 +188,9 @@ function object = parse_object(varargin)
end end
end end
parse_char('}'); parse_char('}');
if(isstruct(object))
object=jstruct2array(object);
end
%%------------------------------------------------------------------------- %%-------------------------------------------------------------------------
...@@ -405,9 +382,8 @@ function str = parseStr(varargin) ...@@ -405,9 +382,8 @@ function str = parseStr(varargin)
%%------------------------------------------------------------------------- %%-------------------------------------------------------------------------
function num = parse_number(varargin) function num = parse_number(varargin)
global pos inStr len isoct global pos inStr isoct
currstr=inStr(pos:end); currstr=inStr(pos:min(pos+30,end));
numstr=0;
if(isoct~=0) if(isoct~=0)
numstr=regexp(currstr,'^\s*-?(?:0|[1-9]\d*)(?:\.\d+)?(?:[eE][+\-]?\d+)?','end'); numstr=regexp(currstr,'^\s*-?(?:0|[1-9]\d*)(?:\.\d+)?(?:[eE][+\-]?\d+)?','end');
[num, one] = sscanf(currstr, '%f', 1); [num, one] = sscanf(currstr, '%f', 1);
...@@ -439,13 +415,6 @@ function val = parse_value(varargin) ...@@ -439,13 +415,6 @@ function val = parse_value(varargin)
return; return;
case '{' case '{'
val = parse_object(varargin{:}); val = parse_object(varargin{:});
if isstruct(val)
if(~isempty(strmatch('x0x5F_ArrayType_',fieldnames(val), 'exact')))
val=jstruct2array(val);
end
elseif isempty(val)
val = struct;
end
return; return;
case {'-','0','1','2','3','4','5','6','7','8','9'} case {'-','0','1','2','3','4','5','6','7','8','9'}
val = parse_number(varargin{:}); val = parse_number(varargin{:});
...@@ -564,4 +533,3 @@ end ...@@ -564,4 +533,3 @@ end
if(endpos==0) if(endpos==0)
error('unmatched "]"'); error('unmatched "]"');
end end
loadubjson.m
+ 7
− 94
View file @ abb57c9f
...@@ -94,15 +94,6 @@ if(jsoncount==1 && iscell(data)) ...@@ -94,15 +94,6 @@ if(jsoncount==1 && iscell(data))
data=data{1}; data=data{1};
end end
if(~isempty(data))
if(isstruct(data)) % data can be a struct array
data=jstruct2array(data);
elseif(iscell(data))
data=jcell2array(data);
end
end
%% %%
function newdata=parse_collection(id,data,obj) function newdata=parse_collection(id,data,obj)
...@@ -114,80 +105,6 @@ if(jsoncount>0 && exist('data','var')) ...@@ -114,80 +105,6 @@ if(jsoncount>0 && exist('data','var'))
end end
end end
%%
function newdata=jcell2array(data)
len=length(data);
newdata=data;
for i=1:len
if(isstruct(data{i}))
newdata{i}=jstruct2array(data{i});
elseif(iscell(data{i}))
newdata{i}=jcell2array(data{i});
end
end
%%-------------------------------------------------------------------------
function newdata=jstruct2array(data)
fn=fieldnames(data);
newdata=data;
len=length(data);
for i=1:length(fn) % depth-first
for j=1:len
if(isstruct(getfield(data(j),fn{i})))
newdata(j)=setfield(newdata(j),fn{i},jstruct2array(getfield(data(j),fn{i})));
end
end
end
if(~isempty(strmatch('x0x5F_ArrayType_',fn)) && ~isempty(strmatch('x0x5F_ArrayData_',fn)))
newdata=cell(len,1);
for j=1:len
ndata=cast(data(j).x0x5F_ArrayData_,data(j).x0x5F_ArrayType_);
iscpx=0;
if(~isempty(strmatch('x0x5F_ArrayIsComplex_',fn)))
if(data(j).x0x5F_ArrayIsComplex_)
iscpx=1;
end
end
if(~isempty(strmatch('x0x5F_ArrayIsSparse_',fn)))
if(data(j).x0x5F_ArrayIsSparse_)
if(~isempty(strmatch('x0x5F_ArraySize_',fn)))
dim=double(data(j).x0x5F_ArraySize_);
if(iscpx && size(ndata,2)==4-any(dim==1))
ndata(:,end-1)=complex(ndata(:,end-1),ndata(:,end));
end
if isempty(ndata)
% All-zeros sparse
ndata=sparse(dim(1),prod(dim(2:end)));
elseif dim(1)==1
% Sparse row vector
ndata=sparse(1,ndata(:,1),ndata(:,2),dim(1),prod(dim(2:end)));
elseif dim(2)==1
% Sparse column vector
ndata=sparse(ndata(:,1),1,ndata(:,2),dim(1),prod(dim(2:end)));
else
% Generic sparse array.
ndata=sparse(ndata(:,1),ndata(:,2),ndata(:,3),dim(1),prod(dim(2:end)));
end
else
if(iscpx && size(ndata,2)==4)
ndata(:,3)=complex(ndata(:,3),ndata(:,4));
end
ndata=sparse(ndata(:,1),ndata(:,2),ndata(:,3));
end
end
elseif(~isempty(strmatch('x0x5F_ArraySize_',fn)))
if(iscpx && size(ndata,2)==2)
ndata=complex(ndata(:,1),ndata(:,2));
end
ndata=reshape(ndata(:),data(j).x0x5F_ArraySize_);
end
newdata{j}=ndata;
end
if(len==1)
newdata=newdata{1};
end
end
%%------------------------------------------------------------------------- %%-------------------------------------------------------------------------
function object = parse_object(varargin) function object = parse_object(varargin)
parse_char('{'); parse_char('{');
...@@ -226,6 +143,9 @@ function object = parse_object(varargin) ...@@ -226,6 +143,9 @@ function object = parse_object(varargin)
if(count==-1) if(count==-1)
parse_char('}'); parse_char('}');
end end
if(isstruct(object))
object=struct2jdata(object);
end
%%------------------------------------------------------------------------- %%-------------------------------------------------------------------------
function [cid,len]=elem_info(type) function [cid,len]=elem_info(type)
...@@ -241,7 +161,7 @@ end ...@@ -241,7 +161,7 @@ end
%%------------------------------------------------------------------------- %%-------------------------------------------------------------------------
function [data adv]=parse_block(type,count,varargin) function [data, adv]=parse_block(type,count,varargin)
global pos inStr isoct fileendian systemendian global pos inStr isoct fileendian systemendian
[cid,len]=elem_info(type); [cid,len]=elem_info(type);
datastr=inStr(pos:pos+len*count-1); datastr=inStr(pos:pos+len*count-1);
...@@ -282,7 +202,7 @@ global pos inStr isoct ...@@ -282,7 +202,7 @@ global pos inStr isoct
end end
if(~isempty(type)) if(~isempty(type))
if(count>=0) if(count>=0)
[object adv]=parse_block(type,count,varargin{:}); [object, adv]=parse_block(type,count,varargin{:});
if(~isempty(dim)) if(~isempty(dim))
object=reshape(object,dim); object=reshape(object,dim);
end end
...@@ -292,7 +212,7 @@ global pos inStr isoct ...@@ -292,7 +212,7 @@ global pos inStr isoct
endpos=matching_bracket(inStr,pos); endpos=matching_bracket(inStr,pos);
[cid,len]=elem_info(type); [cid,len]=elem_info(type);
count=(endpos-pos)/len; count=(endpos-pos)/len;
[object adv]=parse_block(type,count,varargin{:}); [object, adv]=parse_block(type,count,varargin{:});
pos=pos+adv; pos=pos+adv;
parse_char(']'); parse_char(']');
return; return;
...@@ -426,13 +346,6 @@ function val = parse_value(varargin) ...@@ -426,13 +346,6 @@ function val = parse_value(varargin)
return; return;
case '{' case '{'
val = parse_object(varargin{:}); val = parse_object(varargin{:});
if isstruct(val)
if(~isempty(strmatch('x0x5F_ArrayType_',fieldnames(val), 'exact')))
val=jstruct2array(val);
end
elseif isempty(val)
val = struct;
end
return; return;
case {'i','U','I','l','L','d','D'} case {'i','U','I','l','L','d','D'}
val = parse_number(varargin{:}); val = parse_number(varargin{:});
...@@ -511,7 +424,7 @@ while(pos<len) ...@@ -511,7 +424,7 @@ while(pos<len)
end end
error('unmatched quotation mark'); error('unmatched quotation mark');
%%------------------------------------------------------------------------- %%-------------------------------------------------------------------------
function [endpos e1l e1r maxlevel] = matching_bracket(str,pos) function [endpos, e1l, e1r, maxlevel] = matching_bracket(str,pos)
global arraytoken global arraytoken
level=1; level=1;
maxlevel=level; maxlevel=level;
......
struct2jdata.m 0 → 100644
+ 96
− 0
View file @ abb57c9f
function newdata=struct2jdata(data,varargin)
%
% newdata=struct2jdata(data,opt,...)
%
% convert a JData object (in the form of a struct array) into an array
%
% authors:Qianqian Fang (fangq<at> nmr.mgh.harvard.edu)
%
% input:
% data: a struct array. If data contains JData keywords in the first
% level children, these fields are parsed and regrouped into a
% data object (arrays, trees, graphs etc) based on JData
% specification. The JData keywords are
% "_ArrayType_", "_ArraySize_", "_ArrayData_"
% "_ArrayIsSparse_", "_ArrayIsComplex_"
% opt: (optional) a list of 'Param',value pairs for additional options
% The supported options include
% 'Recursive', if set to 1, will apply the conversion to
% every child; 0 to disable
%
% output:
% newdata: the covnerted data if the input data does contain a JData
% structure; otherwise, the same as the input.
%
% examples:
% obj=struct('_ArrayType_','double','_ArraySize_',[2 3],
% '_ArrayIsSparse_',1 ,'_ArrayData_',null);
% ubjdata=struct2jdata(obj);
%
% license:
% BSD or GPL version 3, see LICENSE_{BSD,GPLv3}.txt files for details
%
% -- this function is part of JSONLab toolbox (http://iso2mesh.sf.net/cgi-bin/index.cgi?jsonlab)
%
fn=fieldnames(data);
newdata=data;
len=length(data);
if(jsonopt('Recursive',0,varargin{:})==1)
for i=1:length(fn) % depth-first
for j=1:len
if(isstruct(getfield(data(j),fn{i})))
newdata(j)=setfield(newdata(j),fn{i},jstruct2array(getfield(data(j),fn{i})));
end
end
end
end
if(~isempty(strmatch('x0x5F_ArrayType_',fn)) && ~isempty(strmatch('x0x5F_ArrayData_',fn)))
newdata=cell(len,1);
for j=1:len
ndata=cast(data(j).x0x5F_ArrayData_,data(j).x0x5F_ArrayType_);
iscpx=0;
if(~isempty(strmatch('x0x5F_ArrayIsComplex_',fn)))
if(data(j).x0x5F_ArrayIsComplex_)
iscpx=1;
end
end
if(~isempty(strmatch('x0x5F_ArrayIsSparse_',fn)))
if(data(j).x0x5F_ArrayIsSparse_)
if(~isempty(strmatch('x0x5F_ArraySize_',fn)))
dim=double(data(j).x0x5F_ArraySize_);
if(iscpx && size(ndata,2)==4-any(dim==1))
ndata(:,end-1)=complex(ndata(:,end-1),ndata(:,end));
end
if isempty(ndata)
% All-zeros sparse
ndata=sparse(dim(1),prod(dim(2:end)));
elseif dim(1)==1
% Sparse row vector
ndata=sparse(1,ndata(:,1),ndata(:,2),dim(1),prod(dim(2:end)));
elseif dim(2)==1
% Sparse column vector
ndata=sparse(ndata(:,1),1,ndata(:,2),dim(1),prod(dim(2:end)));
else
% Generic sparse array.
ndata=sparse(ndata(:,1),ndata(:,2),ndata(:,3),dim(1),prod(dim(2:end)));
end
else
if(iscpx && size(ndata,2)==4)
ndata(:,3)=complex(ndata(:,3),ndata(:,4));
end
ndata=sparse(ndata(:,1),ndata(:,2),ndata(:,3));
end
end
elseif(~isempty(strmatch('x0x5F_ArraySize_',fn)))
if(iscpx && size(ndata,2)==2)
ndata=complex(ndata(:,1),ndata(:,2));
end
ndata=reshape(ndata(:),data(j).x0x5F_ArraySize_);
end
newdata{j}=ndata;
end
if(len==1)
newdata=newdata{1};
end
end
\ No newline at end of file
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment