Skip to content
Snippets Groups Projects
Commit eefccf3e authored by Qianqian Fang's avatar Qianqian Fang
Browse files

call jsonencode/decode in jsave/jload, parse embedded jdata struct

parent 9434103a
No related branches found
No related tags found
No related merge requests found
Show whitespace changes
Inline Side-by-side
jdatadecode.m
+ 31
− 2
View file @ eefccf3e
......@@ -4,7 +4,7 @@ function newdata=jdatadecode(data,varargin)
%
% Convert all JData object (in the form of a struct array) into an array
% (accepts JData objects loaded from either loadjson/loadubjson or
% jsondecode for MATLAB R2018a or later)
% jsondecode for MATLAB R2016b or later)
%
% This function implements the JData Specification Draft 3 (Jun. 2020)
% see http://github.com/fangq/jdata for details
......@@ -113,6 +113,9 @@ function newdata=jdatadecode(data,varargin)
for j=1:len
if(isfield(data,N_('_ArrayZipSize_')) && isfield(data,N_('_ArrayZipData_')))
zipmethod='zip';
if(isstruct(data(j).(N_('_ArrayZipSize_'))))
data(j).(N_('_ArrayZipSize_'))=jdatadecode(data(j).(N_('_ArrayZipSize_')),opt);
end
dims=data(j).(N_('_ArrayZipSize_'))(:)';
if(length(dims)==1)
dims=[1 dims];
......@@ -140,6 +143,9 @@ function newdata=jdatadecode(data,varargin)
error('compression method is not supported');
end
else
if(isstruct(data(j).(N_('_ArrayData_'))))
data(j).(N_('_ArrayData_'))=jdatadecode(data(j).(N_('_ArrayData_')),opt);
end
if(isstruct(data(j).(N_('_ArrayData_'))) && isfield(data(j).(N_('_ArrayData_')),N_('_ArrayType_')))
data(j).(N_('_ArrayData_'))=jdatadecode(data(j).(N_('_ArrayData_')),varargin{:});
end
......@@ -149,6 +155,9 @@ function newdata=jdatadecode(data,varargin)
ndata=cast(data(j).(N_('_ArrayData_')),char(data(j).(N_('_ArrayType_'))));
end
if(isfield(data,N_('_ArrayZipSize_')))
if(isstruct(data(j).(N_('_ArrayZipSize_'))))
data(j).(N_('_ArrayZipSize_'))=jdatadecode(data(j).(N_('_ArrayZipSize_')),opt);
end
dims=data(j).(N_('_ArrayZipSize_'))(:)';
if(iscell(dims))
dims=cell2mat(dims);
......@@ -160,6 +169,9 @@ function newdata=jdatadecode(data,varargin)
ndata=permute(ndata,ndims(ndata):-1:1);
end
iscpx=0;
if(isfield(data,N_('_ArrayIsComplex_')) && isstruct(data(j).(N_('_ArrayIsComplex_'))) )
data(j).(N_('_ArrayIsComplex_'))=jdatadecode(data(j).(N_('_ArrayIsComplex_')),opt);
end
if(isfield(data,N_('_ArrayIsComplex_')) && data(j).(N_('_ArrayIsComplex_')) )
iscpx=1;
end
......@@ -170,8 +182,14 @@ function newdata=jdatadecode(data,varargin)
iscol=1;
end
end
if(isfield(data,N_('_ArrayIsSparse_')) && isstruct(data(j).(N_('_ArrayIsSparse_'))) )
data(j).(N_('_ArrayIsSparse_'))=jdatadecode(data(j).(N_('_ArrayIsSparse_')),opt);
end
if(isfield(data,N_('_ArrayIsSparse_')) && data(j).(N_('_ArrayIsSparse_')))
if(isfield(data,N_('_ArraySize_')))
if(isstruct(data(j).(N_('_ArraySize_'))))
data(j).(N_('_ArraySize_'))=jdatadecode(data(j).(N_('_ArraySize_')),opt);
end
dim=data(j).(N_('_ArraySize_'))(:)';
if(iscell(dim))
dim=cell2mat(dim);
......@@ -203,6 +221,9 @@ function newdata=jdatadecode(data,varargin)
ndata=sparse(ndata(1,:),ndata(2,:),ndata(3,:));
end
elseif(isfield(data,N_('_ArrayShape_')))
if(isstruct(data(j).(N_('_ArrayShape_'))))
data(j).(N_('_ArrayShape_'))=jdatadecode(data(j).(N_('_ArrayShape_')),opt);
end
if(iscpx)
if(size(ndata,1)==2)
dim=size(ndata);
......@@ -227,7 +248,11 @@ function newdata=jdatadecode(data,varargin)
else
datasize=size(arraydata);
end
if(isstruct(data(j).(N_('_ArraySize_'))))
data(j).(N_('_ArraySize_'))=jdatadecode(data(j).(N_('_ArraySize_')),opt);
end
arraysize=data.(N_('_ArraySize_'));
if(iscell(arraysize))
arraysize=cell2mat(arraysize);
end
......@@ -278,12 +303,16 @@ function newdata=jdatadecode(data,varargin)
end
ndata=spdiags(reshape(arraydata,min(arraysize),datasize(1)),double(shapeid{2}):-1:-double(shapeid{3}),arraysize(1),arraysize(2));
elseif(strcmpi(shapeid{1},'toeplitz'))
ndata=toeplitz(arraydata(:,1),arraydata(:,2)).';
arraydata=reshape(arraydata,flipud(datasize(:))');
ndata=toeplitz(arraydata(1:arraysize(1),2),arraydata(1:arraysize(2),1));
end
if(opt.fullarrayshape && issparse(ndata))
ndata=cast(full(ndata),data(j).(N_('_ArrayType_')));
end
elseif(isfield(data,N_('_ArraySize_')))
if(isstruct(data(j).(N_('_ArraySize_'))))
data(j).(N_('_ArraySize_'))=jdatadecode(data(j).(N_('_ArraySize_')),opt);
end
if(iscpx)
ndata=complex(ndata(1,:),ndata(2,:));
end
......
......
jdataencode.m
+ 83
− 60
View file @ eefccf3e
......@@ -5,8 +5,11 @@ function jdata=jdataencode(data, varargin)
% jdata=jdataencode(data, options)
% jdata=jdataencode(data, 'Param1',value1, 'Param2',value2,...)
%
% Serialize a MATLAB struct or cell array into a JData-compliant
% structure as defined in the JData spec: http://github.com/fangq/jdata
% Annotate a MATLAB struct or cell array into a JData-compliant data
% structure as defined in the JData spec: http://github.com/fangq/jdata.
% This encoded form servers as an intermediate format that allows unambiguous
% storage, exchange of complex data structures and easy-to-serialize by
% json encoders such as savejson and jsonencode (MATLAB R2016b or newer)
%
% This function implements the JData Specification Draft 3 (Jun. 2020)
% see http://github.com/fangq/jdata for details
......@@ -17,6 +20,11 @@ function jdata=jdataencode(data, varargin)
% data: a structure (array) or cell (array) to be encoded.
% options: (optional) a struct or Param/value pairs for user
% specified options (first in [.|.] is the default)
% AnnotateArray: [0|1] - if set to 1, convert all 1D/2D matrices
% to the annotated JData array format to preserve data types;
% N-D (N>2), complex and sparse arrays are encoded using the
% annotated format by default. Please set this option to 1 if
% you intend to use MATLAB's jsonencode to convert to JSON.
% Base64: [0|1] if set to 1, _ArrayZipData_ is assumed to
% be encoded with base64 format and need to be
% decoded first. This is needed for JSON but not
......@@ -50,6 +58,14 @@ function jdata=jdataencode(data, varargin)
% example:
% jd=jdataencode(struct('a',rand(5)+1i*rand(5),'b',[],'c',sparse(5,5)))
%
% encodedmat=jdataencode(single(magic(5)),'annotatearray',1,'prefix','x')
% jdatadecode(jsondecode(jsonencode(encodedmat))) % serialize by jsonencode
% jdatadecode(loadjson(savejson('',encodedmat))) % serialize by savejson
%
% encodedtoeplitz=jdataencode(uint8(toeplitz([1,2,3,4],[1,5,6])),'usearrayshape',1,'prefix','x')
% jdatadecode(jsondecode(jsonencode(encodedtoeplitz))) % serialize by jsonencode
% jdatadecode(loadjson(savejson('',encodedtoeplitz))) % serialize by savejson
%
% license:
% BSD or GPL version 3, see LICENSE_{BSD,GPLv3}.txt files for details
%
......@@ -77,6 +93,7 @@ opt.mapasstruct=jsonopt('MapAsStruct',0,opt);
opt.usearrayzipsize=jsonopt('UseArrayZipSize',1,opt);
opt.messagepack=jsonopt('MessagePack',0,opt);
opt.usearrayshape=jsonopt('UseArrayShape',0,opt) && exist('bandwidth');
opt.annotatearray=jsonopt('AnnotateArray',0,opt);
jdata=obj2jd(data,opt);
......@@ -166,14 +183,9 @@ newitem=struct(N('_ArrayType_'),class(item),N('_ArraySize_'),size(item));
zipmethod=varargin{1}.compression;
minsize=varargin{1}.compressarraysize;
% no encoding for char arrays or non-sparse real vectors
if(isempty(item) || isa(item,'string') || ischar(item) || varargin{1}.nestarray || ...
((isvector(item) || (ndims(item)==2 && ~varargin{1}.usearrayshape)) ...
&& isreal(item) && ~issparse(item)))
newitem=item;
return;
% 2d numerical (real/complex/sparse) arrays with _ArrayShape_ encoding enabled
elseif(varargin{1}.usearrayshape && ndims(item)==2 && ~isvector(item))
if(varargin{1}.usearrayshape && ndims(item)==2 && ~isvector(item))
encoded=1;
if(~isreal(item))
newitem.(N('_ArrayIsComplex_'))=true;
end
......@@ -214,11 +226,12 @@ elseif(varargin{1}.usearrayshape && ndims(item)==2 && ~isvector(item))
newitem.(N('_ArrayData_'))(1,1:size(item,2))=item(1,:);
newitem.(N('_ArrayData_'))(2,1:size(item,1))=item(:,1).';
else % full matrix
newitem=item;
newitem=rmfield(newitem,N('_ArrayZipSize_'));
encoded=0;
end
% serialize complex data at last
if(isstruct(newitem) && ~isreal(newitem.(N('_ArrayData_'))))
if(encoded && isstruct(newitem) && ~isreal(newitem.(N('_ArrayData_'))))
item=squeeze(zeros([2, size(newitem.(N('_ArrayData_')))]));
item(1,:)=real(newitem.(N('_ArrayData_'))(:));
item(2,:)=imag(newitem.(N('_ArrayData_'))(:));
......@@ -228,6 +241,7 @@ elseif(varargin{1}.usearrayshape && ndims(item)==2 && ~isvector(item))
% wrap _ArrayData_ into a single row vector, and store preprocessed
% size to _ArrayZipSize_ (force varargin{1}.usearrayzipsize=true)
if(encoded)
if(isstruct(newitem) && ~isvector(newitem.(N('_ArrayData_'))))
item=newitem.(N('_ArrayData_'));
item=permute(item,ndims(item):-1:1);
......@@ -235,16 +249,26 @@ elseif(varargin{1}.usearrayshape && ndims(item)==2 && ~isvector(item))
else
newitem=rmfield(newitem,N('_ArrayZipSize_'));
end
newitem.(N('_ArrayData_'))=full(newitem.(N('_ArrayData_')));
else
return
end
end
% no encoding for char arrays or non-sparse real vectors
if(isempty(item) || isa(item,'string') || ischar(item) || varargin{1}.nestarray || ...
((isvector(item) || ndims(item)==2) && isreal(item) && ~issparse(item) && ...
~varargin{1}.annotatearray))
newitem=item;
return;
end
if(isa(item,'logical'))
item=uint8(item);
end
if(isreal(item))
if(issparse(item))
fulldata=full(item(find(item)));
fulldata=full(item(item~=0));
newitem.(N('_ArrayIsSparse_'))=true;
newitem.(N('_ArrayZipSize_'))=[2+(~isvector(item)),length(fulldata)];
if(isvector(item))
......@@ -262,7 +286,7 @@ else
else
newitem.(N('_ArrayIsComplex_'))=true;
if(issparse(item))
fulldata=full(item(find(item)));
fulldata=full(item(item~=0));
newitem.(N('_ArrayIsSparse_'))=true;
newitem.(N('_ArrayZipSize_'))=[3+(~isvector(item)),length(fulldata)];
if(isvector(item))
......@@ -286,7 +310,6 @@ else
newitem.(N('_ArrayData_'))=permute(data,ndims(data):-1:1);
newitem=rmfield(newitem,N('_ArrayZipSize_'));
end
end
if(~isempty(zipmethod) && numel(item)>minsize)
compfun=str2func([zipmethod 'encode']);
......
......
jload.m
+ 19
− 3
View file @ eefccf3e
......@@ -23,8 +23,11 @@ function varargout=jload(filename, varargin)
% ws ['base'|'wsname']: the name of the workspace in which the
% variables are to be saved
% vars [{'var1','var2',...}]: list of variables to be saved
% Header [0|1]: if set to 1, return the metadata of the variables
% header [0|1]: if set to 1, return the metadata of the variables
% stored in the file
% matlab [0|1] if set to 1, use matlab's built-in jsondecode to
% parse the json file and then decode the output by
% jdatadecode; input file must have a suffix of .jdt
%
% all options for loadubjson/loadjson (depends on file suffix)
% can be used to adjust the parsing options
......@@ -61,7 +64,16 @@ elseif(regexp(filename,'\.[mM][sS][gG][pP][kK]$'))
loadfun=@loadmsgpack;
end
if(jsonopt('matlab',0,opt) && exist('jsonencode','builtin'))
jsonstr=fileread(filename);
pos=regexp(jsonstr,'}\n\n\n{"WorkspaceData":','once');
if(isempty(pos))
error('the json file is not generated using matlab''s jsonencode');
end
header=jsondecode(jsonstr(1:pos+1));
else
header=loadfun(filename,'ObjectID',1, varargin{:});
end
if(jsonopt('Header',0,opt))
varargout{1}=header;
......@@ -78,7 +90,11 @@ if(any(isfound==0))
error('specified variable is not found');
end
body=loadfun(filename,'ObjectID',2, varargin{:});
if(jsonopt('matlab',0,opt) && exist('jsonencode','builtin'))
body=jdatadecode(jsondecode(jsonstr(pos+4:end)));
else
body=jsondecode(filename,'ObjectID',2, varargin{:});
end
for i=1:length(varlist)
assignin(ws, varlist{i}, body.WorkspaceData.(varlist{i}));
......
......
jsave.m
+ 28
− 5
View file @ eefccf3e
......@@ -23,9 +23,12 @@ function varargout=jsave(filename, varargin)
% ws ['base'|'wsname']: the name of the workspace in which the
% variables are to be saved
% vars [{'var1','var2',...}]: cell array of variable names to be saved
% matlab [0|1] if set to 1, use matlab's built-in jsonencode to
% store encoded data to a json file; output file
% must have a suffix of .jdt
%
% all options for saveubjson/savejson (depends on file suffix)
% can be used to adjust the output
% can be used to adjust the output unless "'matlab',1" is used
%
% output:
% varlist: a list of variables loaded
......@@ -57,7 +60,6 @@ if(any(isfound==0))
error('specified variable is not found');
end
metadata=struct;
header=struct;
body=struct;
......@@ -100,6 +102,27 @@ if(nargout==1)
varargout{1}=header;
end
if(jsonopt('matlab',0,opt) && exist('jsonencode','builtin'))
if(isempty(regexp(filename,'\.[jJ][sS][oO][nN]$', 'once')))
filename=regexprep(filename,'\.[a-zA-Z]+$','.jdt');
end
output.WorkspaceHeader=jdataencode(header,'prefix','x','base64',1,varargin{:});
headerjson=jsonencode(output);
clear output;
output.WorkspaceData=jdataencode(body,'AnnotateArray',1,'base64',1,...
'Compression','zlib','UseArrayZipSize',1,'MapAsStruct',1,...
'prefix','x',varargin{:});
bodyjson=jsonencode(output);
clear output;
fid=fopen(filename,jsonopt('writemode','w',opt));
fwrite(fid,headerjson);
fwrite(fid,sprintf('\n\n\n'));
fwrite(fid,bodyjson);
fclose(fid);
else
savefun('WorkspaceHeader',header,'filename',filename,varargin{:});
savefun('WorkspaceData',body,'filename',filename,'append',1,...
'compression','zlib','keeptype',1,'array2struct',1,varargin{:});
end
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment