From 3dfa904faec470c66b39bdc522a9d149984a7908 Mon Sep 17 00:00:00 2001
From: Qianqian Fang <fangqq@gmail.com>
Date: Sun, 13 Feb 2022 23:53:42 -0500
Subject: [PATCH] debugged and tested mmap, add mmapinclude and mmapexclude
options
---
filterjsonmmap.m | 53 +++++++++++++++++++++++++++
jsonget.m | 53 +++++++++++++++------------
jsonset.m | 40 +++++++++++++--------
loadbj.m | 62 +++++++++++++++++++-------------
loadjson.m | 80 +++++++++++++++++++++++++++--------------
savejson.m | 2 +-
test/run_jsonlab_test.m | 48 ++++++++++++++++++++++++-
7 files changed, 249 insertions(+), 89 deletions(-)
create mode 100644 filterjsonmmap.m
diff --git a/filterjsonmmap.m b/filterjsonmmap.m
new file mode 100644
index 0000000..1070e10
--- /dev/null
+++ b/filterjsonmmap.m
@@ -0,0 +1,53 @@
+function mmap=filterjsonmmap(mmap, patterns, isinclude)
+%
+% mmap=filterjsonmmap(mmap, patterns, isinclude)
+%
+% filter JSON mmap keys based on inclusive or exclusive string patterns
+%
+% authors:Qianqian Fang (q.fang <at> neu.edu)
+% initially created on 2022/02/13
+%
+% input:
+% mmap: memory-map returned by loadjson/loadbj of the same data
+% important: mmap must be produced from the same file/string,
+% otherwise calling this function may cause data corruption
+% patterns: a string or a cell array of strings, each string will
+% be tested to match the JSONPath keys in mmap
+% isinclude: 1 (default) to include all mmap entries that match at
+% least one of the patterns, and 0 - exclude those that match
+%
+% output:
+% mmap: a filtered JSON mmap
+%
+% examples:
+% str='{"arr":[[1,2],"a",{"c":2}],"obj":{"k":"test"}}';
+% [dat, mmap]=loadjson(str);
+% savejson('',mmap)
+% newmmap=filterjsonmmap(mmap,{'arr.[1]', 'obj.k'});
+% savejson('',newmmap)
+%
+% license:
+% BSD or GPL version 3, see LICENSE_{BSD,GPLv3}.txt files for details
+%
+% -- this function is part of JSONLab toolbox (http://iso2mesh.sf.net/cgi-bin/index.cgi?jsonlab)
+%
+ if(nargin<3)
+ isinclude=1;
+ end
+ if(nargin>1 && ~isempty(patterns))
+ keylist=[mmap{:}];
+ keylist=keylist(1:2:end);
+ if(~iscell(patterns))
+ patterns={patterns};
+ end
+ mask=zeros(1,length(keylist));
+ for i=1:length(patterns)
+ mask=mask+cellfun(@length, strfind(keylist,patterns{i}));
+ end
+ if(isinclude)
+ mmap=mmap(mask>0);
+ else
+ mmap(mask>0)=[];
+ end
+ end
+end
\ No newline at end of file
diff --git a/jsonget.m b/jsonget.m
index 1f5d921..3e0a8e4 100644
--- a/jsonget.m
+++ b/jsonget.m
@@ -14,7 +14,8 @@ function json=jsonget(fname,mmap,varargin)
% important: mmap must be produced from the same file/string,
% otherwise calling this function may cause data corruption
% '$.jsonpath1,2,3,...': a series of strings in the form of JSONPath
-% as the key to each of the record to be retrieved
+% as the key to each of the record to be retrieved; if no paths
+% are given, all items in mmap are retrieved
%
% output:
% json: a cell array, made of elements {'$.jsonpath_i',json_string_i}
@@ -23,8 +24,8 @@ function json=jsonget(fname,mmap,varargin)
% str='[[1,2],"a",{"c":2}]{"k":"test"}';
% [dat, mmap]=loadjson(str);
% savejson('',dat,'filename','mydata.json','compact',1);
-% json=jsonget(str,mmap,'$.[0].[*]','$.[2].c')
-% json=jsonget('mydata.json',mmap,'$.[0].[*]','$.[2].c')
+% json=jsonget(str,mmap,'$.[0]','$.[2].c')
+% json=jsonget('mydata.json',mmap,'$.[0]','$.[2].c')
%
% license:
% BSD or GPL version 3, see LICENSE_{BSD,GPLv3}.txt files for details
@@ -34,18 +35,11 @@ function json=jsonget(fname,mmap,varargin)
if(regexp(fname,'^\s*(?:\[.*\])|(?:\{.*\})\s*$','once'))
inputstr=fname;
-elseif(isoctavemesh)
+elseif(~exist('memmapfile','file'))
if(exist(fname,'file'))
try
fid = fopen(fname,'rb');
- inputstr = fread(fid,'char',inf)';
- fclose(fid);
catch
- try
- inputstr = urlread(['file://',fname]);
- catch
- inputstr = urlread(['file://',fullfile(pwd,fname)]);
- end
end
end
end
@@ -53,18 +47,33 @@ end
mmap=[mmap{:}];
keylist=mmap(1:2:end);
+loc=1:length(keylist);
+if(length(varargin)>=1)
+ [tf,loc]=ismember(varargin,keylist);
+ if(any(tf))
+ keylist=keylist(loc);
+ else
+ keylist={};
+ end
+end
+
json={};
-for i=1:length(varargin)
- if(regexp(varargin{i},'^\$'))
- [tf,loc]=ismember(varargin{i},keylist);
- if(tf)
- rec={'uint8',[1,mmap{loc*2}(2)], 'x'};
- if(exist('inputstr','var'))
- json{end+1}={varargin{i}, inputstr(mmap{loc*2}(1):mmap{loc*2}(1)+mmap{loc*2}(2)-1)};
- else
- fmap=memmapfile(fname,'writable',false, 'offset',mmap{loc*2}(1),'format', rec);
- json{end+1}={varargin{i}, char(fmap.Data(1).x)};
- end
+for i=1:length(keylist)
+ bmap=mmap{loc(i)*2};
+ rec={'uint8',[1,bmap(2)], 'x'};
+ if(exist('inputstr','var'))
+ json{end+1}={keylist{i}, inputstr(bmap(1):bmap(1)+bmap(2)-1)};
+ else
+ if(exist('fid','var') && fid>=0)
+ fseek(fid, bmap(1), -1);
+ json{end+1}={keylist{i}, fread(fid,bmap(1),'uint8=>char')};
+ else
+ fmap=memmapfile(fname,'writable',false, 'offset',bmap(1),'format', rec);
+ json{end+1}={keylist{i}, char(fmap.Data(1).x)};
end
end
end
+
+if(exist('fid','var') && fid>0)
+ fclose(fid);
+end
diff --git a/jsonset.m b/jsonset.m
index a80cac5..27a5fc9 100644
--- a/jsonset.m
+++ b/jsonset.m
@@ -3,7 +3,7 @@ function json=jsonset(fname,mmap,varargin)
% json=jsonset(fname,mmap,'$.jsonpath1',newval1,'$.jsonpath2','newval2',...)
%
% Fast writing of JSON data records to stream or disk using memory-map
-% (mmap) returned by loadjson and JSONPath-like keys
+% (mmap) returned by loadjson/loadbj and JSONPath-like keys
%
% authors:Qianqian Fang (q.fang <at> neu.edu)
% initially created on 2022/02/02
@@ -22,11 +22,20 @@ function json=jsonset(fname,mmap,varargin)
% written
%
% examples:
-% str='[[1,2],"a",{"c":2}]{"k":"test"}';
-% [dat, mmap]=loadjson(str);
-% savejson('',dat,'filename','mydata.json','compact',1);
-% json=jsonset(str,mmap,'$.[2].c','5')
-% json=jsonset('mydata.json',mmap,'$.[2].c','"c":5')
+% % create test data
+% d.arr={[1,2],'a',struct('c',2)}; d.obj=struct('k','test')
+% % convert to json string
+% str=savejson('',d,'compact',1)
+% % parse and return mmap
+% [dat, mmap]=loadjson(str);
+% % display mmap entries
+% savejson('',mmap)
+% % replace value using mmap
+% json=jsonset(str,mmap,'$.arr.[2].c','5')
+% % save same json string to file (must set savebinary 1)
+% savejson('',d,'filename','file.json','compact',1,'savebinary',1);
+% % fast write to file
+% json=jsonset('file.json',mmap,'$.arr.[2].c','5')
%
% license:
% BSD or GPL version 3, see LICENSE_{BSD,GPLv3}.txt files for details
@@ -37,7 +46,9 @@ function json=jsonset(fname,mmap,varargin)
if(regexp(fname,'^\s*(?:\[.*\])|(?:\{.*\})\s*$','once'))
inputstr=fname;
else
- fid=fopen(fname,'wb');
+ if(~exist('memmapfile','file'))
+ fid=fopen(fname,'r+b');
+ end
end
mmap=[mmap{:}];
@@ -55,22 +66,23 @@ for i=1:2:length(varargin)
if(regexp(varargin{i},'^\$'))
[tf,loc]=ismember(varargin{i},keylist);
if(tf)
- rec={'uint8',[1,mmap{loc*2}(2)], 'x'};
+ bmap=mmap{loc*2};
if(ischar(varargin{i+1}))
val=varargin{i+1};
else
val=savejson('',varargin{i+1},'compact',1);
end
- if(length(val)<=rec{1,2}(2))
- val=[val repmat(' ',[1,rec{1,2}(2)-length(val)])];
+ if(length(val)<=bmap(2))
+ val=[val repmat(' ',[1,bmap(2)-length(val)])];
if(exist('inputstr','var'))
- inputstr(mmap{loc*2}(1):mmap{loc*2}(1)+mmap{loc*2}(2)-1)=val;
+ inputstr(bmap(1):bmap(1)+bmap(2)-1)=val;
else
if(exist('memmapfile','file'))
- fmap=memmapfile(fname,'writable',true,'offset',mmap{loc*2}(1),'format', rec);
- fmap.x=val;
+ rec={'uint8', [1 bmap(2)], 'x'};
+ fmap=memmapfile(fname,'writable',true,'offset', bmap(1)-1, 'format', rec, 'repeat',1);
+ fmap.Data.x=uint8(val);
else
- fseek(fid,mmap{loc*2}(1)-1,'bof');
+ fseek(fid,bmap(1)-1,'bof');
fwrite(fid,val);
end
json{end+1}={varargin{i},val};
diff --git a/loadbj.m b/loadbj.m
index a42ff32..a94a515 100644
--- a/loadbj.m
+++ b/loadbj.m
@@ -58,6 +58,15 @@ function [data, mmap] = loadbj(fname,varargin)
% for output format, it is incompatible with all
% previous releases; if old output is desired,
% please set FormatVersion to 1.9 or earlier.
+% MmapOnly [0|1]: if set to 1, this function only returns mmap
+% MMapInclude 'str1' or {'str1','str2',..}: if provided, the
+% returned mmap will be filtered by only keeping
+% entries containing any one of the string patterns
+% provided in a cell
+% MMapExclude 'str1' or {'str1','str2',..}: if provided, the
+% returned mmap will be filtered by removing
+% entries containing any one of the string patterns
+% provided in a cell
%
% output:
% dat: a cell array, where {...} blocks are converted into cell arrays,
@@ -98,6 +107,7 @@ function [data, mmap] = loadbj(fname,varargin)
opt.simplifycellarray=jsonopt('SimplifyCellArray',0,opt);
opt.usemap=jsonopt('UseMap',0,opt);
opt.nameisstring=jsonopt('NameIsString',0,opt);
+ mmaponly=jsonopt('MmapOnly',0,opt);
[os,maxelem,systemendian]=computer;
opt.flipendian_=(systemendian ~= upper(jsonopt('Endian','L',opt)));
@@ -108,22 +118,28 @@ function [data, mmap] = loadbj(fname,varargin)
maxobjid=inf;
end
- mmap={};
opt.jsonpath_='$';
+ if(nargout>1 || mmaponly)
+ mmap={};
+ end
jsoncount=1;
while pos <= inputlen
[cc, pos]=next_char(inputstr, pos);
switch(cc)
case '{'
- if(nargout>1)
+ if(nargout>1 || mmaponly)
+ mmap{end+1}={opt.jsonpath_,pos};
[data{jsoncount}, pos, newmmap] = parse_object(inputstr, pos, opt);
+ mmap{end}{2}=[mmap{end}{2},pos-mmap{end}{2}];
mmap=[mmap(:);newmmap(:)];
else
[data{jsoncount}, pos] = parse_object(inputstr, pos, opt);
end
case '['
- if(nargout>1)
+ if(nargout>1 || mmaponly)
+ mmap{end+1}={opt.jsonpath_,pos};
[data{jsoncount}, pos, newmmap] = parse_array(inputstr, pos, opt);
+ mmap{end}{2}=[mmap{end}{2},pos-mmap{end}{2}];
mmap=[mmap(:);newmmap(:)];
else
[data{jsoncount}, pos] = parse_array(inputstr, pos, opt);
@@ -133,9 +149,9 @@ function [data, mmap] = loadbj(fname,varargin)
otherwise
error_pos('Outer level structure must be an object or an array', inputstr, pos);
end
- if(jsoncount>=maxobjid)
- break;
- end
+ if(jsoncount>=maxobjid)
+ break;
+ end
opt.jsonpath_=sprintf('$%d',jsoncount);
jsoncount=jsoncount+1;
end % while
@@ -148,7 +164,11 @@ function [data, mmap] = loadbj(fname,varargin)
if(jsoncount==1 && iscell(data))
data=data{1};
end
-
+ if(nargout>1 || mmaponly)
+ mmap=mmap';
+ mmap=filterjsonmmap(mmap, jsonopt('MMapExclude',{},opt), 0);
+ mmap=filterjsonmmap(mmap, jsonopt('MMapInclude',{},opt), 1);
+ end
if(jsonopt('JDataDecode',1,varargin{:})==1)
try
data=jdatadecode(data,'Base64',0,'Recursive',1,varargin{:});
@@ -158,6 +178,9 @@ function [data, mmap] = loadbj(fname,varargin)
ME.identifier, ME.message, savejson('',ME.stack));
end
end
+ if(mmaponly)
+ data=mmap;
+ end
end
%%-------------------------------------------------------------------------
@@ -179,7 +202,7 @@ end
function [object, pos, mmap] = parse_array(inputstr, pos, varargin) % JSON array is written in row-major order
if(nargout>2)
- mmap={{[varargin{1}.jsonpath_ '.[*]'],pos}};
+ mmap={};
origpath=varargin{1}.jsonpath_;
end
pos=parse_char(inputstr, pos, '[');
@@ -216,9 +239,6 @@ function [object, pos, mmap] = parse_array(inputstr, pos, varargin) % JSON arra
object=permute(reshape(object,fliplr(dim(:)')),length(dim):-1:1);
end
pos=pos+adv;
- if(nargout>2)
- mmap{1}{2}=[mmap{1}{2},pos-mmap{1}{2}];
- end
return;
else
endpos=match_bracket(inputstr,pos);
@@ -227,9 +247,6 @@ function [object, pos, mmap] = parse_array(inputstr, pos, varargin) % JSON arra
[object, adv]=parse_block(inputstr, pos, type,count,varargin{:});
pos=pos+adv;
pos=parse_char(inputstr, pos, ']');
- if(nargout>2)
- mmap{1}{2}=[mmap{1}{2},pos-mmap{1}{2}+1];
- end
return;
end
end
@@ -238,7 +255,9 @@ function [object, pos, mmap] = parse_array(inputstr, pos, varargin) % JSON arra
while 1
if(nargout>2)
varargin{1}.jsonpath_=[origpath '.' sprintf('[%d]',length(object))];
+ mmap{end+1}={varargin{1}.jsonpath_, pos};
[val, pos, newmmap] = parse_value(inputstr, pos, varargin{:});
+ mmap{end}{2}=[mmap{end}{2}, pos-mmap{end}{2}];
mmap=[mmap(:);newmmap(:)];
else
[val, pos] = parse_value(inputstr, pos, varargin{:});
@@ -277,9 +296,6 @@ function [object, pos, mmap] = parse_array(inputstr, pos, varargin) % JSON arra
if(count==-1)
pos=parse_char(inputstr, pos, ']');
end
- if(nargout>2)
- mmap{1}{2}=[mmap{1}{2},pos-mmap{1}{2}+1];
- end
end
%%-------------------------------------------------------------------------
@@ -413,7 +429,8 @@ end
%%-------------------------------------------------------------------------
function [object, pos, mmap] = parse_object(inputstr, pos, varargin)
if(nargout>2)
- mmap={{varargin{1}.jsonpath_,pos}};
+ mmap={};
+ origpath=varargin{1}.jsonpath_;
end
pos=parse_char(inputstr,pos,'{');
usemap=varargin{1}.usemap;
@@ -446,10 +463,8 @@ function [object, pos, mmap] = parse_object(inputstr, pos, varargin)
error_pos('Name of value at position %d cannot be empty', inputstr, pos);
end
if(nargout>2)
- varargin{1}.jsonpath_=[mmap{1}{1},'.',str];
- mmap{end+1}={varargin{1}.jsonpath_,pos-length(str)-2};
- end
- if(nargout>2)
+ varargin{1}.jsonpath_=[origpath,'.',str];
+ mmap{end+1}={varargin{1}.jsonpath_,pos};
[val, pos,newmmap] = parse_value(inputstr, pos, varargin{:});
mmap{end}{2}=[mmap{end}{2}, pos-mmap{end}{2}];
mmap=[mmap(:);newmmap(:)];
@@ -471,9 +486,6 @@ function [object, pos, mmap] = parse_object(inputstr, pos, varargin)
if(count==-1)
pos=parse_char(inputstr, pos, '}');
end
- if(nargout>2)
- mmap{1}={[mmap{1}{1} '.*'],[mmap{1}{2}, pos-mmap{1}{2}]};
- end
end
%%-------------------------------------------------------------------------
diff --git a/loadjson.m b/loadjson.m
index 08db99f..60d34d8 100644
--- a/loadjson.m
+++ b/loadjson.m
@@ -67,6 +67,15 @@ function [data, mmap] = loadjson(fname,varargin)
% jsondecode, if presents (MATLAB R2016b or Octave
% 6) first. If jsondecode does not exist or failed,
% this function falls back to the jsonlab parser
+% MmapOnly [0|1]: if set to 1, this function only returns mmap
+% MMapInclude 'str1' or {'str1','str2',..}: if provided, the
+% returned mmap will be filtered by only keeping
+% entries containing any one of the string patterns
+% provided in a cell
+% MMapExclude 'str1' or {'str1','str2',..}: if provided, the
+% returned mmap will be filtered by removing
+% entries containing any one of the string patterns
+% provided in a cell
%
% output:
% dat: a cell array, where {...} blocks are converted into cell arrays,
@@ -113,7 +122,6 @@ function [data, mmap] = loadjson(fname,varargin)
error_pos('input file does not exist');
end
- mmap={};
if(jsonopt('BuiltinJSON',0,opt) && exist('jsondecode','builtin'))
try
newstring=regexprep(string,'[\r\n]','');
@@ -143,6 +151,7 @@ function [data, mmap] = loadjson(fname,varargin)
opt.parsestringarray=jsonopt('ParseStringArray',0,opt);
opt.usemap=jsonopt('UseMap',0,opt);
opt.arraydepth_=1;
+ mmaponly=jsonopt('MmapOnly',0,opt);
if(jsonopt('ShowProgress',0,opt)==1)
opt.progressbar_=waitbar(0,'loading ...');
@@ -154,20 +163,27 @@ function [data, mmap] = loadjson(fname,varargin)
maxobjid=inf;
end
opt.jsonpath_='$';
+ if(nargout>1 || mmaponly)
+ mmap={};
+ end
jsoncount=1;
while pos <= inputlen
- [cc,pos]=next_char(inputstr, pos);
+ [cc,pos,w1]=next_char(inputstr, pos);
switch(cc)
case '{'
- if(nargout>1)
+ if(nargout>1 || mmaponly)
+ mmap{end+1}={opt.jsonpath_,pos-w1};
[data{jsoncount},pos,index_esc,newmmap] = parse_object(inputstr, pos, esc, index_esc,opt);
+ mmap{end}{2}=[mmap{end}{2},pos-mmap{end}{2}];
mmap=[mmap(:);newmmap(:)];
else
[data{jsoncount},pos,index_esc] = parse_object(inputstr, pos, esc, index_esc,opt);
end
case '['
- if(nargout>1)
+ if(nargout>1 || mmaponly)
+ mmap{end+1}={opt.jsonpath_,pos-w1};
[data{jsoncount},pos,index_esc,newmmap] = parse_array(inputstr, pos, esc, index_esc,opt);
+ mmap{end}{2}=[mmap{end}{2},pos-mmap{end}{2}];
mmap=[mmap(:);newmmap(:)];
else
[data{jsoncount},pos,index_esc] = parse_array(inputstr, pos, esc, index_esc,opt);
@@ -175,9 +191,9 @@ function [data, mmap] = loadjson(fname,varargin)
otherwise
pos=error_pos('Outer level structure must be an object or an array',inputstr,pos);
end
- if(jsoncount>=maxobjid)
- break;
- end
+ if(jsoncount>=maxobjid)
+ break;
+ end
opt.jsonpath_=sprintf('$%d',jsoncount);
jsoncount=jsoncount+1;
end % while
@@ -190,7 +206,11 @@ function [data, mmap] = loadjson(fname,varargin)
if(jsoncount==1 && iscell(data))
data=data{1};
end
-
+ if(nargout>1 || mmaponly)
+ mmap=mmap';
+ mmap=filterjsonmmap(mmap, jsonopt('MMapExclude',{},opt), 0);
+ mmap=filterjsonmmap(mmap, jsonopt('MMapInclude',{},opt), 1);
+ end
if(jsonopt('JDataDecode',1,varargin{:})==1)
try
data=jdatadecode(data,'Base64',1,'Recursive',1,varargin{:});
@@ -200,7 +220,9 @@ function [data, mmap] = loadjson(fname,varargin)
ME.identifier, ME.message, savejson('',ME.stack));
end
end
-
+ if(mmaponly)
+ data=mmap;
+ end
if(isfield(opt,'progressbar_'))
close(opt.progressbar_);
end
@@ -212,7 +234,7 @@ end
function [object, pos,index_esc, mmap] = parse_array(inputstr, pos, esc, index_esc, varargin) % JSON array is written in row-major order
if(nargout>3)
- mmap={{[varargin{1}.jsonpath_ '.[*]'],pos}};
+ mmap={};
origpath=varargin{1}.jsonpath_;
end
pos=parse_char(inputstr, pos, '[');
@@ -230,9 +252,6 @@ function [object, pos,index_esc, mmap] = parse_array(inputstr, pos, esc, index_e
try
if((varargin{1}.fastarrayparser)>=1 && arraydepth>=varargin{1}.fastarrayparser)
[endpos, maxlevel]=fast_match_bracket(varargin{1}.arraytoken_,varargin{1}.arraytokenidx_,pos);
- if(nargout>3)
- mmap{1}{2}=[mmap{1}{2},endpos-mmap{1}{2}+1];
- end
if(~isempty(endpos))
arraystr=['[' inputstr(pos:endpos)];
arraystr=sscanf_prep(arraystr);
@@ -292,7 +311,9 @@ function [object, pos,index_esc, mmap] = parse_array(inputstr, pos, esc, index_e
varargin{1}.arraydepth_=arraydepth+1;
if(nargout>3)
varargin{1}.jsonpath_=[origpath '.' sprintf('[%d]',length(object))];
- [val, pos,index_esc, newmmap] = parse_value(inputstr, pos, esc, index_esc,varargin{:});
+ mmap{end+1}={varargin{1}.jsonpath_, pos};
+ [val, pos, index_esc, newmmap] = parse_value(inputstr, pos, esc, index_esc,varargin{:});
+ mmap{end}{2}=[mmap{end}{2}, pos-mmap{end}{2}];
mmap=[mmap(:);newmmap(:)];
else
[val, pos,index_esc] = parse_value(inputstr, pos, esc, index_esc,varargin{:});
@@ -339,19 +360,26 @@ function [object, pos,index_esc, mmap] = parse_array(inputstr, pos, esc, index_e
end
%%-------------------------------------------------------------------------
-function pos=parse_char(inputstr, pos, c)
+function [pos, w1, w2]=parse_char(inputstr, pos, c)
+ w1=pos;
+ w2=0;
pos=skip_whitespace(pos, inputstr);
+ w1=pos-w1;
if pos > length(inputstr) || inputstr(pos) ~= c
pos=error_pos(sprintf('Expected %c at position %%d', c),inputstr,pos);
else
pos = pos + 1;
+ w2=pos;
pos=skip_whitespace(pos, inputstr);
+ w2=pos-w2;
end
end
%%-------------------------------------------------------------------------
-function [c, pos] = next_char(inputstr, pos)
+function [c, pos, w1] = next_char(inputstr, pos)
+ w1=pos;
pos=skip_whitespace(pos, inputstr);
+ w1=pos-w1;
if pos > length(inputstr)
c = [];
else
@@ -360,7 +388,10 @@ function [c, pos] = next_char(inputstr, pos)
end
%%-------------------------------------------------------------------------
-function [str, pos,index_esc] = parseStr(inputstr, pos, esc, index_esc, varargin)
+function [str, pos,index_esc, mmap] = parseStr(inputstr, pos, esc, index_esc, varargin)
+ if(nargout>3)
+ mmap={};
+ end
if inputstr(pos) ~= '"'
pos=error_pos('String starting with " expected at position %d',inputstr,pos);
else
@@ -444,8 +475,7 @@ function varargout = parse_value(inputstr, pos, esc, index_esc, varargin)
end
switch(inputstr(pos))
case '"'
- [varargout{1:3}] = parseStr(inputstr, pos, esc, index_esc,varargin{:});
- varargout{3}=index_esc;
+ [varargout{1:nargout}] = parseStr(inputstr, pos, esc, index_esc,varargin{:});
return;
case '['
[varargout{1:nargout}] = parse_array(inputstr, pos, esc, index_esc, varargin{:});
@@ -481,7 +511,8 @@ end
%%-------------------------------------------------------------------------
function [object, pos, index_esc, mmap] = parse_object(inputstr, pos, esc, index_esc, varargin)
if(nargout>3)
- mmap={{varargin{1}.jsonpath_,pos}};
+ mmap={};
+ origpath=varargin{1}.jsonpath_;
end
pos=parse_char(inputstr, pos, '{');
usemap=varargin{1}.usemap;
@@ -497,11 +528,11 @@ function [object, pos, index_esc, mmap] = parse_object(inputstr, pos, esc, index
if isempty(str)
pos=error_pos('Name of value at position %d cannot be empty',inputstr,pos);
end
+ pos=parse_char(inputstr, pos, ':');
if(nargout>3)
- varargin{1}.jsonpath_=[mmap{1}{1},'.',str];
- mmap{end+1}={varargin{1}.jsonpath_,pos-length(str)-2};
+ varargin{1}.jsonpath_=[origpath,'.',str];
+ mmap{end+1}={varargin{1}.jsonpath_,pos};
end
- pos=parse_char(inputstr, pos, ':');
if(nargout>3)
[val, pos,index_esc, newmmap] = parse_value(inputstr, pos, esc, index_esc, varargin{:});
mmap{end}{2}=[mmap{end}{2}, pos-mmap{end}{2}];
@@ -522,9 +553,6 @@ function [object, pos, index_esc, mmap] = parse_object(inputstr, pos, esc, index
end
end
pos=parse_char(inputstr, pos, '}');
- if(nargout>3)
- mmap{1}={[mmap{1}{1} '.*'],[mmap{1}{2}, pos-mmap{1}{2}]};
- end
end
%%-------------------------------------------------------------------------
diff --git a/savejson.m b/savejson.m
index 2bf4098..4ab2cc1 100644
--- a/savejson.m
+++ b/savejson.m
@@ -255,7 +255,7 @@ if(~isempty(filename))
mode='a';
end
if(jsonopt('SaveBinary',0,opt)==1)
- if(isempty(encoding))
+ if(~isempty(encoding))
fid = fopen(filename, [mode 'b'],endian,encoding);
else
fid = fopen(filename, [mode 'b'],endian);
diff --git a/test/run_jsonlab_test.m b/test/run_jsonlab_test.m
index 0c68b95..5907691 100644
--- a/test/run_jsonlab_test.m
+++ b/test/run_jsonlab_test.m
@@ -24,7 +24,7 @@ function run_jsonlab_test(tests)
%
if(nargin==0)
- tests={'js','jso','bj','bjo'};
+ tests={'js','jso','bj','bjo','jmap','bmap'};
end
%%
@@ -285,3 +285,49 @@ if(ismember('bjo',tests))
test_jsonlab('test default float endian for savebj',@savebj,typecast(uint8('1e05'),'single'),'d1e05');
test_jsonlab('test default float endian for saveubjson',@saveubjson,typecast(uint8('12345678'),'double'),'D87654321');
end
+
+%%
+if(ismember('jmap',tests))
+ fprintf(sprintf('%s\n',char(ones(1,79)*61)));
+ fprintf('Test JSON mmap\n');
+ fprintf(sprintf('%s\n',char(ones(1,79)*61)));
+
+ test_jsonlab('mmap of a 1D numerical array',@savejson,loadjson('[1,2,3]','mmaponly',1),'[["$",[1,7]]]','compact',1);
+ test_jsonlab('mmap of a 1D mixed array',@savejson,loadjson('[1,"2",3]','mmaponly',1),'[["$",[1,9]]]','compact',1);
+ test_jsonlab('mmap of a 2D array',@savejson,loadjson('[[1,2,3],[4,5,6]]','mmaponly',1),'[["$",[1,17]]]','compact',1);
+ test_jsonlab('mmap of concatenated json',@savejson,loadjson('[1,2,3][4,5,6]','mmaponly',1),'[["$",[1,7]],["$1",[8,7]]]','compact',1);
+ test_jsonlab('mmap of concatenated json objects',@savejson,loadjson('[1,2,3]{"a":[4,5]}','mmaponly',1),'[["$",[1,7]],["$1",[8,11]],["$1.a",[13,5]]]','compact',1);
+ test_jsonlab('mmap of an array with an object',@savejson,loadjson('[1,2,{"a":3}]','mmaponly',1),...
+ '[["$",[1,13]],["$.[0]",[2,1]],["$.[1]",[4,1]],["$.[2]",[6,7]],["$.[2].a",[11,1]]]','compact',1);
+ test_jsonlab('mmap of an object',@savejson,loadjson('{"a":1,"b":[2,3]}','mmaponly',1),...
+ '[["$",[1,17]],["$.a",[6,1]],["$.b",[12,5]]]','compact',1);
+ test_jsonlab('mmap of object with white-space',@savejson,loadjson('{"a":1 , "b" : [2,3]}','mmaponly',1),...
+ '[["$",[1,23]],["$.a",[6,1]],["$.b",[18,5]]]','compact',1);
+ test_jsonlab('mmapinclude option',@savejson,loadjson('[[1,2,3],{"a":[4,5]}]','mmaponly',1,'mmapinclude','.a'),...
+ '[["$.[1].a",[15,5]]]','compact',1);
+ test_jsonlab('mmapexclude option',@savejson,loadjson('[[1,2,3],{"a":[4,5]}]','mmaponly',1,'mmapexclude',{'[0]','[1]','[2]'}),...
+ '[["$",[1,21]]]','compact',1);
+ test_jsonlab('json with indentation',@savejson,loadjson(savejson({[1,2,3],struct('a',[4,5])}),'mmaponly',1,'mmapinclude','.a'),...
+ '[["$.[1].a",[22,7]]]','compact',1);
+end
+
+%%
+if(ismember('bmap',tests))
+ fprintf(sprintf('%s\n',char(ones(1,79)*61)));
+ fprintf('Test Binary JSON mmap\n');
+ fprintf(sprintf('%s\n',char(ones(1,79)*61)));
+
+ test_jsonlab('mmap of a 1D numerical array',@savejson,loadbj(savebj([1,2,3]),'mmaponly',1),'[["$",[1,9]]]','compact',1);
+ test_jsonlab('mmap of a 1D mixed array',@savejson,loadbj(savebj({1,'2',3}),'mmaponly',1),'[["$",[1,8]],["$.[0]",[2,2]],["$.[1]",[4,2]],["$.[2]",[6,2]]]','compact',1);
+ test_jsonlab('mmap of a 2D array',@savejson,loadbj(savebj([[1,2,3],[4,5,6]]),'mmaponly',1),'[["$",[1,12]]]','compact',1);
+ test_jsonlab('mmap of an array with an object',@savejson,loadbj(savebj({1,2,struct('a',3)}),'mmaponly',1),...
+ '[["$",[1,13]],["$.[0]",[2,2]],["$.[1]",[4,2]],["$.[2]",[6,7]],["$.[2].a",[10,2]]]','compact',1);
+ test_jsonlab('mmap of an object',@savejson,loadbj(savebj(struct('a',1,'b',[2,3])),'mmaponly',1),...
+ '[["$",[1,18]],["$.a",[5,2]],["$.b",[10,8]]]','compact',1);
+ test_jsonlab('mmapinclude option',@savejson,loadbj(savebj({[1,2,3],struct('a',[4,5])}),'mmaponly',1,'mmapinclude','.a'),...
+ '[["$.[1].a",[15,8]]]','compact',1);
+ test_jsonlab('mmapexclude option',@savejson,loadbj(savebj({[1,2,3],struct('a',[4,5])}),'mmaponly',1,'mmapexclude',{'[0]','[1]','[2]'}),...
+ '[["$",[1,24]]]','compact',1);
+ test_jsonlab('json with indentation',@savejson,loadbj(savebj({[1,2,3],struct('a',[4,5])}),'mmaponly',1,'mmapinclude','.a'),...
+ '[["$.[1].a",[15,8]]]','compact',1);
+end
\ No newline at end of file
--
GitLab