diff --git a/AUTHORS.txt b/AUTHORS.txt index 81b715bce7de552c38cfcec352b4755562a26cb7..87c49347448bcbf8c5cc8cb681a18b44fab57c52 100644 --- a/AUTHORS.txt +++ b/AUTHORS.txt @@ -1,13 +1,12 @@ The author of "jsonlab" toolbox is Qianqian Fang. Qianqian -is currently an Associate Professor in the Department of Bioengineering, +is currently an Assistant Professor in the Department of Bioengineering, Northeastern University. Address: Qianqian Fang Department of Bioengineering Northeastern University - ISEC 223 + ISEC 206 360 Huntington Ave, Boston, MA 02115, USA - Office: 503 Holmes Hall Phone[O]: 617-373-3829 URL: http://fanglab.org Email: <q.fang at neu.edu> and <fangqq at gmail.com> @@ -77,4 +76,4 @@ This toolbox contains patches submitted by the following contributors: (PR#41) Feature: Added package.json to be able to intall via npm package manager - Paul Koprowski <https://github.com/pjkoprowski> - (PR#55) Feature: Added Encoding option to loadjson and savejson. \ No newline at end of file + (PR#55) Feature: Added Encoding option to loadjson and savejson. diff --git a/README.rst b/README.rst index ba4ecfb41cbf3fd26d471c751cebe4279050f9a7..e6fd20a908c2444db5810efb3c5ff59c9537920c 100644 --- a/README.rst +++ b/README.rst @@ -62,6 +62,11 @@ comparison v1.9.8 is compatible with Draft 2; v1.9 and previous releases are compatible with Draft 1. JSONLab v2.0 can read all data files generated by v1.9.8, but v1.9.8 can not read the new UBJSON markers introduced in v2.0. +The ``jsave/jload`` functions are experimental. The generated ``.jamm`` files are +renamed binary-JData/UBJSON files; they can be 50% smaller than ``.mat`` files +(if using ``jsave(...,'compression','lzma')``) and can be readily loaded among +a long list of programming environments such as Python, JavaScript and Go. + The saveubjson/loadubjson now supports Binary JData specification (BJData) Draft 1 (https://github.com/fangq/bjdata). The BJData spec is largely compatible with UBJSON spec Draft 12, with the following differences (we are working with @@ -287,6 +292,69 @@ jdatadecode.m newjd=jdatadecode(jd) isequaln(newjd,rawdata) +--------- +examples +--------- + +Under the ``examples`` folder, you can find several scripts to demonstrate the +basic utilities of JSONLab. Running the ``demo_jsonlab_basic.m`` script, you +will see the conversions from MATLAB data structure to JSON text and backward. +In ``jsonlab_selftest.m``, we load complex JSON files downloaded from the Internet +and validate the ``loadjson/savejson`` functions for regression testing purposes. +Similarly, a ``"demo_ubjson_basic.m"`` script is provided to test the ``saveubjson`` +and ``loadubjson`` functions for various matlab data structures, and +``demo_msgpack_basic.m`` is for testing ``savemsgpack`` and ``loadmsgpack`` functions. + +Please run these examples and understand how JSONLab works before you use +it to process your data. + + +================ +Using ``jsave/jload`` to share workspace +================ + +Starting from JSONLab v2.0, we provide a pair of functions, ``jsave/jload`` to store +and retrieve variables from the current workspace, similar to the ``save/load`` +functions in MATLAB and Octave. The files ``jsave/jload`` use is by default +a binary JData file with a suffix ``.jamm``. The file size is comparable +(can be smaller if use ``lzma`` compression) to `.mat` files. This feature +is currently experimental. + +The main benefits of using .jamm file to share matlab variables include + +* a ``.jamm`` file can be 50% smaller than a ``.mat`` file when using + ``jsave(..., "compression","lzma")``; the only drawback is longer saving time. +* a ``.jamm`` file can be readily read/used among many programming environments, including + Python, JavaScript, Go, Java etc, where .mat file support is not available. + Parsers of ``.jamm`` files are largely compatible with UBJSON's parsers available at + http://ubjson.org/?page_id=48 +* a ``.jamm`` file is quasi-human-readable, one can see the internal data fields + even in a command line, for example using ``strings -n 2 file.jamm | astyle``, + making the binary data easy to be understood, shared and reused. +* ``jsave/jload`` can also use MessagePack and JSON formats as the underlying + data storage format, addressing needs from a diverse set of applications. + MessagePack parsers are readily available at https://msgpack.org/ + +For example, to load the ``.jamm`` file in python, one needs to install **py-jdata** +(https://pypi.org/project/jdata/) and py-bjdata (https://pypi.org/project/bjdata/) + +.. code-block:: shell + + pip install jdata + pip install bjdata + +Other built-in Python modules needed include ``json`` and ``numpy``. + +Once these modules are installed, one can open python, and run + +.. code-block:: python + + import jdata as jd + import numpy as np + from collections import OrderedDict + + mydata=jd.loadb('myfile.jamm',object_pairs_hook=OrderedDict); + ---------- jsave.m ---------- @@ -312,22 +380,6 @@ jload.m jload('mydata.json') ---------- -examples ---------- - -Under the ``examples`` folder, you can find several scripts to demonstrate the -basic utilities of JSONLab. Running the ``demo_jsonlab_basic.m`` script, you -will see the conversions from MATLAB data structure to JSON text and backward. -In ``jsonlab_selftest.m``, we load complex JSON files downloaded from the Internet -and validate the ``loadjson/savejson`` functions for regression testing purposes. -Similarly, a ``"demo_ubjson_basic.m"`` script is provided to test the ``saveubjson`` -and ``loadubjson`` functions for various matlab data structures, and -``demo_msgpack_basic.m`` is for testing ``savemsgpack`` and ``loadmsgpack`` functions. - -Please run these examples and understand how JSONLab works before you use -it to process your data. - ======================= Known Issues and TODOs ======================= diff --git a/README.txt b/README.txt index 2c5967acd998e56136a359daa29852d75785da71..a92468e8d4b79888f304a7f186057de9b76cee35 100644 --- a/README.txt +++ b/README.txt @@ -66,6 +66,11 @@ comparison, v1.9.8 is compatible with Draft 2; v1.9 and previous releases are compatible with Draft 1. JSONLab v2.0 can read all data files generated by v1.9.8, but v1.9.8 can not read the new UBJSON markers introduced in v2.0. +The `jsave/jload` functions are experimental. The generated `.jamm` files are +renamed binary-JData/UBJSON files; they can be 50% smaller than `.mat` files +(if using `jsave(...,'compression','lzma')`) and can be readily loaded among +a long list of programming environments such as Python, JavaScript and Go. + The saveubjson/loadubjson now supports Binary JData specification (BJData) Draft 1 (https://github.com/fangq/bjdata). The BJData spec is largely compatible with UBJSON Spec Draft 12, with the following differences (we are working with @@ -187,13 +192,6 @@ a pair of in-memory data converters that translate complex data structures to the easy-to-serialized forms according to the JData specifications. The detailed help information can be found in the `Contents.m` file. -In addition, we also provide a pair of functions, `jsave/jload` to store -and retrieve variables from the current workspace, similar to the `save/load` -functions in MATLAB and Octave. The files `jsave/jload` use is by default -a binary JData file with self-contained metadata. The file size is comparable -(can be smaller if use `lzma` compression) to `.mat` files. This feature -is currently experimental. - In the below section, we simply provide a few examples on how to use each of the core functions for encoding/decoding JSON/UBJSON/MessagePack data @@ -250,6 +248,47 @@ each of the core functions for encoding/decoding JSON/UBJSON/MessagePack data newjd=jdatadecode(jd) isequaln(newjd,rawdata) + +=== examples === + +Under the "examples" folder, you can find several scripts to demonstrate the +basic utilities of JSONLab. Running the "demo_jsonlab_basic.m" script, you +will see the conversions from MATLAB data structure to JSON text and backward. +In "jsonlab_selftest.m", we load complex JSON files downloaded from the Internet +and validate the loadjson/savejson functions for regression testing purposes. +Similarly, a "demo_ubjson_basic.m" script is provided to test the saveubjson +and `loadubjson` functions for various matlab data structures. + +Please run these examples and understand how JSONLab works before you use +it to process your data. + +------------------------------------------------------------------------------- + +IV.Using `jsave/jload` to share workspace + +Starting from JSONLab v2.0, we provide a pair of functions, `jsave/jload` to store +and retrieve variables from the current workspace, similar to the `save/load` +functions in MATLAB and Octave. The files `jsave/jload` use is by default +a binary JData file with a suffix `.jamm`. The file size is comparable +(can be smaller if use `lzma` compression) to `.mat` files. This feature +is currently experimental. + +The main benefits of using .jamm file to share matlab variables include + +* a `.jamm` file can be 50% smaller than a `.mat` file when using \ + `jsave(..., "compression","lzma")`; the only drawback is longer saving time. +* a `.jamm` file can be readily read/used among many programming environments, including \ + Python, JavaScript, Go, Java etc, where .mat file support is not available. \ + Parsers of `.jamm` is largely compatible with UBJSON's parsers available at \ + http://ubjson.org/?page_id=48 +* a `.jamm` file is quasi-human-readable, one can see the internal data fields \ + even in a command line, for example using `strings -n 2 file.jamm | astyle`, \ + making the binary data easy to be understood, shared and reused. +* jsave/jload can also use MessagePack and JSON formats as the underlying \ + data storage format, addressing needs from a diverse set of applications. \ + MessagePack parsers are readily available at https://msgpack.org/ + + === jsave.m === jsave % save workspace to jamdata.jamm @@ -267,22 +306,9 @@ each of the core functions for encoding/decoding JSON/UBJSON/MessagePack data jload('mydata.json') -=== examples === - -Under the "examples" folder, you can find several scripts to demonstrate the -basic utilities of JSONLab. Running the "demo_jsonlab_basic.m" script, you -will see the conversions from MATLAB data structure to JSON text and backward. -In "jsonlab_selftest.m", we load complex JSON files downloaded from the Internet -and validate the loadjson/savejson functions for regression testing purposes. -Similarly, a "demo_ubjson_basic.m" script is provided to test the saveubjson -and `loadubjson` functions for various matlab data structures. - -Please run these examples and understand how JSONLab works before you use -it to process your data. - ------------------------------------------------------------------------------- -IV. Known Issues and TODOs +V. Known Issues and TODOs JSONLab has several known limitations. We are striving to make it more general and robust. Hopefully in a few future releases, the limitations become less. @@ -303,7 +329,7 @@ files, however, it can parse all UBJSON files produced by saveubjson. ------------------------------------------------------------------------------- -V. Contribution and feedback +VI. Contribution and feedback JSONLab is an open-source project. This means you can not only use it and modify it as you wish, but also you can contribute your changes back to JSONLab so @@ -346,7 +372,7 @@ mailing list to report any questions you may have regarding JSONLab: ------------------------------------------------------------------------------- -V. Acknowledgement +VII. Acknowledgement This toolbox contains modified functions from the below toolboxes: diff --git a/encodevarname.m b/encodevarname.m index 5534f8a7ba182c5130a36370aeeb3743904e6c65..a8815e130c0a0e4ffef261b117c1910fc8024be8 100644 --- a/encodevarname.m +++ b/encodevarname.m @@ -38,11 +38,7 @@ function str = encodevarname(str,varargin) % if(~isvarname(str(1))) - if(exist('unicode2native','builtin')) - str=regexprep(str,'^([^A-Za-z])','x0x${sprintf(''%X'',unicode2native($1))}_','once'); - else - str=sprintf('x0x%X_%s',char(str(1))+0,str(2:end)); - end + str=sprintf('x0x%X_%s',char(str(1))+0,str(2:end)); end if(isvarname(str)) return; diff --git a/fast_match_bracket.m b/fast_match_bracket.m index 3134467c8acd50681f9df581264fad3a9abf974c..cfeba3ddd79b415064a21045e888654872a06cfb 100644 --- a/fast_match_bracket.m +++ b/fast_match_bracket.m @@ -53,7 +53,13 @@ if(length(count)==1 && count==']') maxlevel=1; return; end +count=count(1:min(length(count),8)); flag=cumsum(count==brackets(1))-cumsum(count==brackets(2))+1; endpos = find(flag==0,1); +if(isempty(endpos)) + count= key(startpos:end); + flag=cumsum(count==brackets(1))-cumsum(count==brackets(2))+1; + endpos = find(flag==0,1); +end maxlevel=max([1,max(flag(1:endpos))]); endpos = pos(endpos + startpos-1); diff --git a/jload.m b/jload.m index 783a7fb270347c1a0f45485111209e32607db399..0e40b41c2b472669b914c43b386b426a96450f8f 100644 --- a/jload.m +++ b/jload.m @@ -93,7 +93,7 @@ end if(jsonopt('matlab',0,opt) && exist('jsonencode','builtin')) body=jdatadecode(jsondecode(jsonstr(pos+4:end))); else - body=jsondecode(filename,'ObjectID',2, varargin{:}); + body=loadfun(filename,'ObjectID',2, varargin{:}); end for i=1:length(varlist) diff --git a/savejson.m b/savejson.m index c9b9b1c4c9a6cb9286ddf1ffa13cf2d38c276b8e..9fa7c1ffcda493f75c490920369077a3ed1cb665 100644 --- a/savejson.m +++ b/savejson.m @@ -648,7 +648,7 @@ else txt=sprintf(dataformat,txt,padding0,'"_ArrayZipSize_": ',regexprep(mat2str(size(fulldata)),'\s+',','), sep); txt=sprintf(dataformat,txt,padding0,'"_ArrayZipType_": "',dozip, ['"' sep]); compfun=str2func([dozip 'encode']); - txt=sprintf(dataformat,txt,padding0,'"_ArrayZipData_": "',base64encode(compfun(typecast(fulldata(:),'uint8'))),['"' nl]); + txt=sprintf(dataformat,txt,padding0,'"_ArrayZipData_": "',char(base64encode(compfun(typecast(fulldata(:),'uint8')))),['"' nl]); else if(isreal(item)) txt=sprintf(dataformat,txt,padding0,'"_ArrayData_": ',...