From 2e3344c93b6a3165049cead95e710bb7aabfafdd Mon Sep 17 00:00:00 2001 From: Qianqian Fang <fangqq@gmail.com> Date: Sun, 30 Jan 2022 23:14:21 -0500 Subject: [PATCH] [Breaking] use little-endian for all BJData numeric,fix BJData ND array order,prepare 3.0-beta --- AUTHORS.txt | 12 ++- ChangeLog.txt | 24 +++++ Contents.m | 103 +++++++++++++++----- DESCRIPTION | 4 +- LICENSE_BSD.txt | 2 +- LICENSE_GPLv3.txt | 2 +- README.rst | 166 ++++++++++++++++---------------- README.txt | 182 +++++++++++++++++------------------ examples/jsonlab_speedtest.m | 4 +- jdatadecode.m | 2 +- jload.m | 2 +- jsave.m | 5 +- loadbj.m | 35 +++++-- loadjson.m | 23 ++++- loadubjson.m | 7 +- nestbracket2dim.m | 32 +++++- savebj.m | 73 ++++++++++---- savejson.m | 23 ++++- saveubjson.m | 21 ++-- test/run_jsonlab_test.m | 20 +++- 20 files changed, 471 insertions(+), 271 deletions(-) diff --git a/AUTHORS.txt b/AUTHORS.txt index 644c0d2..2f1f23b 100644 --- a/AUTHORS.txt +++ b/AUTHORS.txt @@ -1,4 +1,4 @@ -The author of "jsonlab" toolbox is Qianqian Fang. Qianqian is +The author of "JSONLab" toolbox is Qianqian Fang. Dr. Fang is currently an Associate Professor in the Department of Bioengineering, Northeastern University, USA. @@ -8,10 +8,16 @@ Address: Qianqian Fang ISEC 206 360 Huntington Ave, Boston, MA 02115, USA Phone[O]: 617-373-3829 -URL: http://fanglab.org +URL: http://fanglab.org, http://neurojson.org Email: <q.fang at neu.edu> and <fangqq at gmail.com> +Since 2021, this project has been partially funded by the USA National Institute of Health (NIH) +and National Institute of Neurological Disorders and Stroke (NINDS) under grant +U24-NS124027 (PI Qianqian Fang). It is the reference implementation of the JData +and BJData specifications developed under the NeuroJSON Project (http://neurojson.org). + + The script loadjson.m was built upon previous works (BSD 3-clause license) by - Nedialko Krouchev: http://www.mathworks.com/matlabcentral/fileexchange/25713 @@ -37,7 +43,7 @@ The loadmsgpack.m script was modified from This toolbox contains patches submitted by the following contributors: - Blake Johnson <bjohnso at bbn.com> - part of revision 341 + part of svn revision 341 - Niclas Borlin <Niclas.Borlin at cs.umu.se> various fixes in revision 394, including diff --git a/ChangeLog.txt b/ChangeLog.txt index e1e4c8a..a3b1a65 100644 --- a/ChangeLog.txt +++ b/ChangeLog.txt @@ -5,6 +5,30 @@ ---------------------------------------------------------------------------- JSONlab ChangeLog (key features marked by *): +== JSONlab 2.9.8 (codename: Micronus Prime - beta), FangQ <q.fang (at) neu.edu> == + + 2022-01-30*[ ] [bjdata:breaking] Upgrade savebj/loadbj to BJData v1-draft 2, use little-endian by default + 2022-01-30*[ ] [bjdata:breaking] Fix optimized ND array element order (previously used column-major) + 2022-01-30*[ ] optimize loadjson and loadbj speed + 2022-01-30*[ ] add 'BuiltinJSON' option for savejson/loadjson to call jsonencode/jsondecode + 2022-01-30*[ ] more robust tests on ND array when parsing JSON numerical array construct + 2021-06-23 [632531f] fix inconsistency between singlet integer and float values, close #70 + 2021-06-23 [f7d8226] prevent function calls when parsing array strings using eval, fix #75 + 2021-06-23 [b1ae5fa] fix #73 as a regression to #22 + 2020-09-29 [d0cb3b8] Fix for loading objects. + 2020-07-26 [d0fb684] Add travis badge + 2020-07-25 [708c36c] drop octave 3.2 + 2020-07-25 [436d84e] debug octave 3.2 + 2020-07-25 [0ce96ec] remove windows and osx targets from travis-ci + 2020-07-25 [0d8baa4] fix ruby does not support error on windows + 2020-07-25*[faa7921] enable travis-ci for jsonlab + 2020-07-08 [321ab1a] add Debian and Ubuntu installation commands + 2020-07-08 [e686828] update author info + 2020-07-08*[ce40fdf] supports ND cell array, fix #66 + 2020-07-07 [6a8ce93] fix string encoding over 399 characters, close #65 + 2020-06-14 [5a58faf] fix DESCRIPTION date bug + 2020-06-14 [9d7e94c] match octave description file and upstream version number + 2020-06-14 [a5b6170] fix warning about lz4encode file name == JSONlab 2.0 (codename: Magnus Prime), FangQ <q.fang (at) neu.edu> == diff --git a/Contents.m b/Contents.m index 9dcfa00..a6528ac 100644 --- a/Contents.m +++ b/Contents.m @@ -230,6 +230,10 @@ % JDataDecode [1|0]: if set to 1, call jdatadecode to decode % JData structures defined in the JData % Specification. +% BuiltinJSON [0|1]: if set to 1, this function attempts to call +% jsondecode, if presents (MATLAB R2016b or Octave +% 6) first. If jsondecode does not exist or failed, +% this function falls back to the jsonlab parser % % output: % dat: a cell array, where {...} blocks are converted into cell arrays, @@ -347,6 +351,10 @@ % 'b': big endian, 'l': little-endian) % PreEncode [1|0]: if set to 1, call jdataencode first to preprocess % the input data before saving +% BuiltinJSON [0|1]: if set to 1, this function attempts to call +% jsonencode, if presents (MATLAB R2016b or Octave +% 6) first. If jsonencode does not exist or failed, +% this function falls back to the jsonlab savejson % % opt can be replaced by a list of ('param',value) pairs. The param % string is equivallent to a field in opt and is case sensitive. @@ -375,9 +383,16 @@ % or % data=loadbj(fname,'param1',value1,'param2',value2,...) % -% Parse a Binary JData (BJData v1 Draft-1, defined in https://github.com/OpenJData/bjdata) +% Parse a Binary JData (BJData v1 Draft-2, defined in https://github.com/NeuroJSON/bjdata) % file or memory buffer and convert into a MATLAB data structure % +% By default, this function parses BJD-compliant output. The BJD +% specification is largely similar to UBJSON, with additional data types +% including uint16(u), uint32(m), uint64(M) and half-precision float (h). +% Starting from BJD Draft-2 (JSONLab 3.0 beta or later), all integer and +% floating-point numbers are parsed in Little-Endian as opposed to +% Big-Endian form as in BJD Draft-1/UBJSON Draft-12 (JSONLab 2.1 or older) +% % initially created on 2013/08/01 % % input: @@ -391,16 +406,26 @@ % SimplifyCell [1|0]: if set to 1, loadbj will call cell2mat % for each element of the JSON data, and group % arrays based on the cell2mat rules. -% IntEndian [B|L]: specify the endianness of the integer fields -% in the BJData/UBJSON input data. B - Big-Endian format for -% integers (as required in the UBJSON specification); -% L - input integer fields are in Little-Endian order. +% Endian ['L'|'B']: specify the endianness of the numbers +% in the BJData/UBJSON input data. Default: 'L'. +% +% Starting from JSONLab 2.9, BJData by default uses +% [L] Little-Endian for both integers and floating +% point numbers. This is a major departure from the +% UBJSON specification, where 'B' - Big-Endian - +% format is used for integer fields. UBJSON does +% not specifically define Endianness for +% floating-point numbers, resulting in mixed +% implementations. JSONLab 2.0-2.1 used 'B' for +% integers and floating-points; JSONLab 1.x uses +% 'B' for integers and native-endianness for +% floating-point numbers. % NameIsString [0|1]: for UBJSON Specification Draft 8 or % earlier versions (JSONLab 1.0 final or earlier), % the "name" tag is treated as a string. To load % these UBJSON data, you need to manually set this % flag to 1. -% UseMap [0|1]: if set to 1, loadjson uses a containers.Map to +% UseMap [0|1]: if set to 1, loadbj uses a containers.Map to % store map objects; otherwise use a struct object % ObjectID [0|interger or list]: if set to a positive number, % it returns the specified JSON object by index @@ -436,17 +461,20 @@ % bjd=savebj(rootname,obj,'param1',value1,'param2',value2,...) % % Convert a MATLAB object (cell, struct, array, table, map, handles ...) -% into a Binary JData (BJData v1 Draft-1), Universal Binary JSON (UBJSON, +% into a Binary JData (BJData v1 Draft-2), Universal Binary JSON (UBJSON, % Draft-12) or a MessagePack binary stream % % initially created on 2013/08/17 % % By default, this function creates BJD-compliant output. The BJD % specification is largely similar to UBJSON, with additional data types -% including uint16(u), uint32(m), uint64(M) and half-precision float (h) +% including uint16(u), uint32(m), uint64(M) and half-precision float (h). +% Starting from BJD Draft-2 (JSONLab 3.0 beta or later), all integer and +% floating-point numbers are stored in Little-Endian as opposed to +% Big-Endian form as in BJD Draft-1/UBJSON Draft-12 (JSONLab 2.1 or older) % % Format specifications: -% Binary JData (BJD): https://github.com/fangq/bjdata +% Binary JData (BJD): https://github.com/NeuroJSON/bjdata % UBJSON: https://github.com/ubjson/universal-binary-json % MessagePack: https://github.com/msgpack/msgpack % @@ -524,7 +552,8 @@ % UBJSON [0|1]: 0: (default)-encode data based on BJData Draft 1 % (supports uint16(u)/uint32(m)/uint64(M)/half(h) markers) % 1: encode data based on UBJSON Draft 12 (without -% u/m/M/h markers) +% u/m/M/h markers);all numeric values are stored in +% the Big-Endian byte order according to Draft-12 % FormatVersion [2|float]: set the JSONLab output version; since % v2.0, JSONLab uses JData specification Draft 3 % for output format, it is incompatible with releases @@ -535,7 +564,21 @@ % of the minimum length without losing accuracy (default) % Debug [0|1]: output binary numbers in <%g> format for debugging % Append [0|1]: if set to 1, append a new object at the end of the file. -% Endian ['n'|'b','l']: Endianness of the output file ('n': native, +% Endian ['L'|'B']: specify the endianness of the numbers +% in the BJData/UBJSON input data. Default: 'L'. +% +% Starting from JSONLab 2.9, BJData by default uses +% [L] Little-Endian for both integers and floating +% point numbers. This is a major departure from the +% UBJSON specification, where 'B' - Big-Endian - +% format is used for integer fields. UBJSON does +% not specifically define Endianness for +% floating-point numbers, resulting in mixed +% implementations. JSONLab 2.0-2.1 used 'B' for +% integers and floating-points; JSONLab 1.x uses +% 'B' for integers and native-endianness for +% floating-point numbers. +% FileEndian ['n'|'b','l']: Endianness of the output file ('n': native, % 'b': big endian, 'l': little-endian) % PreEncode [1|0]: if set to 1, call jdataencode first to preprocess % the input data before saving @@ -553,6 +596,7 @@ % 'MeshCreator','FangQ','MeshTitle','T6 Cube',... % 'SpecialData',[nan, inf, -inf]); % savebj(jsonmesh) +% savebj('',jsonmesh,'debug',1) % savebj('',jsonmesh,'meshdata.bjd') % savebj('mesh1',jsonmesh,'FileName','meshdata.msgpk','MessagePack',1) % savebj('',jsonmesh,'ubjson',1) @@ -576,8 +620,9 @@ % This function is an alias to loadbj % % input: -% fname: input file name, if fname contains "{}" or "[]", fname -% will be interpreted as a UBJSON string +% fname: input file name, if the file with such name exists, it will +% be read, otherwise, this function will attempt to parse the +% string in fname as a UBJSON stream % opt: a struct to store parsing options, opt can be replaced by % a list of ('param',value) pairs - the param string is equivallent % to a field in opt. The supported options can be found by typing @@ -606,20 +651,21 @@ % ubj=saveubjson(rootname,obj,opt) % ubj=saveubjson(rootname,obj,'param1',value1,'param2',value2,...) % -% Convert a MATLAB object (cell, struct, array, table, map, handles ...) -% into a Universal Binary JSON (UBJSON, Draft 12) or a MessagePack binary stream +% Convert a MATLAB object (cell, struct, array, table, map, graphs ...) +% into a Universal Binary JSON (UBJSON, Draft-12) or a MessagePack binary stream % % initially created on 2013/08/17 % % Format specifications: -% Binary JData (BJData):https://github.com/fangq/bjdata +% Binary JData (BJData):https://github.com/NeuroJSON/bjdata % UBJSON: https://github.com/ubjson/universal-binary-json % MessagePack: https://github.com/msgpack/msgpack % -% This function is the same as calling "savebj(...,'ubjson',1)". By , -% default this function creates UBJSON-compliant output without the +% This function is the same as calling "savebj(..,'ubjson',1,'endian','B')" +% By default this function creates UBJSON-compliant output without the % newly added uint16(u), uint32(m), uint64(M) and half-precision float (h) -% data types. +% data types and use Big-Endian for all numerical values as in UBJSON +% Draft-12. % % This function by default still enables an optimized ND-array format for efficient % array storage. To ensure the output compatible to UBJSON Draft-12, one should use @@ -641,7 +687,7 @@ % Please type "help savebj" for details for all supported options. % % output: -% json: a binary string in the UBJSON format (see http://ubjson.org) +% ubj: a binary string in the UBJSON format (see http://ubjson.org) % % examples: % jsonmesh=struct('MeshVertex3',[0 0 0;1 0 0;0 1 0;1 1 0;0 0 1;1 0 1;0 1 1;1 1 1],... @@ -720,7 +766,7 @@ % fname: (optional) output file name; if not given, save to 'jamdata.jamm' % if fname has a '.json' or '.jdt' suffix, a text-based % JSON/JData file will be created (slow); if the suffix is '.jamm' or -% '.jdb', a Binary JData (https://github.com/fangq/bjdata/) file will be created. +% '.jdb', a Binary JData (https://github.com/NeuroJSON/bjdata/) file will be created. % opt: (optional) a struct to store parsing options, opt can be replaced by % a list of ('param',value) pairs - the param string is equivallent % to a field in opt. opt can have the following @@ -780,7 +826,7 @@ % parse the json file and then decode the output by % jdatadecode; input file must have a suffix of .jdt % -% all options for loadubjson/loadjson (depends on file suffix) +% all options for loadbj/loadjson (depends on file suffix) % can be used to adjust the parsing options % % output: @@ -1033,7 +1079,7 @@ % BSD or GPL version 3, see LICENSE_{BSD,GPLv3}.txt files for details % -%==== function varargout = lz4hcencode(varargin) ==== +%==== function varargout = lz4encode(varargin) ==== % % output = lz4encode(input) % or @@ -1392,9 +1438,9 @@ % BSD or GPL version 3, see LICENSE_{BSD,GPLv3}.txt files for details % -%==== function [dims, maxlevel, count] = nestbracket2dim(str,brackets) ==== +%==== function [dims, isndarray, maxlevel, count] = nestbracket2dim(str,brackets,testndarray) ==== % -% [dims, maxlevel, count] = nestbracket2dim(str,brackets) +% [dims, isndarray, maxlevel, count] = nestbracket2dim(str,brackets) % % Extracting the dimension vector of a JSON string formatted array % by analyzing the pairs of opening/closing bracket tokenss; this function @@ -1409,12 +1455,17 @@ % if not given, brackets is set to '[]' to find matching square-brackets; % for example, '{}' looks for a matching closing curly-bracket in % the string key(pos(startpos,:end)) +% testndarray: (optional), 1 to test if the input string contains an +% ND array, i.e. with uniform element lengths (recursively) % % output: % dims: the speculated dimension vector with the length matching the maximum % depth of the embedded bracket pairs. When the input string encodes an % N-D array, the dims vector contains all integers; however, returning -% an all-integer dims vector does not mean the array is rectangular. +% an all-integer dims vector does not mean the array is +% rectangular. if testndarray is set to 1, dims returns isndarray +% isndarray: 1 to indicate the input string contains an ND array, +% otherwise, 0 % maxlevel: return the depth of the enclosed brackets in the string, i.e. the % length of the dims vector. % count: the relative depth from the level 0 - scanning from the left diff --git a/DESCRIPTION b/DESCRIPTION index d475894..63efd50 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -1,6 +1,6 @@ Name: jsonlab -Version: 2.0 -Date: 2020-13-06 +Version: 2.9.8 +Date: 2022-30-01 Title: A JSON/UBJSON/MessagePack encoder/decoder for MATLAB/Octave Author: Qianqian Fang <fangqq@gmail.com> Maintainer: Qianqian Fang <fangqq@gmail.com> diff --git a/LICENSE_BSD.txt b/LICENSE_BSD.txt index 26c60e1..18be414 100644 --- a/LICENSE_BSD.txt +++ b/LICENSE_BSD.txt @@ -1,4 +1,4 @@ -Copyright (c) 2011-2020 Qianqian Fang <q.fang at neu.edu> +Copyright (c) 2011-2022 Qianqian Fang <q.fang at neu.edu> Copyright (c) 2014,2016 Bastian Bechtold Copyright (c) 2012, Kota Yamaguchi Copyright (c) 2009, Nedialko Krouchev diff --git a/LICENSE_GPLv3.txt b/LICENSE_GPLv3.txt index e16d62b..42da905 100644 --- a/LICENSE_GPLv3.txt +++ b/LICENSE_GPLv3.txt @@ -3,7 +3,7 @@ = An open-source MATLAB/Octave JSON encoder and decoder = =============================================================================== -Copyright (C) 2011-2020 Qianqian Fang <q.fang <at> neu.edu> +Copyright (C) 2011-2022 Qianqian Fang <q.fang <at> neu.edu> ------------------------------------------------------------------------------- diff --git a/README.rst b/README.rst index 8856093..814a6fe 100644 --- a/README.rst +++ b/README.rst @@ -4,11 +4,14 @@ JSONLab: An open-source MATLAB/Octave JSON encoder and decoder ############################################################################## -* Copyright (C) 2011-2020 Qianqian Fang <q.fang at neu.edu> +* Copyright (C) 2011-2022 Qianqian Fang <q.fang at neu.edu> * License: BSD or GNU General Public License version 3 (GPL v3), see License*.txt -* Version: 2.0 (Magnus Prime) -* JData Specification Version: Draft 3 (http://github.com/fangq/jdata) -* URL: http://openjdata.org/jsonlab +* Version: 2.9.8 (Micronus Prime - Beta) +* JData Specification Version: V1 Draft-3 (http://github.com/NeuroJSON/jdata) +* Binary JData Specification Version: V1 Draft-2 (http://github.com/NeuroJSON/bjdata) +* URL: http://neurojson.org/jsonlab +* Acknowledgement: This project is supported by US National Institute of Health (NIH) + grant `U24-NS124027 <https://taggs.hhs.gov/Detail/AwardDetail?arg_AwardNum=U24NS124027&arg_ProgOfficeCode=137>`_ .. image:: https://travis-ci.com/fangq/jsonlab.svg?branch=master :target: https://travis-ci.com/fangq/jsonlab @@ -24,96 +27,84 @@ Table of Contents What's New ============ -JSONLab v2.0 - code named "Magnus Prime" - is a stable release of JSONLab and -a new milestone towards a stable, complete reference implementation of the -JData Specification (http://openjdata.org) for portable scientific data storage. +We are extremely excited to announce that JSONLab project, as the reference implementation +for the JData and BJData specifications, is official funded by the US National Institute +of Health (NIH) as part of the NeuroJSON project (http://neurojson.org) since 2021. +The goal of the NeuroJSON project is to develop human-readable, +scalable and future-proof neuroimaging data standards and data sharing services. All data +produced from the NeuroJSON project will be using JSON/Binary JData formats as the +underlying serialization standards and use the lightweight JData specification as +general-purpose data annotation standard, all of which have been evolved from the over +a decade development of JSONLab. + +JSONLab v2.9.8 - code named "Micronus - beta" - is a beta-release of JSONLab v3.0 and +contains breaking features for `savebj`/`loadbj`. It implements the latest Binary JData +Specification v1 Draft-2 https://github.com/NeuroJSON/bjdata/, where the default +byte order changed from Big-Endian, as in UBJSON Draft-12/BJData Draft 2, to Little-Endian +format. Previously, JSONLab v2.0 implements BJData Draft-1, which used Big-Endian +as the default byte order for all numerical data. There have been many major updates added to this release since the previous -release v1.9.8 in Oct. 2019. A list of the major changes are summarized below -(with key features marked by *), including the support to ``_ArrayShape_`` to -efficiently encode special matrices and the addition of ``jsave/jload`` to save -and restore variables in MATLAB/Octave like the ``save/load`` commands (experimental): - -- 2020-06-13 [81feef3] skip no-op markers, update documentation -- 2020-06-13 [4904155] jload load data to struct, test if loadbj input is buffer, update error msg -- 2020-06-12 [c334799] change default workspace to caller for jload and jsave -- 2020-06-10 [c883546] fix keeptype single integer bug -- 2020-06-09*[ ] created ``jdata`` and ``bjdata`` python modules to share data with MATLAB -- 2020-06-08*[cbde607] add savebj and loadbj to dedicate to loading and saving bjdata -- 2020-06-08*[e2451e1] add unit testing script, fix issues found in the testing unit -- 2020-06-06 [a44015f] accelerate fast_match_bracket, drop unicode2native for speed -- 2020-06-06 [eefccf3] call jsonencode/decode in jsave/jload, parse embedded jdata struct -- 2020-06-05 [9434103] support Toeplitz matrices, use case-insensitive comparison -- 2020-06-04 [3119ce4] jdatadecode now handles _ArrayOrder_ -- 2020-06-04 [89b844c] remove forced root name, update internal test results -- 2020-06-02*[15ca7ae] add keeptype option to jsave and saveubjson -- 2020-06-02 [7f2cbc4] make jsave and jload work on octave -- 2020-06-01*[8829d6b] apply data compression to strings, new datatype char -- 2020-06-01 [270cbf6] fix loadmsgpack ND array issue -- 2020-06-01*[919f502] add jsave and jload for portable data sharing,update doc -- 2020-05-31 [df3a4fa] debug arrayshape related changes and test all demo scripts -- 2020-05-31*[fc0b285] adding support to _ArrayShape_ to record special matrices -- 2020-05-15*[d88d454] jsonlab is compatible with matlab R2008 -- 2020-05-13 [86efe89] flag to prevent embedding ND array size specifier -- 2020-05-07 [a189a50] use more robust integer type testing -- 2020-05-06*[82f5249] saveubjson now implments BJData spec Draft1,https://github.com/fangq/bjdata -- 2020-05-03 [34bca22] add prj file to compile a matlab package, close #60 -- 2020-05-03 [82dfdcc] handle empty array in loadmsgpack, fix #63, patch by stfnp -- 2020-03-08 [7499bd8] Merge pull request #61 from j2L4e/patch-1 -- 2020-02-09*[6984111] add UseMap option to avoid key name conversion -- 2019-11-16 [e46221a] if _ArraySize_ has a single length, treat as a row vector -- 2019-11-01 [f2bfb65] fix a uint8 upper bound bug -- 2019-10-24 [cc4491d] avoid escaping base64 str, avoid double processing preencoded arrayzipdata -- 2019-10-24 [4dc76ef] make example script compatible with matlab R2010 -- 2019-10-24 [ad8be26] disable underscore escaping in octave,update all tests and outputs -- 2019-10-24 [d4275c6] reduce jsonopt calls to speed up encoding and decoding -- 2019-10-23 [82c9e91] fix invalid jdatadecode example -- 2019-10-23 [398539d] reoptimize for speed -- 2019-10-22*[650b5ec] enable jdataencode in savejson and saveubjson - - -Please note that JSONLab v2.0 is now compliant with JData Spec Draft 3; in -comparison v1.9.8 is compatible with Draft 2; v1.9 and previous releases are -compatible with Draft 1. JSONLab v2.0 can read all data files generated by -v1.9.8, but v1.9.8 can not read the new UBJSON markers introduced in v2.0. - -The newly introduced ``jsave/jload`` functions are in the experimental stage. -They generate ``.jamm`` files which are renamed binary-JData/UBJSON files; -they can be 50% smaller than ``.mat`` files if using ``jsave(...,'compression','lzma')`` -and can be readily opened among a long list of programming environments -such as Python, JavaScript and Go. - -The ``saveubjson/loadubjson`` functions added support to the Binary JData specification (BJData) -v1 Draft-1 (https://github.com/fangq/bjdata) and are now renamed as ``savebj/loadbj`` -(``saveubjson/loadubjson`` are kept for compatibility purposes as aliases to the new -functions). The BJData spec is largely compatible with UBJSON spec Draft 12, with the -following differences (we are working with the UBJSON maintainer to merge -these two specifications): +release v2.0 in June 2020. A list of the major changes are summarized below +(with key features marked by *), including the support to BJData Draft-2 specification, +incorrect optimized ND-array BJData/UBJSON element order, and options to use MATLAB/Octave +built-in jsonencode/jsondecode functions. The octave-jsonlab package has also been +included in the official distributions of Debian Bullseye and Ubuntu 21.04 or newer. + +- 2022-01-30*[ ] [bjdata:breaking] Upgrade savebj/loadbj to BJData v1-draft 2, use little-endian by default +- 2022-01-30*[ ] [bjdata:breaking] Fix optimized ND array element order (previously used column-major) +- 2022-01-30*[ ] optimize loadjson and loadbj speed +- 2022-01-30*[ ] add 'BuiltinJSON' option for savejson/loadjson to call jsonencode/jsondecode +- 2022-01-30*[ ] more robust tests on ND array when parsing JSON numerical array construct +- 2021-06-23 [632531f] fix inconsistency between singlet integer and float values, close #70 +- 2021-06-23 [f7d8226] prevent function calls when parsing array strings using eval, fix #75 +- 2021-06-23 [b1ae5fa] fix #73 as a regression to #22 +- 2021-11-22*[ ] octave-jsonlab is officially in Debian Testing/Bullseye +- 2020-09-29 [d0cb3b8] Fix for loading objects. +- 2020-07-26 [d0fb684] Add travis badge +- 2020-07-25 [708c36c] drop octave 3.2 +- 2020-07-25 [436d84e] debug octave 3.2 +- 2020-07-25 [0ce96ec] remove windows and osx targets from travis-ci +- 2020-07-25 [0d8baa4] fix ruby does not support error on windows +- 2020-07-25*[faa7921] enable travis-ci for jsonlab +- 2020-07-08 [321ab1a] add Debian and Ubuntu installation commands +- 2020-07-08 [e686828] update author info +- 2020-07-08*[ce40fdf] supports ND cell array, fix #66 +- 2020-07-07 [6a8ce93] fix string encoding over 399 characters, close #65 +- 2020-06-14 [5a58faf] fix DESCRIPTION date bug +- 2020-06-14 [9d7e94c] match octave description file and upstream version number +- 2020-06-14 [a5b6170] fix warning about lz4encode file name + + +Please note that the `savejson/loadjson` in both JSONLab v2.0-v3.0 are +compliant with JData Spec Draft 3; the savebj/loadbj` in JSONLab v3.0 is +compatible to BJData spec Draft 2, which contains breaking feature changes +compared to those in JSONLab v2.0. + +The BJData spec was derived from UBJSON spec Draft 12, with the +following breaking differences: - BJData adds 4 new numeric data types: ``uint16 [u]``, ``uint32 [m]``, ``uint64 [M]`` - and ``float16 [h]`` ('''new in JSONLab v2.0''') + and ``float16 [h]`` (supported in JSONLab v2.0 or newer) - BJData supports an optimized ND array container (supported in JSONLab since 2013) - BJData does not convert ``NaN/Inf/-Inf`` to ``null`` (supported in JSONLab since 2013) +- BJData Draft 2 changes the default byte order to Little-Endian instead of Big-Endian (JSONLab 3.0 or later) -To avoid using the new type markers, one should attach ``'UBJSON',1`` in the ``savebj`` -command as +To avoid using the new features, one should attach ``'UBJSON',1`` and ``'Endian','B'`` +in the ``savebj`` command as .. code-block:: matlab - savebj('',data,'FileName','myfile.bjd','UBJSON',1); + savebj('',data,'FileName','myfile.bjd','UBJSON',1, 'Endian','B'); -To read data files generated by JSONLab v1.9 or older versions, you need to attach -option ``'FormatVersion', 1.9`` in all the ``loadjson/savejson`` function calls. - -To convert an older file (JSON/UBJSON) to the new format, you should run +To read BJData data files generated by JSONLab v2.0, you should call .. code-block:: matlab - data=loadjson('my_old_data_file.json','FormatVersion',1.9) - savejson('',data,'FileName','new_file.json') + data=loadbj('my_old_data_file.bjd','Endian','B') -You are strongly encouraged to convert all pre-v1.9.8 generated data files using the new -format. +You are strongly encouraged to convert all pre-v2.9 JSONLab generated BJD or .jamm +files using the new format. ============ @@ -215,7 +206,7 @@ Then open Octave, and type ``pkg load jsonlab`` to enable jsonlab toolbox. Install JSONLab on Debian ---------- -JSONLab is currently available on Debian unstable. To install, you may run +JSONLab is currently available on Debian Bullseye. To install, you may run .. code:: shell @@ -227,7 +218,14 @@ One can alternatively install ``matlab-jsonlab`` if MATLAB is available. Install JSONLab on Ubuntu ---------- -JSONLab is currently available on the below PPA for Ubuntu users: +JSONLab is currently available on Ubuntu 21.04 or newer as package +`octave-jsonlab`. To install, you may run + +.. code:: shell + + sudo apt-get install octave-jsonlab + +For older Ubuntu releases, one can add the below PPA https://launchpad.net/~fangq/+archive/ubuntu/ppa @@ -262,8 +260,8 @@ Using JSONLab JSONLab provides a pair of functions, ``loadjson`` -- a JSON parser, and ``savejson`` -- a MATLAB-to-JSON encoder, to read/write the text-based JSON; and -two equivallent pairs -- ``loadubjson/saveubjson`` for binary -JSON and ``loadmsgpack/savemsgpack`` for MessagePack. The ``load*`` functions +three equivallent pairs -- ``loadbj/savebj`` for binary JData, ``loadubjson/saveubjson`` +for UBJSON and ``loadmsgpack/savemsgpack`` for MessagePack. The ``load*`` functions for the 3 supported data formats share almost the same input parameter format; similarly for the 3 ``save*`` functions (``savejson/saveubjson/savemsgpack``) These encoders and decoders are capable of processing/sharing almost all @@ -497,7 +495,7 @@ JData decoding and converts the enclosed data into Python ``dict``, ``list`` and ``numpy`` objects. Similarly, ``jd.loadb()`` function loads a binary JData/UBJSON file and performs similar conversions. One can directly call ``jd.load()`` to open JSONLab (and derived toolboxes such as **jnifti**: -https://github.com/fangq/jnifti or **jsnirfy**: https://github.com/fangq/jsnirfy) +https://github.com/NeuroJSON/jnifti or **jsnirf**: https://github.com/NeuroJSON/jsnirf) generated files based on their respective file suffix. Similarly, the ``jd.savet()``, ``jd.saveb()`` and ``jd.save`` functions diff --git a/README.txt b/README.txt index ea7cfbd..7074d1c 100644 --- a/README.txt +++ b/README.txt @@ -3,12 +3,14 @@ = An open-source MATLAB/Octave JSON encoder and decoder = =============================================================================== -* Copyright (C) 2011-2020 Qianqian Fang <q.fang at neu.edu> +* Copyright (C) 2011-2022 Qianqian Fang <q.fang at neu.edu> * License: BSD or GNU General Public License version 3 (GPL v3), see License*.txt -* Version: 2.0 (Magnus Prime) -* JData Specification Version: Draft 3 (http://github.com/fangq/jdata) -* URL: http://openjdata.org/jsonlab - +* Version: 2.9.8 (Micronus Prime - Beta) +* JData Specification Version: V1 Draft-3 (http://github.com/NeuroJSON/jdata) +* Binary JData Specification Version: V1 Draft-2 (http://github.com/NeuroJSON/bjdata) +* URL: http://neurojson.org/jsonlab +* Acknowledgement: This project is supported by US National Institute of Health (NIH) \ + grant `U24-NS124027 <https://taggs.hhs.gov/Detail/AwardDetail?arg_AwardNum=U24NS124027&arg_ProgOfficeCode=137>`_ ------------------------------------------------------------------------------- Table of Content: @@ -27,92 +29,80 @@ VIII.Acknowledgement 0. What's New -JSONLab v2.0 - code named "Magnus Prime" - is a stable release of JSONLab and -a new milestone towards a stable, complete reference implementation of the -JData Specification (http://openjdata.org) for portable scientific data storage. +We are extremely excited to announce that JSONLab project, as the reference implementation +for the JData and BJData specifications, is official funded by the US National Institute +of Health (NIH) as part of the NeuroJSON project (http://neurojson.org) since 2021. +The goal of the NeuroJSON project is to develop human-readable, +scalable and future-proof neuroimaging data standards and data sharing services. All data +produced from the NeuroJSON project will be using JSON/Binary JData formats as the +underlying serialization standards and use the lightweight JData specification as +general-purpose data annotation standard, all of which have been evolved from the over +a decade development of JSONLab. + +JSONLab v2.9.8 - code named "Micronus - beta" - is a beta-release of JSONLab v3.0 and +contains breaking features for `savebj`/`loadbj`. It implements the latest Binary JData +Specification v1 Draft-2 https://github.com/NeuroJSON/bjdata/, where the default +byte order changed from Big-Endian, as in UBJSON Draft-12/BJData Draft 2, to Little-Endian +format. Previously, JSONLab v2.0 implements BJData Draft-1, which used Big-Endian +as the default byte order for all numerical data. There have been many major updates added to this release since the previous -release v1.9.8 in Oct. 2019. A list of the major changes are summarized below -(with key features marked by *), including the support to `_ArrayShape_` to -efficiently encode special matrices and the addition of `jsave/jload` to save -and restore variables in MATLAB/Octave like the `save/load` commands (experimental): - -* 2020-06-13 [81feef3] skip no-op markers, update documentation -* 2020-06-13 [4904155] jload load data to struct, test if loadbj input is buffer, update error msg -* 2020-06-12 [c334799] change default workspace to caller for jload and jsave -* 2020-06-10 [c883546] fix keeptype single integer bug -* 2020-06-09*[ ] created `jdata` and `bjdata` python modules to share data with MATLAB -* 2020-06-08*[cbde607] add savebj and loadbj to dedicate to loading and saving bjdata -* 2020-06-08*[e2451e1] add unit testing script, fix issues found in the testing unit -* 2020-06-06 [a44015f] accelerate fast_match_bracket, drop unicode2native for speed -* 2020-06-06 [eefccf3] call jsonencode/decode in jsave/jload, parse embedded jdata struct -* 2020-06-05 [9434103] support Toeplitz matrices, use case-insensitive comparison -* 2020-06-04 [3119ce4] jdatadecode now handles _ArrayOrder_ -* 2020-06-04 [89b844c] remove forced root name, update internal test results -* 2020-06-02*[15ca7ae] add keeptype option to jsave and saveubjson -* 2020-06-02 [7f2cbc4] make jsave and jload work on octave -* 2020-06-01*[8829d6b] apply data compression to strings, new datatype char -* 2020-06-01 [270cbf6] fix loadmsgpack ND array issue -* 2020-06-01*[919f502] add jsave and jload for portable data sharing -* 2020-05-31 [df3a4fa] debug arrayshape related changes and test all demo scripts -* 2020-05-31*[fc0b285] adding support to _ArrayShape_ to record special matrices -* 2020-05-15*[d88d454] jsonlab is compatible with matlab R2008 -* 2020-05-13 [86efe89] flag to prevent embedding ND array size specifier -* 2020-05-07 [a189a50] use more robust integer type testing -* 2020-05-06*[82f5249] saveubjson now implments BJData spec Draft1,https://github.com/fangq/bjdata -* 2020-05-03 [34bca22] add prj file to compile a matlab package, close #60 -* 2020-05-03 [82dfdcc] handle empty array in loadmsgpack, fix #63, patch by stfnp -* 2020-03-08 [7499bd8] Merge pull request #61 from j2L4e/patch-1 -* 2020-02-09*[6984111] add UseMap option to avoid key name conversion -* 2019-11-16 [e46221a] if _ArraySize_ has a single length, treat as a row vector -* 2019-11-01 [f2bfb65] fix a uint8 upper bound bug -* 2019-10-24 [cc4491d] avoid escaping base64 str, avoid double processing preencoded arrayzipdata -* 2019-10-24 [4dc76ef] make example script compatible with matlab R2010 -* 2019-10-24 [ad8be26] disable underscore escaping in octave,update all tests and outputs -* 2019-10-24 [d4275c6] reduce jsonopt calls to speed up encoding and decoding -* 2019-10-23 [82c9e91] fix invalid jdatadecode example -* 2019-10-23 [398539d] reoptimize for speed -* 2019-10-22*[650b5ec] enable jdataencode in savejson and saveubjson - - -Please note that JSONLab v2.0 is now compliant with JData Spec Draft 3; in -comparison, v1.9.8 is compatible with Draft 2; v1.9 and previous releases are -compatible with Draft 1. JSONLab v2.0 can read all data files generated by -v1.9.8, but v1.9.8 can not read the new UBJSON markers introduced in v2.0. - -The newly introduced `jsave/jload` functions are in the experimental stage. -They generate `.jamm` files which are renamed binary-JData/UBJSON files; -they can be 50% smaller than `.mat` files if using `jsave(...,'compression','lzma')` -and can be readily opened among a long list of programming environments -such as Python, JavaScript and Go. - -The `saveubjson/loadubjson` functions added support to the Binary JData specification (BJData) -v1 Draft-1 (https://github.com/fangq/bjdata) and are now renamed as `savebj/loadbj` -(`saveubjson/loadubjson` are kept for compatibility purposes as aliases to the new -functions). The BJData spec is largely compatible with UBJSON spec Draft 12, with the -following differences (we are working with the UBJSON maintainer to merge -these two specifications): - -* BJData adds 4 new numeric data types: `uint16 [u]`, `uint32 [m]`, `uint64 [M]` \ - and `float16 [h]` ('''new in JSONLab v2.0''') +release v2.0 in June 2020. A list of the major changes are summarized below +(with key features marked by *), including the support to BJData Draft-2 specification, +incorrect optimized ND-array BJData/UBJSON element order, and options to use MATLAB/Octave +built-in jsonencode/jsondecode functions. The octave-jsonlab package has also been +included in the official distributions of Debian Bullseye and Ubuntu 21.04 or newer. + +* 2022-01-30*[ ] [bjdata:breaking] Upgrade savebj/loadbj to BJData v1-draft 2, use little-endian by default +* 2022-01-30*[ ] [bjdata:breaking] Fix optimized ND array element order (previously used column-major) +* 2022-01-30*[ ] optimize loadjson and loadbj speed +* 2022-01-30*[ ] add 'BuiltinJSON' option for savejson/loadjson to call jsonencode/jsondecode +* 2022-01-30*[ ] more robust tests on ND array when parsing JSON numerical array construct +* 2021-06-23 [632531f] fix inconsistency between singlet integer and float values, close #70 +* 2021-06-23 [f7d8226] prevent function calls when parsing array strings using eval, fix #75 +* 2021-06-23 [b1ae5fa] fix #73 as a regression to #22 +* 2021-11-22*[ ] octave-jsonlab is officially in Debian Testing/Bullseye +* 2020-09-29 [d0cb3b8] Fix for loading objects. +* 2020-07-26 [d0fb684] Add travis badge +* 2020-07-25 [708c36c] drop octave 3.2 +* 2020-07-25 [436d84e] debug octave 3.2 +* 2020-07-25 [0ce96ec] remove windows and osx targets from travis-ci +* 2020-07-25 [0d8baa4] fix ruby does not support error on windows +* 2020-07-25*[faa7921] enable travis-ci for jsonlab +* 2020-07-08 [321ab1a] add Debian and Ubuntu installation commands +* 2020-07-08 [e686828] update author info +* 2020-07-08*[ce40fdf] supports ND cell array, fix #66 +* 2020-07-07 [6a8ce93] fix string encoding over 399 characters, close #65 +* 2020-06-14 [5a58faf] fix DESCRIPTION date bug +* 2020-06-14 [9d7e94c] match octave description file and upstream version number +* 2020-06-14 [a5b6170] fix warning about lz4encode file name + + +Please note that the `savejson/loadjson` in both JSONLab v2.0-v3.0 are +compliant with JData Spec Draft 3; the savebj/loadbj` in JSONLab v3.0 is +compatible to BJData spec Draft 2, which contains breaking feature changes +compared to those in JSONLab v2.0. + +The BJData spec was derived from UBJSON spec Draft 12, with the +following breaking differences: + +* BJData adds 4 new numeric data types: ``uint16 [u]``, ``uint32 [m]``, ``uint64 [M]`` \ + and ``float16 [h]`` (supported in JSONLab v2.0 or newer) * BJData supports an optimized ND array container (supported in JSONLab since 2013) -* BJData does not convert `NaN/Inf/-Inf` to `null` (supported in JSONLab since 2013) +* BJData does not convert ``NaN/Inf/-Inf`` to ``null`` (supported in JSONLab since 2013) +* BJData Draft 2 changes the default byte order to Little-Endian instead of Big-Endian (JSONLab 3.0 or later) -To avoid using the new type markers, one should attach `'UBJSON',1` in the `savebj` -command as +To avoid using the new features, one should attach ``'UBJSON',1`` and ``'Endian','B'`` +in the ``savebj`` command as - savebj('',data,'FileName','myfile.bjd','UBJSON',1); + savebj('',data,'FileName','myfile.bjd','UBJSON',1, 'Endian','B'); -To read data files generated by JSONLab v1.9 or older versions, you need to attach -option `'FormatVersion', 1.9` in all the `loadjson/savejson` function calls. - -To convert an older file (JSON/UBJSON) to the new format, you should run +To read BJData data files generated by JSONLab v2.0, you should call - data=loadjson('my_old_data_file.json','FormatVersion',1.9) - savejson('',data,'FileName','new_file.json') + data=loadbj('my_old_data_file.bjd','Endian','B') -You are strongly encouraged to convert all pre-v1.9.8 generated data files using the new -format. +You are strongly encouraged to convert all pre-v2.9 JSONLab generated BJD or .jamm +files using the new format. ------------------------------------------------------------------------------- @@ -143,7 +133,7 @@ of scientific data. It has both the flexibility and generality as in other gener file specifications, such as [http://www.hdfgroup.org/HDF5/whatishdf5.html HDF5] but has significantly reduced complexity and excellent readability. -Towards this goal, we have developed the JData Specification (http://github.com/fangq/jdata) +Towards this goal, we have developed the JData Specification (http://github.com/NeuroJSON/jdata) to standardize serializations of complex scientific data structures, such as N-D arrays, sparse/complex-valued arrays, trees, maps, tables and graphs using JSON/binary JSON constructs. The text and binary formatted JData files are @@ -201,7 +191,7 @@ To enable data compression/decompression, you need to install `octave-zmat` usin === Install JSONLab on Debian === -JSONLab is currently available on Debian unstable. To install, you may run +JSONLab is currently available on Debian Bullseye. To install, you may run sudo apt-get install octave-jsonlab @@ -209,8 +199,12 @@ One can alternatively install `matlab-jsonlab` if MATLAB is available. === Install JSONLab on Ubuntu === -JSONLab is currently available on the below PPA for Ubuntu users: +JSONLab is currently available on Ubuntu 21.04 or newer as package +`octave-jsonlab`. To install, you may run + + sudo apt-get install octave-jsonlab +For older Ubuntu releases, one can add the below PPA https://launchpad.net/~fangq/+archive/ubuntu/ppa To install, please run @@ -234,17 +228,17 @@ JSONLab is also available on Arch Linux. You may install it using the below comm III.Using JSONLab -JSONLab provides a pair of functions, `loadjson` -- a JSON parser, and -`savejson` -- a MATLAB-to-JSON encoder, to read/write the text-based JSON; and -two equivallent pairs -- `loadubjson/saveubjson` for binary -JSON and `loadmsgpack/savemsgpack` for MessagePack. The `load*` functions +JSONLab provides a pair of functions, ``loadjson`` -- a JSON parser, and +``savejson`` -- a MATLAB-to-JSON encoder, to read/write the text-based JSON; and +three equivallent pairs -- ``loadbj/savebj`` for binary JData, ``loadubjson/saveubjson`` +for UBJSON and ``loadmsgpack/savemsgpack`` for MessagePack. The ``load*`` functions for the 3 supported data formats share almost the same input parameter format; -similarly for the 3 `save*` functions (`savejson/saveubjson/savemsgpack`). +similarly for the 3 ``save*`` functions (``savejson/saveubjson/savemsgpack``) These encoders and decoders are capable of processing/sharing almost all -data structures supported by MATLAB, thanks to `jdataencode/jdatadecode` - +data structures supported by MATLAB, thanks to ``jdataencode/jdatadecode`` - a pair of in-memory data converters translating complex data structures to the easy-to-serialized forms according to the JData specifications. -The detailed help information can be found in the `Contents.m` file. +The detailed help information can be found in the ``Contents.m`` file. In the below section, we provide a few examples on how to us each of the core functions for encoding/decoding JSON/UBJSON/MessagePack data. @@ -426,7 +420,7 @@ JData decoding and converts the enclosed data into Python `dict`, `list` and `numpy` objects. Similarly, `jd.loadb()` function loads a binary JData/UBJSON file and performs similar conversions. One can directly call `jd.load()` to open JSONLab (and derived toolboxes such as '''jnifti''': -https://github.com/fangq/jnifti or '''jsnirfy''': https://github.com/fangq/jsnirfy) +https://github.com/NeuroJSON/jnifti or '''jsnirf''': https://github.com/NeuroJSON/jsnirf) generated files based on their respective file suffix. Similarly, the `jd.savet()`, `jd.saveb()` and `jd.save` functions diff --git a/examples/jsonlab_speedtest.m b/examples/jsonlab_speedtest.m index 4990fba..b93a075 100644 --- a/examples/jsonlab_speedtest.m +++ b/examples/jsonlab_speedtest.m @@ -8,9 +8,9 @@ tsave=zeros(len,1); tload=zeros(len,1); for i=1:len tic; - json=savejson('data',struct('d1',rand(datalen(i),3),'d2',rand(datalen(i),3)>0.5)); + json=savebj('data',struct('d1',rand(datalen(i),3),'d2',rand(datalen(i),3)>0.5)); tsave(i)=toc; - data=loadjson(json); + data=loadbj(json); tload(i)=toc-tsave(i); fprintf(1,'matrix size: %d\n',datalen(i)); end diff --git a/jdatadecode.m b/jdatadecode.m index a339ccd..1310938 100644 --- a/jdatadecode.m +++ b/jdatadecode.m @@ -123,7 +123,7 @@ function newdata=jdatadecode(data,varargin) if(isfield(data,N_('_ArrayZipType_'))) zipmethod=data(j).(N_('_ArrayZipType_')); end - if(~isempty(strmatch(zipmethod,{'zlib','gzip','lzma','lzip','lz4','lz4hc'}))) + if(ismember(zipmethod,{'zlib','gzip','lzma','lzip','lz4','lz4hc'})) decompfun=str2func([zipmethod 'decode']); arraytype=data(j).(N_('_ArrayType_')); chartype=0; diff --git a/jload.m b/jload.m index 8475d58..83e71fc 100644 --- a/jload.m +++ b/jload.m @@ -31,7 +31,7 @@ function varargout=jload(filename, varargin) % parse the json file and then decode the output by % jdatadecode; input file must have a suffix of .jdt % -% all options for loadubjson/loadjson (depends on file suffix) +% all options for loadbj/loadjson (depends on file suffix) % can be used to adjust the parsing options % % output: diff --git a/jsave.m b/jsave.m index 88064ef..0b448f9 100644 --- a/jsave.m +++ b/jsave.m @@ -14,7 +14,7 @@ function varargout=jsave(filename, varargin) % fname: (optional) output file name; if not given, save to 'jamdata.jamm' % if fname has a '.json' or '.jdt' suffix, a text-based % JSON/JData file will be created (slow); if the suffix is '.jamm' or -% '.jdb', a Binary JData (https://github.com/fangq/bjdata/) file will be created. +% '.jdb', a Binary JData (https://github.com/NeuroJSON/bjdata/) file will be created. % opt: (optional) a struct to store parsing options, opt can be replaced by % a list of ('param',value) pairs - the param string is equivallent % to a field in opt. opt can have the following @@ -65,7 +65,8 @@ body=struct; metadata=struct('CreateDate',datestr(now,29),... 'CreateTime',datestr(now,'hh:mm:ss'),... - 'OriginalName',filename); + 'OriginalName',filename,... + 'BJDataVersion','v1_draft-2'); vers=ver('MATLAB'); if(isempty(vers)) diff --git a/loadbj.m b/loadbj.m index 769e912..cd3b46a 100644 --- a/loadbj.m +++ b/loadbj.m @@ -4,9 +4,16 @@ function data = loadbj(fname,varargin) % or % data=loadbj(fname,'param1',value1,'param2',value2,...) % -% Parse a Binary JData (BJData v1 Draft-1, defined in https://github.com/OpenJData/bjdata) +% Parse a Binary JData (BJData v1 Draft-2, defined in https://github.com/NeuroJSON/bjdata) % file or memory buffer and convert into a MATLAB data structure % +% By default, this function parses BJD-compliant output. The BJD +% specification is largely similar to UBJSON, with additional data types +% including uint16(u), uint32(m), uint64(M) and half-precision float (h). +% Starting from BJD Draft-2 (JSONLab 3.0 beta or later), all integer and +% floating-point numbers are parsed in Little-Endian as opposed to +% Big-Endian form as in BJD Draft-1/UBJSON Draft-12 (JSONLab 2.1 or older) +% % authors:Qianqian Fang (q.fang <at> neu.edu) % initially created on 2013/08/01 % @@ -21,16 +28,26 @@ function data = loadbj(fname,varargin) % SimplifyCell [1|0]: if set to 1, loadbj will call cell2mat % for each element of the JSON data, and group % arrays based on the cell2mat rules. -% IntEndian [B|L]: specify the endianness of the integer fields -% in the BJData/UBJSON input data. B - Big-Endian format for -% integers (as required in the UBJSON specification); -% L - input integer fields are in Little-Endian order. +% Endian ['L'|'B']: specify the endianness of the numbers +% in the BJData/UBJSON input data. Default: 'L'. +% +% Starting from JSONLab 2.9, BJData by default uses +% [L] Little-Endian for both integers and floating +% point numbers. This is a major departure from the +% UBJSON specification, where 'B' - Big-Endian - +% format is used for integer fields. UBJSON does +% not specifically define Endianness for +% floating-point numbers, resulting in mixed +% implementations. JSONLab 2.0-2.1 used 'B' for +% integers and floating-points; JSONLab 1.x uses +% 'B' for integers and native-endianness for +% floating-point numbers. % NameIsString [0|1]: for UBJSON Specification Draft 8 or % earlier versions (JSONLab 1.0 final or earlier), % the "name" tag is treated as a string. To load % these UBJSON data, you need to manually set this % flag to 1. -% UseMap [0|1]: if set to 1, loadjson uses a containers.Map to +% UseMap [0|1]: if set to 1, loadbj uses a containers.Map to % store map objects; otherwise use a struct object % ObjectID [0|interger or list]: if set to a positive number, % it returns the specified JSON object by index @@ -59,7 +76,7 @@ function data = loadbj(fname,varargin) % -- this function is part of JSONLab toolbox (http://iso2mesh.sf.net/cgi-bin/index.cgi?jsonlab) % - if(exist(fname,'file')) + if(length(fname)<4096 && exist(fname,'file')) fid = fopen(fname,'rb'); string = fread(fid,inf,'uint8=>char')'; fclose(fid); @@ -83,7 +100,7 @@ function data = loadbj(fname,varargin) opt.nameisstring=jsonopt('NameIsString',0,opt); [os,maxelem,systemendian]=computer; - opt.flipendian_=(systemendian ~= upper(jsonopt('IntEndian','B',opt))); + opt.flipendian_=(systemendian ~= upper(jsonopt('Endian','L',opt))); objid=jsonopt('ObjectID',0,opt); maxobjid=max(objid); @@ -173,7 +190,7 @@ function [object, pos] = parse_array(inputstr, pos, varargin) % JSON array is w if(count>=0) [object, adv]=parse_block(inputstr, pos, type,count,varargin{:}); if(~isempty(dim)) - object=reshape(object,dim); + object=permute(reshape(object,fliplr(dim(:)')),length(dim):-1:1); end pos=pos+adv; return; diff --git a/loadjson.m b/loadjson.m index 0b50219..cfc1319 100644 --- a/loadjson.m +++ b/loadjson.m @@ -62,6 +62,10 @@ function data = loadjson(fname,varargin) % JDataDecode [1|0]: if set to 1, call jdatadecode to decode % JData structures defined in the JData % Specification. +% BuiltinJSON [0|1]: if set to 1, this function attempts to call +% jsondecode, if presents (MATLAB R2016b or Octave +% 6) first. If jsondecode does not exist or failed, +% this function falls back to the jsonlab parser % % output: % dat: a cell array, where {...} blocks are converted into cell arrays, @@ -102,6 +106,18 @@ function data = loadjson(fname,varargin) else error_pos('input file does not exist'); end + + if(jsonopt('BuiltinJSON',0,opt) && exist('jsondecode','builtin')) + try + newstring=regexprep(string,'[\r\n]',''); + newdata=jsondecode(newstring); + newdata=jdatadecode(newdata,'Base64',1,'Recursive',1,varargin{:}); + data=newdata; + return; + catch + warning('built-in jsondecode function failed to parse the file, fallback to loadjson'); + end + end pos = 1; inputlen = length(string); inputstr = string; arraytokenidx=find(inputstr=='[' | inputstr==']'); @@ -205,8 +221,9 @@ function [object, pos,index_esc] = parse_array(inputstr, pos, esc, index_esc, va % next handle 2D array, these are most common ones if(maxlevel==2 && ~isempty(regexp(arraystr(2:end),'^\s*\[','once'))) + isndarray=nestbracket2dim(arraystr,'[]',1); rowstart=find(arraystr(2:end)=='[',1)+1; - if(rowstart) + if(rowstart && isndarray) [obj, nextidx]=parse2darray(inputstr,pos+rowstart,arraystr); if(nextidx>=length(arraystr)-1) object=obj; @@ -227,8 +244,8 @@ function [object, pos,index_esc] = parse_array(inputstr, pos, esc, index_esc, va % in the future can replace 1d and 2d cases if(maxlevel>2 && ~isempty(regexp(arraystr(2:end),'^\s*\[\s*\[','once'))) astr=arraystr; - dims=nestbracket2dim(astr); - if(any(dims==0) || all(mod(dims(:),1) == 0)) % all dimensions are integers - this can be problematic + [dims,isndarray]=nestbracket2dim(astr); + if(isndarray && (any(dims==0) || all(mod(dims(:),1) == 0))) % all dimensions are integers - this can be problematic astr=arraystr; astr(astr=='[')=''; astr(astr==']')=''; diff --git a/loadubjson.m b/loadubjson.m index e57ef25..b1a9106 100644 --- a/loadubjson.m +++ b/loadubjson.m @@ -12,8 +12,9 @@ function varargout = loadubjson(varargin) % This function is an alias to loadbj % % input: -% fname: input file name, if fname contains "{}" or "[]", fname -% will be interpreted as a UBJSON string +% fname: input file name, if the file with such name exists, it will +% be read, otherwise, this function will attempt to parse the +% string in fname as a UBJSON stream % opt: a struct to store parsing options, opt can be replaced by % a list of ('param',value) pairs - the param string is equivallent % to a field in opt. The supported options can be found by typing @@ -36,4 +37,4 @@ function varargout = loadubjson(varargin) % -- this function is part of JSONLab toolbox (http://iso2mesh.sf.net/cgi-bin/index.cgi?jsonlab) % -[varargout{1:nargout}]=loadbj(varargin{:}); +[varargout{1:nargout}]=loadbj(varargin{:},'endian','B'); diff --git a/nestbracket2dim.m b/nestbracket2dim.m index d1a19bf..1fce8e2 100644 --- a/nestbracket2dim.m +++ b/nestbracket2dim.m @@ -1,6 +1,6 @@ -function [dims, maxlevel, count] = nestbracket2dim(str,brackets) +function [dims, isndarray, maxlevel, count] = nestbracket2dim(str,brackets,testndarray) % -% [dims, maxlevel, count] = nestbracket2dim(str,brackets) +% [dims, isndarray, maxlevel, count] = nestbracket2dim(str,brackets) % % Extracting the dimension vector of a JSON string formatted array % by analyzing the pairs of opening/closing bracket tokenss; this function @@ -16,12 +16,17 @@ function [dims, maxlevel, count] = nestbracket2dim(str,brackets) % if not given, brackets is set to '[]' to find matching square-brackets; % for example, '{}' looks for a matching closing curly-bracket in % the string key(pos(startpos,:end)) +% testndarray: (optional), 1 to test if the input string contains an +% ND array, i.e. with uniform element lengths (recursively) % % output: % dims: the speculated dimension vector with the length matching the maximum % depth of the embedded bracket pairs. When the input string encodes an % N-D array, the dims vector contains all integers; however, returning -% an all-integer dims vector does not mean the array is rectangular. +% an all-integer dims vector does not mean the array is +% rectangular. if testndarray is set to 1, dims returns isndarray +% isndarray: 1 to indicate the input string contains an ND array, +% otherwise, 0 % maxlevel: return the depth of the enclosed brackets in the string, i.e. the % length of the dims vector. % count: the relative depth from the level 0 - scanning from the left @@ -46,8 +51,29 @@ if(nargin<2) end str=str(str==brackets(1) | str==brackets(2) | str==','); count=cumsum(str==brackets(1)) - cumsum(str==brackets(2)); +isndarray=testuniform(count, max(count)); +if(nargin>2 && testndarray) + dims=isndarray; + return; +end maxlevel=max(count); dims=histc(count,1:maxlevel); dims(1:end-1)=dims(1:end-1)*0.5; dims(2:end)=dims(2:end)./dims(1:end-1); dims=fliplr(dims); + +function isndarray=testuniform(count, maxval) +isndarray=false; +if(length(count)>2) + idx=find(count(2:end)==count(1),1); + if(idx==length(count)-2 && count(1)<maxval) + isndarray=testuniform(count(2:end-1), maxval); + return; + end + if(~isempty(idx) && mod(length(count)-1,idx+1)==0) + count2d=reshape(count(1:end-1),idx+1,[]); + if(all(diff(count2d')==0)) + isndarray=true; + end + end +end \ No newline at end of file diff --git a/savebj.m b/savebj.m index 26d9dba..28f7bb3 100644 --- a/savebj.m +++ b/savebj.m @@ -7,7 +7,7 @@ function json=savebj(rootname,obj,varargin) % bjd=savebj(rootname,obj,'param1',value1,'param2',value2,...) % % Convert a MATLAB object (cell, struct, array, table, map, handles ...) -% into a Binary JData (BJData v1 Draft-1), Universal Binary JSON (UBJSON, +% into a Binary JData (BJData v1 Draft-2), Universal Binary JSON (UBJSON, % Draft-12) or a MessagePack binary stream % % author: Qianqian Fang (q.fang <at> neu.edu) @@ -15,10 +15,13 @@ function json=savebj(rootname,obj,varargin) % % By default, this function creates BJD-compliant output. The BJD % specification is largely similar to UBJSON, with additional data types -% including uint16(u), uint32(m), uint64(M) and half-precision float (h) +% including uint16(u), uint32(m), uint64(M) and half-precision float (h). +% Starting from BJD Draft-2 (JSONLab 3.0 beta or later), all integer and +% floating-point numbers are stored in Little-Endian as opposed to +% Big-Endian form as in BJD Draft-1/UBJSON Draft-12 (JSONLab 2.1 or older) % % Format specifications: -% Binary JData (BJD): https://github.com/fangq/bjdata +% Binary JData (BJD): https://github.com/NeuroJSON/bjdata % UBJSON: https://github.com/ubjson/universal-binary-json % MessagePack: https://github.com/msgpack/msgpack % @@ -96,7 +99,8 @@ function json=savebj(rootname,obj,varargin) % UBJSON [0|1]: 0: (default)-encode data based on BJData Draft 1 % (supports uint16(u)/uint32(m)/uint64(M)/half(h) markers) % 1: encode data based on UBJSON Draft 12 (without -% u/m/M/h markers) +% u/m/M/h markers);all numeric values are stored in +% the Big-Endian byte order according to Draft-12 % FormatVersion [2|float]: set the JSONLab output version; since % v2.0, JSONLab uses JData specification Draft 3 % for output format, it is incompatible with releases @@ -107,7 +111,21 @@ function json=savebj(rootname,obj,varargin) % of the minimum length without losing accuracy (default) % Debug [0|1]: output binary numbers in <%g> format for debugging % Append [0|1]: if set to 1, append a new object at the end of the file. -% Endian ['n'|'b','l']: Endianness of the output file ('n': native, +% Endian ['L'|'B']: specify the endianness of the numbers +% in the BJData/UBJSON input data. Default: 'L'. +% +% Starting from JSONLab 2.9, BJData by default uses +% [L] Little-Endian for both integers and floating +% point numbers. This is a major departure from the +% UBJSON specification, where 'B' - Big-Endian - +% format is used for integer fields. UBJSON does +% not specifically define Endianness for +% floating-point numbers, resulting in mixed +% implementations. JSONLab 2.0-2.1 used 'B' for +% integers and floating-points; JSONLab 1.x uses +% 'B' for integers and native-endianness for +% floating-point numbers. +% FileEndian ['n'|'b','l']: Endianness of the output file ('n': native, % 'b': big endian, 'l': little-endian) % PreEncode [1|0]: if set to 1, call jdataencode first to preprocess % the input data before saving @@ -125,6 +143,7 @@ function json=savebj(rootname,obj,varargin) % 'MeshCreator','FangQ','MeshTitle','T6 Cube',... % 'SpecialData',[nan, inf, -inf]); % savebj(jsonmesh) +% savebj('',jsonmesh,'debug',1) % savebj('',jsonmesh,'meshdata.bjd') % savebj('mesh1',jsonmesh,'FileName','meshdata.msgpk','MessagePack',1) % savebj('',jsonmesh,'ubjson',1) @@ -164,13 +183,16 @@ opt.ubjson=bitand(jsonopt('UBJSON',0,opt), ~opt.messagepack); opt.keeptype=jsonopt('KeepType',0,opt); opt.nosubstruct_=0; +[os,maxelem,systemendian]=computer; +opt.flipendian_=(systemendian ~= upper(jsonopt('Endian','L',opt))); + if(jsonopt('PreEncode',1,opt)) obj=jdataencode(obj,'Base64',0,'UseArrayZipSize',opt.messagepack,opt); end dozip=opt.compression; if(~isempty(dozip)) - if(isempty(strmatch(dozip,{'zlib','gzip','lzma','lzip','lz4','lz4hc'}))) + if(~ismember(dozip,{'zlib','gzip','lzma','lzip','lz4','lz4hc'})) error('compression method "%s" is not supported',dozip); end if(exist('zmat','file')~=2 && exist('zmat','file')~=3) @@ -260,15 +282,15 @@ end filename=jsonopt('FileName','',opt); if(~isempty(filename)) encoding = jsonopt('Encoding','',opt); - endian = jsonopt('Endian','n',opt); + fileendian = jsonopt('FileEndian','n',opt); writemode='w'; if(jsonopt('Append',0,opt)) writemode='a'; end if(~exist('OCTAVE_VERSION','builtin')) - fid = fopen(filename, writemode, endian, encoding); + fid = fopen(filename, writemode, fileendian, encoding); else - fid = fopen(filename, writemode, endian); + fid = fopen(filename, writemode, fileendian); end fwrite(fid,json); fclose(fid); @@ -774,7 +796,8 @@ if(isa(mat,'integer') || isinteger(mat) || (~varargin{1}.keeptype && isfloat(mat txt=I_(uint64(mat),varargin{:}); end else - txt=I_a(mat(:),type,size(mat),varargin{:}); + rowmat=permute(mat,ndims(mat):-1:1); + txt=I_a(rowmat(:),type,size(mat),varargin{:}); end else txt=cell2ubjson('',num2cell(mat,2),level,varargin{:}); @@ -787,7 +810,8 @@ elseif(islogical(mat)) if(~isvector(mat) && isnest==1) txt=cell2ubjson('',num2cell(uint8(mat),1),level,varargin{:}); else - txt=I_a(uint8(mat(:)),Imarker(1),size(mat),varargin{:}); + rowmat=permute(mat,ndims(mat):-1:1); + txt=I_a(uint8(rowmat(:)),Imarker(1),size(mat),varargin{:}); end end else @@ -805,7 +829,8 @@ else if(~isvector(mat) && isnest==1) txt=cell2ubjson('',num2cell(mat,1),level,varargin{:}); else - txt=D_a(mat(:),Fmarker(3),size(mat),varargin{:}); + rowmat=permute(mat,ndims(mat):-1:1); + txt=D_a(rowmat(:),Fmarker(3),size(mat),varargin{:}); end end end @@ -841,10 +866,10 @@ end val=I_(uint32(num),varargin{:}); if(val(1)>char(210)) num=uint32(num); - val=[char(210) data2byte(swapbytes(cast(num,'uint32')),'uint8')]; + val=[char(210) data2byte(endiancheck(cast(num,'uint32'),varargin{:}),'uint8')]; elseif(val(1)<char(209)) num=uint16(num); - val=[char(209) data2byte(swapbytes(cast(num,'uint16')),'uint8')]; + val=[char(209) data2byte(endiancheck(cast(num,'uint16'),varargin{:}),'uint8')]; end val(1)=char(val(1)-209+base1); @@ -860,7 +885,7 @@ if(isfield(varargin{1},'inttype_')) if(isdebug) val=[Imarker(varargin{1}.inttype_) sprintf('<%.0f>',num)]; else - val=[Imarker(varargin{1}.inttype_) data2byte(swapbytes(cast(num,cid{varargin{1}.inttype_})),'uint8')]; + val=[Imarker(varargin{1}.inttype_) data2byte(endiancheck(cast(num,cid{varargin{1}.inttype_}),varargin{:}),'uint8')]; end return; end @@ -875,7 +900,7 @@ if(Imarker(1)~='U') end end if(Imarker(1)~='U' && num<0 && num<127) - val=data2byte((swapbytes(cast(num,'uint8')) & 127),'uint8'); + val=data2byte((endiancheck(cast(num,'uint8'),varargin{:}) & 127),'uint8'); return; end key=Imarker; @@ -884,7 +909,7 @@ for i=1:length(cid) if(isdebug) val=[key(i) sprintf('<%.0f>',num)]; else - val=[key(i) data2byte(swapbytes(cast(num,cid{i})),'uint8')]; + val=[key(i) data2byte(endiancheck(cast(num,cid{i}),varargin{:}),'uint8')]; end return; end @@ -902,7 +927,7 @@ isdebug=varargin{1}.debug; if(isdebug) output=sprintf('<%g>',num); else - output=data2byte(swapbytes(num),'uint8'); + output=data2byte(endiancheck(num,varargin{:}),'uint8'); end Fmarker=varargin{1}.FM_; @@ -933,7 +958,7 @@ end % based on UBJSON specs, all integer types are stored in big endian format cid=varargin{1}.IType_; -data=data2byte(swapbytes(cast(num,cid{id})),'uint8'); +data=data2byte(endiancheck(cast(num,cid{id}),varargin{:}),'uint8'); blen=varargin{1}.IByte_(id); @@ -979,7 +1004,7 @@ if(id==0) error('unsupported float array'); end -data=data2byte(swapbytes(cast(num,varargin{1}.FType_{id})),'uint8'); +data=data2byte(endiancheck(cast(num,varargin{1}.FType_{id}),varargin{:}),'uint8'); blen=varargin{1}.FByte_(id); isnest=varargin{1}.nestarray; @@ -1032,3 +1057,11 @@ end function bytes=data2byte(varargin) bytes=typecast(varargin{:}); bytes=char(bytes(:)'); + +%%------------------------------------------------------------------------- +function newdata=endiancheck(data, varargin) +if(varargin{1}.flipendian_) + newdata=swapbytes(data); +else + newdata=data; +end \ No newline at end of file diff --git a/savejson.m b/savejson.m index 3ffc57a..04f752b 100644 --- a/savejson.m +++ b/savejson.m @@ -102,6 +102,10 @@ function json=savejson(rootname,obj,varargin) % 'b': big endian, 'l': little-endian) % PreEncode [1|0]: if set to 1, call jdataencode first to preprocess % the input data before saving +% BuiltinJSON [0|1]: if set to 1, this function attempts to call +% jsonencode, if presents (MATLAB R2016b or Octave +% 6) first. If jsonencode does not exist or failed, +% this function falls back to the jsonlab savejson % % opt can be replaced by a list of ('param',value) pairs. The param % string is equivallent to a field in opt and is case sensitive. @@ -158,13 +162,30 @@ opt.nan=jsonopt('NaN','"_NaN_"',opt); opt.num2cell_=0; opt.nosubstruct_=0; +if(jsonopt('BuiltinJSON',0,opt) && exist('jsonencode','builtin')) + try + obj=jdataencode(obj,'Base64',1,'AnnotateArray',1,'UseArrayZipSize',1,opt); + if(isempty(rootname)) + json=jsonencode(obj); + else + json=jsonencode(struct(rootname,obj)); + end + if(isempty(regexp(json,'^[{\[]', 'once'))) + json=['[',json,']']; + end + return; + catch + warning('built-in jsonencode function failed to encode the data, fallback to savejson'); + end +end + if(jsonopt('PreEncode',1,opt)) obj=jdataencode(obj,'Base64',1,'UseArrayZipSize',0,opt); end dozip=opt.compression; if(~isempty(dozip)) - if(isempty(strmatch(dozip,{'zlib','gzip','lzma','lzip','lz4','lz4hc'}))) + if(~ismember(dozip,{'zlib','gzip','lzma','lzip','lz4','lz4hc'})) error('compression method "%s" is not supported',dozip); end if(exist('zmat','file')~=2 && exist('zmat','file')~=3) diff --git a/saveubjson.m b/saveubjson.m index 61b415c..e0e5652 100644 --- a/saveubjson.m +++ b/saveubjson.m @@ -6,21 +6,22 @@ function ubj=saveubjson(rootname,obj,varargin) % ubj=saveubjson(rootname,obj,opt) % ubj=saveubjson(rootname,obj,'param1',value1,'param2',value2,...) % -% Convert a MATLAB object (cell, struct, array, table, map, handles ...) -% into a Universal Binary JSON (UBJSON, Draft 12) or a MessagePack binary stream +% Convert a MATLAB object (cell, struct, array, table, map, graphs ...) +% into a Universal Binary JSON (UBJSON, Draft-12) or a MessagePack binary stream % % author: Qianqian Fang (q.fang <at> neu.edu) % initially created on 2013/08/17 % % Format specifications: -% Binary JData (BJData):https://github.com/fangq/bjdata +% Binary JData (BJData):https://github.com/NeuroJSON/bjdata % UBJSON: https://github.com/ubjson/universal-binary-json % MessagePack: https://github.com/msgpack/msgpack % -% This function is the same as calling "savebj(...,'ubjson',1)". By , -% default this function creates UBJSON-compliant output without the +% This function is the same as calling "savebj(..,'ubjson',1,'endian','B')" +% By default this function creates UBJSON-compliant output without the % newly added uint16(u), uint32(m), uint64(M) and half-precision float (h) -% data types. +% data types and use Big-Endian for all numerical values as in UBJSON +% Draft-12. % % This function by default still enables an optimized ND-array format for efficient % array storage. To ensure the output compatible to UBJSON Draft-12, one should use @@ -42,7 +43,7 @@ function ubj=saveubjson(rootname,obj,varargin) % Please type "help savebj" for details for all supported options. % % output: -% json: a binary string in the UBJSON format (see http://ubjson.org) +% ubj: a binary string in the UBJSON format (see http://ubjson.org) % % examples: % jsonmesh=struct('MeshVertex3',[0 0 0;1 0 0;0 1 0;1 1 0;0 0 1;1 0 1;0 1 1;1 1 1],... @@ -63,9 +64,9 @@ function ubj=saveubjson(rootname,obj,varargin) % if(nargin==1) - ubj=savebj('',rootname,'ubjson',1); + ubj=savebj('',rootname,'ubjson',1,'endian','B'); elseif(length(varargin)==1 && ischar(varargin{1})) - ubj=savebj(rootname,obj,'FileName',varargin{1},'ubjson',1); + ubj=savebj(rootname,obj,'FileName',varargin{1},'ubjson',1,'endian','B'); else - ubj=savebj(rootname,obj,varargin{:},'ubjson',1); + ubj=savebj(rootname,obj,varargin{:},'ubjson',1,'endian','B'); end diff --git a/test/run_jsonlab_test.m b/test/run_jsonlab_test.m index a271ac5..f2e447e 100644 --- a/test/run_jsonlab_test.m +++ b/test/run_jsonlab_test.m @@ -45,7 +45,7 @@ if(ismember('js',tests)) test_jsonlab('empty cell',@savejson,{},'[]'); test_jsonlab('empty string',@savejson,'','""','compact',1); test_jsonlab('string escape',@savejson,sprintf('jdata\n\b\ashall\tprevail\t"\"\\'),'"jdata\n\b\ashall\tprevail\t\"\"\\"'); - if(exist('isstring')) + if(exist('string')) test_jsonlab('string type',@savejson,string(sprintf('jdata\n\b\ashall\tprevail')),'["jdata\n\b\ashall\tprevail"]','compact',1); test_jsonlab('string array',@savejson,[string('jdata'),string('shall'),string('prevail')],'["jdata","shall","prevail"]','compact',1); end @@ -55,6 +55,8 @@ if(ismember('js',tests)) test_jsonlab('char array',@savejson,['AC';'EG'],'["AC","EG"]','compact',1); test_jsonlab('maps',@savejson,struct('a',1,'b','test'),'{"a":1,"b":"test"}','compact',1); test_jsonlab('2d array',@savejson,[1,2,3;4,5,6],'[[1,2,3],[4,5,6]]','compact',1); + test_jsonlab('non-uniform 2d array',@savejson,{[1,2],[3,4,5],[6,7]},'[[1,2],[3,4,5],[6,7]]','compact',1); + test_jsonlab('non-uniform array with length multiple of first element',@savejson,{[1,2],[3,4,5,6],[7,8]},'[[1,2],[3,4,5,6],[7,8]]','compact',1); test_jsonlab('3d (row-major) nested array',@savejson,reshape(1:(2*3*2),2,3,2),... '[[[1,7],[3,9],[5,11]],[[2,8],[4,10],[6,12]]]','compact',1,'nestarray',1); test_jsonlab('3d (column-major) nested array',@savejson,reshape(1:(2*3*2),2,3,2),... @@ -173,7 +175,7 @@ if(ismember('bj',tests)) test_jsonlab('empty string',@savebj,'','SU<0>','debug',1); test_jsonlab('skip no-op before marker and after value',@savebj,loadbj(char(['NN[NU' char(5) 'NNNU' char(1) ']'])),'[$U#U<2><5><1>','debug',1); test_jsonlab('string escape',@savebj,sprintf('jdata\n\b\ashall\tprevail\t"\"\\'),sprintf('SU<25>jdata\n\b\ashall\tprevail\t\"\"\\'),'debug',1); - if(exist('isstring')) + if(exist('string')) test_jsonlab('string type',@savebj,string(sprintf('jdata\n\b\ashall\tprevail')),sprintf('[SU<21>jdata\n\b\ashall\tprevail]'),'debug',1); test_jsonlab('string array',@savebj,[string('jdata');string('shall');string('prevail')],'[[SU<5>jdataSU<5>shallSU<7>prevail]]','debug',1); end @@ -182,7 +184,7 @@ if(ismember('bj',tests)) test_jsonlab('mixed array',@savebj,{'a',1,0.9},'[CaU<1>D<0.9>]','debug',1); test_jsonlab('char array',@savebj,['AC';'EG'],'[SU<2>ACSU<2>EG]','debug',1); test_jsonlab('maps',@savebj,struct('a',1,'b','test'),'{U<1>aU<1>U<1>bSU<4>test}','debug',1); - test_jsonlab('2d array',@savebj,[1,2,3;4,5,6],'[$U#[$U#U<2><2><3><1><4><2><5><3><6>','debug',1); + test_jsonlab('2d array',@savebj,[1,2,3;4,5,6],'[$U#[$U#U<2><2><3><1><2><3><4><5><6>','debug',1); test_jsonlab('3d (row-major) nested array',@savebj,reshape(1:(2*3*2),2,3,2),... '[[[U<1>U<7>][U<3>U<9>][U<5>U<11>]][[U<2>U<8>][U<4>U<10>][U<6>U<12>]]]','debug',1,'nestarray',1); test_jsonlab('3d (column-major) nested array',@savebj,reshape(1:(2*3*2),2,3,2),... @@ -194,9 +196,9 @@ if(ismember('bj',tests)) test_jsonlab('empty sparse matrix',@savebj,sparse(2,3),... '{U<11>_ArrayType_SU<6>doubleU<11>_ArraySize_[$U#U<2><2><3>U<15>_ArrayIsSparse_TU<11>_ArrayData_Z}','debug',1); test_jsonlab('real sparse matrix',@savebj,sparse([0,3,0,1,4]'),... - '{U<11>_ArrayType_SU<6>doubleU<11>_ArraySize_[$U#U<2><5><1>U<15>_ArrayIsSparse_TU<11>_ArrayData_[$U#[$U#U<2><2><3><2><3><4><1><5><4>}','debug',1); + '{U<11>_ArrayType_SU<6>doubleU<11>_ArraySize_[$U#U<2><5><1>U<15>_ArrayIsSparse_TU<11>_ArrayData_[$U#[$U#U<2><2><3><2><4><5><3><1><4>}','debug',1); test_jsonlab('complex sparse matrix',@savebj,sparse([0,3i,0,1,4i].'),... - '{U<11>_ArrayType_SU<6>doubleU<11>_ArraySize_[$U#U<2><5><1>U<16>_ArrayIsComplex_TU<15>_ArrayIsSparse_TU<11>_ArrayData_[$U#[$U#U<2><3><3><2><0><3><4><1><0><5><0><4>}','debug',1); + '{U<11>_ArrayType_SU<6>doubleU<11>_ArraySize_[$U#U<2><5><1>U<16>_ArrayIsComplex_TU<15>_ArrayIsSparse_TU<11>_ArrayData_[$U#[$U#U<2><3><3><2><4><5><0><1><0><3><0><4>}','debug',1); test_jsonlab('heterogeneous cell',@savebj,{{1,{2,3}},{4,5},{6};{7},{8,9},{10}},... '[[[U<1>[U<2>U<3>]][U<4>U<5>][U<6>]][[U<7>][U<8>U<9>][U<10>]]]','debug',1); test_jsonlab('struct array',@savebj,repmat(struct('i',1.1,'d','str'),[1,2]),... @@ -271,4 +273,12 @@ if(ismember('bjo',tests)) test_jsonlab('H marker for out of bound integer',@savebj,2^64-1,'HU<20>18446744073709551616','debug',1,'ubjson',1); test_jsonlab('do not downcast integers to the shortest format',@savebj,int32(5),'l<5>','debug',1,'keeptype',1); test_jsonlab('do not downcast integer array to the shortest format',@savebj,int32([5,6]),'[$l#U<2><5><6>','debug',1,'keeptype',1); + test_jsonlab('test little endian uint32',@savebj,typecast(uint8('abcd'),'uint32'),'mabcd','endian','L'); + test_jsonlab('test big endian uint32',@savebj,typecast(uint8('abcd'),'uint32'),'mdcba','endian','B'); + test_jsonlab('test little endian double',@savebj,typecast(uint8('01234567'),'double'),'D01234567','endian','L'); + test_jsonlab('test big endian double',@savebj,typecast(uint8('01234567'),'double'),'D76543210','endian','B'); + test_jsonlab('test default int endian for savebj',@savebj,typecast(uint8('jd'),'uint16'),'ujd'); + test_jsonlab('test default int endian for saveubjson',@saveubjson,typecast(uint8('jd'),'uint16'),'Idj'); + test_jsonlab('test default float endian for savebj',@savebj,typecast(uint8('1e05'),'single'),'d1e05'); + test_jsonlab('test default float endian for saveubjson',@saveubjson,typecast(uint8('1e05'),'single'),'d50e1'); end -- GitLab