remove old files from archive, add option to use jsonencode

archive/get_dataset.m

-function [dd, dat] = get_dataset(provider,dataset, varargin)
-if numel(varargin) > 0
-    options = parse_options(varargin);
-else
-    options = ['?limit=1000&offset=0&q=&observations=1&align_periods=1&' ...
-               'dimensions={}'];
-end
-default_api_base_url = 'https://api.db.nomics.world/v22/series/';
-url = join([default_api_base_url, provider, '/', dataset, options]);
-dat = webread(join(url));
-docs = dat.series.docs;
-nseries = dat.dataset.nb_series;
-nrow = 1;
-for id = 1:nseries
-    nrow = nrow + length(docs(id).period);
-end
-dimensions_codes = dat.dataset.dimensions_codes_order;
-ndim = length(dimensions_codes);
-ncol = ndim + 10;
-dd = cell(nrow,ncol);
-dd{1,1} = '@frequency';
-dd{1,2} = 'dataset_code';
-dd{1,3} = 'dataset_name';
-dd{1,4} = 'indexed_at';
-dd{1,5} = 'original_period';
-dd{1,6} = 'period';
-dd{1,7} = 'provider_code';
-dd{1,8} = 'series_code';
-dd{1,9} = 'series name';
-dd{1,10} = 'value';
-for i = 1:ndim
-    dd{1,10+i} = dimensions_codes{i};
-end
-
-row = 2;
-for i = 1:nseries
-    doc = docs(i);
-    x_frequency = doc.x_frequency;
-    dataset_code = doc.dataset_code;
-    dataset_name = doc.dataset_name;
-    series_code = doc.series_code;
-    series_name = doc.series_name;
-    indexed_at = doc.indexed_at;
-    provider_code = doc.provider_code;
-    dimensions = doc.dimensions;
-    period = doc.period;
-    period_start_day = doc.period_start_day;
-    value = doc.value;
-    for j = 1:length(period)
-        dd{row,1} = x_frequency;
-        dd{row,2} = dataset_code;
-        dd{row,3} = dataset_name;
-        dd{row,4} = indexed_at;
-        dd{row,5} = period{j};
-        dd{row,6} = period_start_day{j};
-        dd{row,7} = provider_code;
-        dd{row,8} = series_code;
-        dd{row,9} = series_name;
-        if iscell(value)
-            dd{row,10} = value{j};
-        else
-            dd{row,10} = value(j);
-        end
-        for k = 1:ndim
-            dim_code = dimensions.(dimensions_codes{k});
-            dd{row,10+k} = dim_code;
-        end
-        row = row + 1;
-    end
-end
-
-function options_str = parse_options(arguments)
-i = 1;
-options = struct();
-options.limit = 1000;
-options.dimensions = '{}';
-options.mask = '';
-while i <= length(arguments)
-    if arguments{1} == 'limit'
-        options.limit = arguments{2};
-        i = i + 2;
-    elseif arguments{1} == 'dimensions'
-        if ~isempty(options.mask)
-            error('rdb: you can''t use both option dimension and option mask');
-        end
-    elseif arguments =='mask'
-        if options.dimensions ~= '{}'
-            error('rdb: you can''t use both option dimension and option mask');
-        end
-        options.mask = arguments{2};
-        i = i+2;
-    else
-        disp_usage();
-    end
-end
-
-options_str = sprintf('?limit=%d0&q=&observations=1&align_periods=1&dimensions=%s',...
-                      options.limit, options.dimensions);
-if options.mask
-    options_str = options_str + '&' + options.mask;
-end
\ No newline at end of file

archive/get_dseries.m

-function [dd, dat] = get_dseries(provider,dataset,series_id)
-%url = join(['https://api.db.nomics.world/v22/series/', provider, '/',...
-%            dataset, '/', series_id, '?observations=true'])
-%json = urlread(join(url))
-%save test json
-f = fopen("test.json");
-json = fgets(f);
-fclose(f);
-dat = jsondecode(json);
-dd = ''
-docs = dat.series.docs;
-first_period = docs.period{1};
-if docs.x_frequency == 'annual'
-    first_period = join([first_period, 'A']);
-end
-dd = dseries(docs.value, first_period, docs.series_code, docs.series_name)
-

archive/get_series.m

-function [dd, dat] = get_series(provider,dataset,series_id)
-%url = join(['https://api.db.nomics.world/v22/series/', provider, '/',...
-%            dataset, '/', series_id, '?observations=true'])
-%json = urlread(join(url))
-%save test json
-f = fopen("test.json");
-json = fgets(f);
-fclose(f);
-dat = jsondecode(json);
-dd = ''
-docs = dat.series.docs;
-first_period = docs.period{1};
-if docs.x_frequency == 'annual'
-    first_period = join([first_period, 'A']);
-end
-dd = struct()
-dd.values = docs.value
-dd.periods = docs.period
-dd.code = docs.series_code
-dd.name = docs.series_name
-
-

archive/rdb.m

-function data = rdb(str, varargin)
-if nargin == 0
-    [d, dd] = get_series(str)
-elseif nargin == 1
-    [d, dd] = get_dataset(str, varargin{1})
-elseif nargin == 2
-    [d, dd] = get_series(str, varargin{1}, varargin{2})
-elseif nargin > 1
-    [d, dd] = get_dataset(str, vararin{1}, varargin{2:end})
-end

archive/run_all_tests_matlab.m

-function run_all_tests()
-r = [];
-r = [r; run_this_test('test_fetch_series_by_code')];
-r = [r; run_this_test('test_fetch_series_by_code_mask')];
-r = [r; run_this_test('test_fetch_series_by_code_mask_with_plus_in_dimension_code')];
-r = [r; run_this_test('test_fetch_series_by_id')];
-r = [r; run_this_test('test_fetch_series_by_ids_in_same_dataset')];
-r = [r; run_this_test('test_fetch_series_by_ids_in_different_datasets')];
-r = [r; run_this_test('test_fetch_series_by_dimension')];
-r = [r; run_this_test('test_fetch_series_of_dataset')];
-r = [r; run_this_test('test_fetch_series_by_api_link')];
-r = [r; run_this_test('test_fetch_series_with_na_values')];
-r = [r; run_this_test('test_fetch_series_with_max_nb_series')];
-r = [r; run_this_test('test_fetch_series_with_filter_on_one_series')];
-print_results(r);
-
-
-function o = run_this_test(file)
-try
-    tstart = tic;
-    run([file '.m']);
-    elapsed = toc(tstart);
-    o = {file, true, elapsed};
-catch
-    o = {file, false, NaN};
-end
-
-function print_results(r)
-message = sprintf('Testsuite results:\n');
-for i = 1:size(r, 1)
-    if r{i,2}
-        message = sprintf('%s\n%s\t\t PASS (%ss)', message, r{i,1}, num2str(r{i,3}));
-    else
-        message = sprintf('%s\n%s\t\t FAILED', message, r{i,1});
-    end
-end
-disp(message)

src/utils/iter_filtered_series.m

@@ -51,8 +51,14 @@ for gg = 1:size(grouped_series, 2)
         posted_series_list{series} = posted_series;
     end
     
-    posted_series_list = savejson('',posted_series_list, 'Compact', 1);
-    posted_series_list = regexprep(posted_series_list,{' [[',']]'},{'[',']'});
+    if matlab_ver_less_than('9.8')
+        posted_series_list = savejson('',posted_series_list, 'Compact', 1);
+        posted_series_list = regexprep(posted_series_list,{' [[',']]'},{'[',']'});
+    else
+        posted_series_list = jsonencode(posted_series_list);
+    end
+    
+    posted_series_list = transform_json_request(posted_series_list);
     json_request = sprintf('{"filters":%s,"series":%s}', dbnomics_filters, posted_series_list);
  
     try

[Sébastien Villemot]

I don’t understand this test. MATLAB 9.8 is R2020a, but the doc says that jsondecode was introduced in R2016b (i.e. MATLAB 9.1). [Also note that there was a small behaviour change in R2017b regarding the handling of NaN and Inf, I don’t know if this impacts mdbnomics]

Can you possibly clarify?

[Dóra Kocsis]

Thanks, this is most likely a typo, I fixed it now. Regarding the changes in R2017b, I'm checking if there is an impact.

[Dóra Kocsis]

The change in handling of NaN and Inf doesn't impact mdbnomics, because in the POST request still the original values are pushed where the string NA is not yet converted to NaN.