hapiclient 0.2.7__tar.gz → 0.2.9__tar.gz

{hapiclient-0.2.7 → hapiclient-0.2.9}/CHANGES.txt

@@ -110,4 +110,17 @@ v0.2.7b1:
 	2026-04-08 -- Improve NaN handling https://github.com/hapi-server/client-python/issues/88
 	2026-05-02 -- Logging and testing https://github.com/hapi-server/client-python/issues/89
 v0.2.7:
-	2026--05-02-- Support for Python 2.7 dropped. This is last release with support for 3.5 through 3.8.
+	2026-05-02 -- Support for Python 2.7 dropped. This is last release with support for 3.5 through 3.8.
+v0.2.8b0:
+	2026-05-02 -- Documentation
+v0.2.8b1:
+	2026-05-02 -- Versioning updates
+	2026-05-19 -- Better response error messages
+v0.2.8b2:
+	2026-05-20 -- Refactor
+v0.2.8:
+	2026-06-04 -- Compat. fixes for hapiplot package.
+v0.2.8b0:
+	2026-06-08 -- Show time parsing error message
+v0.2.9:
+	2026-06-08 -- 93-occasional-crashes-due-to-undefined-variable-dt-in-get_csv-hapiclient-version-028

{hapiclient-0.2.7/hapiclient.egg-info → hapiclient-0.2.9}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: hapiclient
-Version: 0.2.7
+Version: 0.2.9
 Summary: Interface to Heliophysics data server API
 Home-page: http://pypi.python.org/pypi/hapiclient/
 Author: Bob Weigel
@@ -28,7 +28,9 @@ Dynamic: summary
 
 **HAPI Client for Python**
 
-Basic usage examples for various HAPI servers are given in [hapi_demo.py](https://github.com/hapi-server/client-python/blob/master/hapi_demo.py) and the Examples section of a Jupyter Notebook hosted on Google Colab: [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/hapi-server/client-python-notebooks/blob/master/hapi_demo.ipynb#examples).
+Basic usage examples for various HAPI servers are given in [hapi_demo.py](https://github.com/hapi-server/client-python/blob/master/hapi_demo.py). See also
+* Examples section of a Jupyter Notebook hosted on Google Colab: [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/hapi-server/client-python-notebooks/blob/master/hapi_demo.ipynb#examples).
+* [PyHC Summer School tutorials](https://github.com/hapi-server/tutorial-python)
 
 # Installation
 

{hapiclient-0.2.7 → hapiclient-0.2.9}/README.md

@@ -3,7 +3,9 @@
 
 **HAPI Client for Python**
 
-Basic usage examples for various HAPI servers are given in [hapi_demo.py](https://github.com/hapi-server/client-python/blob/master/hapi_demo.py) and the Examples section of a Jupyter Notebook hosted on Google Colab: [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/hapi-server/client-python-notebooks/blob/master/hapi_demo.ipynb#examples).
+Basic usage examples for various HAPI servers are given in [hapi_demo.py](https://github.com/hapi-server/client-python/blob/master/hapi_demo.py). See also
+* Examples section of a Jupyter Notebook hosted on Google Colab: [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/hapi-server/client-python-notebooks/blob/master/hapi_demo.ipynb#examples).
+* [PyHC Summer School tutorials](https://github.com/hapi-server/tutorial-python)
 
 # Installation
 

{hapiclient-0.2.7 → hapiclient-0.2.9}/hapiclient/__init__.py

@@ -1,6 +1,9 @@
 # Allow "from hapiclient import hapi"
 from hapiclient.hapi import hapi
 
+# Allow "from hapiclient import request2path"
+from hapiclient.hapi import request2path
+
 # Allow "from hapiclient import hapitime2datetime"
 from hapiclient.hapitime import hapitime2datetime
 
@@ -10,7 +13,7 @@ from hapiclient.hapitime import datetime2hapitime
 # Allow "from hapiclient import HAPIError"
 from hapiclient.util import HAPIError
 
-__version__ = '0.2.7'
+__version__ = '0.2.9'
 
 import sys
 import platform

hapiclient-0.2.9/hapiclient/cache.py

@@ -0,0 +1,255 @@
+def server2dirname(server):
+  """Convert a server URL to a directory name."""
+
+  import re
+
+  urld = re.sub(r"https*://", "", server)
+  urld = re.sub(r'/', '_', urld)
+
+  return urld
+
+
+def cachedir(*args):
+  """HAPI cache directory.
+
+  cachedir() returns os.path.join(tempfile.gettempdir(), 'hapi-data')
+
+  cachedir(basedir, server) returns os.path.join(basedir, server2dirname(server))
+  """
+
+  import os
+  import tempfile
+
+  if len(args) != 0 and len(args) != 2:
+    raise ValueError('cachedir() takes either 0 or 2 arguments.')
+
+  if len(args) == 0:
+    # cachedir()
+    return os.path.join(tempfile.gettempdir(), 'hapi-data')
+
+  if len(args) == 2:
+    # cachedir(base_dir, server)
+    return os.path.join(args[0], server2dirname(args[1]))
+
+
+def request2path(*args):
+  # request2path(server, dataset, parameters, start, stop)
+  # request2path(server, dataset, parameters, start, stop, cachedir)
+  import os
+  import re
+  import platform
+
+  if len(args) == 5:
+    # Use default if cachedir not given.
+    cachedirectory = cachedir()
+  else:
+    cachedirectory = args[5]
+
+  args = list(args)
+
+  # Replace forbidden characters in directory and filename
+  # Replacements assume that there will be no name collisions,
+  # e.g., one parameter named abc-< and another abc-@lt@.
+  # This also introduces an incompatibility between caches on Windows
+  # Unix.
+  if platform.system() == 'Windows':
+    # List and code from responses in
+    # https://stackoverflow.com/q/1976007
+    reps = (
+              ('<', '@lt@'),
+              ('>', '@gt@'),
+              (':', '@colon@'),
+              ('"', '@doublequote@'),
+              ('/', '@forwardslash@'),
+              ('/', '@backslash@'),
+              ('\\|', '@pipe@'),
+              ('\\?', '@questionmark@'),
+              ('\\*', '@asterisk@')
+          )
+
+    for element in reps:
+      args[1] = re.sub(element[0], element[1], args[1])
+      args[2] = re.sub(element[0], element[1], args[2])
+
+  else:
+    args[1] = re.sub('/','@forwardslash@',args[1])
+    args[2] = re.sub('/','@forwardslash@',args[2])
+
+  # To shorten filenames.
+  args[3] = re.sub(r'-|:|\.|Z', '', args[3])
+  args[4] = re.sub(r'-|:|\.|Z', '', args[4])
+
+  # URL subdirectory
+  urldirectory = server2dirname(args[0])
+  fname = '%s_%s_%s_%s' % (args[1], args[2], args[3], args[4])
+
+  return os.path.join(cachedirectory, urldirectory, fname)
+
+
+def meta_cache_paths(SERVER, DATASET, cachedir):
+  """Return dict with metadata cache directory and file names."""
+
+  fname_root = request2path(SERVER, DATASET, '', '', '', cachedir)
+
+  return {
+    'json': fname_root + '.json',
+    'pkl': fname_root + '.pkl'
+  }
+
+
+def meta_cache_read(SERVER, DATASET, opts):
+  """Read metadata from PKL cache. Returns meta dict or None."""
+
+  import os
+  import pickle
+
+  from hapiclient.log import log
+
+  if not opts["usecache"]:
+    log('Not checking metadata cache because usecache is False.')
+    return None
+
+  fnamepkl = meta_cache_paths(SERVER, DATASET, opts['cachedir'])['pkl']
+  if os.path.isfile(fnamepkl):
+    log('Reading %s' % os.path.basename(fnamepkl))
+    with open(fnamepkl, 'rb') as f:
+      return pickle.load(f)
+
+  if opts["usecache"]:
+    log('No metadata cache file found: %s' % os.path.basename(fnamepkl))
+
+  return None
+
+
+def meta_cache_write(meta, SERVER, DATASET, opts):
+  """Write metadata to JSON and PKL cache files."""
+
+  import os
+  import json
+  import pickle
+
+  from hapiclient.log import log
+
+  if not opts["cache"]:
+    return
+
+  paths = meta_cache_paths(SERVER, DATASET, opts['cachedir'])
+  fnamejson, fnamepkl = paths['json'], paths['pkl']
+
+  server_dir = cachedir(opts["cachedir"], SERVER)
+  os.makedirs(server_dir, exist_ok=True)
+
+  log('Writing %s ' % os.path.basename(fnamejson))
+  with open(fnamejson, 'w') as f:
+    json.dump(meta, f, indent=4)
+
+  log('Writing %s ' % os.path.basename(fnamepkl))
+  with open(fnamepkl, 'wb') as f:
+    # protocol=2 used for Python 2.7 compatibility.
+    pickle.dump(meta, f, protocol=2)
+
+
+def data_cache_paths(SERVER, DATASET, PARAMETERS, START, STOP, cachedir):
+  """Return dict with data cache file names."""
+
+  fname_root = request2path(SERVER, DATASET, PARAMETERS, START, STOP, cachedir)
+
+  return {
+    'csv': fname_root + '.csv',
+    'bin': fname_root + '.bin',
+    'npy': fname_root + '.npy',
+    'pkl': fname_root + '.pkl'
+  }
+
+
+def data_cache_read_metax(SERVER, DATASET, PARAMETERS, START, STOP, opts):
+  """Read extended request metadata from PKL cache. Returns meta dict or None."""
+
+  import os
+  import pickle
+
+  from hapiclient.log import log
+
+  if not opts["usecache"]:
+    log('Not checking subsetted metadata cache because usecache is False.')
+    return None
+
+  fnamepklx = data_cache_paths(SERVER, DATASET, PARAMETERS, START, STOP, opts['cachedir'])['pkl']
+  if os.path.isfile(fnamepklx):
+    log('Reading subsetted metadata cache %s' % os.path.basename(fnamepklx))
+    with open(fnamepklx, 'rb') as f:
+      return pickle.load(f)
+  if opts["usecache"]:
+    log('No subsetted metadata cache file found: %s' % os.path.basename(fnamepklx))
+
+  return None
+
+
+def data_cache_read_npy(SERVER, DATASET, PARAMETERS, START, STOP, opts):
+  """Read cached numpy data array. Returns None if not cached."""
+
+  import os
+  import numpy as np
+
+  from hapiclient.log import log
+
+  if not opts["usecache"]:
+    return None
+
+  fnamenpy = data_cache_paths(SERVER, DATASET, PARAMETERS, START, STOP, opts['cachedir'])['npy']
+
+  if not os.path.isfile(fnamenpy):
+    return None
+
+  log('Reading %s ' % os.path.basename(fnamenpy))
+  with open(fnamenpy, 'rb') as f:
+    data = np.load(f)
+
+  return data
+
+
+def data_cache_write(data_result, meta, SERVER, DATASET, PARAMETERS, START, STOP, opts):
+  """Write data array and extended metadata to cache files.
+
+  Also updates meta with file-related x_ fields before writing.
+  """
+
+  import os
+  import pickle
+  import warnings
+  import numpy as np
+
+  from hapiclient.log import log
+
+  data_paths = data_cache_paths(SERVER, DATASET, PARAMETERS, START, STOP, opts['cachedir'])
+  fnamecsv, fnamebin, fnamenpy, fnamepklx = data_paths['csv'], data_paths['bin'], data_paths['npy'], data_paths['pkl']
+
+  meta_paths = meta_cache_paths(SERVER, DATASET, opts['cachedir'])
+  fnamejson, fnamepkl = meta_paths['json'], meta_paths['pkl']
+
+  meta.update({"x_metaFileParsed": fnamepkl})
+  meta.update({"x_dataFileParsed": fnamenpy})
+  meta.update({"x_metaFile": fnamejson})
+  meta.update({"x_dataFile": fnamebin if opts['format'] == 'binary' else fnamecsv})
+
+  if not opts["cache"]:
+    # Need to return after meta is updated.
+    return
+
+  server_dir = cachedir(opts["cachedir"], SERVER)
+  os.makedirs(server_dir, exist_ok=True)
+
+  log('Writing %s' % os.path.basename(fnamepklx))
+  with open(fnamepklx, 'wb') as f:
+    pickle.dump(meta, f, protocol=2)
+
+  log('Writing %s' % os.path.basename(fnamenpy))
+  with warnings.catch_warnings():
+    # Ignore warning that occurs when saving Unicode data.
+    kwargs = {
+        'message': r"Stored array in format 3\.0.*",
+        'category': UserWarning,
+        'module': r"numpy\.lib\.format"
+    }
+    warnings.filterwarnings("ignore", **kwargs)
+    np.save(fnamenpy, data_result)

hapiclient-0.2.9/hapiclient/capabilities.py

@@ -0,0 +1,42 @@
+
+def capabilities(SERVER):
+  """Return the capabilities of a HAPI server.
+
+  Args:
+    SERVER (str): The base URL of the HAPI server.
+
+  Returns:
+    dict: A dictionary containing the capabilities of the server.
+  """
+  from hapiclient.util import urlopen
+
+  caps = urlopen(SERVER + '/capabilities', parse_json=True)
+
+  return caps
+
+
+def get_format(SERVER, format):
+  """Return the transport format to use, accounting for server capabilities.
+
+  If the requested format is not supported by the server, falls back to 'csv'
+  with a warning.
+  """
+
+  from hapiclient.util import error, warning
+
+  cformats = ['csv', 'binary']  # client formats
+  if format not in cformats:
+    msg = 'This client does not handle transport format "%s". Available options: %s'
+    error(msg % (format, ', '.join(cformats)))
+
+  if format != 'csv':
+    caps = capabilities(SERVER)
+    sformats = caps["outputFormats"]  # Server formats
+    if format not in sformats:
+      msg = 'Requested transport format "%s" not avaiable from %s. Will use "csv". Available options: %s'
+      warning(msg % (format, SERVER, ', '.join(sformats)))
+      format = 'csv'
+    if 'binary' not in sformats:
+      format = 'csv'
+
+  return format

hapiclient-0.2.9/hapiclient/catalog.py

@@ -0,0 +1,10 @@
+from hapiclient.log import log
+from hapiclient.util import urlopen
+
+
+def catalog(SERVER):
+  # TODO: Cache
+  url = SERVER + '/catalog'
+  meta = urlopen(url, parse_json=True)
+
+  return meta

hapiclient-0.2.9/hapiclient/data.py

@@ -0,0 +1,271 @@
+def data(SERVER, DATASET, PARAMETERS, START, STOP, opts):
+
+  import os
+  import time
+  from datetime import datetime
+
+  from hapiclient.log import log
+  from hapiclient.util import warning, subset_meta, unicode_check, fix_parameters, missing_length
+  from hapiclient.cache import cachedir, data_cache_read_metax, data_cache_read_npy, data_cache_write
+  from hapiclient.get import get_binary, get_csv
+  from hapiclient.info import info
+  from hapiclient.capabilities import get_format
+
+  unicode_check(DATASET, PARAMETERS)
+  PARAMETERS = fix_parameters(PARAMETERS)
+
+  urld = cachedir(opts["cachedir"], SERVER)
+  if opts['usecache'] or opts['cache']:
+    log('cache subdirectory = %s' % urld)
+
+  if STOP is None:
+    log('STOP was given as None. Getting stopDate for dataset.')
+    meta = info(SERVER, DATASET, None, opts)
+    STOP = meta['stopDate']
+    log('Using STOP = {STOP}')
+
+  tic_totalTime = time.time()
+
+  meta = data_cache_read_metax(SERVER, DATASET, PARAMETERS, START, STOP, opts)
+  metaFromCache = meta is not None
+  if not metaFromCache:
+    meta = info(SERVER, DATASET, None, opts)
+
+  # Add information to metadata so we can figure out the request needed
+  # to generate it. Will also be used for labeling plots by hapiplot().
+  meta.update({"x_server": SERVER})
+  meta.update({"x_dataset": DATASET})
+
+  if opts["cache"]:
+    if not os.path.exists(urld):
+      os.makedirs(urld)
+
+  if opts['dt_chunk'] == 'infer':
+    opts['dt_chunk'] = _dt_chunk_infer(meta, opts)
+
+  if opts['n_chunks'] is not None or opts['dt_chunk'] is not None:
+    chunk_result = _get_chunks(SERVER, DATASET, PARAMETERS, START, STOP, opts, tic_totalTime)
+    if chunk_result is not None:
+      return chunk_result
+
+  if not metaFromCache:
+    meta = subset_meta(meta, PARAMETERS)
+
+  tic = time.time()
+  data_cached = data_cache_read_npy(SERVER, DATASET, PARAMETERS, START, STOP, opts)
+  if data_cached is not None:
+    meta['x_totalTime'] = time.time() - tic_totalTime
+    meta['x_readTime'] = tic - tic_totalTime
+    meta['x_downloadTime'] = 0
+    return data_cached, meta
+
+  opts['format'] = get_format(SERVER, opts['format'])
+
+  # length attribute required for all parameters when serving binary but
+  # is only required for time parameter when serving CSV. This catches
+  # case where server provides binary but is missing a length attribute
+  # in one or more string parameters that were requested. In this case,
+  # there is not enough information to parse binary.
+  if opts['format'] == 'binary' and missing_length(meta, opts):
+    warning('Requesting CSV instead of binary because a string or isotime parameter is missing a length attribute.')
+    opts['format'] = 'csv'
+
+  # Read the data. toc0 is time to download to file or into buffer;
+  # toc is time to parse.
+  if opts['format'] == 'binary':
+    data_result, toc0, toc = get_binary(meta, SERVER, DATASET, PARAMETERS, START, STOP, opts)
+  else:
+    data_result, toc0, toc = get_csv(meta, SERVER, DATASET, PARAMETERS, START, STOP, opts)
+
+  # Extra metadata associated with request will be saved in
+  # a pkl file with same base name as npy data file.
+  meta.update({"x_server": SERVER})
+  meta.update({"x_dataset": DATASET})
+  meta.update({"x_parameters": PARAMETERS})
+  meta.update({"x_time.min": START})
+  meta.update({"x_time.max": STOP})
+  meta.update({"x_requestDate": datetime.now().isoformat()[0:19]})
+  meta.update({"x_cacheDir": urld})
+  meta.update({"x_downloadTime": toc0})
+  meta.update({"x_readTime": toc})
+
+  data_cache_write(data_result, meta, SERVER, DATASET, PARAMETERS, START, STOP, opts)
+
+  meta['x_totalTime'] = time.time() - tic_totalTime
+
+  return data_result, meta
+
+
+def _dt_chunk_infer(meta, opts):
+
+  import isodate
+  from datetime import datetime
+
+  cadence = meta.get('cadence', None)
+
+  # If cadence not given, use 1-day chunks.
+  if cadence is None:
+    cadence = 'PT1M'
+  else:
+    cadence = isodate.parse_duration(cadence)
+    if isinstance(cadence, isodate.Duration):
+      # When a duration does not correspond to an unambiguous
+      # time duration (e.g., P1M), parse_duration returns an
+      # isodate.duration.Duration object. Otherwise, it returns
+      # a datetime.timedelta object.
+      cadence = cadence.totimedelta(start=datetime.now())
+
+  pt1s = isodate.parse_duration('PT1S')
+  pt1h = isodate.parse_duration('PT1H')
+  p1d = isodate.parse_duration('P1D')
+
+  if cadence < pt1s:
+    return 'PT1H'
+  elif pt1s <= cadence <= pt1h:
+    return 'P1D'
+  elif cadence > pt1h:
+    return 'P1M'
+  elif cadence >= p1d:
+    return 'P1Y'
+
+
+def _get_chunks(SERVER, DATASET, PARAMETERS, START, STOP, opts, tic_totalTime):
+
+  import sys
+  import time
+  import isodate
+  import numpy as np
+
+  from datetime import datetime, timedelta
+  from joblib import Parallel, delayed
+  from hapiclient.log import log
+  from hapiclient.hapitime import hapitime2datetime, hapitime_reformat
+
+  def padz(value):
+    return value if 'Z' in value else value + 'Z'
+
+  pSTART = hapitime2datetime(padz(START))[0]
+  pSTOP = hapitime2datetime(padz(STOP))[0]
+
+  if opts['dt_chunk']:
+    pDELTA = isodate.parse_duration(opts['dt_chunk'])
+
+    if opts['dt_chunk'] == 'P1Y':
+      half = isodate.parse_duration('P365D') / 2
+    elif opts['dt_chunk'] == 'P1M':
+      half = isodate.parse_duration('P30D') / 2
+    else:
+      half = pDELTA / 2
+
+    if (pSTOP - pSTART) < half:
+      opts['n_chunks'] = None
+      opts['dt_chunk'] = None
+      return data(SERVER, DATASET, PARAMETERS, START, STOP, opts)
+
+    if opts['dt_chunk'] == 'P1Y':
+      pSTART = datetime(pSTART.year, 1, 1)
+      pSTOP = datetime(pSTOP.year + 1, 1, 1)
+      opts['n_chunks'] = pSTOP.year - pSTART.year
+    elif opts['dt_chunk'] == 'P1M':
+      pSTART = datetime(pSTART.year, pSTART.month, 1)
+      pSTOP = datetime(pSTOP.year, pSTOP.month + 1, 1)
+      opts['n_chunks'] = (pSTOP.year - pSTART.year) * 12 + (pSTOP.month - pSTART.month)
+    elif opts['dt_chunk'] == 'P1D':
+      pSTART = datetime.combine(pSTART.date(), datetime.min.time())
+      pSTOP = datetime.combine(pSTOP.date(), datetime.min.time()) + timedelta(days=1)
+      opts['n_chunks'] = (pSTOP - pSTART).days
+    elif opts['dt_chunk'] == 'PT1H':
+      pSTART = datetime.combine(pSTART.date(), datetime.min.time()) + timedelta(hours=pSTART.hour)
+      pSTOP = datetime.combine(pSTOP.date(), datetime.min.time()) + timedelta(hours=pSTOP.hour + 1)
+      opts['n_chunks'] = int(((pSTOP - pSTART).total_seconds() / 60) / 60)
+  else:
+    pDIFF = pSTOP - pSTART
+    pDELTA = pDIFF / opts['n_chunks']
+
+  n_chunks = opts['n_chunks']
+  opts['n_chunks'] = None
+  opts['dt_chunk'] = None
+
+  backend = 'sequential'
+  if opts['parallel']:
+    # Note that this does not often lead to significant speed-up.
+    # Needs further testing and optimization.
+    backend = 'threading'
+    # multiprocessing not working.
+    #backend = 'multiprocessing'
+    # loky works, but not speed-up.
+    #backend = 'loky'
+
+  log('backend = {}'.format(backend))
+
+  verbose = 0
+  if opts.get('logging'):
+    verbose = 100
+
+  def nhapi(SERVER, DATASET, PARAMETERS, pSTART, pDELTA, i, **opts):
+    START = pSTART + (i * pDELTA)
+    START = str(START.date())+'T'+str(START.time())
+
+    STOP = pSTART + ((i + 1) * pDELTA)
+    STOP = str(STOP.date()) + 'T' + str(STOP.time())
+
+    data_chunk, meta = data(
+        SERVER,
+        DATASET,
+        PARAMETERS,
+        START,
+        STOP,
+        opts
+    )
+    return data_chunk, meta
+
+  resD, resM = zip(
+    *Parallel(n_jobs=opts['n_parallel'], verbose=verbose, backend=backend)(
+      delayed(nhapi)(
+          SERVER,
+          DATASET,
+          PARAMETERS,
+          pSTART,
+          pDELTA,
+          i,
+          **opts
+      ) for i in range(n_chunks)
+    )
+  )
+
+  resD = list(resD)
+
+  tic_trimTime = time.time()
+  if sys.version_info < (3, ):
+    START = hapitime_reformat(str(resD[0]['Time'][0]), START)
+    resD[0] = resD[0][resD[0]['Time'] >= START]
+
+    STOP = hapitime_reformat(str(resD[-1]['Time'][0]), STOP)
+    resD[-1] = resD[-1][resD[-1]['Time'] < STOP]
+  else:
+    START = hapitime_reformat(resD[0]['Time'][0].decode('UTF-8'), START)
+    resD[0] = resD[0][resD[0]['Time'] >= bytes(START, 'UTF-8')]
+
+    STOP = hapitime_reformat(resD[-1]['Time'][0].decode('UTF-8'), STOP)
+    resD[-1] = resD[-1][resD[-1]['Time'] < bytes(STOP, 'UTF-8')]
+  trimTime = time.time() - tic_trimTime
+
+  tic_catTime = time.time()
+  data_concat = np.concatenate(resD)
+  catTime = time.time() - tic_catTime
+
+  meta = resM[0].copy()
+  meta['x_time.max'] = resM[-1]['x_time.max']
+  meta['x_dataFile'] = None
+  meta['x_dataFiles'] = [resM[i]['x_dataFile'] for i in range(len(resM))]
+  meta['x_downloadTime'] = sum([resM[i]['x_downloadTime'] for i in range(len(resM))])
+  meta['x_downloadTimes'] = [resM[i]['x_downloadTime'] for i in range(len(resM))]
+  meta['x_readTime'] = sum([resM[i]['x_readTime'] for i in range(len(resM))])
+  meta['x_readTimes'] = [resM[i]['x_readTime'] for i in range(len(resM))]
+  meta['x_trimTime'] = trimTime
+  meta['x_catTime'] = catTime
+  meta['x_totalTime'] = time.time() - tic_totalTime
+  meta['x_dataFileParsed'] = None
+  meta['x_dataFilesParsed'] = [resM[i]['x_dataFileParsed'] for i in range(len(resM))]
+
+  return data_concat, meta