tonik 0.0.11__tar.gz → 0.1.0__tar.gz

{tonik-0.0.11 → tonik-0.1.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: tonik
-Version: 0.0.11
+Version: 0.1.0
 Summary: Store time series data as HDF5 files and access them through an API.
 Project-URL: Homepage, https://tsc-tools.github.io/tonik
 Project-URL: Issues, https://github.com/tsc-tools/tonik/issues

{tonik-0.0.11 → tonik-0.1.0}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "tonik"
-version = "0.0.11"
+version = "0.1.0"
 authors = [
   { name="Yannik Behr", email="y.behr@gns.cri.nz" },
   { name="Christof Mueller", email="c.mueller@gns.cri.nz" }

{tonik-0.0.11 → tonik-0.1.0}/src/tonik/storage.py

@@ -6,7 +6,7 @@ import re
 import pandas as pd
 import xarray as xr
 
-from .xarray2hdf5 import xarray2hdf5
+from .xarray2netcdf import xarray2netcdf
 from .xarray2zarr import xarray2zarr
 
 LOGGING_CONFIG = {
@@ -102,7 +102,7 @@ class Path(object):
 
     def feature_path(self, feature):
 
-        if self.backend == 'h5netcdf':
+        if self.backend == 'netcdf':
             file_ending = '.nc'
         elif self.backend == 'zarr':
             file_ending = '.zarr'
@@ -154,7 +154,8 @@ class Path(object):
                     format(stack_length, interval, num_periods))
 
         xd_index = dict(datetime=slice(self.starttime, self.endtime))
-        with xr.open_dataset(filename, group='original', engine=self.backend) as ds:
+        engine = 'h5netcdf' if self.backend == 'netcdf' else self.backend
+        with xr.open_dataset(filename, group='original', engine=engine) as ds:
             rq = ds.loc[xd_index].load()
 
         # Stack features
@@ -187,8 +188,8 @@ class Path(object):
         """
         Save a feature to disk
         """
-        if self.backend == 'h5netcdf':
-            xarray2hdf5(data, self.path, **kwargs)
+        if self.backend == 'netcdf':
+            xarray2netcdf(data, self.path, **kwargs)
         elif self.backend == 'zarr':
             xarray2zarr(data, self.path, **kwargs)
 
@@ -214,7 +215,7 @@ class Storage(Path):
     >>> rsam = c("rsam")
     """
 
-    def __init__(self, name, rootdir, starttime=None, endtime=None, create=True, backend='zarr'):
+    def __init__(self, name, rootdir, starttime=None, endtime=None, create=True, backend='netcdf'):
         self.stores = set()
         self.starttime = starttime
         self.endtime = endtime

{tonik-0.0.11 → tonik-0.1.0}/src/tonik/utils.py

@@ -56,3 +56,36 @@ def generate_test_data(dim=1, ndays=30, nfreqs=10,
     xds.attrs['station'] = 'MDR'
     xds.attrs['interval'] = '10min'
     return xds
+
+
+def merge_arrays(xds_old: xr.DataArray, xds_new: xr.DataArray,
+                 resolution: float = None) -> xr.DataArray:
+    """
+    Merge two xarray datasets with the same datetime index.
+
+    Parameters
+    ----------
+    xds_old : xr.DataArray
+        Old array.
+    xds_new : xr.DataArray
+        New array.
+    resolution : float
+        Time resolution in hours.
+
+    Returns
+    -------
+    xr.DataArray
+        Merged array.
+    """
+    xda_old = xds_old.drop_duplicates(
+        'datetime', keep='last')
+    xda_new = xds_new.drop_duplicates(
+        'datetime', keep='last')
+    xda_new = xda_new.combine_first(xda_old)
+    if resolution is not None:
+        new_dates = pd.date_range(
+            xda_new.datetime.values[0],
+            xda_new.datetime.values[-1],
+            freq=f'{resolution}h')
+        xda_new = xda_new.reindex(datetime=new_dates)
+    return xda_new

tonik-0.0.11/src/tonik/xarray2hdf5.py → tonik-0.1.0/src/tonik/xarray2netcdf.py

@@ -1,15 +1,18 @@
-from datetime import datetime
 import logging
 import os
+from datetime import datetime
 from warnings import filterwarnings
 
-from cftime import num2date, date2num
 import h5netcdf
 import numpy as np
+import xarray as xr
+from cftime import date2num, num2date
+
+from .utils import merge_arrays
 
 
-def xarray2hdf5(xArray, fdir, rootGroupName="original", timedim="datetime",
-                archive_starttime=datetime(2000, 1, 1), resolution=None):
+def xarray2netcdf(xArray, fdir, rootGroupName="original", timedim="datetime",
+                  archive_starttime=datetime(2000, 1, 1), resolution=None):
     """
     Store an xarray dataset as an HDF5 file.
 
@@ -31,23 +34,35 @@ def xarray2hdf5(xArray, fdir, rootGroupName="original", timedim="datetime",
         determined from the data.
     """
     filterwarnings(action='ignore', category=DeprecationWarning,
-               message='`np.bool` is a deprecated alias')
+                   message='`np.bool` is a deprecated alias')
 
-    starttime = xArray[timedim].values[0].astype('datetime64[us]').astype(datetime)
-    starttime = min(starttime, archive_starttime)
+    data_starttime = xArray[timedim].values[0].astype(
+        'datetime64[us]').astype(datetime)
+    starttime = min(data_starttime, archive_starttime)
     if resolution is None:
         resolution = (np.diff(xArray[timedim])/np.timedelta64(1, 'h'))[0]
 
     for featureName in list(xArray.data_vars.keys()):
-        h5file = os.path.join(fdir, featureName +'.nc')
+        h5file = os.path.join(fdir, featureName + '.nc')
+        mode = 'w'
+        if os.path.isfile(h5file):
+            if archive_starttime > data_starttime:
+                xds_existing = xr.open_dataset(
+                    h5file, group='original', engine='h5netcdf')
+                xda_new = merge_arrays(
+                    xds_existing[featureName], xArray[featureName],
+                    resolution=resolution)
+                xds_existing.close()
+                xda_new.to_netcdf(h5file, group='original',
+                                  mode='w', engine='h5netcdf')
+                continue
+            mode = 'a'
 
-        mode = 'a' if os.path.isfile(h5file) else 'w'
-        
         with h5netcdf.File(h5file, mode) as h5f:
             try:
                 rootGrp = _create_h5_Structure(rootGroupName, featureName,
                                                h5f, xArray, starttime, timedim)
-            except ValueError: # group already exists, append
+            except ValueError:  # group already exists, append
                 rootGrp = h5f[rootGroupName]
 
             # determine indices
@@ -75,7 +90,8 @@ def xarray2hdf5(xArray, fdir, rootGroupName="original", timedim="datetime",
             try:
                 _setMetaInfo(featureName, h5f, xArray)
             except KeyError as e:
-                logging.warning(f"Could not set all meta info for {featureName}: {e}")
+                logging.warning(
+                    f"Could not set all meta info for {featureName}: {e}")
 
 
 def _create_h5_Structure(defaultGroupName, featureName, h5f, xArray, starttime, timedim):
@@ -85,15 +101,16 @@ def _create_h5_Structure(defaultGroupName, featureName, h5f, xArray, starttime,
     coordinates.attrs['units'] = 'hours since 1970-01-01 00:00:00.0'
     coordinates.attrs['calendar'] = 'gregorian'
     rootGrp.attrs['starttime'] = str(starttime)
-    for label, size in xArray.dims.items(): 
+    for label, size in xArray.dims.items():
         if not np.issubdtype(xArray[label].dtype, np.datetime64):
-            rootGrp.dimensions[label] = size 
+            rootGrp.dimensions[label] = size
             coordinates = rootGrp.create_variable(label, (label,), float)
             coordinates[:] = xArray[label].values
     # Note: xArray.dims returns a dictionary of dimensions that are not necesarily
     # in the right order; xArray[featureName].dims returns a tuple with dimension
     # names in the correct order
-    rootGrp.create_variable(featureName, tuple(xArray[featureName].dims), dtype=float, fillvalue=0.)
+    rootGrp.create_variable(featureName, tuple(
+        xArray[featureName].dims), dtype=float, fillvalue=0.)
     return rootGrp
 
 
@@ -102,4 +119,3 @@ def _setMetaInfo(featureName, h5f, xArray):
     h5f.attrs['latitude'] = -42
     h5f.attrs['longitude'] = 168
     h5f.attrs['datatype'] = featureName
- 

{tonik-0.0.11 → tonik-0.1.0}/src/tonik/xarray2zarr.py

@@ -3,10 +3,28 @@ import os
 
 import xarray as xr
 
+from .utils import merge_arrays
+
 logger = logging.getLogger(__name__)
 
 
-def xarray2zarr(xds, path, mode='a'):
+def xarray2zarr(xds: xr.Dataset, path: str, mode: str = 'a'):
+    """
+    Write xarray dataset to zarr files.
+
+    Parameters
+    ----------
+    xds : xr.Dataset
+        Dataset to write.
+    path : str
+        Path to write the dataset.
+    mode : str, optional
+        Write mode, by default 'a'.
+
+    Returns
+    -------
+    None
+    """
     for feature in xds.data_vars.keys():
         fout = os.path.join(path, feature + '.zarr')
         if not os.path.exists(fout) or mode == 'w':
@@ -15,8 +33,8 @@ def xarray2zarr(xds, path, mode='a'):
         else:
             xds_existing = xr.open_zarr(fout, group='original')
             if xds_existing.datetime[0] > xds.datetime[0] or xds_existing.datetime[-1] > xds.datetime[-1]:
-                xds_new = xr.merge([xds_existing[feature], xds[feature]])
-                xds_new.to_zarr(fout, group='original', mode='w')
+                xda_new = merge_arrays(xds_existing[feature], xds[feature])
+                xda_new.to_zarr(fout, group='original', mode='w')
             else:
                 try:
                     overlap = xds_existing.datetime.where(
@@ -34,9 +52,5 @@ def xarray2zarr(xds, path, mode='a'):
                     msg += "Attempting to merge the two datasets."
                     logger.error(msg)
                     # remove duplicate datetime entries
-                    xda_existing = xds_existing[feature].drop_duplicates(
-                        'datetime', keep='last')
-                    xda_new = xds[feature].drop_duplicates(
-                        'datetime', keep='last')
-                    xda_new = xda_new.combine_first(xda_existing)
+                    xda_new = merge_arrays(xds_existing[feature], xds[feature])
                     xda_new.to_zarr(fout, group='original', mode='w')

{tonik-0.0.11 → tonik-0.1.0}/tests/backend_speed_test.py

@@ -29,7 +29,7 @@ def test_backend_speed():
     execution_time_zarr = timeit.timeit(lambda: write_read('zarr'), number=5)
     logger.info('Write and read with zarr took {} seconds.'.format(
         execution_time_zarr/5))
-    execution_time_h5 = timeit.timeit(lambda: write_read('h5netcdf'), number=5)
+    execution_time_h5 = timeit.timeit(lambda: write_read('netcdf'), number=5)
     logger.info('Write and read with h5 took {} seconds.'.format(
         execution_time_h5/5))
 

{tonik-0.0.11 → tonik-0.1.0}/tests/test_group.py

@@ -1,7 +1,5 @@
-from datetime import datetime
-import json
 import os
-import tempfile
+from datetime import datetime
 
 import numpy as np
 import pandas as pd
@@ -13,7 +11,7 @@ from tonik.utils import generate_test_data
 
 def test_group(tmp_path_factory):
     rootdir = tmp_path_factory.mktemp('data')
-    g = Storage('test_experiment', rootdir, backend='h5netcdf')
+    g = Storage('test_experiment', rootdir, backend='netcdf')
     c = g.get_substore('site1', 'sensor1', 'channel1')
     assert c.path == os.path.join(
         rootdir, 'test_experiment/site1/sensor1/channel1')
@@ -41,7 +39,7 @@ def test_non_existant_feature(tmp_path_factory):
 
 def test_from_directory(tmp_path_factory):
     rootdir = tmp_path_factory.mktemp('data')
-    g = Storage('test_experiment', rootdir, backend='h5netcdf')
+    g = Storage('test_experiment', rootdir, backend='netcdf')
     c = g.get_substore('site1', 'sensor1', 'channel1')
     assert c.path == os.path.join(rootdir, 'test_experiment', 'site1',
                                   'sensor1', 'channel1')

{tonik-0.0.11 → tonik-0.1.0}/tests/test_save.py

@@ -6,20 +6,20 @@ import pytest
 import xarray as xr
 
 from tonik import Storage, generate_test_data
-from tonik.xarray2hdf5 import xarray2hdf5
+from tonik.xarray2netcdf import xarray2netcdf
 
 
-def test_xarray2hdf5(tmp_path_factory):
+def test_xarray2netcdf(tmp_path_factory):
     """
     Test writing xarray data to hdf5.
     """
     xdf = generate_test_data(
         dim=2, ndays=3, tstart=datetime(2022, 7, 18, 0, 0, 0))
-    temp_dir = tmp_path_factory.mktemp('test_xarray2hdf5')
+    temp_dir = tmp_path_factory.mktemp('test_xarray2netcdf')
     g = Storage('test_experiment', rootdir=temp_dir,
                 starttime=datetime.fromisoformat(xdf.attrs['starttime']),
                 endtime=datetime.fromisoformat(xdf.attrs['endtime']),
-                backend='h5netcdf')
+                backend='netcdf')
     c = g.get_substore('MDR', '00', 'HHZ')
     c.save(xdf)
 
@@ -34,14 +34,14 @@ def test_xarray2hdf5(tmp_path_factory):
     assert dt < np.timedelta64(1, 'us')
 
 
-def test_xarray2hdf5_archive_starttime(tmp_path_factory):
+def test_xarray2netcdf_archive_starttime(tmp_path_factory):
     xdf = generate_test_data(
         dim=1, ndays=3, tstart=datetime(2022, 7, 18, 0, 0, 0))
-    temp_dir = tmp_path_factory.mktemp('test_xarray2hdf5')
+    temp_dir = tmp_path_factory.mktemp('test_xarray2netcdf')
     g = Storage('test_experiment', rootdir=temp_dir,
                 starttime=datetime(2000, 1, 1),
                 endtime=datetime.fromisoformat(xdf.attrs['endtime']),
-                backend='h5netcdf')
+                backend='netcdf')
     c = g.get_substore('MDR', '00', 'HHZ')
     c.save(xdf, archive_starttime=datetime(2022, 1, 1))
 
@@ -53,14 +53,32 @@ def test_xarray2hdf5_archive_starttime(tmp_path_factory):
     assert xdf_test.loc['2000-01-01':'2022-07-17T23:50:00'].shape[0] == nitems
 
 
-def test_xarray2hdf5_resolution(tmp_path_factory):
+def test_xarray2netcdf_merge_arrays(tmp_path_factory):
+    temp_dir = tmp_path_factory.mktemp('test_xarray2netcdf')
+    start = datetime(2022, 7, 18, 8, 0, 0)
+    end = datetime(2022, 7, 19, 12, 0, 0)
+    xdf1 = generate_test_data(dim=1, ndays=1, tstart=start, add_nans=False)
+    xdf2 = generate_test_data(dim=1, ndays=1, tstart=end, add_nans=False)
+    g = Storage('test_experiment', rootdir=temp_dir,
+                starttime=start, endtime=end + timedelta(days=1),
+                backend='netcdf')
+    c = g.get_substore('MDR', '00', 'HHZ')
+    c.save(xdf2, archive_starttime=datetime(2022, 8, 1))
+    c.save(xdf1, archive_starttime=datetime(2022, 8, 1))
+    xdf_test = c('rsam')
+    assert xdf_test.isnull().sum() == 24
+    assert xdf_test.loc['2022-07-18T08:00:00'] == xdf1['rsam'].loc['2022-07-18T08:00:00']
+    assert xdf_test.loc['2022-07-20T11:50:00'] == xdf2['rsam'].loc['2022-07-20T11:50:00']
+
+
+def test_xarray2netcdf_resolution(tmp_path_factory):
     xdf = generate_test_data(dim=1, ndays=1, tstart=datetime(2022, 7, 18, 0, 0, 0),
                              add_nans=False)
-    temp_dir = tmp_path_factory.mktemp('test_xarray2hdf5')
+    temp_dir = tmp_path_factory.mktemp('test_xarray2netcdf')
     g = Storage('test_experiment', rootdir=temp_dir,
                 starttime=datetime(2000, 1, 1),
                 endtime=datetime.fromisoformat(xdf.attrs['endtime']),
-                backend='h5netcdf')
+                backend='netcdf')
     c = g.get_substore('MDR', '00', 'HHZ')
     c.save(xdf, resolution=0.1, archive_starttime=datetime(2022, 7, 18))
 
@@ -69,18 +87,18 @@ def test_xarray2hdf5_resolution(tmp_path_factory):
     assert np.isnan(xdf_test.loc['2022-07-18T00:06:00'].data)
 
 
-def test_xarray2hdf5_with_gaps(tmp_path_factory):
+def test_xarray2netcdf_with_gaps(tmp_path_factory):
     """
     Test writing xarray data to hdf5 with gaps.
     """
-    temp_dir = tmp_path_factory.mktemp('test_xarray2hdf5')
+    temp_dir = tmp_path_factory.mktemp('test_xarray2netcdf')
     start = datetime(2022, 7, 18, 8, 0, 0)
     end = datetime(2022, 7, 19, 12, 0, 0)
     xdf1 = generate_test_data(dim=1, ndays=1, tstart=start, add_nans=False)
     xdf2 = generate_test_data(dim=1, ndays=1, tstart=end, add_nans=False)
     g = Storage('test_experiment', rootdir=temp_dir,
                 starttime=start, endtime=end + timedelta(days=1),
-                backend='h5netcdf')
+                backend='netcdf')
     c = g.get_substore('MDR', '00', 'HHZ')
     c.save(xdf1)
     c.save(xdf2)
@@ -89,23 +107,23 @@ def test_xarray2hdf5_with_gaps(tmp_path_factory):
 
 
 @pytest.mark.xfail(raises=OSError)
-def test_xarray2hdf5_multi_access(tmp_path_factory):
+def test_xarray2netcdf_multi_access(tmp_path_factory):
     """
     Test writing xarray data to hdf5 while the file is open. This is currently
     not working with NetCDF4. See the following discussions for reference:
     https://github.com/pydata/xarray/issues/2887
     https://stackoverflow.com/questions/49701623/is-there-a-way-to-release-the-file-lock-for-a-xarray-dataset
     """
-    temp_dir = tmp_path_factory.mktemp('test_xarray2hdf5')
+    temp_dir = tmp_path_factory.mktemp('test_xarray2netcdf')
     xdf1 = generate_test_data(
         dim=1, ndays=1, tstart=datetime(2022, 7, 18, 8, 0, 0))
     xdf2 = generate_test_data(
         dim=1, ndays=1, tstart=datetime(2022, 7, 19, 12, 0, 0))
 
-    xarray2hdf5(xdf1, temp_dir)
+    xarray2netcdf(xdf1, temp_dir)
     xdf_dummy = xr.open_dataset(os.path.join(temp_dir, 'rsam.nc'),
                                 group='original', engine='h5netcdf')
-    xarray2hdf5(xdf2, temp_dir)
+    xarray2netcdf(xdf2, temp_dir)
 
 
 def test_xarray2zarr(tmp_path_factory):