tonik 0.0.12__tar.gz → 0.1.1__tar.gz

{tonik-0.0.12 → tonik-0.1.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: tonik
-Version: 0.0.12
+Version: 0.1.1
 Summary: Store time series data as HDF5 files and access them through an API.
 Project-URL: Homepage, https://tsc-tools.github.io/tonik
 Project-URL: Issues, https://github.com/tsc-tools/tonik/issues

{tonik-0.0.12 → tonik-0.1.1}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "tonik"
-version = "0.0.12"
+version = "0.1.1"
 authors = [
   { name="Yannik Behr", email="y.behr@gns.cri.nz" },
   { name="Christof Mueller", email="c.mueller@gns.cri.nz" }

{tonik-0.0.12 → tonik-0.1.1}/src/tonik/storage.py

@@ -6,7 +6,7 @@ import re
 import pandas as pd
 import xarray as xr
 
-from .xarray2hdf5 import xarray2hdf5
+from .xarray2netcdf import xarray2netcdf
 from .xarray2zarr import xarray2zarr
 
 LOGGING_CONFIG = {
@@ -76,6 +76,7 @@ class Path(object):
         self.name = name
         self.create = create
         self.backend = backend
+        self.engine = 'h5netcdf' if self.backend == 'netcdf' else self.backend
         self.path = os.path.join(parentdir, name)
         if create:
             try:
@@ -102,7 +103,7 @@ class Path(object):
 
     def feature_path(self, feature):
 
-        if self.backend == 'h5netcdf':
+        if self.backend == 'netcdf':
             file_ending = '.nc'
         elif self.backend == 'zarr':
             file_ending = '.zarr'
@@ -154,7 +155,7 @@ class Path(object):
                     format(stack_length, interval, num_periods))
 
         xd_index = dict(datetime=slice(self.starttime, self.endtime))
-        with xr.open_dataset(filename, group='original', engine=self.backend) as ds:
+        with xr.open_dataset(filename, group='original', engine=self.engine) as ds:
             rq = ds.loc[xd_index].load()
 
         # Stack features
@@ -187,11 +188,19 @@ class Path(object):
         """
         Save a feature to disk
         """
-        if self.backend == 'h5netcdf':
-            xarray2hdf5(data, self.path, **kwargs)
+        if self.backend == 'netcdf':
+            xarray2netcdf(data, self.path, **kwargs)
         elif self.backend == 'zarr':
             xarray2zarr(data, self.path, **kwargs)
 
+    def shape(self, feature):
+        """
+        Get shape of a feature on disk
+        """
+        filename = self.feature_path(feature)
+        with xr.open_dataset(filename, group='original', engine=self.engine) as ds:
+            return ds.sizes
+
 
 class Storage(Path):
     """
@@ -214,7 +223,7 @@ class Storage(Path):
     >>> rsam = c("rsam")
     """
 
-    def __init__(self, name, rootdir, starttime=None, endtime=None, create=True, backend='zarr'):
+    def __init__(self, name, rootdir, starttime=None, endtime=None, create=True, backend='netcdf'):
         self.stores = set()
         self.starttime = starttime
         self.endtime = endtime

{tonik-0.0.12 → tonik-0.1.1}/src/tonik/utils.py

@@ -56,3 +56,36 @@ def generate_test_data(dim=1, ndays=30, nfreqs=10,
     xds.attrs['station'] = 'MDR'
     xds.attrs['interval'] = '10min'
     return xds
+
+
+def merge_arrays(xds_old: xr.DataArray, xds_new: xr.DataArray,
+                 resolution: float = None) -> xr.DataArray:
+    """
+    Merge two xarray datasets with the same datetime index.
+
+    Parameters
+    ----------
+    xds_old : xr.DataArray
+        Old array.
+    xds_new : xr.DataArray
+        New array.
+    resolution : float
+        Time resolution in hours.
+
+    Returns
+    -------
+    xr.DataArray
+        Merged array.
+    """
+    xda_old = xds_old.drop_duplicates(
+        'datetime', keep='last')
+    xda_new = xds_new.drop_duplicates(
+        'datetime', keep='last')
+    xda_new = xda_new.combine_first(xda_old)
+    if resolution is not None:
+        new_dates = pd.date_range(
+            xda_new.datetime.values[0],
+            xda_new.datetime.values[-1],
+            freq=f'{resolution}h')
+        xda_new = xda_new.reindex(datetime=new_dates)
+    return xda_new

tonik-0.0.12/src/tonik/xarray2hdf5.py → tonik-0.1.1/src/tonik/xarray2netcdf.py

@@ -1,15 +1,19 @@
-from datetime import datetime
 import logging
 import os
+from datetime import datetime
 from warnings import filterwarnings
 
-from cftime import num2date, date2num
 import h5netcdf
 import numpy as np
+import xarray as xr
+from cftime import date2num, num2date
+
+from .utils import merge_arrays
 
 
-def xarray2hdf5(xArray, fdir, rootGroupName="original", timedim="datetime",
-                archive_starttime=datetime(2000, 1, 1), resolution=None):
+def xarray2netcdf(xArray, fdir, rootGroupName="original", timedim="datetime",
+                  archive_starttime=datetime(2000, 1, 1), resolution=None,
+                  mode='a'):
     """
     Store an xarray dataset as an HDF5 file.
 
@@ -31,23 +35,35 @@ def xarray2hdf5(xArray, fdir, rootGroupName="original", timedim="datetime",
         determined from the data.
     """
     filterwarnings(action='ignore', category=DeprecationWarning,
-               message='`np.bool` is a deprecated alias')
+                   message='`np.bool` is a deprecated alias')
 
-    starttime = xArray[timedim].values[0].astype('datetime64[us]').astype(datetime)
-    starttime = min(starttime, archive_starttime)
+    data_starttime = xArray[timedim].values[0].astype(
+        'datetime64[us]').astype(datetime)
+    starttime = min(data_starttime, archive_starttime)
     if resolution is None:
         resolution = (np.diff(xArray[timedim])/np.timedelta64(1, 'h'))[0]
 
     for featureName in list(xArray.data_vars.keys()):
-        h5file = os.path.join(fdir, featureName +'.nc')
+        h5file = os.path.join(fdir, featureName + '.nc')
+        _mode = 'w'
+        if os.path.isfile(h5file) and mode == 'a':
+            if archive_starttime > data_starttime:
+                xds_existing = xr.open_dataset(
+                    h5file, group='original', engine='h5netcdf')
+                xda_new = merge_arrays(
+                    xds_existing[featureName], xArray[featureName],
+                    resolution=resolution)
+                xds_existing.close()
+                xda_new.to_netcdf(h5file, group='original',
+                                  mode='w', engine='h5netcdf')
+                continue
+            _mode = 'a'
 
-        mode = 'a' if os.path.isfile(h5file) else 'w'
-        
-        with h5netcdf.File(h5file, mode) as h5f:
+        with h5netcdf.File(h5file, _mode) as h5f:
             try:
                 rootGrp = _create_h5_Structure(rootGroupName, featureName,
                                                h5f, xArray, starttime, timedim)
-            except ValueError: # group already exists, append
+            except ValueError:  # group already exists, append
                 rootGrp = h5f[rootGroupName]
 
             # determine indices
@@ -75,7 +91,8 @@ def xarray2hdf5(xArray, fdir, rootGroupName="original", timedim="datetime",
             try:
                 _setMetaInfo(featureName, h5f, xArray)
             except KeyError as e:
-                logging.warning(f"Could not set all meta info for {featureName}: {e}")
+                logging.warning(
+                    f"Could not set all meta info for {featureName}: {e}")
 
 
 def _create_h5_Structure(defaultGroupName, featureName, h5f, xArray, starttime, timedim):
@@ -85,15 +102,16 @@ def _create_h5_Structure(defaultGroupName, featureName, h5f, xArray, starttime,
     coordinates.attrs['units'] = 'hours since 1970-01-01 00:00:00.0'
     coordinates.attrs['calendar'] = 'gregorian'
     rootGrp.attrs['starttime'] = str(starttime)
-    for label, size in xArray.dims.items(): 
+    for label, size in xArray.dims.items():
         if not np.issubdtype(xArray[label].dtype, np.datetime64):
-            rootGrp.dimensions[label] = size 
+            rootGrp.dimensions[label] = size
             coordinates = rootGrp.create_variable(label, (label,), float)
             coordinates[:] = xArray[label].values
     # Note: xArray.dims returns a dictionary of dimensions that are not necesarily
     # in the right order; xArray[featureName].dims returns a tuple with dimension
     # names in the correct order
-    rootGrp.create_variable(featureName, tuple(xArray[featureName].dims), dtype=float, fillvalue=0.)
+    rootGrp.create_variable(featureName, tuple(
+        xArray[featureName].dims), dtype=float, fillvalue=0.)
     return rootGrp
 
 
@@ -102,4 +120,3 @@ def _setMetaInfo(featureName, h5f, xArray):
     h5f.attrs['latitude'] = -42
     h5f.attrs['longitude'] = 168
     h5f.attrs['datatype'] = featureName
- 

{tonik-0.0.12 → tonik-0.1.1}/src/tonik/xarray2zarr.py

@@ -3,31 +3,9 @@ import os
 
 import xarray as xr
 
-logger = logging.getLogger(__name__)
-
-
-def merge_arrays(xds_old: xr.DataArray, xds_new: xr.DataArray) -> xr.DataArray:
-    """
-    Merge two xarray datasets with the same datetime index.
+from .utils import merge_arrays
 
-    Parameters
-    ----------
-    xds_old : xr.DataArray
-        Old array.
-    xds_new : xr.DataArray
-        New array.
-
-    Returns
-    -------
-    xr.DataArray
-        Merged array.
-    """
-    xda_old = xds_old.drop_duplicates(
-        'datetime', keep='last')
-    xda_new = xds_new.drop_duplicates(
-        'datetime', keep='last')
-    xda_new = xda_new.combine_first(xda_old)
-    return xda_new
+logger = logging.getLogger(__name__)
 
 
 def xarray2zarr(xds: xr.Dataset, path: str, mode: str = 'a'):

{tonik-0.0.12 → tonik-0.1.1}/tests/backend_speed_test.py

@@ -29,7 +29,7 @@ def test_backend_speed():
     execution_time_zarr = timeit.timeit(lambda: write_read('zarr'), number=5)
     logger.info('Write and read with zarr took {} seconds.'.format(
         execution_time_zarr/5))
-    execution_time_h5 = timeit.timeit(lambda: write_read('h5netcdf'), number=5)
+    execution_time_h5 = timeit.timeit(lambda: write_read('netcdf'), number=5)
     logger.info('Write and read with h5 took {} seconds.'.format(
         execution_time_h5/5))
 

{tonik-0.0.12 → tonik-0.1.1}/tests/test_group.py

@@ -1,7 +1,5 @@
-from datetime import datetime
-import json
 import os
-import tempfile
+from datetime import datetime
 
 import numpy as np
 import pandas as pd
@@ -13,7 +11,7 @@ from tonik.utils import generate_test_data
 
 def test_group(tmp_path_factory):
     rootdir = tmp_path_factory.mktemp('data')
-    g = Storage('test_experiment', rootdir, backend='h5netcdf')
+    g = Storage('test_experiment', rootdir, backend='netcdf')
     c = g.get_substore('site1', 'sensor1', 'channel1')
     assert c.path == os.path.join(
         rootdir, 'test_experiment/site1/sensor1/channel1')
@@ -41,7 +39,7 @@ def test_non_existant_feature(tmp_path_factory):
 
 def test_from_directory(tmp_path_factory):
     rootdir = tmp_path_factory.mktemp('data')
-    g = Storage('test_experiment', rootdir, backend='h5netcdf')
+    g = Storage('test_experiment', rootdir, backend='netcdf')
     c = g.get_substore('site1', 'sensor1', 'channel1')
     assert c.path == os.path.join(rootdir, 'test_experiment', 'site1',
                                   'sensor1', 'channel1')
@@ -138,3 +136,21 @@ def test_rolling_window(tmp_path_factory):
     np.testing.assert_array_almost_equal(
         np.array(rolling_mean), rsam_rolling.values[num_windows:], 6
     )
+
+
+def test_shape(tmp_path_factory):
+    rootdir = tmp_path_factory.mktemp('data')
+    g = Storage('volcanoes', rootdir=rootdir)
+    tstart = datetime(2016, 1, 1)
+    xdf = generate_test_data(dim=2, intervals=20, tstart=tstart)
+    g.save(xdf, mode='w', archive_starttime=tstart)
+    rsam_shape = g.shape('ssam')
+    assert rsam_shape['datetime'] == 20
+    assert rsam_shape['frequency'] == 10
+    assert rsam_shape['fbfrequency'] == 10
+
+    g1 = Storage('volcanoes', rootdir=rootdir, backend='zarr')
+    g1.save(xdf, mode='w')
+    rsam_shape = g1.shape('ssam')
+    assert rsam_shape['datetime'] == 20
+    assert rsam_shape['frequency'] == 10

{tonik-0.0.12 → tonik-0.1.1}/tests/test_save.py

@@ -6,20 +6,20 @@ import pytest
 import xarray as xr
 
 from tonik import Storage, generate_test_data
-from tonik.xarray2hdf5 import xarray2hdf5
+from tonik.xarray2netcdf import xarray2netcdf
 
 
-def test_xarray2hdf5(tmp_path_factory):
+def test_xarray2netcdf(tmp_path_factory):
     """
     Test writing xarray data to hdf5.
     """
     xdf = generate_test_data(
         dim=2, ndays=3, tstart=datetime(2022, 7, 18, 0, 0, 0))
-    temp_dir = tmp_path_factory.mktemp('test_xarray2hdf5')
+    temp_dir = tmp_path_factory.mktemp('test_xarray2netcdf')
     g = Storage('test_experiment', rootdir=temp_dir,
                 starttime=datetime.fromisoformat(xdf.attrs['starttime']),
                 endtime=datetime.fromisoformat(xdf.attrs['endtime']),
-                backend='h5netcdf')
+                backend='netcdf')
     c = g.get_substore('MDR', '00', 'HHZ')
     c.save(xdf)
 
@@ -34,14 +34,14 @@ def test_xarray2hdf5(tmp_path_factory):
     assert dt < np.timedelta64(1, 'us')
 
 
-def test_xarray2hdf5_archive_starttime(tmp_path_factory):
+def test_xarray2netcdf_archive_starttime(tmp_path_factory):
     xdf = generate_test_data(
         dim=1, ndays=3, tstart=datetime(2022, 7, 18, 0, 0, 0))
-    temp_dir = tmp_path_factory.mktemp('test_xarray2hdf5')
+    temp_dir = tmp_path_factory.mktemp('test_xarray2netcdf')
     g = Storage('test_experiment', rootdir=temp_dir,
                 starttime=datetime(2000, 1, 1),
                 endtime=datetime.fromisoformat(xdf.attrs['endtime']),
-                backend='h5netcdf')
+                backend='netcdf')
     c = g.get_substore('MDR', '00', 'HHZ')
     c.save(xdf, archive_starttime=datetime(2022, 1, 1))
 
@@ -53,14 +53,32 @@ def test_xarray2hdf5_archive_starttime(tmp_path_factory):
     assert xdf_test.loc['2000-01-01':'2022-07-17T23:50:00'].shape[0] == nitems
 
 
-def test_xarray2hdf5_resolution(tmp_path_factory):
+def test_xarray2netcdf_merge_arrays(tmp_path_factory):
+    temp_dir = tmp_path_factory.mktemp('test_xarray2netcdf')
+    start = datetime(2022, 7, 18, 8, 0, 0)
+    end = datetime(2022, 7, 19, 12, 0, 0)
+    xdf1 = generate_test_data(dim=1, ndays=1, tstart=start, add_nans=False)
+    xdf2 = generate_test_data(dim=1, ndays=1, tstart=end, add_nans=False)
+    g = Storage('test_experiment', rootdir=temp_dir,
+                starttime=start, endtime=end + timedelta(days=1),
+                backend='netcdf')
+    c = g.get_substore('MDR', '00', 'HHZ')
+    c.save(xdf2, archive_starttime=datetime(2022, 8, 1))
+    c.save(xdf1, archive_starttime=datetime(2022, 8, 1))
+    xdf_test = c('rsam')
+    assert xdf_test.isnull().sum() == 24
+    assert xdf_test.loc['2022-07-18T08:00:00'] == xdf1['rsam'].loc['2022-07-18T08:00:00']
+    assert xdf_test.loc['2022-07-20T11:50:00'] == xdf2['rsam'].loc['2022-07-20T11:50:00']
+
+
+def test_xarray2netcdf_resolution(tmp_path_factory):
     xdf = generate_test_data(dim=1, ndays=1, tstart=datetime(2022, 7, 18, 0, 0, 0),
                              add_nans=False)
-    temp_dir = tmp_path_factory.mktemp('test_xarray2hdf5')
+    temp_dir = tmp_path_factory.mktemp('test_xarray2netcdf')
     g = Storage('test_experiment', rootdir=temp_dir,
                 starttime=datetime(2000, 1, 1),
                 endtime=datetime.fromisoformat(xdf.attrs['endtime']),
-                backend='h5netcdf')
+                backend='netcdf')
     c = g.get_substore('MDR', '00', 'HHZ')
     c.save(xdf, resolution=0.1, archive_starttime=datetime(2022, 7, 18))
 
@@ -69,18 +87,18 @@ def test_xarray2hdf5_resolution(tmp_path_factory):
     assert np.isnan(xdf_test.loc['2022-07-18T00:06:00'].data)
 
 
-def test_xarray2hdf5_with_gaps(tmp_path_factory):
+def test_xarray2netcdf_with_gaps(tmp_path_factory):
     """
     Test writing xarray data to hdf5 with gaps.
     """
-    temp_dir = tmp_path_factory.mktemp('test_xarray2hdf5')
+    temp_dir = tmp_path_factory.mktemp('test_xarray2netcdf')
     start = datetime(2022, 7, 18, 8, 0, 0)
     end = datetime(2022, 7, 19, 12, 0, 0)
     xdf1 = generate_test_data(dim=1, ndays=1, tstart=start, add_nans=False)
     xdf2 = generate_test_data(dim=1, ndays=1, tstart=end, add_nans=False)
     g = Storage('test_experiment', rootdir=temp_dir,
                 starttime=start, endtime=end + timedelta(days=1),
-                backend='h5netcdf')
+                backend='netcdf')
     c = g.get_substore('MDR', '00', 'HHZ')
     c.save(xdf1)
     c.save(xdf2)
@@ -89,23 +107,23 @@ def test_xarray2hdf5_with_gaps(tmp_path_factory):
 
 
 @pytest.mark.xfail(raises=OSError)
-def test_xarray2hdf5_multi_access(tmp_path_factory):
+def test_xarray2netcdf_multi_access(tmp_path_factory):
     """
     Test writing xarray data to hdf5 while the file is open. This is currently
     not working with NetCDF4. See the following discussions for reference:
     https://github.com/pydata/xarray/issues/2887
     https://stackoverflow.com/questions/49701623/is-there-a-way-to-release-the-file-lock-for-a-xarray-dataset
     """
-    temp_dir = tmp_path_factory.mktemp('test_xarray2hdf5')
+    temp_dir = tmp_path_factory.mktemp('test_xarray2netcdf')
     xdf1 = generate_test_data(
         dim=1, ndays=1, tstart=datetime(2022, 7, 18, 8, 0, 0))
     xdf2 = generate_test_data(
         dim=1, ndays=1, tstart=datetime(2022, 7, 19, 12, 0, 0))
 
-    xarray2hdf5(xdf1, temp_dir)
+    xarray2netcdf(xdf1, temp_dir)
     xdf_dummy = xr.open_dataset(os.path.join(temp_dir, 'rsam.nc'),
                                 group='original', engine='h5netcdf')
-    xarray2hdf5(xdf2, temp_dir)
+    xarray2netcdf(xdf2, temp_dir)
 
 
 def test_xarray2zarr(tmp_path_factory):