tonik 0.0.11__py3-none-any.whl → 0.1.0__py3-none-any.whl

tonik/storage.py

@@ -6,7 +6,7 @@ import re
 import pandas as pd
 import xarray as xr
 
-from .xarray2hdf5 import xarray2hdf5
+from .xarray2netcdf import xarray2netcdf
 from .xarray2zarr import xarray2zarr
 
 LOGGING_CONFIG = {
@@ -102,7 +102,7 @@ class Path(object):
 
     def feature_path(self, feature):
 
-        if self.backend == 'h5netcdf':
+        if self.backend == 'netcdf':
             file_ending = '.nc'
         elif self.backend == 'zarr':
             file_ending = '.zarr'
@@ -154,7 +154,8 @@ class Path(object):
                     format(stack_length, interval, num_periods))
 
         xd_index = dict(datetime=slice(self.starttime, self.endtime))
-        with xr.open_dataset(filename, group='original', engine=self.backend) as ds:
+        engine = 'h5netcdf' if self.backend == 'netcdf' else self.backend
+        with xr.open_dataset(filename, group='original', engine=engine) as ds:
             rq = ds.loc[xd_index].load()
 
         # Stack features
@@ -187,8 +188,8 @@ class Path(object):
         """
         Save a feature to disk
         """
-        if self.backend == 'h5netcdf':
-            xarray2hdf5(data, self.path, **kwargs)
+        if self.backend == 'netcdf':
+            xarray2netcdf(data, self.path, **kwargs)
         elif self.backend == 'zarr':
             xarray2zarr(data, self.path, **kwargs)
 
@@ -214,7 +215,7 @@ class Storage(Path):
     >>> rsam = c("rsam")
     """
 
-    def __init__(self, name, rootdir, starttime=None, endtime=None, create=True, backend='zarr'):
+    def __init__(self, name, rootdir, starttime=None, endtime=None, create=True, backend='netcdf'):
         self.stores = set()
         self.starttime = starttime
         self.endtime = endtime

tonik/utils.py

@@ -56,3 +56,36 @@ def generate_test_data(dim=1, ndays=30, nfreqs=10,
     xds.attrs['station'] = 'MDR'
     xds.attrs['interval'] = '10min'
     return xds
+
+
+def merge_arrays(xds_old: xr.DataArray, xds_new: xr.DataArray,
+                 resolution: float = None) -> xr.DataArray:
+    """
+    Merge two xarray datasets with the same datetime index.
+
+    Parameters
+    ----------
+    xds_old : xr.DataArray
+        Old array.
+    xds_new : xr.DataArray
+        New array.
+    resolution : float
+        Time resolution in hours.
+
+    Returns
+    -------
+    xr.DataArray
+        Merged array.
+    """
+    xda_old = xds_old.drop_duplicates(
+        'datetime', keep='last')
+    xda_new = xds_new.drop_duplicates(
+        'datetime', keep='last')
+    xda_new = xda_new.combine_first(xda_old)
+    if resolution is not None:
+        new_dates = pd.date_range(
+            xda_new.datetime.values[0],
+            xda_new.datetime.values[-1],
+            freq=f'{resolution}h')
+        xda_new = xda_new.reindex(datetime=new_dates)
+    return xda_new

tonik/{xarray2hdf5.py → xarray2netcdf.py}

@@ -1,15 +1,18 @@
-from datetime import datetime
 import logging
 import os
+from datetime import datetime
 from warnings import filterwarnings
 
-from cftime import num2date, date2num
 import h5netcdf
 import numpy as np
+import xarray as xr
+from cftime import date2num, num2date
+
+from .utils import merge_arrays
 
 
-def xarray2hdf5(xArray, fdir, rootGroupName="original", timedim="datetime",
-                archive_starttime=datetime(2000, 1, 1), resolution=None):
+def xarray2netcdf(xArray, fdir, rootGroupName="original", timedim="datetime",
+                  archive_starttime=datetime(2000, 1, 1), resolution=None):
     """
     Store an xarray dataset as an HDF5 file.
 
@@ -31,23 +34,35 @@ def xarray2hdf5(xArray, fdir, rootGroupName="original", timedim="datetime",
         determined from the data.
     """
     filterwarnings(action='ignore', category=DeprecationWarning,
-               message='`np.bool` is a deprecated alias')
+                   message='`np.bool` is a deprecated alias')
 
-    starttime = xArray[timedim].values[0].astype('datetime64[us]').astype(datetime)
-    starttime = min(starttime, archive_starttime)
+    data_starttime = xArray[timedim].values[0].astype(
+        'datetime64[us]').astype(datetime)
+    starttime = min(data_starttime, archive_starttime)
     if resolution is None:
         resolution = (np.diff(xArray[timedim])/np.timedelta64(1, 'h'))[0]
 
     for featureName in list(xArray.data_vars.keys()):
-        h5file = os.path.join(fdir, featureName +'.nc')
+        h5file = os.path.join(fdir, featureName + '.nc')
+        mode = 'w'
+        if os.path.isfile(h5file):
+            if archive_starttime > data_starttime:
+                xds_existing = xr.open_dataset(
+                    h5file, group='original', engine='h5netcdf')
+                xda_new = merge_arrays(
+                    xds_existing[featureName], xArray[featureName],
+                    resolution=resolution)
+                xds_existing.close()
+                xda_new.to_netcdf(h5file, group='original',
+                                  mode='w', engine='h5netcdf')
+                continue
+            mode = 'a'
 
-        mode = 'a' if os.path.isfile(h5file) else 'w'
-        
         with h5netcdf.File(h5file, mode) as h5f:
             try:
                 rootGrp = _create_h5_Structure(rootGroupName, featureName,
                                                h5f, xArray, starttime, timedim)
-            except ValueError: # group already exists, append
+            except ValueError:  # group already exists, append
                 rootGrp = h5f[rootGroupName]
 
             # determine indices
@@ -75,7 +90,8 @@ def xarray2hdf5(xArray, fdir, rootGroupName="original", timedim="datetime",
             try:
                 _setMetaInfo(featureName, h5f, xArray)
             except KeyError as e:
-                logging.warning(f"Could not set all meta info for {featureName}: {e}")
+                logging.warning(
+                    f"Could not set all meta info for {featureName}: {e}")
 
 
 def _create_h5_Structure(defaultGroupName, featureName, h5f, xArray, starttime, timedim):
@@ -85,15 +101,16 @@ def _create_h5_Structure(defaultGroupName, featureName, h5f, xArray, starttime,
     coordinates.attrs['units'] = 'hours since 1970-01-01 00:00:00.0'
     coordinates.attrs['calendar'] = 'gregorian'
     rootGrp.attrs['starttime'] = str(starttime)
-    for label, size in xArray.dims.items(): 
+    for label, size in xArray.dims.items():
         if not np.issubdtype(xArray[label].dtype, np.datetime64):
-            rootGrp.dimensions[label] = size 
+            rootGrp.dimensions[label] = size
             coordinates = rootGrp.create_variable(label, (label,), float)
             coordinates[:] = xArray[label].values
     # Note: xArray.dims returns a dictionary of dimensions that are not necesarily
     # in the right order; xArray[featureName].dims returns a tuple with dimension
     # names in the correct order
-    rootGrp.create_variable(featureName, tuple(xArray[featureName].dims), dtype=float, fillvalue=0.)
+    rootGrp.create_variable(featureName, tuple(
+        xArray[featureName].dims), dtype=float, fillvalue=0.)
     return rootGrp
 
 
@@ -102,4 +119,3 @@ def _setMetaInfo(featureName, h5f, xArray):
     h5f.attrs['latitude'] = -42
     h5f.attrs['longitude'] = 168
     h5f.attrs['datatype'] = featureName
- 

tonik/xarray2zarr.py

@@ -3,10 +3,28 @@ import os
 
 import xarray as xr
 
+from .utils import merge_arrays
+
 logger = logging.getLogger(__name__)
 
 
-def xarray2zarr(xds, path, mode='a'):
+def xarray2zarr(xds: xr.Dataset, path: str, mode: str = 'a'):
+    """
+    Write xarray dataset to zarr files.
+
+    Parameters
+    ----------
+    xds : xr.Dataset
+        Dataset to write.
+    path : str
+        Path to write the dataset.
+    mode : str, optional
+        Write mode, by default 'a'.
+
+    Returns
+    -------
+    None
+    """
     for feature in xds.data_vars.keys():
         fout = os.path.join(path, feature + '.zarr')
         if not os.path.exists(fout) or mode == 'w':
@@ -15,8 +33,8 @@ def xarray2zarr(xds, path, mode='a'):
         else:
             xds_existing = xr.open_zarr(fout, group='original')
             if xds_existing.datetime[0] > xds.datetime[0] or xds_existing.datetime[-1] > xds.datetime[-1]:
-                xds_new = xr.merge([xds_existing[feature], xds[feature]])
-                xds_new.to_zarr(fout, group='original', mode='w')
+                xda_new = merge_arrays(xds_existing[feature], xds[feature])
+                xda_new.to_zarr(fout, group='original', mode='w')
             else:
                 try:
                     overlap = xds_existing.datetime.where(
@@ -34,9 +52,5 @@ def xarray2zarr(xds, path, mode='a'):
                     msg += "Attempting to merge the two datasets."
                     logger.error(msg)
                     # remove duplicate datetime entries
-                    xda_existing = xds_existing[feature].drop_duplicates(
-                        'datetime', keep='last')
-                    xda_new = xds[feature].drop_duplicates(
-                        'datetime', keep='last')
-                    xda_new = xda_new.combine_first(xda_existing)
+                    xda_new = merge_arrays(xds_existing[feature], xds[feature])
                     xda_new.to_zarr(fout, group='original', mode='w')

{tonik-0.0.11.dist-info → tonik-0.1.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: tonik
-Version: 0.0.11
+Version: 0.1.0
 Summary: Store time series data as HDF5 files and access them through an API.
 Project-URL: Homepage, https://tsc-tools.github.io/tonik
 Project-URL: Issues, https://github.com/tsc-tools/tonik/issues

tonik-0.1.0.dist-info/RECORD

@@ -0,0 +1,12 @@
+tonik/__init__.py,sha256=ZBVGh4dm_l9xwiBGb33O5QV9MfZeNiEd3DBDAm6DiHk,511
+tonik/api.py,sha256=8YS0WCMlm5xs2N6V8n9BiP87ywJC_BUzYsPvWR9KWJk,6499
+tonik/storage.py,sha256=Oh3BBQL13yLYHlgk8-a-wstmlUC2vNJ0yi_fnyETK_g,11237
+tonik/utils.py,sha256=3nSRU_GnV6arP4e63YHn4oEV8XbqzVAW8FCvQVIwGdg,2757
+tonik/xarray2netcdf.py,sha256=ey7lY4czlMREUt56SzE3quXNzeqXxV0Ru_EatsJpynA,5088
+tonik/xarray2zarr.py,sha256=jTrHFpjN3lEDIJEI6RyPvvbW2jZsfYwGE_LQaZ6dwx8,2099
+tonik/package_data/index.html,sha256=GKDClUhIam_fAYbNfzAolORhSCG3ae1wW3VjWCg4PMk,2732
+tonik-0.1.0.dist-info/METADATA,sha256=VtdNFXyUplO5s_TmR8xEgW0NdMtKgI4Zl2rWLhxpnK0,1938
+tonik-0.1.0.dist-info/WHEEL,sha256=1yFddiXMmvYK7QYTqtRNtX66WJ0Mz8PYEiEUoOUUxRY,87
+tonik-0.1.0.dist-info/entry_points.txt,sha256=VnGfC5qAzpntEHAb5pooUEpYABSgOfQoNhCEtLDJyf8,45
+tonik-0.1.0.dist-info/licenses/LICENSE,sha256=OXLcl0T2SZ8Pmy2_dmlvKuetivmyPd5m1q-Gyd-zaYY,35149
+tonik-0.1.0.dist-info/RECORD,,

tonik-0.0.11.dist-info/RECORD

@@ -1,12 +0,0 @@
-tonik/__init__.py,sha256=ZBVGh4dm_l9xwiBGb33O5QV9MfZeNiEd3DBDAm6DiHk,511
-tonik/api.py,sha256=8YS0WCMlm5xs2N6V8n9BiP87ywJC_BUzYsPvWR9KWJk,6499
-tonik/storage.py,sha256=Ts6jKLqwcDQvPIea-swBCpnEjQr_xnAjOl-hkUelTn4,11165
-tonik/utils.py,sha256=_TxXf9o9fOvtuOvGO6-ww9F5m0QelHyfQzQw8RGjTV4,1868
-tonik/xarray2hdf5.py,sha256=cekO9vo9ZRlr0VndswJjPC27CEVD3TpRVKLAJ-aAO0g,4465
-tonik/xarray2zarr.py,sha256=0bWMP_PZ0FLr9AlxKrC5M4aJRh_VGkUfJ-A-BGgKqM8,1979
-tonik/package_data/index.html,sha256=GKDClUhIam_fAYbNfzAolORhSCG3ae1wW3VjWCg4PMk,2732
-tonik-0.0.11.dist-info/METADATA,sha256=cXj6-wI7M6kWVvcffHe3W9BJQVdJfkNigzcrd0s5XlQ,1939
-tonik-0.0.11.dist-info/WHEEL,sha256=1yFddiXMmvYK7QYTqtRNtX66WJ0Mz8PYEiEUoOUUxRY,87
-tonik-0.0.11.dist-info/entry_points.txt,sha256=VnGfC5qAzpntEHAb5pooUEpYABSgOfQoNhCEtLDJyf8,45
-tonik-0.0.11.dist-info/licenses/LICENSE,sha256=OXLcl0T2SZ8Pmy2_dmlvKuetivmyPd5m1q-Gyd-zaYY,35149
-tonik-0.0.11.dist-info/RECORD,,