tonik 0.0.12__py3-none-any.whl → 0.1.1__py3-none-any.whl

tonik/storage.py

@@ -6,7 +6,7 @@ import re
 import pandas as pd
 import xarray as xr
 
-from .xarray2hdf5 import xarray2hdf5
+from .xarray2netcdf import xarray2netcdf
 from .xarray2zarr import xarray2zarr
 
 LOGGING_CONFIG = {
@@ -76,6 +76,7 @@ class Path(object):
         self.name = name
         self.create = create
         self.backend = backend
+        self.engine = 'h5netcdf' if self.backend == 'netcdf' else self.backend
         self.path = os.path.join(parentdir, name)
         if create:
             try:
@@ -102,7 +103,7 @@ class Path(object):
 
     def feature_path(self, feature):
 
-        if self.backend == 'h5netcdf':
+        if self.backend == 'netcdf':
             file_ending = '.nc'
         elif self.backend == 'zarr':
             file_ending = '.zarr'
@@ -154,7 +155,7 @@ class Path(object):
                     format(stack_length, interval, num_periods))
 
         xd_index = dict(datetime=slice(self.starttime, self.endtime))
-        with xr.open_dataset(filename, group='original', engine=self.backend) as ds:
+        with xr.open_dataset(filename, group='original', engine=self.engine) as ds:
             rq = ds.loc[xd_index].load()
 
         # Stack features
@@ -187,11 +188,19 @@ class Path(object):
         """
         Save a feature to disk
         """
-        if self.backend == 'h5netcdf':
-            xarray2hdf5(data, self.path, **kwargs)
+        if self.backend == 'netcdf':
+            xarray2netcdf(data, self.path, **kwargs)
         elif self.backend == 'zarr':
             xarray2zarr(data, self.path, **kwargs)
 
+    def shape(self, feature):
+        """
+        Get shape of a feature on disk
+        """
+        filename = self.feature_path(feature)
+        with xr.open_dataset(filename, group='original', engine=self.engine) as ds:
+            return ds.sizes
+
 
 class Storage(Path):
     """
@@ -214,7 +223,7 @@ class Storage(Path):
     >>> rsam = c("rsam")
     """
 
-    def __init__(self, name, rootdir, starttime=None, endtime=None, create=True, backend='zarr'):
+    def __init__(self, name, rootdir, starttime=None, endtime=None, create=True, backend='netcdf'):
         self.stores = set()
         self.starttime = starttime
         self.endtime = endtime

tonik/utils.py

@@ -56,3 +56,36 @@ def generate_test_data(dim=1, ndays=30, nfreqs=10,
     xds.attrs['station'] = 'MDR'
     xds.attrs['interval'] = '10min'
     return xds
+
+
+def merge_arrays(xds_old: xr.DataArray, xds_new: xr.DataArray,
+                 resolution: float = None) -> xr.DataArray:
+    """
+    Merge two xarray datasets with the same datetime index.
+
+    Parameters
+    ----------
+    xds_old : xr.DataArray
+        Old array.
+    xds_new : xr.DataArray
+        New array.
+    resolution : float
+        Time resolution in hours.
+
+    Returns
+    -------
+    xr.DataArray
+        Merged array.
+    """
+    xda_old = xds_old.drop_duplicates(
+        'datetime', keep='last')
+    xda_new = xds_new.drop_duplicates(
+        'datetime', keep='last')
+    xda_new = xda_new.combine_first(xda_old)
+    if resolution is not None:
+        new_dates = pd.date_range(
+            xda_new.datetime.values[0],
+            xda_new.datetime.values[-1],
+            freq=f'{resolution}h')
+        xda_new = xda_new.reindex(datetime=new_dates)
+    return xda_new

tonik/{xarray2hdf5.py → xarray2netcdf.py}

@@ -1,15 +1,19 @@
-from datetime import datetime
 import logging
 import os
+from datetime import datetime
 from warnings import filterwarnings
 
-from cftime import num2date, date2num
 import h5netcdf
 import numpy as np
+import xarray as xr
+from cftime import date2num, num2date
+
+from .utils import merge_arrays
 
 
-def xarray2hdf5(xArray, fdir, rootGroupName="original", timedim="datetime",
-                archive_starttime=datetime(2000, 1, 1), resolution=None):
+def xarray2netcdf(xArray, fdir, rootGroupName="original", timedim="datetime",
+                  archive_starttime=datetime(2000, 1, 1), resolution=None,
+                  mode='a'):
     """
     Store an xarray dataset as an HDF5 file.
 
@@ -31,23 +35,35 @@ def xarray2hdf5(xArray, fdir, rootGroupName="original", timedim="datetime",
         determined from the data.
     """
     filterwarnings(action='ignore', category=DeprecationWarning,
-               message='`np.bool` is a deprecated alias')
+                   message='`np.bool` is a deprecated alias')
 
-    starttime = xArray[timedim].values[0].astype('datetime64[us]').astype(datetime)
-    starttime = min(starttime, archive_starttime)
+    data_starttime = xArray[timedim].values[0].astype(
+        'datetime64[us]').astype(datetime)
+    starttime = min(data_starttime, archive_starttime)
     if resolution is None:
         resolution = (np.diff(xArray[timedim])/np.timedelta64(1, 'h'))[0]
 
     for featureName in list(xArray.data_vars.keys()):
-        h5file = os.path.join(fdir, featureName +'.nc')
+        h5file = os.path.join(fdir, featureName + '.nc')
+        _mode = 'w'
+        if os.path.isfile(h5file) and mode == 'a':
+            if archive_starttime > data_starttime:
+                xds_existing = xr.open_dataset(
+                    h5file, group='original', engine='h5netcdf')
+                xda_new = merge_arrays(
+                    xds_existing[featureName], xArray[featureName],
+                    resolution=resolution)
+                xds_existing.close()
+                xda_new.to_netcdf(h5file, group='original',
+                                  mode='w', engine='h5netcdf')
+                continue
+            _mode = 'a'
 
-        mode = 'a' if os.path.isfile(h5file) else 'w'
-        
-        with h5netcdf.File(h5file, mode) as h5f:
+        with h5netcdf.File(h5file, _mode) as h5f:
             try:
                 rootGrp = _create_h5_Structure(rootGroupName, featureName,
                                                h5f, xArray, starttime, timedim)
-            except ValueError: # group already exists, append
+            except ValueError:  # group already exists, append
                 rootGrp = h5f[rootGroupName]
 
             # determine indices
@@ -75,7 +91,8 @@ def xarray2hdf5(xArray, fdir, rootGroupName="original", timedim="datetime",
             try:
                 _setMetaInfo(featureName, h5f, xArray)
             except KeyError as e:
-                logging.warning(f"Could not set all meta info for {featureName}: {e}")
+                logging.warning(
+                    f"Could not set all meta info for {featureName}: {e}")
 
 
 def _create_h5_Structure(defaultGroupName, featureName, h5f, xArray, starttime, timedim):
@@ -85,15 +102,16 @@ def _create_h5_Structure(defaultGroupName, featureName, h5f, xArray, starttime,
     coordinates.attrs['units'] = 'hours since 1970-01-01 00:00:00.0'
     coordinates.attrs['calendar'] = 'gregorian'
     rootGrp.attrs['starttime'] = str(starttime)
-    for label, size in xArray.dims.items(): 
+    for label, size in xArray.dims.items():
         if not np.issubdtype(xArray[label].dtype, np.datetime64):
-            rootGrp.dimensions[label] = size 
+            rootGrp.dimensions[label] = size
             coordinates = rootGrp.create_variable(label, (label,), float)
             coordinates[:] = xArray[label].values
     # Note: xArray.dims returns a dictionary of dimensions that are not necesarily
     # in the right order; xArray[featureName].dims returns a tuple with dimension
     # names in the correct order
-    rootGrp.create_variable(featureName, tuple(xArray[featureName].dims), dtype=float, fillvalue=0.)
+    rootGrp.create_variable(featureName, tuple(
+        xArray[featureName].dims), dtype=float, fillvalue=0.)
     return rootGrp
 
 
@@ -102,4 +120,3 @@ def _setMetaInfo(featureName, h5f, xArray):
     h5f.attrs['latitude'] = -42
     h5f.attrs['longitude'] = 168
     h5f.attrs['datatype'] = featureName
- 

tonik/xarray2zarr.py

@@ -3,31 +3,9 @@ import os
 
 import xarray as xr
 
-logger = logging.getLogger(__name__)
-
-
-def merge_arrays(xds_old: xr.DataArray, xds_new: xr.DataArray) -> xr.DataArray:
-    """
-    Merge two xarray datasets with the same datetime index.
+from .utils import merge_arrays
 
-    Parameters
-    ----------
-    xds_old : xr.DataArray
-        Old array.
-    xds_new : xr.DataArray
-        New array.
-
-    Returns
-    -------
-    xr.DataArray
-        Merged array.
-    """
-    xda_old = xds_old.drop_duplicates(
-        'datetime', keep='last')
-    xda_new = xds_new.drop_duplicates(
-        'datetime', keep='last')
-    xda_new = xda_new.combine_first(xda_old)
-    return xda_new
+logger = logging.getLogger(__name__)
 
 
 def xarray2zarr(xds: xr.Dataset, path: str, mode: str = 'a'):

{tonik-0.0.12.dist-info → tonik-0.1.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: tonik
-Version: 0.0.12
+Version: 0.1.1
 Summary: Store time series data as HDF5 files and access them through an API.
 Project-URL: Homepage, https://tsc-tools.github.io/tonik
 Project-URL: Issues, https://github.com/tsc-tools/tonik/issues

tonik-0.1.1.dist-info/RECORD

@@ -0,0 +1,12 @@
+tonik/__init__.py,sha256=ZBVGh4dm_l9xwiBGb33O5QV9MfZeNiEd3DBDAm6DiHk,511
+tonik/api.py,sha256=8YS0WCMlm5xs2N6V8n9BiP87ywJC_BUzYsPvWR9KWJk,6499
+tonik/storage.py,sha256=oy7Nz89zeiZchsqpouttzWhSL6UgenUQZiaMX6SdpsI,11499
+tonik/utils.py,sha256=3nSRU_GnV6arP4e63YHn4oEV8XbqzVAW8FCvQVIwGdg,2757
+tonik/xarray2netcdf.py,sha256=aZSSMADmS0DpnprT5TBqERGxrycZTPTNuQhxm1rOsh0,5135
+tonik/xarray2zarr.py,sha256=jTrHFpjN3lEDIJEI6RyPvvbW2jZsfYwGE_LQaZ6dwx8,2099
+tonik/package_data/index.html,sha256=GKDClUhIam_fAYbNfzAolORhSCG3ae1wW3VjWCg4PMk,2732
+tonik-0.1.1.dist-info/METADATA,sha256=kSVqgQLr_k3j7s3mfcAnY5nt4LrA-xplhYFiQFLnUIw,1938
+tonik-0.1.1.dist-info/WHEEL,sha256=1yFddiXMmvYK7QYTqtRNtX66WJ0Mz8PYEiEUoOUUxRY,87
+tonik-0.1.1.dist-info/entry_points.txt,sha256=VnGfC5qAzpntEHAb5pooUEpYABSgOfQoNhCEtLDJyf8,45
+tonik-0.1.1.dist-info/licenses/LICENSE,sha256=OXLcl0T2SZ8Pmy2_dmlvKuetivmyPd5m1q-Gyd-zaYY,35149
+tonik-0.1.1.dist-info/RECORD,,

tonik-0.0.12.dist-info/RECORD

@@ -1,12 +0,0 @@
-tonik/__init__.py,sha256=ZBVGh4dm_l9xwiBGb33O5QV9MfZeNiEd3DBDAm6DiHk,511
-tonik/api.py,sha256=8YS0WCMlm5xs2N6V8n9BiP87ywJC_BUzYsPvWR9KWJk,6499
-tonik/storage.py,sha256=Ts6jKLqwcDQvPIea-swBCpnEjQr_xnAjOl-hkUelTn4,11165
-tonik/utils.py,sha256=_TxXf9o9fOvtuOvGO6-ww9F5m0QelHyfQzQw8RGjTV4,1868
-tonik/xarray2hdf5.py,sha256=cekO9vo9ZRlr0VndswJjPC27CEVD3TpRVKLAJ-aAO0g,4465
-tonik/xarray2zarr.py,sha256=8osAiQSmiMTEGVoqWaCT7kqPvuzpGVZalNt4OhxmpCY,2619
-tonik/package_data/index.html,sha256=GKDClUhIam_fAYbNfzAolORhSCG3ae1wW3VjWCg4PMk,2732
-tonik-0.0.12.dist-info/METADATA,sha256=eHgl8-z2yNDzEuxHdgHbqDNxONe1HE-igHQ5gXLFAtY,1939
-tonik-0.0.12.dist-info/WHEEL,sha256=1yFddiXMmvYK7QYTqtRNtX66WJ0Mz8PYEiEUoOUUxRY,87
-tonik-0.0.12.dist-info/entry_points.txt,sha256=VnGfC5qAzpntEHAb5pooUEpYABSgOfQoNhCEtLDJyf8,45
-tonik-0.0.12.dist-info/licenses/LICENSE,sha256=OXLcl0T2SZ8Pmy2_dmlvKuetivmyPd5m1q-Gyd-zaYY,35149
-tonik-0.0.12.dist-info/RECORD,,