tonik 0.1.3__py3-none-any.whl → 0.1.4__py3-none-any.whl

tonik/storage.py

@@ -1,9 +1,7 @@
 import logging
 import logging.config
 import os
-import re
 
-import pandas as pd
 import xarray as xr
 
 from .xarray2netcdf import xarray2netcdf
@@ -113,69 +111,27 @@ class Path(object):
             self.children[feature] = Path(feature + file_ending, self.path)
         return _feature_path
 
-    def __call__(self, feature, stack_length=None, interval='10min'):
+    def __call__(self, feature, group='original'):
         """
         Request a particular feature
 
         :param feature: Feature name
         :type feature: str
-        :param stack_length: length of moving average in time
-        :type stack_length: str
 
         """
-        if self.endtime <= self.starttime:
+        if self.endtime < self.starttime:
             raise ValueError('Startime has to be smaller than endtime.')
 
         filename = self.feature_path(feature)
 
         logger.debug(
             f"Reading feature {feature} between {self.starttime} and {self.endtime}")
-        num_periods = None
-        if stack_length is not None:
-            valid_stack_units = ['W', 'D', 'h', 'T', 'min', 'S']
-            if re.match(r'\d*\s*(\w*)', stack_length).group(1) not in valid_stack_units:
-                raise ValueError(
-                    'Stack length should be one of: {}'.
-                    format(', '.join(valid_stack_units))
-                )
-
-            if pd.to_timedelta(stack_length) < pd.to_timedelta(interval):
-                raise ValueError('Stack length {} is less than interval {}'.
-                                 format(stack_length, interval))
-
-            # Rewind starttime to account for stack length
-            self.starttime -= pd.to_timedelta(stack_length)
-
-            num_periods = (pd.to_timedelta(stack_length) /
-                           pd.to_timedelta(interval))
-            if not num_periods.is_integer():
-                raise ValueError(
-                    'Stack length {} / interval {} = {}, but it needs'
-                    ' to be a whole number'.
-                    format(stack_length, interval, num_periods))
 
         xd_index = dict(datetime=slice(self.starttime, self.endtime))
-        with xr.open_dataset(filename, group='original', engine=self.engine) as ds:
+        with xr.open_dataset(filename, group=group, engine=self.engine) as ds:
             rq = ds[feature].loc[xd_index].load()
             rq.attrs = ds.attrs
 
-        # Stack features
-        if stack_length is not None:
-            logger.debug("Stacking feature...")
-            try:
-                xdf = rq.rolling(datetime=int(num_periods),
-                                 center=False,
-                                 min_periods=1).mean()
-                # Return requested timeframe to that defined in initialisation
-                self.starttime += pd.to_timedelta(stack_length)
-                xdf_new = xdf.loc[self.starttime:self.endtime]
-                xdf_new = xdf_new.rename(feature)
-            except ValueError as e:
-                logger.error(e)
-                logger.error('Stack length {} is not valid for feature {}'.
-                             format(stack_length, feature))
-            else:
-                return xdf_new
         return rq
 
     def load(self, *args, **kwargs):

tonik/xarray2netcdf.py

@@ -11,7 +11,7 @@ from cftime import date2num, num2date
 from .utils import merge_arrays
 
 
-def xarray2netcdf(xArray, fdir, rootGroupName="original", timedim="datetime",
+def xarray2netcdf(xArray, fdir, group="original", timedim="datetime",
                   archive_starttime=datetime(2000, 1, 1), resolution=None,
                   mode='a'):
     """
@@ -23,7 +23,7 @@ def xarray2netcdf(xArray, fdir, rootGroupName="original", timedim="datetime",
         Data to store.
     fdir : str
         Directory to store data under.
-    rootGroupName : str
+    group : str
         Hdf5 group name.
     timedim : str
         Name of time dimension.
@@ -49,22 +49,22 @@ def xarray2netcdf(xArray, fdir, rootGroupName="original", timedim="datetime",
         if os.path.isfile(h5file) and mode == 'a':
             if archive_starttime > data_starttime:
                 xds_existing = xr.open_dataset(
-                    h5file, group='original', engine='h5netcdf')
+                    h5file, group=group, engine='h5netcdf')
                 xda_new = merge_arrays(
                     xds_existing[featureName], xArray[featureName],
                     resolution=resolution)
                 xds_existing.close()
-                xda_new.to_netcdf(h5file, group='original',
+                xda_new.to_netcdf(h5file, group=group,
                                   mode='w', engine='h5netcdf')
                 continue
             _mode = 'a'
 
         with h5netcdf.File(h5file, _mode) as h5f:
             try:
-                rootGrp = _create_h5_Structure(rootGroupName, featureName,
+                rootGrp = _create_h5_Structure(group, featureName,
                                                h5f, xArray, starttime, timedim)
             except ValueError:  # group already exists, append
-                rootGrp = h5f[rootGroupName]
+                rootGrp = h5f[group]
 
             # determine indices
             new_time = date2num(xArray[timedim].values.astype('datetime64[us]').astype(datetime),

tonik/xarray2zarr.py

@@ -2,13 +2,14 @@ import logging
 import os
 
 import xarray as xr
+from zarr.errors import PathNotFoundError
 
 from .utils import merge_arrays
 
 logger = logging.getLogger(__name__)
 
 
-def xarray2zarr(xds: xr.Dataset, path: str, mode: str = 'a'):
+def xarray2zarr(xds: xr.Dataset, path: str, mode: str = 'a', group='original'):
     """
     Write xarray dataset to zarr files.
 
@@ -20,6 +21,8 @@ def xarray2zarr(xds: xr.Dataset, path: str, mode: str = 'a'):
         Path to write the dataset.
     mode : str, optional
         Write mode, by default 'a'.
+    group : str, optional
+        Group name, by default 'original'
 
     Returns
     -------
@@ -29,28 +32,32 @@ def xarray2zarr(xds: xr.Dataset, path: str, mode: str = 'a'):
         fout = os.path.join(path, feature + '.zarr')
         if not os.path.exists(fout) or mode == 'w':
             xds[feature].to_zarr(
-                fout, group='original', mode='w')
+                fout, group=group, mode='w')
         else:
-            xds_existing = xr.open_zarr(fout, group='original')
+            try:
+                xds_existing = xr.open_zarr(fout, group=group)
+            except PathNotFoundError:
+                xds[feature].to_zarr(fout, group=group, mode='a')
+                continue
             if xds_existing.datetime[0] > xds.datetime[0] or xds_existing.datetime[-1] > xds.datetime[-1]:
                 xda_new = merge_arrays(xds_existing[feature], xds[feature])
-                xda_new.to_zarr(fout, group='original', mode='w')
+                xda_new.to_zarr(fout, group=group, mode='w')
             else:
                 try:
                     overlap = xds_existing.datetime.where(
                         xds_existing.datetime == xds.datetime)
                     if overlap.size > 0:
                         xds[feature].loc[dict(datetime=overlap)].to_zarr(
-                            fout, group='original', mode='r+', region='auto')
+                            fout, group=group, mode='r+', region='auto')
                         xds[feature].drop_sel(datetime=overlap).to_zarr(
-                            fout, group='original', mode='a', append_dim="datetime")
+                            fout, group=group, mode='a', append_dim="datetime")
                     else:
                         xds[feature].to_zarr(
-                            fout, group='original', append_dim='datetime')
+                            fout, group=group, append_dim='datetime')
                 except Exception as e:
                     msg = f"Appending {feature} to {fout} failed: {e}\n"
                     msg += "Attempting to merge the two datasets."
                     logger.error(msg)
                     # remove duplicate datetime entries
                     xda_new = merge_arrays(xds_existing[feature], xds[feature])
-                    xda_new.to_zarr(fout, group='original', mode='w')
+                    xda_new.to_zarr(fout, group=group, mode='w')

{tonik-0.1.3.dist-info → tonik-0.1.4.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: tonik
-Version: 0.1.3
+Version: 0.1.4
 Summary: Store time series data as HDF5 files and access them through an API.
 Project-URL: Homepage, https://tsc-tools.github.io/tonik
 Project-URL: Issues, https://github.com/tsc-tools/tonik/issues

tonik-0.1.4.dist-info/RECORD

@@ -0,0 +1,12 @@
+tonik/__init__.py,sha256=ZBVGh4dm_l9xwiBGb33O5QV9MfZeNiEd3DBDAm6DiHk,511
+tonik/api.py,sha256=8YS0WCMlm5xs2N6V8n9BiP87ywJC_BUzYsPvWR9KWJk,6499
+tonik/storage.py,sha256=GNJ6w9VHOeTR_ZJMZ-Ipqe3nFK2I91fkHYwg1k9bEuo,9470
+tonik/utils.py,sha256=3nSRU_GnV6arP4e63YHn4oEV8XbqzVAW8FCvQVIwGdg,2757
+tonik/xarray2netcdf.py,sha256=Bjt7kytnrlBgZrVNmRGir9TfNqyvW_ZNrwYBNvxiWio,5199
+tonik/xarray2zarr.py,sha256=EhEo5kqzrKyXR37RX2zYtgOAviZdRqstZi_4ZtUmSDc,2342
+tonik/package_data/index.html,sha256=GKDClUhIam_fAYbNfzAolORhSCG3ae1wW3VjWCg4PMk,2732
+tonik-0.1.4.dist-info/METADATA,sha256=GkFoQugVoozfs2jTWNchQujP5RDATzeo1s7rqkx63GQ,1938
+tonik-0.1.4.dist-info/WHEEL,sha256=1yFddiXMmvYK7QYTqtRNtX66WJ0Mz8PYEiEUoOUUxRY,87
+tonik-0.1.4.dist-info/entry_points.txt,sha256=VnGfC5qAzpntEHAb5pooUEpYABSgOfQoNhCEtLDJyf8,45
+tonik-0.1.4.dist-info/licenses/LICENSE,sha256=OXLcl0T2SZ8Pmy2_dmlvKuetivmyPd5m1q-Gyd-zaYY,35149
+tonik-0.1.4.dist-info/RECORD,,

tonik-0.1.3.dist-info/RECORD

@@ -1,12 +0,0 @@
-tonik/__init__.py,sha256=ZBVGh4dm_l9xwiBGb33O5QV9MfZeNiEd3DBDAm6DiHk,511
-tonik/api.py,sha256=8YS0WCMlm5xs2N6V8n9BiP87ywJC_BUzYsPvWR9KWJk,6499
-tonik/storage.py,sha256=DNIfNb3oCgICcUI_MADQjQdUqkmZJIu42zCYGRs9ers,11512
-tonik/utils.py,sha256=3nSRU_GnV6arP4e63YHn4oEV8XbqzVAW8FCvQVIwGdg,2757
-tonik/xarray2netcdf.py,sha256=w8ubAWa2yCnk9-JQ_e7CHdfEjvjcK1pXol8Qy_PT2hY,5241
-tonik/xarray2zarr.py,sha256=jTrHFpjN3lEDIJEI6RyPvvbW2jZsfYwGE_LQaZ6dwx8,2099
-tonik/package_data/index.html,sha256=GKDClUhIam_fAYbNfzAolORhSCG3ae1wW3VjWCg4PMk,2732
-tonik-0.1.3.dist-info/METADATA,sha256=DaGP_RFy6luf75DQMr9UA3cfJXlOVJbdHFPU09RCE_4,1938
-tonik-0.1.3.dist-info/WHEEL,sha256=1yFddiXMmvYK7QYTqtRNtX66WJ0Mz8PYEiEUoOUUxRY,87
-tonik-0.1.3.dist-info/entry_points.txt,sha256=VnGfC5qAzpntEHAb5pooUEpYABSgOfQoNhCEtLDJyf8,45
-tonik-0.1.3.dist-info/licenses/LICENSE,sha256=OXLcl0T2SZ8Pmy2_dmlvKuetivmyPd5m1q-Gyd-zaYY,35149
-tonik-0.1.3.dist-info/RECORD,,