tonik 0.1.16__py3-none-any.whl → 0.1.18__py3-none-any.whl

tonik/api.py

@@ -26,8 +26,9 @@ InventoryReturnType = Union[list, dict]
 
 class TonikAPI:
 
-    def __init__(self, rootdir) -> None:
+    def __init__(self, rootdir, backend='netcdf') -> None:
         self.rootdir = rootdir
+        self.backend = backend
         self.app = FastAPI()
 
         # -- allow any origin to query API
@@ -72,7 +73,7 @@ class TonikAPI:
         _et = self.preprocess_datetime(endtime)
         g = Storage(group, rootdir=self.rootdir,
                     starttime=_st, endtime=_et,
-                    create=False)
+                    create=False, backend=self.backend)
         c = g
         if subdir:
             c = g.get_substore(*subdir)
@@ -147,7 +148,8 @@ class TonikAPI:
         return freq, dates, spec
 
     async def inventory(self, group: str, subdir: SubdirType = None, tree: bool = True) -> InventoryReturnType:
-        sg = Storage(group, rootdir=self.rootdir, create=False)
+        sg = Storage(group, rootdir=self.rootdir,
+                     create=False, backend=self.backend)
         try:
             c = sg.get_substore(*subdir)
         except TypeError:
@@ -168,7 +170,8 @@ class TonikAPI:
         _st = self.preprocess_datetime(starttime)
         _et = self.preprocess_datetime(endtime)
         sg = Storage(group, rootdir=self.rootdir,
-                     starttime=_st, endtime=_et, create=False)
+                     starttime=_st, endtime=_et, create=False,
+                     backend=self.backend)
         try:
             c = sg.get_substore(*subdir)
         except TypeError:
@@ -183,10 +186,11 @@ class TonikAPI:
 def main(argv=None):
     parser = ArgumentParser()
     parser.add_argument("--rootdir", default='/tmp')
+    parser.add_argument("--backend", default='netcdf')
     parser.add_argument("-p", "--port", default=8003, type=int)
     parser.add_argument("--host", default='0.0.0.0')
     args = parser.parse_args(argv)
-    ta = TonikAPI(args.rootdir)
+    ta = TonikAPI(args.rootdir, backend=args.backend)
     uvicorn.run(ta.app, host=args.host, port=args.port)
 
 

tonik/storage.py

@@ -112,7 +112,7 @@ class Path(object):
             self.children[feature] = Path(feature + file_ending, self.path)
         return _feature_path
 
-    def __call__(self, feature, group='original', attributes_only=False):
+    def __call__(self, feature, group='original', metadata=False):
         """
         Request a particular feature
 
@@ -122,9 +122,9 @@ class Path(object):
         """
         filename = self.feature_path(feature)
 
-        if attributes_only:
-            with xr.open_dataset(filename, group=group, engine=self.engine) as ds:
-                return ds.attrs
+        if metadata:
+            with xr.open_dataset(filename, group='meta', engine=self.engine) as ds:
+                return ds
 
         if self.endtime < self.starttime:
             raise ValueError('Startime has to be smaller than endtime.')

tonik/xarray2netcdf.py

@@ -1,6 +1,6 @@
 import logging
 import os
-from datetime import datetime
+from datetime import datetime, timezone
 from warnings import filterwarnings
 
 import h5netcdf
@@ -40,6 +40,7 @@ def xarray2netcdf(xArray, fdir, group="original", timedim="datetime",
     data_starttime = xArray[timedim].values[0].astype(
         'datetime64[us]').astype(datetime)
     starttime = min(data_starttime, archive_starttime)
+    now = datetime.now(tz=timezone.utc)
     if resolution is None:
         resolution = (np.diff(xArray[timedim])/np.timedelta64(1, 'h'))[0]
 
@@ -61,11 +62,16 @@ def xarray2netcdf(xArray, fdir, group="original", timedim="datetime",
 
         with h5netcdf.File(h5file, _mode) as h5f:
             try:
-                rootGrp = _create_h5_Structure(group, featureName,
-                                               h5f, xArray, starttime, timedim)
+                rootGrp = _create_root_group(group, featureName,
+                                             h5f, xArray, starttime, timedim)
             except ValueError:  # group already exists, append
                 rootGrp = h5f[group]
 
+            try:
+                metaGrp = _create_meta_group(h5f, resolution)
+            except ValueError:  # group already exists, append
+                metaGrp = h5f['meta']
+
             # determine indices
             new_time = date2num(xArray[timedim].values.astype('datetime64[us]').astype(datetime),
                                 units=rootGrp[timedim].attrs['units'],
@@ -73,6 +79,7 @@ def xarray2netcdf(xArray, fdir, group="original", timedim="datetime",
             t0 = date2num(starttime,
                           units=rootGrp[timedim].attrs['units'],
                           calendar=rootGrp[timedim].attrs['calendar'])
+
             indices = np.rint((new_time - t0)/resolution).astype(int)
             if not np.all(indices >= 0):
                 raise ValueError("Data starts before the archive start time")
@@ -86,18 +93,21 @@ def xarray2netcdf(xArray, fdir, group="original", timedim="datetime",
                 data[:, indices] = xArray[featureName].values
             else:
                 data[indices] = xArray[featureName].values
-            rootGrp.attrs['endtime'] = str(num2date(times[-1], units=rootGrp[timedim].attrs['units'],
-                                                    calendar=rootGrp[timedim].attrs['calendar']))
-            rootGrp.attrs['resolution'] = resolution
-            rootGrp.attrs['resolution_units'] = 'h'
-            try:
-                _setMetaInfo(featureName, rootGrp, xArray)
-            except KeyError as e:
-                logging.warning(
-                    f"Could not set all meta info for {featureName}: {e}")
+            now_time = date2num(now, units=metaGrp['update_log'].attrs['units'],
+                                calendar=metaGrp['update_log'].attrs['calendar'])
+            ulog = metaGrp['update_log']
+            ldata = metaGrp['last_datapoint']
+            metaGrp.resize_dimension('update', ulog.shape[0] + 1)
+            ulog[-1] = now_time
+            metaGrp.resize_dimension('endtime', ldata.shape[0] + 1)
+            ldata[-1] = times[-1]
+            old_resolution = metaGrp['resolution'][()]
+            if old_resolution != resolution:
+                raise ValueError(f"Resolution mismatch for {featureName}: "
+                                 f"{old_resolution} != {resolution}")
 
 
-def _create_h5_Structure(defaultGroupName, featureName, h5f, xArray, starttime, timedim):
+def _create_root_group(defaultGroupName, featureName, h5f, xArray, starttime, timedim):
     rootGrp = h5f.create_group(defaultGroupName)
     rootGrp.dimensions[timedim] = None
     coordinates = rootGrp.create_variable(timedim, (timedim,), float)
@@ -114,10 +124,33 @@ def _create_h5_Structure(defaultGroupName, featureName, h5f, xArray, starttime,
     # names in the correct order
     rootGrp.create_variable(featureName, tuple(
         xArray[featureName].dims), dtype=float, fillvalue=0.)
+    _set_attributes(featureName, rootGrp, xArray)
     return rootGrp
 
 
-def _setMetaInfo(featureName, rootGrp, xArray):
+def _set_attributes(featureName, rootGrp, xArray):
+    """
+    Set attributes for the root group. Attributes are assumed to not change
+    over time. If they do, they should be stored in the 'meta' group.
+    """
     for key, value in xArray.attrs.items():
         rootGrp.attrs[key] = value
     rootGrp.attrs['feature'] = featureName
+
+
+def _create_meta_group(h5f, resolution):
+    """
+    Create meta group to track processing history.
+    """
+    metaGrp = h5f.create_group('meta')
+    metaGrp.dimensions['update'] = None
+    ulog = metaGrp.create_variable('update_log', ('update',), float)
+    ulog.attrs['units'] = 'hours since 1970-01-01 00:00:00.0'
+    ulog.attrs['calendar'] = 'gregorian'
+    metaGrp.dimensions['endtime'] = None
+    ldata = metaGrp.create_variable('last_datapoint', ('endtime',), float)
+    ldata.attrs['units'] = 'hours since 1970-01-01 00:00:00.0'
+    ldata.attrs['calendar'] = 'gregorian'
+    res = metaGrp.create_variable("resolution", (), dtype=float)
+    res[()] = resolution
+    return metaGrp

tonik/xarray2zarr.py

@@ -1,3 +1,4 @@
+from datetime import datetime, timezone
 import logging
 import os
 
@@ -18,7 +19,7 @@ logger = logging.getLogger(__name__)
 def get_chunks(xda: xr.DataArray, chunks: int = 1,
                timedim: str = 'datetime') -> dict:
     """
-    Determine the chunk size for the datetime dimension. Other dimensions are assumed to be 
+    Determine the chunk size for the datetime dimension. Other dimensions are assumed to be
     small enough to not require chunking.
 
     Parameters
@@ -45,7 +46,7 @@ def fill_time_gaps_between_datasets(xds_existing: xr.DataArray, xds_new: xr.Data
     ----------
     xds_existing : xr.Dataset
         Existing dataset on disk
-    xds_new : xr.Dataset  
+    xds_new : xr.Dataset
         New dataset to append
     timedim : str
         Name of the time dimension
@@ -106,6 +107,22 @@ def _build_append_payload_full_chunks(payload: xr.DataArray, mode: str,
                                       chunklen: int, timedim: str = "datetime") -> xr.DataArray:
     """
     Construct the sequence to append so that the final total length is a multiple of `chunklen`
+
+    Parameters
+    ----------
+    payload : xr.DataArray
+        DataArray to append
+    mode : str
+        'a' for append, 'p' for prepend
+    chunklen : int
+        Chunk length in number of time steps
+    timedim : str
+        Name of the time dimension
+
+    Returns
+    -------
+    xr.DataArray
+        Padded DataArray with length a multiple of chunklen
     """
     if mode not in ['a', 'p']:
         raise ValueError(
@@ -141,7 +158,58 @@ def _build_append_payload_full_chunks(payload: xr.DataArray, mode: str,
             payload = xr.concat([pad_da, payload], dim=timedim)
         payload = payload.chunk({timedim: chunklen})
     return payload
-#
+
+
+def _update_meta_data(fout: str,
+                      last_datapoint: np.datetime64,
+                      resolution: float | None = None,
+                      meta_group: str = "meta") -> None:
+    """
+    Append current update time (and last_datapoint) to meta group.
+
+    Parameters
+    ----------
+    fout : str
+        Base zarr store path (per-variable .zarr directory).
+    last_datapoint : np.datetime64
+        Latest data time in the feature.
+    resolution : float | None
+        Optional time resolution (hours) to store once.
+    meta_group : str
+        Group name for metadata.
+    """
+
+    now = np.datetime64(datetime.now(tz=timezone.utc), 'ns')
+    new_update = xr.DataArray([now],
+                              coords={'update': [now]},
+                              dims=['update'],
+                              name='update_log')
+    new_last = xr.DataArray([last_datapoint],
+                            coords={'endtime': [now]},
+                            dims=['endtime'],
+                            name='last_datapoint')
+
+    try:
+        meta = xr.open_zarr(fout, group=meta_group, chunks=None)
+        # Existing vars -> concatenate
+        update_old = meta.get('update_log')
+        last_old = meta.get('last_datapoint')
+        res_da_old = meta.get('resolution').values[()]
+        new_update = xr.concat([update_old, new_update], dim='update')
+        new_last = xr.concat([last_old, new_last], dim='endtime')
+        if resolution != res_da_old:
+            raise ValueError(f"Resolution mismatch for {fout}: "
+                             f"{res_da_old} != {resolution}")
+        res_da = xr.DataArray(resolution, name='resolution')
+    except Exception:
+        # First creation
+        res_da = xr.DataArray(
+            resolution, name='resolution') if resolution is not None else None
+
+    vars = {'update_log': new_update, 'last_datapoint': new_last}
+    if res_da is not None:
+        vars['resolution'] = res_da
+    xr.Dataset(vars).to_zarr(fout, group=meta_group, mode='w')
 
 
 def xarray2zarr(xds: xr.Dataset, path: str, mode: str = 'a', group='original',
@@ -182,6 +250,9 @@ def xarray2zarr(xds: xr.Dataset, path: str, mode: str = 'a', group='original',
         fout = os.path.join(path, feature + '.zarr')
         # nchunks = get_chunks(xds[feature], chunks)
         nchunks = chunks
+        last_dp = xds[feature][timedim].values[-1]
+        _update_meta_data(fout, last_dp, resolution=float(
+            get_dt(xds[timedim]) / pd.Timedelta(1, 'h')))
         try:
             xds_existing = xr.open_zarr(fout, group=group)
             has_store = True
@@ -204,6 +275,7 @@ def xarray2zarr(xds: xr.Dataset, path: str, mode: str = 'a', group='original',
             combined = xda_new.combine_first(xds_existing[feature]).compute()
             combined.chunk({timedim: nchunks}).to_zarr(fout, group=group, mode='w',
                                                        write_empty_chunks=True)
+
         elif xds_existing[timedim][-1] < xds[timedim][0]:
             # append
             xda_new = fill_time_gaps_between_datasets(xds_existing[feature].isel({timedim: -1}),
@@ -212,12 +284,14 @@ def xarray2zarr(xds: xr.Dataset, path: str, mode: str = 'a', group='original',
                 xda_new, 'a', nchunks)
             xda_new.to_zarr(fout, group=group, mode='a',
                             append_dim=timedim)
+
         elif xds_existing[timedim][0] > xds[timedim][0] and xds_existing[timedim][-1] < xds[timedim][-1]:
             # existing datetimes are contained in new array
             xda_new = _build_append_payload_full_chunks(
                 xds[feature], 'a', nchunks)
             xda_new.to_zarr(fout, group=group, mode='w',
                             write_empty_chunks=True)
+
         else:
             overlap = xds_existing[timedim].where(
                 xds_existing[timedim] == xds[timedim])

{tonik-0.1.16.dist-info → tonik-0.1.18.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: tonik
-Version: 0.1.16
+Version: 0.1.18
 Summary: Store time series data as HDF5 files and access them through an API.
 Project-URL: Homepage, https://tsc-tools.github.io/tonik
 Project-URL: Issues, https://github.com/tsc-tools/tonik/issues
@@ -21,15 +21,17 @@ Requires-Dist: python-json-logger>=2.0
 Requires-Dist: s3fs
 Requires-Dist: uvicorn[standard]>=0.22
 Requires-Dist: xarray[accel,io,parallel]
-Requires-Dist: zarr[remote-tests]<3; python_version < '3.11'
-Requires-Dist: zarr[remote-tests]>=3.0.3; python_version >= '3.11'
+Requires-Dist: zarr
 Provides-Extra: dev
+Requires-Dist: build; extra == 'dev'
 Requires-Dist: httpx; extra == 'dev'
 Requires-Dist: ipykernel; extra == 'dev'
 Requires-Dist: mkdocs; extra == 'dev'
 Requires-Dist: mkdocs-jupyter; extra == 'dev'
 Requires-Dist: mkdocstrings[python]; extra == 'dev'
 Requires-Dist: pytest; extra == 'dev'
+Requires-Dist: twine; extra == 'dev'
+Requires-Dist: zarr[remote-tests]; extra == 'dev'
 Description-Content-Type: text/markdown
 
 # Tonik

tonik-0.1.18.dist-info/RECORD

@@ -0,0 +1,14 @@
+tonik/__init__.py,sha256=dov-nMeGFBzLspmj4rWKjC4r736vmaPDgMEkHSUfP98,523
+tonik/api.py,sha256=vW0ykOo5iGAV0_WuOepdrnUyFp83F7KyJTd43ksLmUk,7985
+tonik/grafana_annotations.py,sha256=ZU9Cy-HT4vvMfYIQzD9WboaDVOCBDv__NmXbk1qKWJo,5838
+tonik/storage.py,sha256=jcCVx2N8J1ZBKM73k-OaxB0uxukn4VAM_-CCaCeAKwk,10589
+tonik/utils.py,sha256=vRFMoCU7dbfnnm5RALBR-XrpPGDFtQoeTDzxFiYf3bo,7522
+tonik/xarray2netcdf.py,sha256=hO3adraANvSuvmMJj0moDRAKXK01uTpAuxHPKv5xMwY,6486
+tonik/xarray2zarr.py,sha256=aPaN-gSI0YFAOeD30ex6fq6vtzrNO7YSKraitrUn6Fc,11222
+tonik/package_data/index.html,sha256=ZCZ-BtGRERsL-6c_dfY43qd2WAaggH7xereennGL6ww,4372
+tonik/package_data/whakaari_labels.json,sha256=96UZSq41yXgAJxuKivLBKlRTw-33jkjh7AGKTsDQ9Yg,3993
+tonik-0.1.18.dist-info/METADATA,sha256=bRaWxrTariy4xjpcKTrqDk0_GqtHOReVYIjbWz1f1hQ,2207
+tonik-0.1.18.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+tonik-0.1.18.dist-info/entry_points.txt,sha256=y82XyTeQddM87gCTzgSQaTlKF3VFicO4hhClHUv6j1A,127
+tonik-0.1.18.dist-info/licenses/LICENSE,sha256=OXLcl0T2SZ8Pmy2_dmlvKuetivmyPd5m1q-Gyd-zaYY,35149
+tonik-0.1.18.dist-info/RECORD,,

tonik-0.1.16.dist-info/RECORD

@@ -1,14 +0,0 @@
-tonik/__init__.py,sha256=dov-nMeGFBzLspmj4rWKjC4r736vmaPDgMEkHSUfP98,523
-tonik/api.py,sha256=XDKiz1AzYNBOwYfaRxpMgqGRDAPJEE6wWJyBxuYPRLc,7751
-tonik/grafana_annotations.py,sha256=ZU9Cy-HT4vvMfYIQzD9WboaDVOCBDv__NmXbk1qKWJo,5838
-tonik/storage.py,sha256=vFxIrY92cSYOYOpNXHxCAjdXgkrRytaRDpudtK0glmg,10608
-tonik/utils.py,sha256=vRFMoCU7dbfnnm5RALBR-XrpPGDFtQoeTDzxFiYf3bo,7522
-tonik/xarray2netcdf.py,sha256=gDNT6nxnRbXPeRqZ3URW5oXY3Nfh3TCrfueE-eUrIoY,5181
-tonik/xarray2zarr.py,sha256=SSchDqy5oyYrIG4smV8fslsUg2UPSyyQjUA5ZlP1P4I,8630
-tonik/package_data/index.html,sha256=ZCZ-BtGRERsL-6c_dfY43qd2WAaggH7xereennGL6ww,4372
-tonik/package_data/whakaari_labels.json,sha256=96UZSq41yXgAJxuKivLBKlRTw-33jkjh7AGKTsDQ9Yg,3993
-tonik-0.1.16.dist-info/METADATA,sha256=EOwmXNC5b6IJsnTLMelBZ3vL1ljkfZwhM8Hoz6iHiZQ,2191
-tonik-0.1.16.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-tonik-0.1.16.dist-info/entry_points.txt,sha256=y82XyTeQddM87gCTzgSQaTlKF3VFicO4hhClHUv6j1A,127
-tonik-0.1.16.dist-info/licenses/LICENSE,sha256=OXLcl0T2SZ8Pmy2_dmlvKuetivmyPd5m1q-Gyd-zaYY,35149
-tonik-0.1.16.dist-info/RECORD,,