tonik 0.1.20__tar.gz → 0.1.21__tar.gz

{tonik-0.1.20 → tonik-0.1.21}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: tonik
-Version: 0.1.20
+Version: 0.1.21
 Summary: Store time series data as HDF5 files and access them through an API.
 Project-URL: Homepage, https://tsc-tools.github.io/tonik
 Project-URL: Issues, https://github.com/tsc-tools/tonik/issues

{tonik-0.1.20 → tonik-0.1.21}/pixi.lock

@@ -3681,8 +3681,8 @@ packages:
   timestamp: 1763054914403
 - pypi: ./
   name: tonik
-  version: 0.1.20
-  sha256: 34ecbc02a23b22c1e089cf36f78c90c6c214a3e88b9d272f69e349157f1728c6
+  version: 0.1.21
+  sha256: 8df82fc10e56aba3de54b9483f427f92a415574614dd09716f8cadc9c33a2510
   requires_dist:
   - h5py>=3.8
   - datashader>=0.14

{tonik-0.1.20 → tonik-0.1.21}/pyproject.toml

@@ -12,7 +12,7 @@ exclude = [
 
 [project]
 name = "tonik"
-version = "0.1.20"
+version = "0.1.21"
 authors = [
   { name="Yannik Behr", email="y.behr@gns.cri.nz" },
   { name="Christof Mueller", email="c.mueller@gns.cri.nz" }
@@ -64,6 +64,7 @@ grafana_annotations = "tonik.grafana_annotations:main"
 
 [tool.pytest.ini_options]
 log_cli = true
+addopts = "-s"
 
 [tool.hatch.envs.test]
 dependencies = [

{tonik-0.1.20 → tonik-0.1.21}/src/tonik/storage.py

@@ -18,7 +18,7 @@ LOGGING_CONFIG = {
             "datefmt": "%Y-%m-%d %H:%M:%S",  # How to display dates
         },
         "json": {  # The formatter name
-            "()": "pythonjsonlogger.jsonlogger.JsonFormatter",  # The class to instantiate!
+            "()": "pythonjsonlogger.json.JsonFormatter",  # The class to instantiate!
             # Json is more complex, but easier to read, display all attributes!
             "format": """
                     asctime: %(asctime)s

{tonik-0.1.20 → tonik-0.1.21}/src/tonik/xarray2zarr.py

@@ -267,7 +267,7 @@ def xarray2zarr(xds: xr.Dataset, path: str, mode: str = 'a', group='original',
             continue
 
         if xds_existing[timedim][0] > xds[timedim][-1]:
-            # prepend
+            logger.debug("Prepending data to existing zarr store.")
             xda_new = fill_time_gaps_between_datasets(xds_existing[feature].isel({timedim: 0}),
                                                       xds[feature], mode='p')
             xda_new = _build_append_payload_full_chunks(
@@ -277,7 +277,7 @@ def xarray2zarr(xds: xr.Dataset, path: str, mode: str = 'a', group='original',
                                                        write_empty_chunks=True)
 
         elif xds_existing[timedim][-1] < xds[timedim][0]:
-            # append
+            logger.debug("Appending data to existing zarr store.")
             xda_new = fill_time_gaps_between_datasets(xds_existing[feature].isel({timedim: -1}),
                                                       xds[feature], mode='a')
             xda_new = _build_append_payload_full_chunks(
@@ -286,13 +286,19 @@ def xarray2zarr(xds: xr.Dataset, path: str, mode: str = 'a', group='original',
                             append_dim=timedim)
 
         elif xds_existing[timedim][0] > xds[timedim][0] and xds_existing[timedim][-1] < xds[timedim][-1]:
-            # existing datetimes are contained in new array
+            logger.debug(
+                "Data in zarr store contained in new data. Rewriting zarr store.")
             xda_new = _build_append_payload_full_chunks(
                 xds[feature], 'a', nchunks)
             xda_new.to_zarr(fout, group=group, mode='w',
                             write_empty_chunks=True)
 
         else:
+            logger.debug("Data in zarr store overlaps with new data.")
+            logger.debug(
+                f"Endtime of existing data: {xds_existing[timedim][-1].values}")
+            logger.debug(f"Starttime of new data: {xds[timedim][0].values}")
+            xds_existing = xds_existing.drop_duplicates(timedim, keep='last')
             overlap = xds_existing[timedim].where(
                 xds_existing[timedim] == xds[timedim])
             xds[feature].loc[{timedim: overlap}].to_zarr(

{tonik-0.1.20 → tonik-0.1.21}/tests/test_save.py

@@ -169,6 +169,24 @@ def test_xarray2netcdf_multi_access(tmp_path_factory):
                                 group='original', engine='h5netcdf')
     xarray2netcdf(xdf2, temp_dir)
 
+@pytest.mark.slow
+def test_netcdf_attribute_bug(tmp_path_factory):
+    """
+    Test to replicate behaviour when attribute is updated more than
+    2^16 times.
+    """
+    temp_dir = tmp_path_factory.mktemp('test_netcdf_attribute_bug')
+    g = Storage('test_experiment', rootdir=temp_dir, backend='netcdf')
+    c = g.get_substore('MDR', '00', 'HHZ')
+    tstart = datetime(2022, 7, 18, 8, 0, 0)
+    for i in range(70000):
+        if i % 1000 == 0:
+            print(f'Iteration {i}')
+        xdf = generate_test_data(tstart=tstart, dim=1, intervals=3, freq='1h')
+        xdf.attrs['last_update'] = str(tstart + timedelta(hours=3))
+        tstart += timedelta(days=1)
+        c.save(xdf)
+
 
 def test_xarray2zarr(tmp_path_factory):
     xdf = generate_test_data(