tonik 0.1.20__py3-none-any.whl → 0.1.21__py3-none-any.whl

tonik/storage.py

@@ -18,7 +18,7 @@ LOGGING_CONFIG = {
             "datefmt": "%Y-%m-%d %H:%M:%S",  # How to display dates
         },
         "json": {  # The formatter name
-            "()": "pythonjsonlogger.jsonlogger.JsonFormatter",  # The class to instantiate!
+            "()": "pythonjsonlogger.json.JsonFormatter",  # The class to instantiate!
             # Json is more complex, but easier to read, display all attributes!
             "format": """
                     asctime: %(asctime)s

tonik/xarray2zarr.py

@@ -267,7 +267,7 @@ def xarray2zarr(xds: xr.Dataset, path: str, mode: str = 'a', group='original',
             continue
 
         if xds_existing[timedim][0] > xds[timedim][-1]:
-            # prepend
+            logger.debug("Prepending data to existing zarr store.")
             xda_new = fill_time_gaps_between_datasets(xds_existing[feature].isel({timedim: 0}),
                                                       xds[feature], mode='p')
             xda_new = _build_append_payload_full_chunks(
@@ -277,7 +277,7 @@ def xarray2zarr(xds: xr.Dataset, path: str, mode: str = 'a', group='original',
                                                        write_empty_chunks=True)
 
         elif xds_existing[timedim][-1] < xds[timedim][0]:
-            # append
+            logger.debug("Appending data to existing zarr store.")
             xda_new = fill_time_gaps_between_datasets(xds_existing[feature].isel({timedim: -1}),
                                                       xds[feature], mode='a')
             xda_new = _build_append_payload_full_chunks(
@@ -286,13 +286,19 @@ def xarray2zarr(xds: xr.Dataset, path: str, mode: str = 'a', group='original',
                             append_dim=timedim)
 
         elif xds_existing[timedim][0] > xds[timedim][0] and xds_existing[timedim][-1] < xds[timedim][-1]:
-            # existing datetimes are contained in new array
+            logger.debug(
+                "Data in zarr store contained in new data. Rewriting zarr store.")
             xda_new = _build_append_payload_full_chunks(
                 xds[feature], 'a', nchunks)
             xda_new.to_zarr(fout, group=group, mode='w',
                             write_empty_chunks=True)
 
         else:
+            logger.debug("Data in zarr store overlaps with new data.")
+            logger.debug(
+                f"Endtime of existing data: {xds_existing[timedim][-1].values}")
+            logger.debug(f"Starttime of new data: {xds[timedim][0].values}")
+            xds_existing = xds_existing.drop_duplicates(timedim, keep='last')
             overlap = xds_existing[timedim].where(
                 xds_existing[timedim] == xds[timedim])
             xds[feature].loc[{timedim: overlap}].to_zarr(

{tonik-0.1.20.dist-info → tonik-0.1.21.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: tonik
-Version: 0.1.20
+Version: 0.1.21
 Summary: Store time series data as HDF5 files and access them through an API.
 Project-URL: Homepage, https://tsc-tools.github.io/tonik
 Project-URL: Issues, https://github.com/tsc-tools/tonik/issues

{tonik-0.1.20.dist-info → tonik-0.1.21.dist-info}/RECORD

@@ -1,14 +1,14 @@
 tonik/__init__.py,sha256=dov-nMeGFBzLspmj4rWKjC4r736vmaPDgMEkHSUfP98,523
 tonik/api.py,sha256=vW0ykOo5iGAV0_WuOepdrnUyFp83F7KyJTd43ksLmUk,7985
 tonik/grafana_annotations.py,sha256=ZU9Cy-HT4vvMfYIQzD9WboaDVOCBDv__NmXbk1qKWJo,5838
-tonik/storage.py,sha256=jcCVx2N8J1ZBKM73k-OaxB0uxukn4VAM_-CCaCeAKwk,10589
+tonik/storage.py,sha256=bYBl3JPpH8D3iIFOj5AZQXc4M8txAbwFKt4xTfdotgg,10583
 tonik/utils.py,sha256=GwAXfGFQWhlsLThQvSux1SooRkW-iIkJP99JMH72t5Y,11791
 tonik/xarray2netcdf.py,sha256=nq6RHk5ciaAg1bxNDiyHPRdAts1C7fj7jtDbaLaSTWM,6497
-tonik/xarray2zarr.py,sha256=Dg9_b6Zwj_UQMhOez6wrPeHn0rUffUHqbv-e4pP_t3w,11233
+tonik/xarray2zarr.py,sha256=kRhgDdo8CDT1ceszwQEQNfXdgbnmL5nNejUzaMnyFXM,11707
 tonik/package_data/index.html,sha256=ZCZ-BtGRERsL-6c_dfY43qd2WAaggH7xereennGL6ww,4372
 tonik/package_data/whakaari_labels.json,sha256=96UZSq41yXgAJxuKivLBKlRTw-33jkjh7AGKTsDQ9Yg,3993
-tonik-0.1.20.dist-info/METADATA,sha256=CU6bp4tr1Z6dAPxPGLXkuP38N10SonJKzMwLakr38Xc,2207
-tonik-0.1.20.dist-info/WHEEL,sha256=WLgqFyCfm_KASv4WHyYy0P3pM_m7J5L9k2skdKLirC8,87
-tonik-0.1.20.dist-info/entry_points.txt,sha256=y82XyTeQddM87gCTzgSQaTlKF3VFicO4hhClHUv6j1A,127
-tonik-0.1.20.dist-info/licenses/LICENSE,sha256=OXLcl0T2SZ8Pmy2_dmlvKuetivmyPd5m1q-Gyd-zaYY,35149
-tonik-0.1.20.dist-info/RECORD,,
+tonik-0.1.21.dist-info/METADATA,sha256=rEp5KTN5xDizNLajJkP3i6lvGI02hrR-sqqfIRfn4M0,2207
+tonik-0.1.21.dist-info/WHEEL,sha256=WLgqFyCfm_KASv4WHyYy0P3pM_m7J5L9k2skdKLirC8,87
+tonik-0.1.21.dist-info/entry_points.txt,sha256=y82XyTeQddM87gCTzgSQaTlKF3VFicO4hhClHUv6j1A,127
+tonik-0.1.21.dist-info/licenses/LICENSE,sha256=OXLcl0T2SZ8Pmy2_dmlvKuetivmyPd5m1q-Gyd-zaYY,35149
+tonik-0.1.21.dist-info/RECORD,,