tonik 0.1.19__py3-none-any.whl → 0.1.21__py3-none-any.whl

tonik/storage.py

@@ -18,7 +18,7 @@ LOGGING_CONFIG = {
             "datefmt": "%Y-%m-%d %H:%M:%S",  # How to display dates
         },
         "json": {  # The formatter name
-            "()": "pythonjsonlogger.jsonlogger.JsonFormatter",  # The class to instantiate!
+            "()": "pythonjsonlogger.json.JsonFormatter",  # The class to instantiate!
             # Json is more complex, but easier to read, display all attributes!
             "format": """
                     asctime: %(asctime)s

tonik/utils.py

@@ -1,4 +1,4 @@
-from typing import List
+from typing import List, Union
 from datetime import datetime, timezone, timedelta
 
 import numpy as np
@@ -13,12 +13,40 @@ def generate_test_data(dim=1, ndays=30, nfreqs=10,
                        freq_names=None, add_nans=True):
     """
     Generate a 1D or 2D feature for testing.
+
+    Parameters
+    ----------
+    dim : int
+        Dimension of the data (1 or 2).
+    ndays : int
+        Number of days to generate data for.
+    nfreqs : int
+        Number of frequencies (only for dim=2).
+    tstart : datetime
+        Start time of the data.
+    freq : str
+        Frequency of the data (e.g., '10min').
+    intervals : int
+        Number of intervals to generate. If None, calculated from ndays and freq.
+    feature_names : list
+        Names of the features to generate.
+    seed : int
+        Random seed for reproducibility.
+    freq_names : list
+        Names of the frequency dimensions (only for dim=2).
+    add_nans : bool
+        Whether to add NaN values to the data.
+
+    Returns
+    -------
+    xr.Dataset
+        Generated test dataset.
     """
     assert dim < 3
     assert dim > 0
 
     if intervals is None:
-        nints = ndays * 6 * 24
+        nints = ndays * int(pd.Timedelta('1h')/pd.Timedelta(freq)) * 24
     else:
         nints = intervals
     dates = pd.date_range(tstart, freq=freq, periods=nints)
@@ -59,6 +87,116 @@ def generate_test_data(dim=1, ndays=30, nfreqs=10,
     return xds
 
 
+def round_datetime(dt: datetime, interval: Union[int, float, timedelta]) -> datetime:
+    """
+    Find closest multiple of interval to given time.
+
+    Parameters:
+    -----------
+    dt : datetime
+        The datetime to round.
+    interval : Union[int, float, timedelta]
+        The interval to which to round the datetime.
+
+    Returns:
+    --------
+    datetime
+        The rounded datetime.
+    """
+    # Normalize interval to whole seconds (supports float/timedelta inputs)
+    if isinstance(interval, timedelta):
+        interval_sec = int(interval.total_seconds())
+    else:
+        interval_sec = int(interval)
+
+    if interval_sec <= 0:
+        raise ValueError("interval must be positive (seconds)")
+
+    # Accept ObsPy UTCDateTime transparently (preserve type on return)
+    _is_obspy = False
+    try:
+        from obspy import UTCDateTime as _UTCDateTime  # type: ignore
+        if isinstance(dt, _UTCDateTime):
+            _is_obspy = True
+            dt_py = dt.datetime  # Python datetime in UTC
+        else:
+            dt_py = dt
+    except Exception:
+        dt_py = dt
+
+    epoch = (
+        datetime(1970, 1, 1)
+        if dt_py.tzinfo is None
+        else datetime(1970, 1, 1, tzinfo=dt_py.tzinfo)
+    )
+
+    # Compute integer seconds since epoch to avoid float precision issues
+    seconds = int((dt_py - epoch).total_seconds())
+    floored = (seconds + 0.5 * interval_sec) % interval_sec
+    rounded = epoch + timedelta(seconds=seconds + 0.5 * interval_sec - floored)
+
+    if _is_obspy:
+        from obspy import UTCDateTime as _UTCDateTime  # type: ignore
+        return _UTCDateTime(rounded)
+
+    return rounded
+
+
+def floor_datetime(dt: datetime, interval: Union[int, float, timedelta]) -> datetime:
+    """
+    Floor a datetime to the latest multiple of a given interval.
+
+    Assumes ``dt`` represents a UTC time (naive or tz-aware is fine) and
+    aligns against the Unix epoch 1970-01-01T00:00:00Z. The interval is in
+    seconds (int/float) or a timedelta. Returns a datetime with the same
+    "naive vs aware" form as ``dt``.
+
+    Examples
+    --------
+    >>> from datetime import datetime
+    >>> floor_datetime(datetime.fromisoformat('2025-11-27T10:12:43'), 600)
+    datetime.datetime(2025, 11, 27, 10, 10)
+    """
+
+    # Normalize interval to whole seconds (supports float/timedelta inputs)
+    if isinstance(interval, timedelta):
+        interval_sec = int(interval.total_seconds())
+    else:
+        interval_sec = int(interval)
+
+    if interval_sec <= 0:
+        raise ValueError("interval must be positive (seconds)")
+
+    # Accept ObsPy UTCDateTime transparently (preserve type on return)
+    _is_obspy = False
+    try:
+        from obspy import UTCDateTime as _UTCDateTime  # type: ignore
+        if isinstance(dt, _UTCDateTime):
+            _is_obspy = True
+            dt_py = dt.datetime  # Python datetime in UTC
+        else:
+            dt_py = dt
+    except Exception:
+        dt_py = dt
+
+    epoch = (
+        datetime(1970, 1, 1)
+        if dt_py.tzinfo is None
+        else datetime(1970, 1, 1, tzinfo=dt_py.tzinfo)
+    )
+
+    # Compute integer seconds since epoch to avoid float precision issues
+    seconds = int((dt_py - epoch).total_seconds())
+    floored = seconds - (seconds % interval_sec)
+    rounded = epoch + timedelta(seconds=floored)
+
+    if _is_obspy:
+        from obspy import UTCDateTime as _UTCDateTime  # type: ignore
+        return _UTCDateTime(rounded)
+
+    return rounded
+
+
 def get_dt(times):
     """
     Infer the sampling of the time dimension.

tonik/xarray2zarr.py

@@ -267,7 +267,7 @@ def xarray2zarr(xds: xr.Dataset, path: str, mode: str = 'a', group='original',
             continue
 
         if xds_existing[timedim][0] > xds[timedim][-1]:
-            # prepend
+            logger.debug("Prepending data to existing zarr store.")
             xda_new = fill_time_gaps_between_datasets(xds_existing[feature].isel({timedim: 0}),
                                                       xds[feature], mode='p')
             xda_new = _build_append_payload_full_chunks(
@@ -277,7 +277,7 @@ def xarray2zarr(xds: xr.Dataset, path: str, mode: str = 'a', group='original',
                                                        write_empty_chunks=True)
 
         elif xds_existing[timedim][-1] < xds[timedim][0]:
-            # append
+            logger.debug("Appending data to existing zarr store.")
             xda_new = fill_time_gaps_between_datasets(xds_existing[feature].isel({timedim: -1}),
                                                       xds[feature], mode='a')
             xda_new = _build_append_payload_full_chunks(
@@ -286,13 +286,19 @@ def xarray2zarr(xds: xr.Dataset, path: str, mode: str = 'a', group='original',
                             append_dim=timedim)
 
         elif xds_existing[timedim][0] > xds[timedim][0] and xds_existing[timedim][-1] < xds[timedim][-1]:
-            # existing datetimes are contained in new array
+            logger.debug(
+                "Data in zarr store contained in new data. Rewriting zarr store.")
             xda_new = _build_append_payload_full_chunks(
                 xds[feature], 'a', nchunks)
             xda_new.to_zarr(fout, group=group, mode='w',
                             write_empty_chunks=True)
 
         else:
+            logger.debug("Data in zarr store overlaps with new data.")
+            logger.debug(
+                f"Endtime of existing data: {xds_existing[timedim][-1].values}")
+            logger.debug(f"Starttime of new data: {xds[timedim][0].values}")
+            xds_existing = xds_existing.drop_duplicates(timedim, keep='last')
             overlap = xds_existing[timedim].where(
                 xds_existing[timedim] == xds[timedim])
             xds[feature].loc[{timedim: overlap}].to_zarr(

{tonik-0.1.19.dist-info → tonik-0.1.21.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: tonik
-Version: 0.1.19
+Version: 0.1.21
 Summary: Store time series data as HDF5 files and access them through an API.
 Project-URL: Homepage, https://tsc-tools.github.io/tonik
 Project-URL: Issues, https://github.com/tsc-tools/tonik/issues

{tonik-0.1.19.dist-info → tonik-0.1.21.dist-info}/RECORD

@@ -1,14 +1,14 @@
 tonik/__init__.py,sha256=dov-nMeGFBzLspmj4rWKjC4r736vmaPDgMEkHSUfP98,523
 tonik/api.py,sha256=vW0ykOo5iGAV0_WuOepdrnUyFp83F7KyJTd43ksLmUk,7985
 tonik/grafana_annotations.py,sha256=ZU9Cy-HT4vvMfYIQzD9WboaDVOCBDv__NmXbk1qKWJo,5838
-tonik/storage.py,sha256=jcCVx2N8J1ZBKM73k-OaxB0uxukn4VAM_-CCaCeAKwk,10589
-tonik/utils.py,sha256=vRFMoCU7dbfnnm5RALBR-XrpPGDFtQoeTDzxFiYf3bo,7522
+tonik/storage.py,sha256=bYBl3JPpH8D3iIFOj5AZQXc4M8txAbwFKt4xTfdotgg,10583
+tonik/utils.py,sha256=GwAXfGFQWhlsLThQvSux1SooRkW-iIkJP99JMH72t5Y,11791
 tonik/xarray2netcdf.py,sha256=nq6RHk5ciaAg1bxNDiyHPRdAts1C7fj7jtDbaLaSTWM,6497
-tonik/xarray2zarr.py,sha256=Dg9_b6Zwj_UQMhOez6wrPeHn0rUffUHqbv-e4pP_t3w,11233
+tonik/xarray2zarr.py,sha256=kRhgDdo8CDT1ceszwQEQNfXdgbnmL5nNejUzaMnyFXM,11707
 tonik/package_data/index.html,sha256=ZCZ-BtGRERsL-6c_dfY43qd2WAaggH7xereennGL6ww,4372
 tonik/package_data/whakaari_labels.json,sha256=96UZSq41yXgAJxuKivLBKlRTw-33jkjh7AGKTsDQ9Yg,3993
-tonik-0.1.19.dist-info/METADATA,sha256=IdmsIGrgcA1LV2Ji59HlyeN9ykMT4n6iId6YcPO936Q,2207
-tonik-0.1.19.dist-info/WHEEL,sha256=WLgqFyCfm_KASv4WHyYy0P3pM_m7J5L9k2skdKLirC8,87
-tonik-0.1.19.dist-info/entry_points.txt,sha256=y82XyTeQddM87gCTzgSQaTlKF3VFicO4hhClHUv6j1A,127
-tonik-0.1.19.dist-info/licenses/LICENSE,sha256=OXLcl0T2SZ8Pmy2_dmlvKuetivmyPd5m1q-Gyd-zaYY,35149
-tonik-0.1.19.dist-info/RECORD,,
+tonik-0.1.21.dist-info/METADATA,sha256=rEp5KTN5xDizNLajJkP3i6lvGI02hrR-sqqfIRfn4M0,2207
+tonik-0.1.21.dist-info/WHEEL,sha256=WLgqFyCfm_KASv4WHyYy0P3pM_m7J5L9k2skdKLirC8,87
+tonik-0.1.21.dist-info/entry_points.txt,sha256=y82XyTeQddM87gCTzgSQaTlKF3VFicO4hhClHUv6j1A,127
+tonik-0.1.21.dist-info/licenses/LICENSE,sha256=OXLcl0T2SZ8Pmy2_dmlvKuetivmyPd5m1q-Gyd-zaYY,35149
+tonik-0.1.21.dist-info/RECORD,,