downsampler 0.4.0__tar.gz → 0.4.2__tar.gz

{downsampler-0.4.0/src/downsampler.egg-info → downsampler-0.4.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: downsampler
-Version: 0.4.0
+Version: 0.4.2
 Summary: Timeseries DataFrame downsampling with LTTB, aggregation methods, gap handling, and fidelity testing
 Author-email: Eelco Doornbos <eelco.doornbos@knmi.nl>
 License-Expression: MIT

{downsampler-0.4.0 → downsampler-0.4.2}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "downsampler"
-version = "0.4.0"
+version = "0.4.2"
 description = "Timeseries DataFrame downsampling with LTTB, aggregation methods, gap handling, and fidelity testing"
 readme = "README.md"
 license = "MIT"

{downsampler-0.4.0 → downsampler-0.4.2}/src/downsampler/__init__.py

@@ -31,6 +31,8 @@ Example:
     >>> result = downsample_dataframe(df, target_cadence='5min', config=config)
 """
 
+import importlib.metadata as _metadata
+
 from downsampler.config import (
     AggregationMethod,
     EdgeHandling,
@@ -66,7 +68,13 @@ from downsampler.ranged import (
     DataFetcher,
 )
 
-__version__ = "0.3.1"
+# Single source of truth is pyproject.toml; read it back from the installed
+# distribution so this constant can never drift from the released version
+# (it sat at 0.3.1 through the 0.4.0 release).
+try:
+    __version__ = _metadata.version("downsampler")
+except _metadata.PackageNotFoundError:  # running from a source tree, not installed
+    __version__ = "unknown"
 
 __all__ = [
     # Config

{downsampler-0.4.0 → downsampler-0.4.2}/src/downsampler/lttb.py

@@ -107,7 +107,74 @@ def downsample_lttb(
             df_in.index, _output_columns(df_in, target_column, include_columns)
         )
 
-    return concatenate_with_gap_markers(resampled_segments)
+    result = concatenate_with_gap_markers(resampled_segments)
+    return _mark_uncovered_edges(result, df_in.index, df_valid.index, gap_threshold)
+
+
+def _mark_uncovered_edges(
+    result: pd.DataFrame,
+    covered: pd.DatetimeIndex,
+    valid: pd.DatetimeIndex,
+    gap_threshold: pd.Timedelta,
+    offset: pd.Timedelta = pd.Timedelta('0.1s'),
+) -> pd.DataFrame:
+    """Mark a hole that touches the frame's edge instead of its interior.
+
+    **Tactical, and written to be deleted.** Step 7 of the caller's
+    ``docs/plans/gap-representation-claimed-windows.md`` retires this whole
+    function once a partition stores its claimed window, at which point the
+    exact extent is known and no heuristic is needed. Do not build on it.
+
+    ``downsample_lttb`` opens with ``dropna``, which collapses the frame's
+    *coverage* onto its *valid extent*: a day covering 00:00-23:59 whose data
+    stops at 22:01 becomes, to everything downstream, a day that simply ends at
+    22:01. Markers then come only from ``concatenate_with_gap_markers``, which
+    inserts them *between* segments -- so a batch with one surviving segment
+    emits none anywhere, by construction. A gap straddling midnight is a
+    trailing hole in day N and a leading hole in day N+1, and neither is
+    "between two segments", so it vanishes and a plot draws a confident
+    straight line across twelve missing hours.
+
+    The padding is the evidence, and it is still in ``covered``. Where the
+    frame's coverage runs past its valid data by enough to matter, a marker
+    goes just outside the valid extent -- ``first_valid - offset`` /
+    ``last_valid + offset``, mirroring the interior convention and staying
+    strictly inside the batch window so the pipeline's trim-to-range keeps it.
+
+    **Why half the threshold.** A hole touching the frame edge is a *lower
+    bound* on the true gap; the rest of it lives in the neighbouring batch.
+    Testing each side at the full threshold would miss any gap the boundary
+    bisects. At ``gap_threshold / 2`` the two halves sum to the threshold, so
+    no gap at or above the threshold can escape by straddling.
+
+    Known residual, accepted: a straddling gap in ``[threshold/2, threshold)``
+    whose other side is exactly zero gets marked where an interior gap of the
+    same width would be interpolated across. Narrow, and conservative in
+    direction -- a break shown where a line could have been drawn.
+
+    This is not needed for ``downsample_m4``, which never drops NaN rows: a
+    padded edge is selected as its own buckets' extrema and reaches the output
+    as marker rows already. Measured 2026-09-05, in both the padded and the
+    inherited-marker shapes.
+    """
+    if len(covered) == 0 or len(valid) == 0:
+        return result
+
+    half = gap_threshold / 2
+    marker_times = []
+    if valid[0] - covered[0] >= half:
+        marker_times.append(valid[0] - offset)
+    if covered[-1] - valid[-1] >= half:
+        marker_times.append(valid[-1] + offset)
+
+    if not marker_times:
+        return result
+
+    markers = pd.DataFrame(
+        {col: [np.nan] * len(marker_times) for col in result.columns},
+        index=pd.DatetimeIndex(marker_times, name=result.index.name),
+    )
+    return pd.concat([result, markers]).sort_index()
 
 
 def _output_columns(

{downsampler-0.4.0 → downsampler-0.4.2/src/downsampler.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: downsampler
-Version: 0.4.0
+Version: 0.4.2
 Summary: Timeseries DataFrame downsampling with LTTB, aggregation methods, gap handling, and fidelity testing
 Author-email: Eelco Doornbos <eelco.doornbos@knmi.nl>
 License-Expression: MIT

{downsampler-0.4.0 → downsampler-0.4.2}/src/downsampler.egg-info/SOURCES.txt

@@ -25,5 +25,6 @@ tests/test_edges.py
 tests/test_fidelity.py
 tests/test_gaps.py
 tests/test_lttb.py
+tests/test_lttb_edge_markers.py
 tests/test_m4.py
 tests/test_ranged.py

downsampler-0.4.2/tests/test_lttb_edge_markers.py

@@ -0,0 +1,119 @@
+"""Edge markers for holes that reach a frame's boundary.
+
+``downsample_lttb`` opens with ``dropna``, which collapses a frame's *coverage*
+onto its *valid extent*. A day covering 00:00-23:59 whose data stops at 22:01
+becomes, downstream, a day that simply ends at 22:01 — and since markers are
+inserted only *between* segments, a batch left with one segment emits none at
+all. A gap straddling midnight is a trailing hole in day N and a leading hole
+in day N+1, neither of which is "between two segments", so it disappears and a
+renderer draws a straight line across it.
+
+Reported from a live ``ace_mag_rtsw_archive_lttb_PT15M`` plot showing exactly
+that: a confident interpolated line across twelve missing hours on
+2024-01-03/04. ``_mark_uncovered_edges`` reads the coverage back off the input
+index, which ``dropna`` never touched.
+"""
+
+import numpy as np
+import pandas as pd
+import pytest
+
+from downsampler.lttb import downsample_lttb
+
+
+def day(start: str, periods: int = 1440) -> pd.DatetimeIndex:
+    return pd.date_range(start, periods=periods, freq="1min", tz="UTC")
+
+
+def padded(index: pd.DatetimeIndex, *nan_spans: tuple[str, str]) -> pd.DataFrame:
+    """A NaN-padded master: cadence preserved, invalid samples blanked."""
+    df = pd.DataFrame({"value": np.sin(np.arange(len(index)) / 100.0)}, index=index)
+    for lo, hi in nan_spans:
+        span = (index >= pd.Timestamp(lo, tz="UTC")) & (index < pd.Timestamp(hi, tz="UTC"))
+        df.loc[span, "value"] = np.nan
+    return df
+
+
+def markers(result: pd.DataFrame) -> list[pd.Timestamp]:
+    return list(result.index[result["value"].isna()])
+
+
+class TestTheReportedDefect:
+    """The real ACE 2024-01-03/04 shape, one day at a time."""
+
+    def test_trailing_hole_is_marked(self):
+        """Day N: valid to 22:01, padded to 23:59. A marker just past the
+        valid extent tells the reader the day did not simply end there."""
+        df = padded(day("2024-01-03"), ("2024-01-03T22:02", "2024-01-04"))
+        result = downsample_lttb(df, target_column="value", target_cadence="PT15M")
+        assert markers(result) == [
+            pd.Timestamp("2024-01-03T22:01:00.1", tz="UTC")
+        ]
+
+    def test_leading_hole_is_marked(self):
+        """Day N+1: padded from midnight, valid from 10:11."""
+        df = padded(day("2024-01-04"), ("2024-01-04", "2024-01-04T10:11"))
+        result = downsample_lttb(df, target_column="value", target_cadence="PT15M")
+        assert markers(result) == [
+            pd.Timestamp("2024-01-04T10:10:59.9", tz="UTC")
+        ]
+
+    def test_the_two_halves_compose(self):
+        """Read together, the day-batched output declares the same single hole
+        the unbatched run does — the invariant the defect broke."""
+        whole = padded(
+            pd.date_range("2024-01-03", periods=2880, freq="1min", tz="UTC"),
+            ("2024-01-03T22:02", "2024-01-04T10:11"),
+        )
+        one_pass = downsample_lttb(whole, target_column="value", target_cadence="PT15M")
+        split = pd.concat([
+            downsample_lttb(whole[whole.index.day == d], target_column="value",
+                            target_cadence="PT15M")
+            for d in (3, 4)
+        ]).sort_index()
+
+        def hole(out):
+            real = out.index[out["value"].notna()]
+            gap = max(zip(real[:-1], real[1:]), key=lambda p: p[1] - p[0])
+            return gap, any(gap[0] < m < gap[1] for m in markers(out))
+
+        assert hole(split) == hole(one_pass)
+
+
+class TestNoSpuriousMarkers:
+    """§2.4 of the plan: a gapless year through the real ladder produced zero
+    spurious markers, and that must survive this change."""
+
+    def test_a_complete_frame_is_not_marked(self):
+        df = padded(day("2024-01-03"))
+        result = downsample_lttb(df, target_column="value", target_cadence="PT15M")
+        assert markers(result) == []
+
+    @pytest.mark.parametrize("pad_minutes, marked", [
+        (1, False),    # a single dropped sample is not a gap
+        (5, False),
+        (14, False),   # just under half the 30-minute threshold
+        (15, True),    # exactly half
+        (60, True),
+    ])
+    def test_half_the_threshold_is_the_cutoff(self, pad_minutes, marked):
+        """A hole at the edge is a lower bound on the true gap, the rest being
+        in the neighbouring batch. Half the threshold on each side sums to the
+        threshold, so no gap at or above it escapes by straddling — and nothing
+        below it is marked, which is what keeps small dropouts vanishing on
+        zoom-out instead of proliferating markers up the ladder."""
+        index = day("2024-01-03")
+        cut = index[-pad_minutes]
+        df = padded(index, (str(cut), "2024-01-04"))
+        result = downsample_lttb(df, target_column="value", target_cadence="PT15M")
+        assert bool(markers(result)) is marked
+
+    def test_a_wholly_invalid_frame_still_takes_the_existing_path(self):
+        """``gap_marker_frame`` already brackets that case; the edge rule must
+        not double-mark it."""
+        df = padded(day("2024-01-03"), ("2024-01-03", "2024-01-04"))
+        result = downsample_lttb(df, target_column="value", target_cadence="PT15M")
+        assert markers(result) == [
+            pd.Timestamp("2024-01-03", tz="UTC"),
+            pd.Timestamp("2024-01-03T23:59", tz="UTC"),
+        ]