downsampler 0.3.0__tar.gz → 0.3.1__tar.gz

{downsampler-0.3.0/src/downsampler.egg-info → downsampler-0.3.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: downsampler
-Version: 0.3.0
+Version: 0.3.1
 Summary: Timeseries DataFrame downsampling with LTTB, aggregation methods, gap handling, and fidelity testing
 Author-email: Eelco Doornbos <eelco.doornbos@knmi.nl>
 License-Expression: MIT

{downsampler-0.3.0 → downsampler-0.3.1}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "downsampler"
-version = "0.3.0"
+version = "0.3.1"
 description = "Timeseries DataFrame downsampling with LTTB, aggregation methods, gap handling, and fidelity testing"
 readme = "README.md"
 license = "MIT"

{downsampler-0.3.0 → downsampler-0.3.1}/src/downsampler/__init__.py

@@ -65,7 +65,7 @@ from downsampler.ranged import (
     DataFetcher,
 )
 
-__version__ = "0.2.0"
+__version__ = "0.3.1"
 
 __all__ = [
     # Config

{downsampler-0.3.0 → downsampler-0.3.1}/src/downsampler/gaps.py

@@ -331,6 +331,12 @@ def interpolate_small_gaps(
     in the original data are preserved — only the newly created rows get
     interpolated values.
 
+    Each column is interpolated **from its own non-NaN samples**, and any new
+    timestamp outside a column's valid-sample range is left NaN (``np.interp``
+    ``left``/``right`` = NaN, i.e. no edge-clamp and no extrapolation). So a
+    column with a hole larger than ``gap_threshold`` keeps an honest gap at the
+    filled timestamps rather than borrowing another column's edge value.
+
     Args:
         df: DataFrame with DatetimeIndex.
         gap_threshold: Gaps at or above this duration are "real" gaps and
@@ -349,41 +355,63 @@ def interpolate_small_gaps(
         source_cadence = estimate_cadence(df)
 
     jitter_threshold = source_cadence * 1.5
-    deltas = df.index.to_series().diff()
-
     numeric_cols = get_numeric_columns(df)
-    # Convert original timestamps to float64 for np.interp
-    orig_timestamps = df.index.astype(np.int64).astype(np.float64)
 
-    new_rows = []
+    # 1. Row skeleton: existing rows plus synthetic rows inside *index* gaps
+    #    smaller than gap_threshold, at source_cadence, so LTTB sees a
+    #    continuous grid across small gaps. Values are filled per-column below.
+    deltas = df.index.to_series().diff()
+    extra_times: list[pd.Timestamp] = []
     for i in range(1, len(df)):
         delta = deltas.iloc[i]
         if delta > jitter_threshold and delta < gap_threshold:
-            t_before = df.index[i - 1]
-            t_after = df.index[i]
-            # Generate timestamps at source_cadence intervals within the gap
-            new_times = pd.date_range(
-                start=t_before + source_cadence,
-                end=t_after - source_cadence * 0.5,  # don't duplicate t_after
-                freq=source_cadence,
+            extra_times.extend(
+                pd.date_range(
+                    start=df.index[i - 1] + source_cadence,
+                    end=df.index[i] - source_cadence * 0.5,  # don't duplicate t_after
+                    freq=source_cadence,
+                )
             )
-            if len(new_times) == 0:
-                continue
-
-            new_ts_float = new_times.astype(np.int64).astype(np.float64)
-            row_data = {col: np.interp(new_ts_float, orig_timestamps, df[col].values)
-                        for col in numeric_cols}
-            chunk = pd.DataFrame(row_data, index=new_times)
-            # Add non-numeric columns as NaN (they can't be interpolated)
-            for col in df.columns:
-                if col not in numeric_cols:
-                    chunk[col] = np.nan
-            new_rows.append(chunk)
-
-    if not new_rows:
-        return df.copy()
+    if extra_times:
+        out = df.reindex(df.index.append(pd.DatetimeIndex(extra_times)).sort_values())
+    else:
+        out = df.copy()
+
+    # 2. Per-column, fill each column's *own* small gaps from its *own* samples,
+    #    leaving holes wider than gap_threshold NaN. Interpolation is inside-only
+    #    (no extrapolation → no edge-clamp) and each column is independent (no
+    #    cross-column bleed).
+    for col in numeric_cols:
+        out[col] = _interpolate_column_small_gaps(out[col], gap_threshold)
+
+    return out
+
+
+def _interpolate_column_small_gaps(
+    s: pd.Series,
+    gap_threshold: pd.Timedelta,
+) -> pd.Series:
+    """Time-interpolate a single column's gaps narrower than gap_threshold.
+
+    Points outside the column's valid range stay NaN (no extrapolation), and
+    any run of NaNs whose bounding valid samples are gap_threshold or more
+    apart is re-blanked so wide holes remain honest gaps.
+    """
+    valid = s.notna()
+    if valid.sum() < 2:
+        return s
+
+    filled = s.interpolate(method="time", limit_area="inside")
+
+    # Re-blank interpolated points that fall inside a wide (>= gap_threshold)
+    # hole between two consecutive valid samples.
+    idx = s.index
+    valid_pos = np.flatnonzero(valid.to_numpy())
+    for a, b in zip(valid_pos[:-1], valid_pos[1:]):
+        if b > a + 1 and (idx[b] - idx[a]) >= gap_threshold:
+            filled.iloc[a + 1:b] = np.nan
 
-    return pd.concat([df] + new_rows).sort_index()
+    return filled
 
 
 def concatenate_with_gap_markers(

{downsampler-0.3.0 → downsampler-0.3.1}/src/downsampler/lttb.py

@@ -107,11 +107,23 @@ def _lttb_single_segment(
 ) -> pd.DataFrame | None:
     """Apply LTTB to a single contiguous segment.
 
+    LTTB selects real source timetags for the *target* column; every other
+    requested column is then **selected** at those timetags rather than
+    interpolated onto them. ``lttbc.downsample`` returns exact input points,
+    so the selected ``time_num`` values map back to the segment rows
+    unambiguously, and each ride-along column keeps its real measured value
+    (or the small-gap-interpolated value the preprocessing stage placed there).
+    This makes it impossible to fabricate a value: earlier code interpolated
+    include columns with ``np.interp``, which clamps to a column's edge value
+    for any timetag outside that column's valid range — stamping a foreign
+    constant across every gap. Selection cannot do that.
+
     Args:
         df: Input DataFrame (no gaps).
         target_column: Column to optimize for.
         target_cadence: Target cadence.
-        include_columns: Additional columns to include.
+        include_columns: Additional columns to carry (selected, not
+            interpolated). If None, all numeric columns are carried.
 
     Returns:
         Downsampled DataFrame or None if cannot process.
@@ -136,46 +148,29 @@ def _lttb_single_segment(
         return None
 
     # Apply LTTB downsampling (lttbc uses separate x, y arrays)
-    x_down, y_down = lttbc.downsample(
+    x_down, _ = lttbc.downsample(
         df_clean['time_num'].values,
         df_clean[target_column].values,
         n_out
     )
-    df_resampled = pd.DataFrame(
-        {'time_num': x_down, target_column: y_down}
-    )
 
-    # Reconstruct the datetime index
-    df_resampled.index = (
-        timeref +
-        pd.to_timedelta(df_resampled['time_num'], unit='min')
-    )
+    # Row-select the segment rows LTTB chose. ``x_down`` are exact members of
+    # the input ``time_num`` (lttbc returns real input points), so ``isin``
+    # selects exactly those rows — carrying every column's real value and the
+    # original datetime index, no interpolation.
+    selected = df_work[df_work['time_num'].isin(x_down)]
 
-    # Interpolate other columns to LTTB-selected time points. Each column
-    # interpolates from its own non-NaN samples — np.interp propagates NaN
-    # from any NaN in fp, which would blank values near unrelated holes.
+    keep = [target_column]
     for col in df.columns:
-        if col in ['time', 'time_num', target_column]:
+        if col in ('time', 'time_num', target_column):
             continue
         if include_columns is not None and col not in include_columns:
             continue
         if not pd.api.types.is_numeric_dtype(df[col]):
             continue
+        keep.append(col)
 
-        col_valid = df_work[['time_num', col]].dropna()
-        if col_valid.empty:
-            df_resampled[col] = np.nan
-        else:
-            df_resampled[col] = np.interp(
-                x=df_resampled['time_num'].values,
-                xp=col_valid['time_num'].values,
-                fp=col_valid[col].values
-            )
-
-    # Clean up
-    df_resampled = df_resampled.drop(['time_num'], axis=1)
-
-    return df_resampled
+    return selected[keep].copy()
 
 
 def downsample_lttb_with_config(

{downsampler-0.3.0 → downsampler-0.3.1/src/downsampler.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: downsampler
-Version: 0.3.0
+Version: 0.3.1
 Summary: Timeseries DataFrame downsampling with LTTB, aggregation methods, gap handling, and fidelity testing
 Author-email: Eelco Doornbos <eelco.doornbos@knmi.nl>
 License-Expression: MIT

{downsampler-0.3.0 → downsampler-0.3.1}/tests/test_gaps.py

@@ -239,22 +239,36 @@ class TestInterpolateSmallGaps:
         # Small gap filled (2 new points), large gap left alone
         assert len(result) == 8
 
-    def test_preserves_original_nans(self):
-        """Test that pre-existing NaN values are not overwritten."""
-        times = pd.date_range('2024-01-01', periods=5, freq='1min')
-        df = pd.DataFrame(
-            {'value': [0.0, np.nan, 2.0, 3.0, 4.0]},
-            index=times,
+    def test_small_inplace_gap_filled_wide_preserved(self):
+        """Small in-place NaN holes are bridged per-column; wide holes stay NaN.
+
+        (New 0.3.1 contract: small-gap filling is a per-column preprocessing
+        step, so a hole narrower than gap_threshold in an otherwise-dense column
+        is interpolated from that column's own samples, while a hole at or above
+        the threshold is preserved as an honest gap.)
+        """
+        times = pd.date_range('2024-01-01', periods=20, freq='1min')
+        value = np.arange(20, dtype=float)
+        value[3] = np.nan                 # 1-min hole  -> small, should fill
+        value[8:15] = np.nan              # 7-min hole  -> < 10min, should fill
+        df = pd.DataFrame({'value': value}, index=times)
+
+        small = interpolate_small_gaps(
+            df, gap_threshold=pd.Timedelta('10min'),
+            source_cadence=pd.Timedelta('1min'),
         )
-
-        result = interpolate_small_gaps(
-            df,
-            gap_threshold=pd.Timedelta('10min'),
+        # Both holes are narrower than 10 min -> bridged (linear: value == index).
+        assert small['value'].isna().sum() == 0
+        assert small.iloc[3]['value'] == pytest.approx(3.0)
+        assert small.iloc[10]['value'] == pytest.approx(10.0)
+
+        # With a tighter threshold the 7-min hole is now "wide" -> preserved.
+        wide = interpolate_small_gaps(
+            df, gap_threshold=pd.Timedelta('5min'),
             source_cadence=pd.Timedelta('1min'),
         )
-
-        # Original NaN should still be NaN
-        assert np.isnan(result.iloc[1]['value'])
+        assert not np.isnan(wide.iloc[3]['value'])       # 1-min hole still filled
+        assert wide['value'].iloc[8:14].isna().all()     # 7-min hole preserved
 
     def test_auto_cadence_estimation(self):
         """Test that source_cadence is auto-estimated when None."""

{downsampler-0.3.0 → downsampler-0.3.1}/tests/test_lttb.py

@@ -236,12 +236,77 @@ class TestLttbNanEncodedGaps:
             source_cadence=pd.Timedelta('1min'),
         )
 
-        # 'other' is constant 100 outside its hole; interpolation across
-        # the hole must bridge it rather than propagate NaN.
+        # 'other' is constant 100 outside its hole; the small-gap preprocessing
+        # bridges it per-column rather than propagating NaN.
         assert result['other'].isna().sum() == 0
         assert (result['other'] == 100.0).all()
 
 
+class TestLttbIncludeSelection:
+    """Include columns are row-SELECTED at LTTB timetags, never interpolated.
+
+    Regression coverage for the np.interp edge-clamp bug: interpolating an
+    include column onto the target's timetags stamped the column's edge value
+    across every timetag outside its valid range (a constant foreign peak at
+    each gap). Selection cannot fabricate values.
+    """
+
+    def test_high_window_does_not_bleed_into_quiet_period(self):
+        """A column valid only in a late high window must not appear earlier."""
+        idx = pd.date_range('2024-01-01 00:00', periods=24 * 60, freq='1min')
+        signal = 350 + 10 * np.sin(np.arange(len(idx)) / 30.0)  # dense all day
+        other = np.full(len(idx), np.nan)
+        hi = (idx >= '2024-01-01 20:00') & (idx <= '2024-01-01 21:00')
+        other[hi] = 602.9  # the ONLY place 'other' has data
+        df = pd.DataFrame({'signal': signal, 'other': other}, index=idx)
+
+        result = downsample_lttb(
+            df, target_column='signal', target_cadence='PT5M',
+            source_cadence=pd.Timedelta('1min'),
+        )
+
+        # No 'other' value may appear at a timetag outside its valid window,
+        # and certainly not the clamped constant 602.9 in the quiet morning.
+        nonnull = result['other'].dropna()
+        assert (nonnull.index >= pd.Timestamp('2024-01-01 20:00')).all()
+        assert (nonnull.index <= pd.Timestamp('2024-01-01 21:00')).all()
+        morning = result.loc[result.index < pd.Timestamp('2024-01-01 19:00'), 'other']
+        assert morning.isna().all()
+
+    def test_ridealong_wide_hole_stays_nan(self):
+        """A hole wider than gap_threshold in a ride-along column stays NaN."""
+        idx = pd.date_range('2024-01-01', periods=240, freq='1min')
+        signal = 500 + 10 * np.sin(np.arange(240) / 20.0)  # complete target
+        other = np.full(240, 100.0)
+        other[60:150] = np.nan  # 90-min hole >> 30-min gap_threshold (PT15M)
+        df = pd.DataFrame({'signal': signal, 'other': other}, index=idx)
+
+        result = downsample_lttb(
+            df, target_column='signal', target_cadence='PT15M',
+            source_cadence=pd.Timedelta('1min'),
+        )
+
+        in_hole = result[(result.index > idx[60]) & (result.index < idx[149])]
+        assert not in_hole.empty          # target is dense there, so timetags exist
+        assert in_hole['other'].isna().all()
+
+    def test_include_values_are_exact_source_selections(self):
+        """Each carried include value equals the real source sample (not interp)."""
+        idx = pd.date_range('2024-01-01', periods=240, freq='1min')
+        rng = np.random.default_rng(0)
+        signal = 500 + 50 * np.sin(np.arange(240) / 15.0) + rng.normal(0, 3, 240)
+        other = 100 + rng.normal(0, 10, 240)  # noisy, dense (no gaps)
+        df = pd.DataFrame({'signal': signal, 'other': other}, index=idx)
+
+        result = downsample_lttb(
+            df, target_column='signal', target_cadence='PT15M',
+            source_cadence=pd.Timedelta('1min'),
+        )
+
+        for t, v in result['other'].dropna().items():
+            assert v == pytest.approx(df.loc[t, 'other'])
+
+
 class TestLttbShortSegments:
     """Short data islands must survive instead of being dropped."""