downsampler 0.2.0__tar.gz → 0.3.1__tar.gz

{downsampler-0.2.0/src/downsampler.egg-info → downsampler-0.3.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: downsampler
-Version: 0.2.0
+Version: 0.3.1
 Summary: Timeseries DataFrame downsampling with LTTB, aggregation methods, gap handling, and fidelity testing
 Author-email: Eelco Doornbos <eelco.doornbos@knmi.nl>
 License-Expression: MIT
@@ -22,12 +22,6 @@ Requires-Dist: pandas>=1.3
 Requires-Dist: lttbc>=0.3
 Requires-Dist: scipy>=1.7
 Requires-Dist: requests>=2.32.5
-Provides-Extra: test
-Requires-Dist: pytest>=7.0; extra == "test"
-Requires-Dist: pytest-cov>=4.0; extra == "test"
-Provides-Extra: dev
-Requires-Dist: downsampler[test]; extra == "dev"
-Requires-Dist: marimo; extra == "dev"
 Dynamic: license-file
 
 # downsampler
@@ -57,6 +51,13 @@ A Python package for time series DataFrame downsampling with LTTB, M4, multiple
 pip install downsampler
 ```
 
+> **Note (Linux + Python 3.11):** the `lttbc` dependency's prebuilt cp311
+> Linux wheel was compiled against NumPy 1.x and fails to import under
+> NumPy 2 (`numpy.core.multiarray failed to import`). Force a source build:
+> `pip install --no-binary lttbc downsampler`. Other Python versions and
+> macOS have no prebuilt wheel and build from source automatically. With
+> uv, this repo's `[tool.uv] no-binary-package` setting handles it.
+
 ## Quick Start
 
 ### Basic Downsampling
@@ -237,7 +238,7 @@ print(summary_table(results))
 | `gap_threshold` | str/Timedelta | "auto" | Min duration for gaps |
 | `edge_handling` | EdgeHandling | KEEP | How to handle edges |
 | `edge_window` | int | 2 | Points at each edge |
-| `min_points_per_segment` | int | 3 | Min points for processing |
+| `min_points_per_segment` | int | 1 | Min points per segment; smaller segments are dropped |
 | `min_completeness` | float | 0.9 | Min fraction of expected points per bucket |
 | `source_cadence` | str/Timedelta | None | Source data cadence (estimated if None) |
 

{downsampler-0.2.0 → downsampler-0.3.1}/README.md

@@ -25,6 +25,13 @@ A Python package for time series DataFrame downsampling with LTTB, M4, multiple
 pip install downsampler
 ```
 
+> **Note (Linux + Python 3.11):** the `lttbc` dependency's prebuilt cp311
+> Linux wheel was compiled against NumPy 1.x and fails to import under
+> NumPy 2 (`numpy.core.multiarray failed to import`). Force a source build:
+> `pip install --no-binary lttbc downsampler`. Other Python versions and
+> macOS have no prebuilt wheel and build from source automatically. With
+> uv, this repo's `[tool.uv] no-binary-package` setting handles it.
+
 ## Quick Start
 
 ### Basic Downsampling
@@ -205,7 +212,7 @@ print(summary_table(results))
 | `gap_threshold` | str/Timedelta | "auto" | Min duration for gaps |
 | `edge_handling` | EdgeHandling | KEEP | How to handle edges |
 | `edge_window` | int | 2 | Points at each edge |
-| `min_points_per_segment` | int | 3 | Min points for processing |
+| `min_points_per_segment` | int | 1 | Min points per segment; smaller segments are dropped |
 | `min_completeness` | float | 0.9 | Min fraction of expected points per bucket |
 | `source_cadence` | str/Timedelta | None | Source data cadence (estimated if None) |
 

{downsampler-0.2.0 → downsampler-0.3.1}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "downsampler"
-version = "0.2.0"
+version = "0.3.1"
 description = "Timeseries DataFrame downsampling with LTTB, aggregation methods, gap handling, and fidelity testing"
 readme = "README.md"
 license = "MIT"
@@ -30,13 +30,22 @@ dependencies = [
     "requests>=2.32.5",
 ]
 
-[project.optional-dependencies]
+[dependency-groups]
 test = ["pytest>=7.0", "pytest-cov>=4.0"]
-dev = ["downsampler[test]", "marimo"]
+dev = [{include-group = "test"}, "marimo"]
 
 [project.urls]
 Homepage = "https://gitlab.com/KNMI-OSS/spaceweather/libs/downsampler"
 Repository = "https://gitlab.com/KNMI-OSS/spaceweather/libs/downsampler"
 
+[tool.uv]
+# lttbc's prebuilt cp311 Linux wheel was compiled against NumPy 1.x and
+# crashes on import under NumPy 2 ("numpy.core.multiarray failed to
+# import"). Building from source compiles it against the installed NumPy.
+# Other Python versions ship no Linux wheel and always build from source.
+# Same workaround as in spaceweather-data-pipelines; drop when lttbc
+# publishes NumPy-2 wheels.
+no-binary-package = ["lttbc"]
+
 [tool.setuptools.packages.find]
 where = ["src"]

{downsampler-0.2.0 → downsampler-0.3.1}/src/downsampler/__init__.py

@@ -65,7 +65,7 @@ from downsampler.ranged import (
     DataFetcher,
 )
 
-__version__ = "0.2.0"
+__version__ = "0.3.1"
 
 __all__ = [
     # Config

{downsampler-0.2.0 → downsampler-0.3.1}/src/downsampler/config.py

@@ -46,7 +46,9 @@ class DownsampleConfig:
             "auto" means 2x the target cadence.
         edge_handling: Strategy for handling edge points.
         edge_window: Number of points at each edge to consider as edge points.
-        min_points_per_segment: Minimum points required in a segment for processing.
+        min_points_per_segment: Minimum points required in a segment for
+            processing. Default 1: even single-point data islands between
+            gaps are kept, so intermittent data survives downsampling.
         source_cadence: Expected cadence of the source data. Used by LTTB to
             interpolate small gaps and by aggregators for completeness
             calculation. If None, estimated from data.
@@ -64,7 +66,7 @@ class DownsampleConfig:
     gap_threshold: Union[str, pd.Timedelta] = "auto"
     edge_handling: EdgeHandling = EdgeHandling.KEEP
     edge_window: int = 2
-    min_points_per_segment: int = 3
+    min_points_per_segment: int = 1
     source_cadence: Union[str, pd.Timedelta, None] = None
     min_completeness: float = 0.9
 

{downsampler-0.2.0 → downsampler-0.3.1}/src/downsampler/gaps.py

@@ -68,9 +68,14 @@ def groupby_gaps(
         2
     """
     df_work = df.copy()
-    deltas = df_work.index.diff()[1:]
-    gap_indices = (deltas >= timedelta_max_gap).cumsum()
-    df_work['gap_index'] = [0, *gap_indices]
+    if len(df_work) == 0:
+        # [0, *gap_indices] below would fabricate a spurious row on an
+        # empty frame; an empty gap_index column yields zero groups.
+        df_work['gap_index'] = np.array([], dtype=np.int64)
+    else:
+        deltas = df_work.index.diff()[1:]
+        gap_indices = (deltas >= timedelta_max_gap).cumsum()
+        df_work['gap_index'] = [0, *gap_indices]
     dfs_out = df_work.groupby('gap_index')
     return dfs_out
 
@@ -326,6 +331,12 @@ def interpolate_small_gaps(
     in the original data are preserved — only the newly created rows get
     interpolated values.
 
+    Each column is interpolated **from its own non-NaN samples**, and any new
+    timestamp outside a column's valid-sample range is left NaN (``np.interp``
+    ``left``/``right`` = NaN, i.e. no edge-clamp and no extrapolation). So a
+    column with a hole larger than ``gap_threshold`` keeps an honest gap at the
+    filled timestamps rather than borrowing another column's edge value.
+
     Args:
         df: DataFrame with DatetimeIndex.
         gap_threshold: Gaps at or above this duration are "real" gaps and
@@ -344,41 +355,63 @@ def interpolate_small_gaps(
         source_cadence = estimate_cadence(df)
 
     jitter_threshold = source_cadence * 1.5
-    deltas = df.index.to_series().diff()
-
     numeric_cols = get_numeric_columns(df)
-    # Convert original timestamps to float64 for np.interp
-    orig_timestamps = df.index.astype(np.int64).astype(np.float64)
 
-    new_rows = []
+    # 1. Row skeleton: existing rows plus synthetic rows inside *index* gaps
+    #    smaller than gap_threshold, at source_cadence, so LTTB sees a
+    #    continuous grid across small gaps. Values are filled per-column below.
+    deltas = df.index.to_series().diff()
+    extra_times: list[pd.Timestamp] = []
     for i in range(1, len(df)):
         delta = deltas.iloc[i]
         if delta > jitter_threshold and delta < gap_threshold:
-            t_before = df.index[i - 1]
-            t_after = df.index[i]
-            # Generate timestamps at source_cadence intervals within the gap
-            new_times = pd.date_range(
-                start=t_before + source_cadence,
-                end=t_after - source_cadence * 0.5,  # don't duplicate t_after
-                freq=source_cadence,
+            extra_times.extend(
+                pd.date_range(
+                    start=df.index[i - 1] + source_cadence,
+                    end=df.index[i] - source_cadence * 0.5,  # don't duplicate t_after
+                    freq=source_cadence,
+                )
             )
-            if len(new_times) == 0:
-                continue
-
-            new_ts_float = new_times.astype(np.int64).astype(np.float64)
-            row_data = {col: np.interp(new_ts_float, orig_timestamps, df[col].values)
-                        for col in numeric_cols}
-            chunk = pd.DataFrame(row_data, index=new_times)
-            # Add non-numeric columns as NaN (they can't be interpolated)
-            for col in df.columns:
-                if col not in numeric_cols:
-                    chunk[col] = np.nan
-            new_rows.append(chunk)
-
-    if not new_rows:
-        return df.copy()
+    if extra_times:
+        out = df.reindex(df.index.append(pd.DatetimeIndex(extra_times)).sort_values())
+    else:
+        out = df.copy()
+
+    # 2. Per-column, fill each column's *own* small gaps from its *own* samples,
+    #    leaving holes wider than gap_threshold NaN. Interpolation is inside-only
+    #    (no extrapolation → no edge-clamp) and each column is independent (no
+    #    cross-column bleed).
+    for col in numeric_cols:
+        out[col] = _interpolate_column_small_gaps(out[col], gap_threshold)
+
+    return out
+
+
+def _interpolate_column_small_gaps(
+    s: pd.Series,
+    gap_threshold: pd.Timedelta,
+) -> pd.Series:
+    """Time-interpolate a single column's gaps narrower than gap_threshold.
+
+    Points outside the column's valid range stay NaN (no extrapolation), and
+    any run of NaNs whose bounding valid samples are gap_threshold or more
+    apart is re-blanked so wide holes remain honest gaps.
+    """
+    valid = s.notna()
+    if valid.sum() < 2:
+        return s
+
+    filled = s.interpolate(method="time", limit_area="inside")
+
+    # Re-blank interpolated points that fall inside a wide (>= gap_threshold)
+    # hole between two consecutive valid samples.
+    idx = s.index
+    valid_pos = np.flatnonzero(valid.to_numpy())
+    for a, b in zip(valid_pos[:-1], valid_pos[1:]):
+        if b > a + 1 and (idx[b] - idx[a]) >= gap_threshold:
+            filled.iloc[a + 1:b] = np.nan
 
-    return pd.concat([df] + new_rows).sort_index()
+    return filled
 
 
 def concatenate_with_gap_markers(

{downsampler-0.2.0 → downsampler-0.3.1}/src/downsampler/lttb.py

@@ -1,6 +1,5 @@
 """LTTB (Largest Triangle Three Buckets) downsampling with gap handling."""
 
-import logging
 import pandas as pd
 import numpy as np
 import lttbc
@@ -17,7 +16,7 @@ def downsample_lttb(
     target_cadence: str | pd.Timedelta,
     include_columns: list[str] | None = None,
     gap_threshold: pd.Timedelta | None = None,
-    min_points_per_segment: int = 3,
+    min_points_per_segment: int = 1,
     source_cadence: pd.Timedelta | None = None,
 ) -> pd.DataFrame:
     """Perform LTTB downsampling on a pandas DataFrame.
@@ -26,10 +25,16 @@ def downsample_lttb(
     preserves visual characteristics of the data by selecting points that
     maximize the area of triangles formed with adjacent buckets.
 
-    Small gaps (below gap_threshold) are filled by linear interpolation
-    before downsampling so LTTB receives continuous segments. Large gaps
-    (at/above gap_threshold) split the data into segments processed
-    independently, with NaN marker rows inserted between them.
+    Rows where the target column is NaN are treated as absent data, so
+    fill-encoded gaps (NaN rows present at nominal cadence) behave exactly
+    like missing-row gaps. Small gaps (below gap_threshold) are filled by
+    linear interpolation before downsampling so LTTB receives continuous
+    segments. Large gaps (at/above gap_threshold) split the data into
+    segments processed independently, with NaN marker rows inserted between
+    them. Segments always keep their first and last points, so gap
+    boundaries in the output match the source data exactly. Segments too
+    short for triangle selection are passed through as their first/last
+    points rather than dropped.
 
     Args:
         df_in: Input DataFrame with DatetimeIndex.
@@ -60,8 +65,16 @@ def downsample_lttb(
     if gap_threshold is None:
         gap_threshold = 2 * target_cadence
 
+    # A row without a valid target sample carries no signal for LTTB. Drop
+    # such rows before gap detection so that fill-encoded gaps (NaN rows at
+    # nominal cadence, as produced by CDF/NetCDF fill values) segment the
+    # data exactly like missing-row gaps.
+    df_valid = df_in.dropna(subset=[target_column])
+    if df_valid.empty:
+        return pd.DataFrame(columns=df_in.columns)
+
     # Interpolate small gaps so LTTB receives continuous input
-    df_interp = interpolate_small_gaps(df_in, gap_threshold, source_cadence)
+    df_interp = interpolate_small_gaps(df_valid, gap_threshold, source_cadence)
 
     # Split at large gaps and process each segment
     segments = split_at_gaps(df_interp, gap_threshold)
@@ -94,21 +107,31 @@ def _lttb_single_segment(
 ) -> pd.DataFrame | None:
     """Apply LTTB to a single contiguous segment.
 
+    LTTB selects real source timetags for the *target* column; every other
+    requested column is then **selected** at those timetags rather than
+    interpolated onto them. ``lttbc.downsample`` returns exact input points,
+    so the selected ``time_num`` values map back to the segment rows
+    unambiguously, and each ride-along column keeps its real measured value
+    (or the small-gap-interpolated value the preprocessing stage placed there).
+    This makes it impossible to fabricate a value: earlier code interpolated
+    include columns with ``np.interp``, which clamps to a column's edge value
+    for any timetag outside that column's valid range — stamping a foreign
+    constant across every gap. Selection cannot do that.
+
     Args:
         df: Input DataFrame (no gaps).
         target_column: Column to optimize for.
         target_cadence: Target cadence.
-        include_columns: Additional columns to include.
+        include_columns: Additional columns to carry (selected, not
+            interpolated). If None, all numeric columns are carried.
 
     Returns:
         Downsampled DataFrame or None if cannot process.
     """
-    # Compute number of output points
-    n_out = compute_output_points(df.index[0], df.index[-1], target_cadence)
-
-    if n_out < 3:
-        logging.warning("Cannot perform LTTB downsampling on less than 3 points")
-        return None
+    # Compute number of output points. Segments spanning less than the
+    # target cadence still keep their first and last points (lttbc handles
+    # n_out <= 2 gracefully) so short data islands survive downsampling.
+    n_out = max(2, compute_output_points(df.index[0], df.index[-1], target_cadence))
 
     # Set up the data - convert time to numeric for LTTB algorithm
     df_work = df.copy()
@@ -116,48 +139,38 @@ def _lttb_single_segment(
     timeunit = '1min'
     df_work['time_num'] = (df_work.index - timeref) / pd.to_timedelta(timeunit)
 
-    # Prepare data for LTTB (time_num, target_column)
+    # Prepare data for LTTB (time_num, target_column). The caller already
+    # drops NaN-target rows; this guard protects against lttbc silently
+    # converting any remaining NaN to 0.0 (a fabricated value).
     df_clean = df_work[['time_num', target_column]].dropna()
 
-    if len(df_clean) < 3:
-        logging.warning("Insufficient non-NaN data points for LTTB")
+    if df_clean.empty:
         return None
 
     # Apply LTTB downsampling (lttbc uses separate x, y arrays)
-    x_down, y_down = lttbc.downsample(
+    x_down, _ = lttbc.downsample(
         df_clean['time_num'].values,
         df_clean[target_column].values,
         n_out
     )
-    df_resampled = pd.DataFrame(
-        {'time_num': x_down, target_column: y_down}
-    )
 
-    # Reconstruct the datetime index
-    df_resampled.index = (
-        timeref +
-        pd.to_timedelta(df_resampled['time_num'], unit='min')
-    )
+    # Row-select the segment rows LTTB chose. ``x_down`` are exact members of
+    # the input ``time_num`` (lttbc returns real input points), so ``isin``
+    # selects exactly those rows — carrying every column's real value and the
+    # original datetime index, no interpolation.
+    selected = df_work[df_work['time_num'].isin(x_down)]
 
-    # Interpolate other columns to LTTB-selected time points
+    keep = [target_column]
     for col in df.columns:
-        if col in ['time', 'time_num', target_column]:
+        if col in ('time', 'time_num', target_column):
             continue
         if include_columns is not None and col not in include_columns:
             continue
         if not pd.api.types.is_numeric_dtype(df[col]):
             continue
+        keep.append(col)
 
-        df_resampled[col] = np.interp(
-            x=df_resampled['time_num'].values,
-            xp=df_work['time_num'].values,
-            fp=df_work[col].values
-        )
-
-    # Clean up
-    df_resampled = df_resampled.drop(['time_num'], axis=1)
-
-    return df_resampled
+    return selected[keep].copy()
 
 
 def downsample_lttb_with_config(

{downsampler-0.2.0 → downsampler-0.3.1}/src/downsampler/m4.py

@@ -24,7 +24,7 @@ def downsample_m4(
     target_cadence: str | pd.Timedelta,
     include_columns: list[str] | None = None,
     gap_threshold: pd.Timedelta | None = None,
-    min_points_per_segment: int = 3,
+    min_points_per_segment: int = 1,
     deduplicate: bool = True,
     collinearity_threshold: float | None = None,
     target_points_per_bucket: float = 3.0

{downsampler-0.2.0 → downsampler-0.3.1/src/downsampler.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: downsampler
-Version: 0.2.0
+Version: 0.3.1
 Summary: Timeseries DataFrame downsampling with LTTB, aggregation methods, gap handling, and fidelity testing
 Author-email: Eelco Doornbos <eelco.doornbos@knmi.nl>
 License-Expression: MIT
@@ -22,12 +22,6 @@ Requires-Dist: pandas>=1.3
 Requires-Dist: lttbc>=0.3
 Requires-Dist: scipy>=1.7
 Requires-Dist: requests>=2.32.5
-Provides-Extra: test
-Requires-Dist: pytest>=7.0; extra == "test"
-Requires-Dist: pytest-cov>=4.0; extra == "test"
-Provides-Extra: dev
-Requires-Dist: downsampler[test]; extra == "dev"
-Requires-Dist: marimo; extra == "dev"
 Dynamic: license-file
 
 # downsampler
@@ -57,6 +51,13 @@ A Python package for time series DataFrame downsampling with LTTB, M4, multiple
 pip install downsampler
 ```
 
+> **Note (Linux + Python 3.11):** the `lttbc` dependency's prebuilt cp311
+> Linux wheel was compiled against NumPy 1.x and fails to import under
+> NumPy 2 (`numpy.core.multiarray failed to import`). Force a source build:
+> `pip install --no-binary lttbc downsampler`. Other Python versions and
+> macOS have no prebuilt wheel and build from source automatically. With
+> uv, this repo's `[tool.uv] no-binary-package` setting handles it.
+
 ## Quick Start
 
 ### Basic Downsampling
@@ -237,7 +238,7 @@ print(summary_table(results))
 | `gap_threshold` | str/Timedelta | "auto" | Min duration for gaps |
 | `edge_handling` | EdgeHandling | KEEP | How to handle edges |
 | `edge_window` | int | 2 | Points at each edge |
-| `min_points_per_segment` | int | 3 | Min points for processing |
+| `min_points_per_segment` | int | 1 | Min points per segment; smaller segments are dropped |
 | `min_completeness` | float | 0.9 | Min fraction of expected points per bucket |
 | `source_cadence` | str/Timedelta | None | Source data cadence (estimated if None) |
 

downsampler-0.3.1/src/downsampler.egg-info/requires.txt

@@ -0,0 +1,5 @@
+numpy>=2.0
+pandas>=1.3
+lttbc>=0.3
+scipy>=1.7
+requests>=2.32.5

{downsampler-0.2.0 → downsampler-0.3.1}/tests/test_gaps.py

@@ -239,22 +239,36 @@ class TestInterpolateSmallGaps:
         # Small gap filled (2 new points), large gap left alone
         assert len(result) == 8
 
-    def test_preserves_original_nans(self):
-        """Test that pre-existing NaN values are not overwritten."""
-        times = pd.date_range('2024-01-01', periods=5, freq='1min')
-        df = pd.DataFrame(
-            {'value': [0.0, np.nan, 2.0, 3.0, 4.0]},
-            index=times,
+    def test_small_inplace_gap_filled_wide_preserved(self):
+        """Small in-place NaN holes are bridged per-column; wide holes stay NaN.
+
+        (New 0.3.1 contract: small-gap filling is a per-column preprocessing
+        step, so a hole narrower than gap_threshold in an otherwise-dense column
+        is interpolated from that column's own samples, while a hole at or above
+        the threshold is preserved as an honest gap.)
+        """
+        times = pd.date_range('2024-01-01', periods=20, freq='1min')
+        value = np.arange(20, dtype=float)
+        value[3] = np.nan                 # 1-min hole  -> small, should fill
+        value[8:15] = np.nan              # 7-min hole  -> < 10min, should fill
+        df = pd.DataFrame({'value': value}, index=times)
+
+        small = interpolate_small_gaps(
+            df, gap_threshold=pd.Timedelta('10min'),
+            source_cadence=pd.Timedelta('1min'),
         )
-
-        result = interpolate_small_gaps(
-            df,
-            gap_threshold=pd.Timedelta('10min'),
+        # Both holes are narrower than 10 min -> bridged (linear: value == index).
+        assert small['value'].isna().sum() == 0
+        assert small.iloc[3]['value'] == pytest.approx(3.0)
+        assert small.iloc[10]['value'] == pytest.approx(10.0)
+
+        # With a tighter threshold the 7-min hole is now "wide" -> preserved.
+        wide = interpolate_small_gaps(
+            df, gap_threshold=pd.Timedelta('5min'),
             source_cadence=pd.Timedelta('1min'),
         )
-
-        # Original NaN should still be NaN
-        assert np.isnan(result.iloc[1]['value'])
+        assert not np.isnan(wide.iloc[3]['value'])       # 1-min hole still filled
+        assert wide['value'].iloc[8:14].isna().all()     # 7-min hole preserved
 
     def test_auto_cadence_estimation(self):
         """Test that source_cadence is auto-estimated when None."""

downsampler-0.3.1/tests/test_lttb.py

@@ -0,0 +1,408 @@
+"""Tests for LTTB downsampling."""
+
+import pytest
+import pandas as pd
+import numpy as np
+
+from downsampler.lttb import downsample_lttb, downsample_lttb_with_config
+from downsampler.config import DownsampleConfig, AggregationMethod
+
+
+class TestDownsampleLttb:
+    """Tests for LTTB downsampling function."""
+
+    def test_basic_lttb(self, sine_df):
+        """Test basic LTTB downsampling."""
+        result = downsample_lttb(
+            sine_df,
+            target_column='signal',
+            target_cadence='PT10S'
+        )
+
+        assert len(result) < len(sine_df)
+        assert 'signal' in result.columns
+
+    def test_preserves_extreme_values(self, sine_df):
+        """Test that LTTB preserves extreme values reasonably well."""
+        result = downsample_lttb(
+            sine_df,
+            target_column='signal',
+            target_cadence='PT10S'
+        )
+
+        # Check that max/min are close to original
+        orig_max = sine_df['signal'].max()
+        orig_min = sine_df['signal'].min()
+        result_max = result['signal'].max()
+        result_min = result['signal'].min()
+
+        # Allow 10% tolerance
+        assert abs(result_max - orig_max) < 0.1 * abs(orig_max)
+        assert abs(result_min - orig_min) < 0.1 * abs(orig_min - orig_max)
+
+    def test_include_columns(self, sine_df):
+        """Test including additional columns."""
+        result = downsample_lttb(
+            sine_df,
+            target_column='signal',
+            target_cadence='PT10S',
+            include_columns=['signal', 'noise']
+        )
+
+        assert 'signal' in result.columns
+        assert 'noise' in result.columns
+
+    def test_gap_handling(self, gappy_df):
+        """Test LTTB with gappy data."""
+        # Add a target column
+        gappy_df['signal'] = np.sin(np.linspace(0, 4 * np.pi, len(gappy_df)))
+
+        result = downsample_lttb(
+            gappy_df,
+            target_column='signal',
+            target_cadence='PT5M',
+            gap_threshold=pd.Timedelta('30min')
+        )
+
+        # Should produce output from both segments
+        assert len(result) > 0
+
+    def test_insufficient_points(self):
+        """Test handling of insufficient points."""
+        small_df = pd.DataFrame(
+            {'value': [1, 2]},
+            index=pd.date_range('2024-01-01', periods=2, freq='1s')
+        )
+
+        result = downsample_lttb(
+            small_df,
+            target_column='value',
+            target_cadence='PT10S',
+            min_points_per_segment=3
+        )
+
+        # Should return empty or minimal result
+        assert len(result) == 0
+
+
+class TestLttbGapHandling:
+    """Tests for LTTB gap handling behavior."""
+
+    def test_lttb_inserts_nan_markers_at_large_gaps(self):
+        """Test that LTTB output contains NaN markers between segments."""
+        # Two segments with a large gap between them
+        times1 = pd.date_range('2024-01-01 00:00', periods=100, freq='1s')
+        times2 = pd.date_range('2024-01-01 01:00', periods=100, freq='1s')
+        t1 = np.linspace(0, 4 * np.pi, 100)
+        t2 = np.linspace(0, 4 * np.pi, 100)
+        df = pd.concat([
+            pd.DataFrame({'signal': np.sin(t1)}, index=times1),
+            pd.DataFrame({'signal': np.sin(t2)}, index=times2),
+        ])
+
+        result = downsample_lttb(
+            df,
+            target_column='signal',
+            target_cadence='PT10S',
+            gap_threshold=pd.Timedelta('5min'),
+        )
+
+        # Should have NaN markers
+        assert result['signal'].isna().sum() > 0
+
+    def test_lttb_interpolates_small_gaps(self):
+        """Test that small gaps are filled before LTTB processes."""
+        # Data with a small 5-second gap (below gap_threshold of 30s)
+        times = list(pd.date_range('2024-01-01 00:00', periods=50, freq='1s'))
+        # Remove 3 points to create a small gap
+        del times[25:28]
+        t = np.linspace(0, 4 * np.pi, len(times))
+        df = pd.DataFrame({'signal': np.sin(t)}, index=pd.DatetimeIndex(times))
+
+        result = downsample_lttb(
+            df,
+            target_column='signal',
+            target_cadence='PT5S',
+            gap_threshold=pd.Timedelta('30s'),
+            source_cadence=pd.Timedelta('1s'),
+        )
+
+        # Should NOT have NaN markers (gap was small and interpolated)
+        assert result['signal'].isna().sum() == 0
+        assert len(result) > 0
+
+    def test_lttb_source_cadence_via_config(self):
+        """Test that source_cadence is passed through config."""
+        times1 = pd.date_range('2024-01-01 00:00', periods=100, freq='1s')
+        times2 = pd.date_range('2024-01-01 01:00', periods=100, freq='1s')
+        t = np.linspace(0, 4 * np.pi, 100)
+        df = pd.concat([
+            pd.DataFrame({'signal': np.sin(t)}, index=times1),
+            pd.DataFrame({'signal': np.sin(t)}, index=times2),
+        ])
+
+        config = DownsampleConfig(
+            method=AggregationMethod.LTTB,
+            lttb_target_column='signal',
+            source_cadence='PT1S',
+        )
+
+        result = downsample_lttb_with_config(df, 'PT10S', config)
+        assert len(result) > 0
+
+
+class TestLttbNanEncodedGaps:
+    """Fill-encoded gaps (NaN rows) must behave like missing-row gaps."""
+
+    @staticmethod
+    def _make_gappy(encoding: str) -> pd.DataFrame:
+        """4 hours of 1-min data with a 45-min gap from 01:00 to 01:45."""
+        idx = pd.date_range('2024-01-01', periods=240, freq='1min')
+        values = 500 + 10 * np.sin(np.arange(240) / 20.0)
+        other = 100 + np.cos(np.arange(240) / 10.0)
+        df = pd.DataFrame({'signal': values, 'other': other}, index=idx)
+        gap_rows = idx[60:105]
+        if encoding == 'nan_rows':
+            df.loc[gap_rows, ['signal', 'other']] = np.nan
+            return df
+        return df.drop(gap_rows)
+
+    @pytest.mark.parametrize('encoding', ['nan_rows', 'missing_rows'])
+    def test_gap_boundaries_preserved_exactly(self, encoding):
+        """Last point before and first point after a gap must survive."""
+        df = self._make_gappy(encoding)
+
+        result = downsample_lttb(
+            df, target_column='signal', target_cadence='PT15M',
+            source_cadence=pd.Timedelta('1min'),
+        )
+
+        # LTTB keeps first/last of each segment: the valid samples
+        # adjacent to the gap must be in the output.
+        assert pd.Timestamp('2024-01-01 00:59') in result.index
+        assert pd.Timestamp('2024-01-01 01:45') in result.index
+
+    def test_nan_rows_equivalent_to_missing_rows(self):
+        """Both gap encodings must produce identical output."""
+        result_nan = downsample_lttb(
+            self._make_gappy('nan_rows'),
+            target_column='signal', target_cadence='PT15M',
+            source_cadence=pd.Timedelta('1min'),
+        )
+        result_missing = downsample_lttb(
+            self._make_gappy('missing_rows'),
+            target_column='signal', target_cadence='PT15M',
+            source_cadence=pd.Timedelta('1min'),
+        )
+
+        pd.testing.assert_frame_equal(result_nan, result_missing)
+
+    def test_nan_gap_produces_marker(self):
+        """A NaN-encoded gap must yield a NaN marker row in the output."""
+        result = downsample_lttb(
+            self._make_gappy('nan_rows'),
+            target_column='signal', target_cadence='PT15M',
+            source_cadence=pd.Timedelta('1min'),
+        )
+
+        markers = result[result['signal'].isna()]
+        assert len(markers) == 1
+        # Marker sits just after the last point of the pre-gap segment
+        assert pd.Timestamp('2024-01-01 00:59') < markers.index[0]
+        assert markers.index[0] < pd.Timestamp('2024-01-01 01:45')
+
+    def test_all_nan_target_returns_empty(self):
+        """A frame whose target column is entirely NaN yields no output."""
+        idx = pd.date_range('2024-01-01', periods=100, freq='1min')
+        df = pd.DataFrame({'signal': np.nan, 'other': 1.0}, index=idx)
+
+        result = downsample_lttb(df, target_column='signal', target_cadence='PT15M')
+
+        assert len(result) == 0
+        assert list(result.columns) == ['signal', 'other']
+
+    def test_include_column_own_nan_holes(self):
+        """Include columns interpolate from their own valid samples only."""
+        idx = pd.date_range('2024-01-01', periods=240, freq='1min')
+        df = pd.DataFrame({
+            'signal': 500 + 10 * np.sin(np.arange(240) / 20.0),
+            'other': 100.0,
+        }, index=idx)
+        # Hole in 'other' only — target column is complete
+        df.loc[idx[100:110], 'other'] = np.nan
+
+        result = downsample_lttb(
+            df, target_column='signal', target_cadence='PT15M',
+            source_cadence=pd.Timedelta('1min'),
+        )
+
+        # 'other' is constant 100 outside its hole; the small-gap preprocessing
+        # bridges it per-column rather than propagating NaN.
+        assert result['other'].isna().sum() == 0
+        assert (result['other'] == 100.0).all()
+
+
+class TestLttbIncludeSelection:
+    """Include columns are row-SELECTED at LTTB timetags, never interpolated.
+
+    Regression coverage for the np.interp edge-clamp bug: interpolating an
+    include column onto the target's timetags stamped the column's edge value
+    across every timetag outside its valid range (a constant foreign peak at
+    each gap). Selection cannot fabricate values.
+    """
+
+    def test_high_window_does_not_bleed_into_quiet_period(self):
+        """A column valid only in a late high window must not appear earlier."""
+        idx = pd.date_range('2024-01-01 00:00', periods=24 * 60, freq='1min')
+        signal = 350 + 10 * np.sin(np.arange(len(idx)) / 30.0)  # dense all day
+        other = np.full(len(idx), np.nan)
+        hi = (idx >= '2024-01-01 20:00') & (idx <= '2024-01-01 21:00')
+        other[hi] = 602.9  # the ONLY place 'other' has data
+        df = pd.DataFrame({'signal': signal, 'other': other}, index=idx)
+
+        result = downsample_lttb(
+            df, target_column='signal', target_cadence='PT5M',
+            source_cadence=pd.Timedelta('1min'),
+        )
+
+        # No 'other' value may appear at a timetag outside its valid window,
+        # and certainly not the clamped constant 602.9 in the quiet morning.
+        nonnull = result['other'].dropna()
+        assert (nonnull.index >= pd.Timestamp('2024-01-01 20:00')).all()
+        assert (nonnull.index <= pd.Timestamp('2024-01-01 21:00')).all()
+        morning = result.loc[result.index < pd.Timestamp('2024-01-01 19:00'), 'other']
+        assert morning.isna().all()
+
+    def test_ridealong_wide_hole_stays_nan(self):
+        """A hole wider than gap_threshold in a ride-along column stays NaN."""
+        idx = pd.date_range('2024-01-01', periods=240, freq='1min')
+        signal = 500 + 10 * np.sin(np.arange(240) / 20.0)  # complete target
+        other = np.full(240, 100.0)
+        other[60:150] = np.nan  # 90-min hole >> 30-min gap_threshold (PT15M)
+        df = pd.DataFrame({'signal': signal, 'other': other}, index=idx)
+
+        result = downsample_lttb(
+            df, target_column='signal', target_cadence='PT15M',
+            source_cadence=pd.Timedelta('1min'),
+        )
+
+        in_hole = result[(result.index > idx[60]) & (result.index < idx[149])]
+        assert not in_hole.empty          # target is dense there, so timetags exist
+        assert in_hole['other'].isna().all()
+
+    def test_include_values_are_exact_source_selections(self):
+        """Each carried include value equals the real source sample (not interp)."""
+        idx = pd.date_range('2024-01-01', periods=240, freq='1min')
+        rng = np.random.default_rng(0)
+        signal = 500 + 50 * np.sin(np.arange(240) / 15.0) + rng.normal(0, 3, 240)
+        other = 100 + rng.normal(0, 10, 240)  # noisy, dense (no gaps)
+        df = pd.DataFrame({'signal': signal, 'other': other}, index=idx)
+
+        result = downsample_lttb(
+            df, target_column='signal', target_cadence='PT15M',
+            source_cadence=pd.Timedelta('1min'),
+        )
+
+        for t, v in result['other'].dropna().items():
+            assert v == pytest.approx(df.loc[t, 'other'])
+
+
+class TestLttbShortSegments:
+    """Short data islands must survive instead of being dropped."""
+
+    def test_short_segment_keeps_first_and_last(self):
+        """A segment spanning less than the target cadence keeps its extent."""
+        # 10 minutes of data at 1-min cadence, target PT1H: old behavior
+        # dropped this entirely (n_out < 3).
+        idx = pd.date_range('2024-01-01', periods=10, freq='1min')
+        df = pd.DataFrame({'signal': np.linspace(1.0, 2.0, 10)}, index=idx)
+
+        result = downsample_lttb(df, target_column='signal', target_cadence='PT1H')
+
+        assert idx[0] in result.index
+        assert idx[-1] in result.index
+
+    def test_intermittent_islands_all_survive(self):
+        """Several short islands between gaps each keep points and markers."""
+        frames = []
+        for hour in [0, 3, 6]:  # gaps well above the 2x PT1H auto threshold
+            idx = pd.date_range(f'2024-01-01 {hour:02d}:00', periods=8, freq='1min')
+            frames.append(pd.DataFrame(
+                {'signal': np.full(8, float(hour + 1))}, index=idx
+            ))
+        df = pd.concat(frames)
+
+        result = downsample_lttb(df, target_column='signal', target_cadence='PT1H')
+
+        # All three islands present with their values
+        for hour in [0, 3, 6]:
+            assert (result['signal'] == hour + 1).any()
+        # Two NaN markers between three segments
+        assert result['signal'].isna().sum() == 2
+
+    def test_single_isolated_point_survives(self):
+        """A lone valid sample between gaps is kept, bracketed by markers."""
+        idx1 = pd.date_range('2024-01-01 00:00', periods=30, freq='1min')
+        idx2 = pd.date_range('2024-01-01 02:00', periods=30, freq='1min')
+        lone = pd.Timestamp('2024-01-01 01:00')
+        df = pd.concat([
+            pd.DataFrame({'signal': 1.0}, index=idx1),
+            pd.DataFrame({'signal': 99.0}, index=[lone]),
+            pd.DataFrame({'signal': 2.0}, index=idx2),
+        ])
+
+        result = downsample_lttb(
+            df, target_column='signal', target_cadence='PT5M',
+            source_cadence=pd.Timedelta('1min'),
+        )
+
+        assert lone in result.index
+        assert result.loc[lone, 'signal'] == 99.0
+        # Markers on both sides of the lone point
+        pos = result.index.get_loc(lone)
+        assert np.isnan(result['signal'].iloc[pos - 1])
+        assert np.isnan(result['signal'].iloc[pos + 1])
+
+    def test_min_points_per_segment_still_filters(self):
+        """An explicit min_points_per_segment above 1 still discards."""
+        idx1 = pd.date_range('2024-01-01 00:00', periods=30, freq='1min')
+        idx2 = pd.date_range('2024-01-01 02:00', periods=2, freq='1min')
+        df = pd.concat([
+            pd.DataFrame({'signal': 1.0}, index=idx1),
+            pd.DataFrame({'signal': 2.0}, index=idx2),
+        ])
+
+        result = downsample_lttb(
+            df, target_column='signal', target_cadence='PT5M',
+            min_points_per_segment=3,
+            source_cadence=pd.Timedelta('1min'),
+        )
+
+        assert not (result['signal'] == 2.0).any()
+
+
+class TestDownsampleLttbWithConfig:
+    """Tests for LTTB downsampling with config."""
+
+    def test_with_config(self, sine_df):
+        """Test LTTB with full configuration."""
+        config = DownsampleConfig(
+            method=AggregationMethod.LTTB,
+            lttb_target_column='signal',
+            min_points_per_segment=5
+        )
+
+        result = downsample_lttb_with_config(
+            sine_df,
+            'PT10S',
+            config
+        )
+
+        assert len(result) > 0
+
+    def test_missing_target_column_raises(self, sine_df):
+        """Test that missing target column raises error."""
+        config = DownsampleConfig(method=AggregationMethod.LTTB)
+
+        with pytest.raises(ValueError, match="lttb_target_column"):
+            downsample_lttb_with_config(sine_df, 'PT10S', config)

downsampler-0.2.0/src/downsampler.egg-info/requires.txt

@@ -1,13 +0,0 @@
-numpy>=2.0
-pandas>=1.3
-lttbc>=0.3
-scipy>=1.7
-requests>=2.32.5
-
-[dev]
-downsampler[test]
-marimo
-
-[test]
-pytest>=7.0
-pytest-cov>=4.0

downsampler-0.2.0/tests/test_lttb.py

@@ -1,178 +0,0 @@
-"""Tests for LTTB downsampling."""
-
-import pytest
-import pandas as pd
-import numpy as np
-
-from downsampler.lttb import downsample_lttb, downsample_lttb_with_config
-from downsampler.config import DownsampleConfig, AggregationMethod
-
-
-class TestDownsampleLttb:
-    """Tests for LTTB downsampling function."""
-
-    def test_basic_lttb(self, sine_df):
-        """Test basic LTTB downsampling."""
-        result = downsample_lttb(
-            sine_df,
-            target_column='signal',
-            target_cadence='PT10S'
-        )
-
-        assert len(result) < len(sine_df)
-        assert 'signal' in result.columns
-
-    def test_preserves_extreme_values(self, sine_df):
-        """Test that LTTB preserves extreme values reasonably well."""
-        result = downsample_lttb(
-            sine_df,
-            target_column='signal',
-            target_cadence='PT10S'
-        )
-
-        # Check that max/min are close to original
-        orig_max = sine_df['signal'].max()
-        orig_min = sine_df['signal'].min()
-        result_max = result['signal'].max()
-        result_min = result['signal'].min()
-
-        # Allow 10% tolerance
-        assert abs(result_max - orig_max) < 0.1 * abs(orig_max)
-        assert abs(result_min - orig_min) < 0.1 * abs(orig_min - orig_max)
-
-    def test_include_columns(self, sine_df):
-        """Test including additional columns."""
-        result = downsample_lttb(
-            sine_df,
-            target_column='signal',
-            target_cadence='PT10S',
-            include_columns=['signal', 'noise']
-        )
-
-        assert 'signal' in result.columns
-        assert 'noise' in result.columns
-
-    def test_gap_handling(self, gappy_df):
-        """Test LTTB with gappy data."""
-        # Add a target column
-        gappy_df['signal'] = np.sin(np.linspace(0, 4 * np.pi, len(gappy_df)))
-
-        result = downsample_lttb(
-            gappy_df,
-            target_column='signal',
-            target_cadence='PT5M',
-            gap_threshold=pd.Timedelta('30min')
-        )
-
-        # Should produce output from both segments
-        assert len(result) > 0
-
-    def test_insufficient_points(self):
-        """Test handling of insufficient points."""
-        small_df = pd.DataFrame(
-            {'value': [1, 2]},
-            index=pd.date_range('2024-01-01', periods=2, freq='1s')
-        )
-
-        result = downsample_lttb(
-            small_df,
-            target_column='value',
-            target_cadence='PT10S',
-            min_points_per_segment=3
-        )
-
-        # Should return empty or minimal result
-        assert len(result) == 0
-
-
-class TestLttbGapHandling:
-    """Tests for LTTB gap handling behavior."""
-
-    def test_lttb_inserts_nan_markers_at_large_gaps(self):
-        """Test that LTTB output contains NaN markers between segments."""
-        # Two segments with a large gap between them
-        times1 = pd.date_range('2024-01-01 00:00', periods=100, freq='1s')
-        times2 = pd.date_range('2024-01-01 01:00', periods=100, freq='1s')
-        t1 = np.linspace(0, 4 * np.pi, 100)
-        t2 = np.linspace(0, 4 * np.pi, 100)
-        df = pd.concat([
-            pd.DataFrame({'signal': np.sin(t1)}, index=times1),
-            pd.DataFrame({'signal': np.sin(t2)}, index=times2),
-        ])
-
-        result = downsample_lttb(
-            df,
-            target_column='signal',
-            target_cadence='PT10S',
-            gap_threshold=pd.Timedelta('5min'),
-        )
-
-        # Should have NaN markers
-        assert result['signal'].isna().sum() > 0
-
-    def test_lttb_interpolates_small_gaps(self):
-        """Test that small gaps are filled before LTTB processes."""
-        # Data with a small 5-second gap (below gap_threshold of 30s)
-        times = list(pd.date_range('2024-01-01 00:00', periods=50, freq='1s'))
-        # Remove 3 points to create a small gap
-        del times[25:28]
-        t = np.linspace(0, 4 * np.pi, len(times))
-        df = pd.DataFrame({'signal': np.sin(t)}, index=pd.DatetimeIndex(times))
-
-        result = downsample_lttb(
-            df,
-            target_column='signal',
-            target_cadence='PT5S',
-            gap_threshold=pd.Timedelta('30s'),
-            source_cadence=pd.Timedelta('1s'),
-        )
-
-        # Should NOT have NaN markers (gap was small and interpolated)
-        assert result['signal'].isna().sum() == 0
-        assert len(result) > 0
-
-    def test_lttb_source_cadence_via_config(self):
-        """Test that source_cadence is passed through config."""
-        times1 = pd.date_range('2024-01-01 00:00', periods=100, freq='1s')
-        times2 = pd.date_range('2024-01-01 01:00', periods=100, freq='1s')
-        t = np.linspace(0, 4 * np.pi, 100)
-        df = pd.concat([
-            pd.DataFrame({'signal': np.sin(t)}, index=times1),
-            pd.DataFrame({'signal': np.sin(t)}, index=times2),
-        ])
-
-        config = DownsampleConfig(
-            method=AggregationMethod.LTTB,
-            lttb_target_column='signal',
-            source_cadence='PT1S',
-        )
-
-        result = downsample_lttb_with_config(df, 'PT10S', config)
-        assert len(result) > 0
-
-
-class TestDownsampleLttbWithConfig:
-    """Tests for LTTB downsampling with config."""
-
-    def test_with_config(self, sine_df):
-        """Test LTTB with full configuration."""
-        config = DownsampleConfig(
-            method=AggregationMethod.LTTB,
-            lttb_target_column='signal',
-            min_points_per_segment=5
-        )
-
-        result = downsample_lttb_with_config(
-            sine_df,
-            'PT10S',
-            config
-        )
-
-        assert len(result) > 0
-
-    def test_missing_target_column_raises(self, sine_df):
-        """Test that missing target column raises error."""
-        config = DownsampleConfig(method=AggregationMethod.LTTB)
-
-        with pytest.raises(ValueError, match="lttb_target_column"):
-            downsample_lttb_with_config(sine_df, 'PT10S', config)