downsampler 0.3.1__tar.gz → 0.4.0__tar.gz

{downsampler-0.3.1/src/downsampler.egg-info → downsampler-0.4.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: downsampler
-Version: 0.3.1
+Version: 0.4.0
 Summary: Timeseries DataFrame downsampling with LTTB, aggregation methods, gap handling, and fidelity testing
 Author-email: Eelco Doornbos <eelco.doornbos@knmi.nl>
 License-Expression: MIT

{downsampler-0.3.1 → downsampler-0.4.0}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "downsampler"
-version = "0.3.1"
+version = "0.4.0"
 description = "Timeseries DataFrame downsampling with LTTB, aggregation methods, gap handling, and fidelity testing"
 readme = "README.md"
 license = "MIT"

{downsampler-0.3.1 → downsampler-0.4.0}/src/downsampler/__init__.py

@@ -47,6 +47,7 @@ from downsampler.gaps import (
     split_at_gaps,
     wrap_in_nans,
     mark_gaps_in_dataframe,
+    gap_marker_frame,
     interpolate_small_gaps,
     concatenate_with_gap_markers,
 )
@@ -87,6 +88,7 @@ __all__ = [
     "split_at_gaps",
     "wrap_in_nans",
     "mark_gaps_in_dataframe",
+    "gap_marker_frame",
     "interpolate_small_gaps",
     "concatenate_with_gap_markers",
     # LTTB

{downsampler-0.3.1 → downsampler-0.4.0}/src/downsampler/gaps.py

@@ -414,6 +414,43 @@ def _interpolate_column_small_gaps(
     return filled
 
 
+def gap_marker_frame(
+    index: pd.DatetimeIndex,
+    columns: list[str],
+) -> pd.DataFrame:
+    """NaN marker rows bracketing a window that yielded no usable points.
+
+    Returned in place of an empty frame when the input covers a real time
+    window but every sample in it is unusable — e.g. a batch whose target
+    column is entirely NaN (fill-encoded), or one where every segment was
+    dropped as too short. Returning nothing there would be indistinguishable
+    from "this period was never processed": a plot drawn from the stored
+    output connects straight across the window, showing a confident straight
+    line (or, zoomed out, a quiet period) where the truth is missing data.
+
+    Marking both ends of the window rather than a single point makes the gap's
+    *extent* explicit, so consumers see where invalid data began and ended
+    instead of a single break of unknown width. A one-sample window yields one
+    marker row.
+
+    Args:
+        index: DatetimeIndex of the input that produced no output.
+        columns: Output columns the caller would otherwise have produced.
+
+    Returns:
+        DataFrame of all-NaN rows at the first and last input timestamps, or
+        an empty frame when ``index`` is empty (nothing is knowable then).
+    """
+    if len(index) == 0:
+        return pd.DataFrame(columns=columns)
+
+    marker_times = [index[0]] if len(index) == 1 else [index[0], index[-1]]
+    return pd.DataFrame(
+        {col: [np.nan] * len(marker_times) for col in columns},
+        index=pd.DatetimeIndex(marker_times, name=index.name),
+    )
+
+
 def concatenate_with_gap_markers(
     segments: list[pd.DataFrame],
     offset: pd.Timedelta = pd.Timedelta('0.1s'),

{downsampler-0.3.1 → downsampler-0.4.0}/src/downsampler/lttb.py

@@ -5,7 +5,12 @@ import numpy as np
 import lttbc
 
 from downsampler.config import DownsampleConfig, EdgeHandling
-from downsampler.gaps import split_at_gaps, interpolate_small_gaps, concatenate_with_gap_markers
+from downsampler.gaps import (
+    split_at_gaps,
+    interpolate_small_gaps,
+    concatenate_with_gap_markers,
+    gap_marker_frame,
+)
 from downsampler.edges import apply_edge_handling
 from downsampler.utils import parse_cadence, get_numeric_columns, compute_output_points, filter_columns
 
@@ -71,7 +76,11 @@ def downsample_lttb(
     # data exactly like missing-row gaps.
     df_valid = df_in.dropna(subset=[target_column])
     if df_valid.empty:
-        return pd.DataFrame(columns=df_in.columns)
+        # The window is covered but wholly unusable — mark it rather than
+        # returning nothing, so the gap survives into the stored output.
+        return gap_marker_frame(
+            df_in.index, _output_columns(df_in, target_column, include_columns)
+        )
 
     # Interpolate small gaps so LTTB receives continuous input
     df_interp = interpolate_small_gaps(df_valid, gap_threshold, source_cadence)
@@ -94,11 +103,36 @@ def downsample_lttb(
             resampled_segments.append(resampled)
 
     if not resampled_segments:
-        return pd.DataFrame(columns=df_in.columns)
+        return gap_marker_frame(
+            df_in.index, _output_columns(df_in, target_column, include_columns)
+        )
 
     return concatenate_with_gap_markers(resampled_segments)
 
 
+def _output_columns(
+    df: pd.DataFrame,
+    target_column: str,
+    include_columns: list[str] | None = None,
+) -> list[str]:
+    """Columns an LTTB run emits: the target plus carried numeric columns.
+
+    Shared by the normal path and the all-gap marker path so that a batch
+    which produced only markers has the same schema as one that produced
+    data — a storage writer must not see the column set change with content.
+    """
+    keep = [target_column]
+    for col in df.columns:
+        if col in ('time', 'time_num', target_column):
+            continue
+        if include_columns is not None and col not in include_columns:
+            continue
+        if not pd.api.types.is_numeric_dtype(df[col]):
+            continue
+        keep.append(col)
+    return keep
+
+
 def _lttb_single_segment(
     df: pd.DataFrame,
     target_column: str,
@@ -160,17 +194,7 @@ def _lttb_single_segment(
     # original datetime index, no interpolation.
     selected = df_work[df_work['time_num'].isin(x_down)]
 
-    keep = [target_column]
-    for col in df.columns:
-        if col in ('time', 'time_num', target_column):
-            continue
-        if include_columns is not None and col not in include_columns:
-            continue
-        if not pd.api.types.is_numeric_dtype(df[col]):
-            continue
-        keep.append(col)
-
-    return selected[keep].copy()
+    return selected[_output_columns(df, target_column, include_columns)].copy()
 
 
 def downsample_lttb_with_config(

{downsampler-0.3.1 → downsampler-0.4.0}/src/downsampler/m4.py

@@ -14,7 +14,11 @@ import pandas as pd
 import numpy as np
 
 from downsampler.config import DownsampleConfig
-from downsampler.gaps import split_at_gaps, concatenate_with_gap_markers
+from downsampler.gaps import (
+    split_at_gaps,
+    concatenate_with_gap_markers,
+    gap_marker_frame,
+)
 from downsampler.edges import apply_edge_handling
 from downsampler.utils import parse_cadence, get_numeric_columns, compute_output_points, filter_columns
 
@@ -92,7 +96,14 @@ def downsample_m4(
             resampled_segments.append(resampled)
 
     if not resampled_segments:
-        return pd.DataFrame(columns=df_in.columns)
+        # Covered but unusable (every segment too short, or all-NaN input):
+        # mark the window so the gap survives into the stored output rather
+        # than reading as "never processed".
+        if include_columns is None:
+            columns = get_numeric_columns(df_in)
+        else:
+            columns = [c for c in include_columns if c in df_in.columns]
+        return gap_marker_frame(df_in.index, columns)
 
     return concatenate_with_gap_markers(resampled_segments)
 

{downsampler-0.3.1 → downsampler-0.4.0/src/downsampler.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: downsampler
-Version: 0.3.1
+Version: 0.4.0
 Summary: Timeseries DataFrame downsampling with LTTB, aggregation methods, gap handling, and fidelity testing
 Author-email: Eelco Doornbos <eelco.doornbos@knmi.nl>
 License-Expression: MIT

{downsampler-0.3.1 → downsampler-0.4.0}/tests/test_gaps.py

@@ -10,6 +10,7 @@ from downsampler.gaps import (
     split_at_gaps,
     iter_segments,
     wrap_in_nans,
+    gap_marker_frame,
     mark_gaps_in_dataframe,
     has_gaps,
     count_gaps,
@@ -392,3 +393,30 @@ class TestConcatenateWithGapMarkers:
 
         expected_marker_time = seg1.index[-1] + offset
         assert result.index[1] == expected_marker_time
+
+
+class TestGapMarkerFrame:
+    """Marker rows standing in for a window that produced no usable output."""
+
+    def test_marks_both_ends(self):
+        idx = pd.date_range('2024-01-01', periods=1440, freq='1min')
+        result = gap_marker_frame(idx, ['a', 'b'])
+
+        assert list(result.columns) == ['a', 'b']
+        assert result.isna().all().all()
+        assert list(result.index) == [idx[0], idx[-1]]
+
+    def test_single_sample_window(self):
+        idx = pd.date_range('2024-01-01', periods=1, freq='1min')
+        assert len(gap_marker_frame(idx, ['a'])) == 1
+
+    def test_empty_index_yields_empty_frame(self):
+        """No input means no known window — nothing to assert about it."""
+        result = gap_marker_frame(pd.DatetimeIndex([]), ['a'])
+
+        assert len(result) == 0
+        assert list(result.columns) == ['a']
+
+    def test_preserves_index_name(self):
+        idx = pd.date_range('2024-01-01', periods=10, freq='1min', name='time')
+        assert gap_marker_frame(idx, ['a']).index.name == 'time'

{downsampler-0.3.1 → downsampler-0.4.0}/tests/test_lttb.py

@@ -81,8 +81,10 @@ class TestDownsampleLttb:
             min_points_per_segment=3
         )
 
-        # Should return empty or minimal result
-        assert len(result) == 0
+        # The segment is dropped as too short, but the window it covered is
+        # marked rather than silently vanishing.
+        assert result['value'].isna().all()
+        assert list(result.index) == [small_df.index[0], small_df.index[-1]]
 
 
 class TestLttbGapHandling:
@@ -211,15 +213,61 @@ class TestLttbNanEncodedGaps:
         assert pd.Timestamp('2024-01-01 00:59') < markers.index[0]
         assert markers.index[0] < pd.Timestamp('2024-01-01 01:45')
 
-    def test_all_nan_target_returns_empty(self):
-        """A frame whose target column is entirely NaN yields no output."""
+    def test_all_nan_target_marks_the_window(self):
+        """An all-NaN target marks its window instead of vanishing.
+
+        Returning nothing would make a covered-but-unusable period
+        indistinguishable from one that was never processed, and a plot drawn
+        from the stored output would connect straight across it.
+        """
         idx = pd.date_range('2024-01-01', periods=100, freq='1min')
         df = pd.DataFrame({'signal': np.nan, 'other': 1.0}, index=idx)
 
         result = downsample_lttb(df, target_column='signal', target_cadence='PT15M')
 
-        assert len(result) == 0
         assert list(result.columns) == ['signal', 'other']
+        assert result.isna().all().all()
+        # Both ends marked, so the gap's extent is explicit.
+        assert list(result.index) == [idx[0], idx[-1]]
+
+    def test_all_nan_target_single_sample_marks_once(self):
+        """A one-sample window needs only one marker."""
+        idx = pd.date_range('2024-01-01', periods=1, freq='1min')
+        df = pd.DataFrame({'signal': [np.nan], 'other': [1.0]}, index=idx)
+
+        result = downsample_lttb(df, target_column='signal', target_cadence='PT15M')
+
+        assert len(result) == 1
+        assert result.index[0] == idx[0]
+
+    def test_empty_input_still_returns_empty(self):
+        """With no input there is no window to mark."""
+        df = pd.DataFrame(
+            {'signal': [], 'other': []},
+            index=pd.DatetimeIndex([], name='time'),
+        )
+
+        result = downsample_lttb(df, target_column='signal', target_cadence='PT15M')
+
+        assert len(result) == 0
+
+    def test_marked_window_matches_normal_batch_schema(self):
+        """A marker-only batch has the same columns as a batch with data.
+
+        A storage writer must not see the column set change with content.
+        """
+        idx = pd.date_range('2024-01-01', periods=100, freq='1min')
+        good = pd.DataFrame(
+            {'signal': np.sin(np.arange(100) / 5.0), 'other': 1.0, 'label': 'x'},
+            index=idx,
+        )
+        bad = good.copy()
+        bad['signal'] = np.nan
+
+        normal = downsample_lttb(good, target_column='signal', target_cadence='PT15M')
+        marked = downsample_lttb(bad, target_column='signal', target_cadence='PT15M')
+
+        assert list(marked.columns) == list(normal.columns)
 
     def test_include_column_own_nan_holes(self):
         """Include columns interpolate from their own valid samples only."""

{downsampler-0.3.1 → downsampler-0.4.0}/tests/test_m4.py

@@ -154,8 +154,9 @@ class TestDownsampleM4:
             min_points_per_segment=3
         )
 
-        # Should return empty result
-        assert len(result) == 0
+        # The segment is dropped as too short, but its window is marked.
+        assert result['value'].isna().all()
+        assert len(result) == 2
 
     def test_output_has_irregular_timestamps(self, sine_df):
         """Test that M4 produces irregular timestamps (not aligned to grid)."""