downsampler 0.1.0__py3-none-any.whl

downsampler/lttb.py

@@ -0,0 +1,207 @@
+"""LTTB (Largest Triangle Three Buckets) downsampling with gap handling."""
+
+import logging
+import pandas as pd
+import numpy as np
+import lttb
+
+from downsampler.config import DownsampleConfig, GapHandling, EdgeHandling
+from downsampler.gaps import split_at_gaps
+from downsampler.edges import apply_edge_handling
+from downsampler.utils import parse_cadence, get_numeric_columns, compute_output_points
+
+
+def downsample_lttb(
+    df_in: pd.DataFrame,
+    target_column: str,
+    target_cadence: str | pd.Timedelta,
+    include_columns: list[str] | None = None,
+    gap_threshold: pd.Timedelta | None = None,
+    min_points_per_segment: int = 3
+) -> pd.DataFrame:
+    """Perform LTTB downsampling on a pandas DataFrame.
+
+    LTTB (Largest Triangle Three Buckets) is a downsampling algorithm that
+    preserves visual characteristics of the data by selecting points that
+    maximize the area of triangles formed with adjacent buckets.
+
+    Args:
+        df_in: Input DataFrame with DatetimeIndex.
+        target_column: Column to optimize visual fidelity for.
+        target_cadence: Target cadence as ISO duration string or Timedelta.
+        include_columns: Additional columns to include in output (interpolated
+            to LTTB-selected time points). If None, includes all numeric columns.
+        gap_threshold: Minimum duration to consider as a gap. If None, uses
+            2x target_cadence.
+        min_points_per_segment: Minimum points required per segment.
+
+    Returns:
+        DataFrame downsampled using LTTB algorithm.
+
+    Example:
+        >>> df = pd.DataFrame(
+        ...     {'signal': np.sin(np.linspace(0, 10*np.pi, 1000))},
+        ...     index=pd.date_range('2024-01-01', periods=1000, freq='1s')
+        ... )
+        >>> result = downsample_lttb(df, 'signal', 'PT10S')
+        >>> len(result) < len(df)
+        True
+    """
+    target_cadence = parse_cadence(target_cadence)
+
+    if gap_threshold is None:
+        gap_threshold = 2 * target_cadence
+
+    # Split at gaps and process each segment
+    segments = split_at_gaps(df_in, gap_threshold)
+
+    resampled_segments = []
+    for segment in segments:
+        if len(segment) < min_points_per_segment:
+            continue
+
+        resampled = _lttb_single_segment(
+            segment,
+            target_column,
+            target_cadence,
+            include_columns
+        )
+        if resampled is not None and len(resampled) > 0:
+            resampled_segments.append(resampled)
+
+    if not resampled_segments:
+        return pd.DataFrame(columns=df_in.columns)
+
+    return pd.concat(resampled_segments, axis=0).sort_index()
+
+
+def _lttb_single_segment(
+    df: pd.DataFrame,
+    target_column: str,
+    target_cadence: pd.Timedelta,
+    include_columns: list[str] | None = None
+) -> pd.DataFrame | None:
+    """Apply LTTB to a single contiguous segment.
+
+    Args:
+        df: Input DataFrame (no gaps).
+        target_column: Column to optimize for.
+        target_cadence: Target cadence.
+        include_columns: Additional columns to include.
+
+    Returns:
+        Downsampled DataFrame or None if cannot process.
+    """
+    # Compute number of output points
+    n_out = compute_output_points(df.index[0], df.index[-1], target_cadence)
+
+    if n_out < 3:
+        logging.warning("Cannot perform LTTB downsampling on less than 3 points")
+        return None
+
+    # Set up the data - convert time to numeric for LTTB algorithm
+    df_work = df.copy()
+    timeref = df.index[0]
+    timeunit = '1min'
+    df_work['time_num'] = (df_work.index - timeref) / pd.to_timedelta(timeunit)
+
+    # Prepare data for LTTB (time_num, target_column)
+    data = df_work[['time_num', target_column]].dropna().values
+
+    if len(data) < 3:
+        logging.warning("Insufficient non-NaN data points for LTTB")
+        return None
+
+    # Apply LTTB downsampling
+    result = lttb.downsample(data, n_out)
+    df_resampled = pd.DataFrame(
+        result,
+        columns=['time_num', target_column]
+    )
+
+    # Reconstruct the datetime index
+    df_resampled.index = (
+        timeref +
+        pd.to_timedelta(df_resampled['time_num'], unit='min')
+    )
+
+    # Determine which columns to interpolate
+    if include_columns is None:
+        cols_to_interp = get_numeric_columns(df)
+    else:
+        cols_to_interp = include_columns
+
+    # Interpolate other columns to LTTB-selected time points
+    for col in df.columns:
+        if col in ['time', 'time_num', target_column]:
+            continue
+        if include_columns is not None and col not in include_columns:
+            continue
+        if not pd.api.types.is_numeric_dtype(df[col]):
+            continue
+
+        df_resampled[col] = np.interp(
+            x=df_resampled['time_num'].values,
+            xp=df_work['time_num'].values,
+            fp=df_work[col].values
+        )
+
+    # Clean up
+    df_resampled = df_resampled.drop(['time_num'], axis=1)
+
+    return df_resampled
+
+
+def downsample_lttb_with_config(
+    df: pd.DataFrame,
+    target_cadence: str | pd.Timedelta,
+    config: DownsampleConfig
+) -> pd.DataFrame:
+    """Apply LTTB downsampling with full configuration.
+
+    Args:
+        df: Input DataFrame with DatetimeIndex.
+        target_cadence: Target cadence.
+        config: Downsampling configuration.
+
+    Returns:
+        Downsampled DataFrame.
+
+    Raises:
+        ValueError: If lttb_target_column is not specified in config.
+    """
+    target_cadence = parse_cadence(target_cadence)
+
+    if config.lttb_target_column is None:
+        raise ValueError("lttb_target_column must be specified for LTTB method")
+
+    # Determine gap threshold
+    gap_threshold = config.get_gap_threshold(target_cadence)
+
+    # Determine include columns
+    include_columns = config.include_columns if config.include_columns else None
+
+    # Apply LTTB
+    result = downsample_lttb(
+        df_in=df,
+        target_column=config.lttb_target_column,
+        target_cadence=target_cadence,
+        include_columns=include_columns,
+        gap_threshold=gap_threshold,
+        min_points_per_segment=config.min_points_per_segment
+    )
+
+    # Apply edge handling
+    if len(result) > 0:
+        result = apply_edge_handling(
+            result,
+            config.edge_handling,
+            config.edge_window
+        )
+
+    # Filter out excluded columns
+    if config.exclude_columns:
+        cols_to_drop = [c for c in config.exclude_columns if c in result.columns]
+        result = result.drop(columns=cols_to_drop)
+
+    return result

downsampler/utils.py

@@ -0,0 +1,150 @@
+"""Utility functions for downsampler."""
+
+import pandas as pd
+import numpy as np
+from scipy.interpolate import interp1d
+
+
+def estimate_cadence(df: pd.DataFrame) -> pd.Timedelta:
+    """Estimate the cadence of a time series DataFrame.
+
+    Uses the median of time differences to be robust to gaps.
+
+    Args:
+        df: DataFrame with DatetimeIndex.
+
+    Returns:
+        Estimated cadence as a Timedelta.
+
+    Raises:
+        ValueError: If DataFrame has fewer than 2 rows.
+    """
+    if len(df) < 2:
+        raise ValueError("DataFrame must have at least 2 rows to estimate cadence")
+
+    deltas = pd.Series(df.index).diff().dropna()
+    return deltas.median()
+
+
+def parse_cadence(cadence: str | pd.Timedelta) -> pd.Timedelta:
+    """Parse a cadence specification into a Timedelta.
+
+    Args:
+        cadence: Either a Timedelta or an ISO 8601 duration string
+            (e.g., "PT1H", "P1D").
+
+    Returns:
+        The cadence as a Timedelta.
+    """
+    if isinstance(cadence, pd.Timedelta):
+        return cadence
+    return pd.to_timedelta(cadence)
+
+
+def get_numeric_columns(df: pd.DataFrame) -> list[str]:
+    """Get list of numeric columns in a DataFrame.
+
+    Args:
+        df: Input DataFrame.
+
+    Returns:
+        List of column names that have numeric dtype.
+    """
+    return [col for col in df.columns if pd.api.types.is_numeric_dtype(df[col])]
+
+
+def filter_columns(
+    df: pd.DataFrame,
+    include: list[str] | None = None,
+    exclude: list[str] | None = None
+) -> list[str]:
+    """Filter columns based on include/exclude lists.
+
+    Args:
+        df: Input DataFrame.
+        include: Columns to include. If empty or None, all columns are included.
+        exclude: Columns to exclude.
+
+    Returns:
+        Filtered list of column names.
+    """
+    columns = list(df.columns)
+
+    if include:
+        columns = [c for c in columns if c in include]
+
+    if exclude:
+        columns = [c for c in columns if c not in exclude]
+
+    return columns
+
+
+def setup_interpolator(
+    dataframe: pd.DataFrame,
+    field: str,
+    kind: str = 'linear'
+) -> callable:
+    """Set up an interpolation function for a DataFrame column.
+
+    Args:
+        dataframe: Input DataFrame with DatetimeIndex.
+        field: Column name to interpolate.
+        kind: Type of interpolation ('linear', 'cubic', etc.).
+
+    Returns:
+        A function that takes a timestamp and returns the interpolated value.
+    """
+    reftime = dataframe.index[0]
+    raw_interpolator = interp1d(
+        x=((dataframe.index - reftime) / pd.to_timedelta(1, 'min')),
+        y=dataframe[field],
+        kind=kind,
+        bounds_error=False,
+        fill_value=np.nan
+    )
+
+    def func(t):
+        result = raw_interpolator((t - reftime) / pd.to_timedelta(1, 'min'))
+        if np.shape(result) == ():  # Convert to float if result is scalar
+            result = float(result)
+        return result
+
+    return func
+
+
+def setup_interpolators(
+    dataframe: pd.DataFrame,
+    kind: str = 'linear'
+) -> dict[str, callable]:
+    """Set up interpolation functions for all numeric columns in a DataFrame.
+
+    Args:
+        dataframe: Input DataFrame with DatetimeIndex.
+        kind: Type of interpolation ('linear', 'cubic', etc.).
+
+    Returns:
+        Dictionary mapping column names to interpolation functions.
+    """
+    interpolators = {}
+    fields = get_numeric_columns(dataframe)
+    for field in fields:
+        interpolators[field] = setup_interpolator(dataframe, field, kind)
+    return interpolators
+
+
+def compute_output_points(
+    start: pd.Timestamp,
+    end: pd.Timestamp,
+    target_cadence: pd.Timedelta
+) -> int:
+    """Compute the number of output points for a given time range and cadence.
+
+    Args:
+        start: Start timestamp.
+        end: End timestamp.
+        target_cadence: Target cadence.
+
+    Returns:
+        Number of output points.
+    """
+    return int(1 + (end - start) / target_cadence)

downsampler-0.1.0.dist-info/METADATA

@@ -0,0 +1,246 @@
+Metadata-Version: 2.4
+Name: downsampler
+Version: 0.1.0
+Summary: Timeseries DataFrame downsampling with LTTB, aggregation methods, gap handling, and fidelity testing
+Author-email: Eelco Doornbos <eelco.doornbos@knmi.nl>
+License: MIT
+Project-URL: Homepage, https://gitlab.com/KNMI-OSS/spaceweather/downsampler
+Project-URL: Repository, https://gitlab.com/KNMI-OSS/spaceweather/downsampler
+Keywords: timeseries,downsampling,lttb,pandas,visualization
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Science/Research
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Topic :: Scientific/Engineering
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: numpy>=1.20
+Requires-Dist: pandas>=1.3
+Requires-Dist: lttb>=0.3
+Requires-Dist: scipy>=1.7
+Provides-Extra: viz
+Requires-Dist: matplotlib>=3.5; extra == "viz"
+Requires-Dist: altair>=5.0; extra == "viz"
+Provides-Extra: test
+Requires-Dist: pytest>=7.0; extra == "test"
+Requires-Dist: pytest-cov>=4.0; extra == "test"
+Provides-Extra: dev
+Requires-Dist: downsampler[test,viz]; extra == "dev"
+Dynamic: license-file
+
+# downsampler
+
+A Python package for time series DataFrame downsampling with LTTB, multiple aggregation methods, gap handling, and fidelity testing.
+
+## Features
+
+- **Multiple downsampling methods**: LTTB (visual fidelity), mean, median, min, max
+- **Gap-aware processing**: Automatically detects and handles gaps in time series
+- **Edge handling**: Flag, discard, or keep edge points
+- **Multi-aggregate output**: Generate min/mean/max columns in a single call
+- **Deferred fetching**: Lazy data loading with automatic edge buffering
+- **Fidelity testing**: Compare methods and measure visual accuracy
+
+## Installation
+
+```bash
+pip install downsampler
+```
+
+Or with visualization support:
+
+```bash
+pip install downsampler[viz]
+```
+
+## Quick Start
+
+### Basic Downsampling
+
+```python
+import pandas as pd
+from downsampler import downsample
+
+# Create sample data
+df = pd.DataFrame(
+    {'temperature': range(1000)},
+    index=pd.date_range('2024-01-01', periods=1000, freq='1s')
+)
+
+# Downsample to 1-minute cadence (default: mean)
+result = downsample(df, target_cadence='PT1M')
+```
+
+### Using Different Methods
+
+```python
+from downsampler import downsample, DownsampleConfig, AggregationMethod
+
+# Mean (default)
+result = downsample(df, '10min')
+
+# Maximum
+result = downsample(df, '10min', method='max')
+
+# LTTB for visual fidelity
+config = DownsampleConfig(
+    method=AggregationMethod.LTTB,
+    lttb_target_column='temperature'
+)
+result = downsample(df, '10min', config=config)
+```
+
+### Multi-Aggregate Downsampling
+
+Create min/mean/max columns for visualization with error bands:
+
+```python
+from downsampler import downsample_multi_aggregate
+
+result = downsample_multi_aggregate(
+    df,
+    target_cadence='1min',
+    variables=['temperature', 'pressure'],
+    aggregations=['min', 'mean', 'max']
+)
+# Result has columns: temperature_min, temperature_mean, temperature_max, etc.
+```
+
+### Handling Gaps
+
+```python
+from downsampler import DownsampleConfig, GapHandling
+
+config = DownsampleConfig(
+    gap_handling=GapHandling.SEGMENT,  # Process segments independently
+    gap_threshold='5min'  # Gaps > 5 min trigger segmentation
+)
+result = downsample(df, '1min', config=config)
+```
+
+### Deferred Data Fetching
+
+For data that needs to be fetched from an external source:
+
+```python
+from downsampler.deferred import deferred_downsample
+
+def fetch_from_api(start, end):
+    # Your data fetching logic here
+    return pd.DataFrame(...)
+
+result = deferred_downsample(
+    fetcher=fetch_from_api,
+    output_start=pd.Timestamp('2024-01-01'),
+    output_end=pd.Timestamp('2024-01-02'),
+    target_cadence='1H'
+)
+```
+
+### Fidelity Comparison
+
+Compare different methods to find the best one for your data:
+
+```python
+from downsampler.fidelity import FidelityComparison
+
+comp = FidelityComparison(original_df, 'signal')
+results = comp.compare_methods('10s')
+
+for r in results:
+    print(f"{r.method.value}: RMSE={r.metrics.rmse:.4f}")
+
+# Get recommendation
+config = comp.recommend_settings('10s', priority='visual')
+```
+
+## Configuration Options
+
+### DownsampleConfig
+
+| Parameter | Type | Default | Description |
+|-----------|------|---------|-------------|
+| `method` | AggregationMethod | MEAN | Downsampling method |
+| `lttb_target_column` | str | None | Column to optimize for LTTB |
+| `include_columns` | list[str] | [] | Columns to include (empty = all) |
+| `exclude_columns` | list[str] | [] | Columns to exclude |
+| `gap_handling` | GapHandling | SEGMENT | How to handle gaps |
+| `gap_threshold` | str/Timedelta | "auto" | Min duration for gaps |
+| `edge_handling` | EdgeHandling | FLAG | How to handle edges |
+| `edge_window` | int | 2 | Points at each edge |
+| `min_points_per_segment` | int | 3 | Min points for processing |
+
+### Aggregation Methods
+
+- `MEAN`: Arithmetic mean (best for general use)
+- `MEDIAN`: Median (robust to outliers)
+- `MIN`: Minimum value (preserves lows)
+- `MAX`: Maximum value (preserves highs)
+- `LTTB`: Largest Triangle Three Buckets (best visual fidelity)
+
+### Gap Handling
+
+- `SEGMENT`: Split at gaps, process independently (recommended)
+- `INTERPOLATE`: Fill gaps before processing
+- `IGNORE`: Treat as continuous data
+
+### Edge Handling
+
+- `KEEP`: Keep edge points as-is
+- `FLAG`: Add `_is_edge` column
+- `DISCARD`: Remove edge points
+
+## Examples
+
+See the `examples/` directory for complete examples:
+
+- `basic_downsampling.py`: Core downsampling features
+- `multi_aggregate.py`: Creating min/mean/max columns
+- `deferred_fetch.py`: Lazy data loading
+- `fidelity_comparison.marimo.py`: Interactive comparison notebook
+
+## API Reference
+
+### Core Functions
+
+```python
+downsample(df, target_cadence, config=None, **kwargs) -> DataFrame
+downsample_multi_aggregate(df, target_cadence, variables, aggregations, ...) -> DataFrame
+```
+
+### Gap Functions
+
+```python
+find_gap_indices(df, timedelta_max_gap) -> Series
+groupby_gaps(df, timedelta_max_gap) -> DataFrameGroupBy
+split_at_gaps(df, timedelta_max_gap) -> list[DataFrame]
+mark_gaps_in_dataframe(df, nominal_timedelta, ...) -> DataFrame
+```
+
+### Deferred Functions
+
+```python
+deferred_downsample(fetcher, output_start, output_end, target_cadence, ...) -> DataFrame
+batch_deferred_downsample(fetcher, ..., batch_size) -> DataFrame
+LazyDownsampler(fetcher, cache_buffer) -> LazyDownsampler
+```
+
+### Fidelity Functions
+
+```python
+compute_metrics(original, downsampled, column) -> FidelityMetrics
+FidelityComparison(original_df, column) -> FidelityComparison
+plot_comparison(original, downsampled, column, backend) -> Figure
+```
+
+## License
+
+MIT License - see LICENSE file for details.
+
+## Contributing
+
+Contributions are welcome! Please feel free to submit issues and pull requests.

downsampler-0.1.0.dist-info/RECORD

@@ -0,0 +1,18 @@
+downsampler/__init__.py,sha256=VbLTpckyt-beR7XNugIFmB4ki7A9UwoMQglLnsQBjDs,2104
+downsampler/aggregators.py,sha256=QXraUXjX4s335Fz9AXq5dIRnIl1GOKVViO1u1IVYQAY,10989
+downsampler/config.py,sha256=YU2Z4JEcb-xlHdanHef-lGSMwrjyaFDALLGtV1FwXxs,2613
+downsampler/core.py,sha256=j9dDSO_D_IfVGwwGIVw3Zg4e8RThSR0UkKY8I5xzfDw,5727
+downsampler/deferred.py,sha256=-uFRpHV5bGb06uXehhO0BPmjNt7uyvzPtTmEYMTlCRs,11859
+downsampler/edges.py,sha256=gzXX9bScf6rx2fEUts3oNVmAYAiE2hmrXIbvqTxLEsI,5863
+downsampler/gaps.py,sha256=-yxk5dNtBORevTCA5_LZz-fxGAdsUaoPabos7jagUHg,9969
+downsampler/lttb.py,sha256=3fWf4pOC61GRsmF_z-clOnPJ8W-XS9auI73eKpwMVaI,6471
+downsampler/utils.py,sha256=yu_AXEMyy8F3LP4a9QoWeravNGYR9m56viC9menlYAU,3914
+downsampler/fidelity/__init__.py,sha256=Ta4wNREv06A53avBizTzPfORK7VzEgkyAjekcZNdsSk,658
+downsampler/fidelity/comparison.py,sha256=EP-l-Dpnkp9MJVoNXqj6a7-Y5MlL8UrBD9eBWJhbnDY,11673
+downsampler/fidelity/metrics.py,sha256=Ls1yWPuDaB5WoAHYNKSsA92hrodo6eCyOVUcFYEAbzM,6687
+downsampler/fidelity/visualization.py,sha256=j6NcdZud6JM8CoMJjajSzGY3qZF1CfpDH8rNnc80-1g,10251
+downsampler-0.1.0.dist-info/licenses/LICENSE,sha256=F0hoUNYeL_QW7JqUFG8CHNuxBnmU0dzLf_xB78xb9gI,1061
+downsampler-0.1.0.dist-info/METADATA,sha256=secDw53t09IzwiagKz98QhjzLPgstivWgcFBYLwPGNM,7006
+downsampler-0.1.0.dist-info/WHEEL,sha256=qELbo2s1Yzl39ZmrAibXA2jjPLUYfnVhUNTlyF1rq0Y,92
+downsampler-0.1.0.dist-info/top_level.txt,sha256=LjlUQA5drUOEpNbPWlA_dx5GT8Xp45GZp1Yhs_r_DSQ,12
+downsampler-0.1.0.dist-info/RECORD,,

downsampler-0.1.0.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (80.10.1)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

downsampler-0.1.0.dist-info/licenses/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 KNMI
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

downsampler-0.1.0.dist-info/top_level.txt

@@ -0,0 +1 @@
+downsampler