downsampler 0.1.0__py3-none-any.whl

downsampler/fidelity/visualization.py

@@ -0,0 +1,359 @@
+"""Visualization helpers for fidelity comparison.
+
+Provides functions for plotting original vs downsampled data and
+comparing different methods.
+"""
+
+from typing import Any
+import pandas as pd
+import numpy as np
+
+from downsampler.fidelity.comparison import ComparisonResult
+
+
+def plot_comparison(
+    original: pd.DataFrame,
+    downsampled: pd.DataFrame,
+    column: str,
+    backend: str = "matplotlib",
+    title: str | None = None,
+    **kwargs
+) -> Any:
+    """Plot original and downsampled data for visual comparison.
+
+    Args:
+        original: Original DataFrame.
+        downsampled: Downsampled DataFrame.
+        column: Column to plot.
+        backend: Plotting backend ('matplotlib' or 'altair').
+        title: Optional plot title.
+        **kwargs: Additional arguments passed to the plotting function.
+
+    Returns:
+        Plot object (matplotlib Figure or Altair Chart).
+
+    Example:
+        >>> fig = plot_comparison(original_df, downsampled_df, 'signal')
+        >>> fig.savefig('comparison.png')
+    """
+    if backend == "matplotlib":
+        return _plot_matplotlib(original, downsampled, column, title, **kwargs)
+    elif backend == "altair":
+        return _plot_altair(original, downsampled, column, title, **kwargs)
+    else:
+        raise ValueError(f"Unknown backend: {backend}")
+
+
+def _plot_matplotlib(
+    original: pd.DataFrame,
+    downsampled: pd.DataFrame,
+    column: str,
+    title: str | None = None,
+    figsize: tuple = (12, 6),
+    **kwargs
+) -> Any:
+    """Create matplotlib comparison plot."""
+    try:
+        import matplotlib.pyplot as plt
+    except ImportError:
+        raise ImportError("matplotlib is required for this function. Install with: pip install matplotlib")
+
+    fig, ax = plt.subplots(figsize=figsize)
+
+    # Plot original
+    ax.plot(
+        original.index,
+        original[column],
+        label='Original',
+        alpha=0.7,
+        linewidth=0.5,
+        color='blue'
+    )
+
+    # Plot downsampled
+    ax.plot(
+        downsampled.index,
+        downsampled[column],
+        label='Downsampled',
+        alpha=0.9,
+        linewidth=1.5,
+        color='red',
+        marker='.',
+        markersize=3
+    )
+
+    ax.set_xlabel('Time')
+    ax.set_ylabel(column)
+    ax.legend()
+
+    if title:
+        ax.set_title(title)
+    else:
+        reduction = len(original) / len(downsampled) if len(downsampled) > 0 else float('inf')
+        ax.set_title(f'Downsampling Comparison ({reduction:.1f}x reduction)')
+
+    fig.tight_layout()
+    return fig
+
+
+def _plot_altair(
+    original: pd.DataFrame,
+    downsampled: pd.DataFrame,
+    column: str,
+    title: str | None = None,
+    width: int = 800,
+    height: int = 400,
+    **kwargs
+) -> Any:
+    """Create Altair comparison plot."""
+    try:
+        import altair as alt
+    except ImportError:
+        raise ImportError("altair is required for this function. Install with: pip install altair")
+
+    # Prepare data
+    orig_data = original[[column]].reset_index()
+    orig_data.columns = ['time', column]
+    orig_data['source'] = 'Original'
+
+    ds_data = downsampled[[column]].reset_index()
+    ds_data.columns = ['time', column]
+    ds_data['source'] = 'Downsampled'
+
+    combined = pd.concat([orig_data, ds_data])
+
+    chart = alt.Chart(combined).mark_line().encode(
+        x='time:T',
+        y=f'{column}:Q',
+        color='source:N',
+        strokeWidth=alt.condition(
+            alt.datum.source == 'Downsampled',
+            alt.value(2),
+            alt.value(0.5)
+        ),
+        opacity=alt.condition(
+            alt.datum.source == 'Downsampled',
+            alt.value(1),
+            alt.value(0.7)
+        )
+    ).properties(
+        width=width,
+        height=height,
+        title=title or 'Downsampling Comparison'
+    )
+
+    return chart
+
+
+def plot_method_comparison(
+    results: list[ComparisonResult],
+    metric: str = "rmse",
+    backend: str = "matplotlib",
+    **kwargs
+) -> Any:
+    """Plot comparison of different methods by a specific metric.
+
+    Args:
+        results: List of ComparisonResult objects.
+        metric: Metric to compare ('rmse', 'mae', 'pearson_r', etc.).
+        backend: Plotting backend ('matplotlib' or 'altair').
+        **kwargs: Additional arguments passed to the plotting function.
+
+    Returns:
+        Plot object.
+    """
+    if backend == "matplotlib":
+        return _plot_method_comparison_matplotlib(results, metric, **kwargs)
+    elif backend == "altair":
+        return _plot_method_comparison_altair(results, metric, **kwargs)
+    else:
+        raise ValueError(f"Unknown backend: {backend}")
+
+
+def _plot_method_comparison_matplotlib(
+    results: list[ComparisonResult],
+    metric: str = "rmse",
+    figsize: tuple = (10, 6),
+    **kwargs
+) -> Any:
+    """Create matplotlib method comparison bar chart."""
+    try:
+        import matplotlib.pyplot as plt
+    except ImportError:
+        raise ImportError("matplotlib is required")
+
+    # Extract data
+    methods = [r.method.value for r in results]
+    values = [getattr(r.metrics, metric) for r in results]
+
+    fig, ax = plt.subplots(figsize=figsize)
+    bars = ax.bar(methods, values, color='steelblue')
+
+    # Highlight best value
+    if metric in ['rmse', 'mae', 'max_error', 'peak_error']:
+        best_idx = np.argmin(values)
+    else:
+        best_idx = np.argmax(values)
+
+    bars[best_idx].set_color('green')
+
+    ax.set_xlabel('Method')
+    ax.set_ylabel(metric.upper())
+    ax.set_title(f'Method Comparison by {metric.upper()}')
+
+    plt.xticks(rotation=45, ha='right')
+    fig.tight_layout()
+
+    return fig
+
+
+def _plot_method_comparison_altair(
+    results: list[ComparisonResult],
+    metric: str = "rmse",
+    width: int = 400,
+    height: int = 300,
+    **kwargs
+) -> Any:
+    """Create Altair method comparison bar chart."""
+    try:
+        import altair as alt
+    except ImportError:
+        raise ImportError("altair is required")
+
+    data = pd.DataFrame([
+        {'method': r.method.value, metric: getattr(r.metrics, metric)}
+        for r in results
+    ])
+
+    chart = alt.Chart(data).mark_bar().encode(
+        x=alt.X('method:N', sort='-y'),
+        y=f'{metric}:Q',
+        color=alt.Color('method:N', legend=None)
+    ).properties(
+        width=width,
+        height=height,
+        title=f'Method Comparison by {metric.upper()}'
+    )
+
+    return chart
+
+
+class MarimoHelper:
+    """Helper class for creating interactive Marimo notebook comparisons.
+
+    Example usage in a Marimo notebook:
+        >>> from downsampler.fidelity import MarimoHelper
+        >>> helper = MarimoHelper()
+        >>> ui, output = helper.interactive_comparison(
+        ...     original_df, 'signal',
+        ...     cadences=['1min', '5min', '10min'],
+        ...     methods=['mean', 'lttb']
+        ... )
+        >>> # Display ui and output in separate cells
+    """
+
+    @staticmethod
+    def interactive_comparison(
+        original: pd.DataFrame,
+        column: str,
+        cadences: list[str],
+        methods: list[str] | None = None
+    ) -> tuple[Any, Any]:
+        """Create interactive comparison UI for Marimo notebooks.
+
+        Args:
+            original: Original DataFrame.
+            column: Column to compare.
+            cadences: List of cadence options.
+            methods: List of method options. If None, uses all methods.
+
+        Returns:
+            Tuple of (ui_element, output_function) for Marimo.
+        """
+        try:
+            import marimo as mo
+        except ImportError:
+            raise ImportError(
+                "marimo is required for interactive comparisons. "
+                "Install with: pip install marimo"
+            )
+
+        from downsampler.config import AggregationMethod
+        from downsampler.core import downsample
+        from downsampler.fidelity.metrics import compute_metrics
+
+        if methods is None:
+            methods = [m.value for m in AggregationMethod]
+
+        # Create UI elements
+        cadence_select = mo.ui.dropdown(
+            options=cadences,
+            value=cadences[0],
+            label="Target Cadence"
+        )
+
+        method_select = mo.ui.dropdown(
+            options=methods,
+            value=methods[0],
+            label="Method"
+        )
+
+        ui = mo.vstack([cadence_select, method_select])
+
+        def compute_output():
+            cadence = cadence_select.value
+            method = AggregationMethod(method_select.value)
+
+            from downsampler.config import DownsampleConfig
+
+            config = DownsampleConfig(
+                method=method,
+                lttb_target_column=column if method == AggregationMethod.LTTB else None
+            )
+
+            downsampled = downsample(original, cadence, config)
+            metrics = compute_metrics(original, downsampled, column)
+
+            # Create plot
+            fig = plot_comparison(original, downsampled, column, backend="matplotlib")
+
+            return mo.vstack([
+                mo.md(f"### Results for {method.value} at {cadence}"),
+                mo.md(f"**Reduction:** {len(original)/len(downsampled):.1f}x"),
+                mo.md(f"**RMSE:** {metrics.rmse:.6f}"),
+                mo.md(f"**Correlation:** {metrics.pearson_r:.4f}"),
+                fig
+            ])
+
+        return ui, compute_output
+
+    @staticmethod
+    def comparison_table(
+        original: pd.DataFrame,
+        column: str,
+        cadences: list[str],
+        methods: list[str] | None = None
+    ) -> pd.DataFrame:
+        """Generate a comparison table for multiple cadences and methods.
+
+        Args:
+            original: Original DataFrame.
+            column: Column to compare.
+            cadences: List of cadences to compare.
+            methods: List of methods. If None, uses all methods.
+
+        Returns:
+            DataFrame with comparison metrics.
+        """
+        from downsampler.fidelity.comparison import FidelityComparison
+        from downsampler.config import AggregationMethod
+
+        if methods is None:
+            method_enums = list(AggregationMethod)
+        else:
+            method_enums = [AggregationMethod(m) for m in methods]
+
+        comp = FidelityComparison(original, column)
+        results = comp.compare_grid(cadences, method_enums)
+
+        return comp.summary_table(results)

downsampler/gaps.py

@@ -0,0 +1,310 @@
+"""Gap detection and handling for time series data."""
+
+import pandas as pd
+import numpy as np
+from typing import Iterator
+
+
+def find_gap_indices(
+    df: pd.DataFrame,
+    timedelta_max_gap: pd.Timedelta
+) -> pd.Series:
+    """Find gaps in a DataFrame and return their locations and durations.
+
+    Identifies gaps in the DataFrame's DatetimeIndex that are equal to or
+    longer than the specified threshold.
+
+    Args:
+        df: DataFrame with DatetimeIndex.
+        timedelta_max_gap: Minimum duration to consider as a gap.
+
+    Returns:
+        Series where indices are the integer positions of gap starts in the
+        DataFrame, and values are the gap durations as multiples of
+        timedelta_max_gap.
+
+    Example:
+        >>> df = pd.DataFrame(
+        ...     {'value': [1, 2, 3]},
+        ...     index=pd.to_datetime(['2024-01-01 00:00', '2024-01-01 00:01', '2024-01-01 00:10'])
+        ... )
+        >>> gaps = find_gap_indices(df, pd.Timedelta('5min'))
+        >>> len(gaps)  # One gap found
+        1
+    """
+    deltas = pd.Series(df.index).diff()[1:]
+    gaps = deltas[deltas >= timedelta_max_gap] / timedelta_max_gap
+    return gaps
+
+
+def groupby_gaps(
+    df: pd.DataFrame,
+    timedelta_max_gap: pd.Timedelta
+) -> pd.api.typing.DataFrameGroupBy:
+    """Split a DataFrame at gaps and return a groupby object.
+
+    Finds gaps in the DataFrame and returns a groupby object where each
+    group is a contiguous segment between gaps.
+
+    Args:
+        df: DataFrame with DatetimeIndex.
+        timedelta_max_gap: Minimum duration to consider as a gap.
+
+    Returns:
+        DataFrameGroupBy object where each group is a contiguous segment.
+
+    Note:
+        This function modifies the input DataFrame by adding a 'gap_index'
+        column. Use .copy() if you need to preserve the original.
+
+    Example:
+        >>> df = pd.DataFrame(
+        ...     {'value': [1, 2, 3, 4, 5]},
+        ...     index=pd.to_datetime([
+        ...         '2024-01-01 00:00', '2024-01-01 00:01', '2024-01-01 00:02',
+        ...         '2024-01-01 00:10', '2024-01-01 00:11'
+        ...     ])
+        ... )
+        >>> groups = groupby_gaps(df.copy(), pd.Timedelta('5min'))
+        >>> len(list(groups))  # Two segments
+        2
+    """
+    deltas = df.index.diff()[1:]
+    gap_indices = (deltas >= timedelta_max_gap).cumsum()
+    df['gap_index'] = [0, *gap_indices]
+    dfs_out = df.groupby('gap_index')
+    return dfs_out
+
+
+def split_at_gaps(
+    df: pd.DataFrame,
+    timedelta_max_gap: pd.Timedelta
+) -> list[pd.DataFrame]:
+    """Split a DataFrame at gaps into a list of DataFrames.
+
+    Finds gaps in the DataFrame and returns a list of DataFrames,
+    each representing a contiguous segment between gaps.
+
+    Args:
+        df: DataFrame with DatetimeIndex.
+        timedelta_max_gap: Minimum duration to consider as a gap.
+
+    Returns:
+        List of DataFrames, one for each contiguous segment.
+
+    Example:
+        >>> df = pd.DataFrame(
+        ...     {'value': [1, 2, 3, 4, 5]},
+        ...     index=pd.to_datetime([
+        ...         '2024-01-01 00:00', '2024-01-01 00:01', '2024-01-01 00:02',
+        ...         '2024-01-01 00:10', '2024-01-01 00:11'
+        ...     ])
+        ... )
+        >>> segments = split_at_gaps(df, pd.Timedelta('5min'))
+        >>> len(segments)
+        2
+        >>> len(segments[0])
+        3
+        >>> len(segments[1])
+        2
+    """
+    df_work = df.copy()
+    groups = groupby_gaps(df_work, timedelta_max_gap)
+    return [group.drop(columns=['gap_index']) for _, group in groups]
+
+
+def iter_segments(
+    df: pd.DataFrame,
+    timedelta_max_gap: pd.Timedelta,
+    min_points: int = 1
+) -> Iterator[pd.DataFrame]:
+    """Iterate over contiguous segments in a DataFrame.
+
+    Yields DataFrames representing contiguous segments between gaps,
+    optionally filtering out segments with too few points.
+
+    Args:
+        df: DataFrame with DatetimeIndex.
+        timedelta_max_gap: Minimum duration to consider as a gap.
+        min_points: Minimum number of points required in a segment.
+
+    Yields:
+        DataFrames for each contiguous segment with at least min_points.
+
+    Example:
+        >>> df = pd.DataFrame(
+        ...     {'value': [1, 2, 3, 4, 5]},
+        ...     index=pd.to_datetime([
+        ...         '2024-01-01 00:00', '2024-01-01 00:01', '2024-01-01 00:02',
+        ...         '2024-01-01 00:10', '2024-01-01 00:11'
+        ...     ])
+        ... )
+        >>> for segment in iter_segments(df, pd.Timedelta('5min'), min_points=3):
+        ...     print(len(segment))
+        3
+    """
+    df_work = df.copy()
+    groups = groupby_gaps(df_work, timedelta_max_gap)
+
+    for _, group in groups:
+        segment = group.drop(columns=['gap_index'])
+        if len(segment) >= min_points:
+            yield segment
+
+
+def wrap_in_nans(
+    df_in: pd.DataFrame,
+    offset: str = 'PT0.1S',
+    where: str = 'both'
+) -> pd.DataFrame:
+    """Add NaN boundary rows before and/or after a DataFrame.
+
+    Returns a copy of the DataFrame with rows of NaN values added at the
+    start and/or end. This is useful for creating visual breaks in plots.
+
+    Args:
+        df_in: Input DataFrame with DatetimeIndex.
+        offset: Time offset for the NaN rows as ISO 8601 duration string.
+        where: Where to add NaN rows: 'start', 'end', or 'both'.
+
+    Returns:
+        DataFrame with NaN boundary rows added.
+
+    Example:
+        >>> df = pd.DataFrame(
+        ...     {'value': [1, 2, 3]},
+        ...     index=pd.date_range('2024-01-01', periods=3, freq='1min')
+        ... )
+        >>> wrapped = wrap_in_nans(df, offset='PT1S')
+        >>> len(wrapped)
+        5
+        >>> np.isnan(wrapped.iloc[0]['value'])
+        True
+    """
+    df_out = df_in.copy()
+    data_nans = {col: np.nan for col in df_out.columns}
+    offset_timedelta = pd.to_timedelta(offset)
+
+    # Add gap before start
+    if where in ('start', 'both'):
+        df_new_record_before = pd.DataFrame(
+            data=data_nans,
+            index=[df_out.index[0] - offset_timedelta]
+        )
+        df_out = pd.concat([df_new_record_before, df_out])
+
+    # Add gap after end
+    if where in ('end', 'both'):
+        df_new_record_after = pd.DataFrame(
+            data=data_nans,
+            index=[df_out.index[-1] + offset_timedelta]
+        )
+        df_out = pd.concat([df_out, df_new_record_after])
+
+    return df_out
+
+
+def mark_gaps_in_dataframe(
+    df: pd.DataFrame,
+    nominal_timedelta: pd.Timedelta = pd.to_timedelta(1, 'min'),
+    nominal_start_time: pd.Timestamp | None = None,
+    nominal_end_time: pd.Timestamp | None = None
+) -> pd.DataFrame:
+    """Insert NaN records at gaps to create visual breaks in plots.
+
+    Looks for gaps in the DataFrame and inserts NaN records to ensure
+    that plotting libraries will show breaks at gap locations.
+
+    Args:
+        df: DataFrame with DatetimeIndex.
+        nominal_timedelta: Expected cadence of the time series.
+        nominal_start_time: If provided, add NaN before first record if
+            it's after this time.
+        nominal_end_time: If provided, add NaN after last record if
+            it's before this time.
+
+    Returns:
+        DataFrame with NaN records inserted at gap locations.
+
+    Example:
+        >>> df = pd.DataFrame(
+        ...     {'value': [1, 2, 3]},
+        ...     index=pd.to_datetime([
+        ...         '2024-01-01 00:00', '2024-01-01 00:01', '2024-01-01 00:10'
+        ...     ])
+        ... )
+        >>> marked = mark_gaps_in_dataframe(df, pd.Timedelta('1min'))
+        >>> len(marked) > len(df)
+        True
+    """
+    deltas = pd.Series(df.index).diff()[1:]
+    gaps = deltas[deltas > nominal_timedelta] / nominal_timedelta
+
+    df_gapfilled = df.copy()
+    data_nans = {col: np.nan for col in df.columns}
+
+    for i, gap in gaps.items():
+        # Add a np.nan record after the start of each gap,
+        # to force breaks in plotted lines
+        time_gap_start = df.index[i - 1] + nominal_timedelta
+        df_new_record = pd.DataFrame(data=data_nans, index=[time_gap_start])
+        df_gapfilled = pd.concat([df_gapfilled, df_new_record]).sort_index()
+
+        # For gaps longer than 1 record, also add a np.nan record before the
+        # end of the gap
+        if gap > 2:
+            time_gap_end = df.index[i] - nominal_timedelta
+            df_new_record = pd.DataFrame(data=data_nans, index=[time_gap_end])
+            df_gapfilled = pd.concat([df_gapfilled, df_new_record]).sort_index()
+
+    # Add gap before start
+    if nominal_start_time is not None:
+        if df.index[0] > nominal_start_time:
+            df_new_record_before = pd.DataFrame(
+                data=data_nans,
+                index=[df.index[0] - nominal_timedelta]
+            )
+            df_gapfilled = pd.concat([df_new_record_before, df_gapfilled])
+
+    # Add gap after end
+    if nominal_end_time is not None:
+        if df.index[-1] < nominal_end_time:
+            df_new_record_after = pd.DataFrame(
+                data=data_nans,
+                index=[df.index[-1] + nominal_timedelta]
+            )
+            df_gapfilled = pd.concat([df_gapfilled, df_new_record_after])
+
+    return df_gapfilled
+
+
+def has_gaps(df: pd.DataFrame, threshold: pd.Timedelta) -> bool:
+    """Check if a DataFrame has gaps larger than the threshold.
+
+    Args:
+        df: DataFrame with DatetimeIndex.
+        threshold: Minimum duration to consider as a gap.
+
+    Returns:
+        True if gaps are found, False otherwise.
+    """
+    if len(df) < 2:
+        return False
+    deltas = df.index.diff()[1:]
+    return (deltas >= threshold).any()
+
+
+def count_gaps(df: pd.DataFrame, threshold: pd.Timedelta) -> int:
+    """Count the number of gaps in a DataFrame.
+
+    Args:
+        df: DataFrame with DatetimeIndex.
+        threshold: Minimum duration to consider as a gap.
+
+    Returns:
+        Number of gaps found.
+    """
+    if len(df) < 2:
+        return 0
+    deltas = df.index.diff()[1:]
+    return (deltas >= threshold).sum()