sensor-modeling 0.2.0__py3-none-any.whl

sensor_modeling/utils/missing.py

@@ -0,0 +1,188 @@
+"""Utilities for handling missing sensor data."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import Any
+
+import pandas as pd
+
+
+@dataclass
+class MissingDataResult:
+    """Container for imputed data and uncertainty metadata.
+
+    Attributes
+    ----------
+    data
+        Data after applying the selected missing-data strategy.
+    original_missing_mask
+        Boolean mask identifying cells that were missing in the input.
+    imputed_mask
+        Boolean mask identifying cells that were missing originally and were
+        filled by the chosen strategy.
+    remaining_missing_mask
+        Boolean mask identifying cells still missing after processing.
+    long_gap_mask
+        Boolean mask marking gaps longer than ``max_gap`` when provided.
+    summary
+        Per-column metadata about missingness, imputation, and longest gaps.
+    """
+
+    data: pd.DataFrame
+    original_missing_mask: pd.DataFrame
+    imputed_mask: pd.DataFrame
+    remaining_missing_mask: pd.DataFrame
+    long_gap_mask: pd.DataFrame
+    summary: dict[str, dict[str, Any]]
+
+
+def _gap_lengths(mask: pd.Series) -> pd.Series:
+    """Return the length of each missing run for every position."""
+    lengths = pd.Series(0, index=mask.index, dtype=int)
+    run_length = 0
+    run_index: list[Any] = []
+    for idx, is_missing in mask.items():
+        if bool(is_missing):
+            run_length += 1
+            run_index.append(idx)
+            continue
+        if run_index:
+            lengths.loc[run_index] = run_length
+            run_length = 0
+            run_index = []
+    if run_index:
+        lengths.loc[run_index] = run_length
+    return lengths
+
+
+def _long_gap_mask(df: pd.DataFrame, max_gap: int | None) -> pd.DataFrame:
+    """Return a boolean mask for missing runs exceeding ``max_gap``."""
+    if max_gap is None:
+        return pd.DataFrame(False, index=df.index, columns=df.columns)
+    mask = df.isna()
+    long_gap = pd.DataFrame(False, index=df.index, columns=df.columns)
+    for col in df.columns:
+        lengths = _gap_lengths(mask[col])
+        long_gap[col] = lengths > max_gap
+    return long_gap
+
+
+def _apply_numeric_interpolation(df: pd.DataFrame, max_gap: int | None) -> pd.DataFrame:
+    """Interpolate numeric columns while leaving unsupported dtypes untouched."""
+    result = df.copy()
+    numeric_cols = result.select_dtypes(include="number").columns
+    if len(numeric_cols) > 0:
+        result.loc[:, numeric_cols] = result.loc[:, numeric_cols].interpolate(
+            method="linear",
+            limit=max_gap,
+            limit_area="inside",
+        )
+    return result
+
+
+def _build_summary(
+    original_missing: pd.DataFrame,
+    imputed_mask: pd.DataFrame,
+    remaining_missing: pd.DataFrame,
+) -> dict[str, dict[str, Any]]:
+    """Summarize missingness and imputation per column."""
+    summary: dict[str, dict[str, Any]] = {}
+    for col in original_missing.columns:
+        gap_lengths = _gap_lengths(original_missing[col])
+        summary[col] = {
+            "original_missing": int(original_missing[col].sum()),
+            "imputed": int(imputed_mask[col].sum()),
+            "remaining_missing": int(remaining_missing[col].sum()),
+            "longest_gap": int(gap_lengths.max()) if len(gap_lengths) else 0,
+            "missing_ratio": float(original_missing[col].mean()),
+        }
+    return summary
+
+
+def handle_missing_data(
+    df: pd.DataFrame,
+    strategy: str = "forward_fill",
+    *,
+    max_gap: int | None = None,
+    add_indicators: bool = False,
+) -> MissingDataResult:
+    """Apply a missing-data strategy and return imputation metadata.
+
+    Parameters
+    ----------
+    df
+        Sensor data indexed by time.
+    strategy
+        One of ``"forward_fill"``, ``"interpolate"``, ``"gap_aware"``,
+        ``"drop"``, or ``"flag"``.
+    max_gap
+        Maximum consecutive missing samples to fill. Longer gaps remain missing
+        for ``"forward_fill"``, ``"interpolate"``, and ``"gap_aware"``.
+    add_indicators
+        When ``True``, append ``"<column>_was_missing"`` boolean columns.
+    """
+
+    if max_gap is not None and max_gap < 1:
+        raise ValueError("max_gap must be at least 1 when provided")
+
+    original = df.copy()
+    original_missing = original.isna()
+    long_gap = _long_gap_mask(original, max_gap)
+    data = original.copy()
+
+    if strategy in {"forward_fill", "ffill"}:
+        data = data.ffill(limit=max_gap)
+    elif strategy == "interpolate":
+        data = _apply_numeric_interpolation(data, max_gap)
+        non_numeric = data.columns.difference(
+            data.select_dtypes(include="number").columns
+        )
+        if len(non_numeric) > 0:
+            data.loc[:, non_numeric] = data.loc[:, non_numeric].ffill(limit=max_gap)
+    elif strategy == "gap_aware":
+        data = _apply_numeric_interpolation(data, max_gap)
+        data = data.ffill(limit=max_gap).bfill(limit=max_gap)
+        data = data.mask(long_gap)
+    elif strategy == "drop":
+        data = data.dropna()
+    elif strategy == "flag":
+        pass
+    else:
+        raise ValueError(f"Unsupported missing-data strategy: {strategy}")
+
+    if add_indicators:
+        for col in original.columns:
+            data[f"{col}_was_missing"] = original_missing[col]
+
+    if strategy == "drop":
+        kept_missing = original_missing.loc[data.index]
+        imputed_mask = pd.DataFrame(False, index=data.index, columns=original.columns)
+        remaining_missing = data[original.columns].isna()
+        long_gap = long_gap.loc[data.index]
+        summary = _build_summary(kept_missing, imputed_mask, remaining_missing)
+    else:
+        imputed_mask = original_missing & ~data[original.columns].isna()
+        remaining_missing = data[original.columns].isna()
+        summary = _build_summary(original_missing, imputed_mask, remaining_missing)
+
+    return MissingDataResult(
+        data=data,
+        original_missing_mask=original_missing
+        if strategy != "drop"
+        else original_missing.loc[data.index],
+        imputed_mask=imputed_mask,
+        remaining_missing_mask=remaining_missing,
+        long_gap_mask=long_gap,
+        summary=summary,
+    )
+
+
+def forward_fill(df: pd.DataFrame) -> pd.DataFrame:
+    """Forward-fill missing values in a DataFrame."""
+    return handle_missing_data(df, strategy="forward_fill").data
+
+
+def interpolate_linear(df: pd.DataFrame) -> pd.DataFrame:
+    """Linearly interpolate interior missing values in numeric columns."""
+    return handle_missing_data(df, strategy="interpolate").data

sensor_modeling/utils/plotting.py

@@ -0,0 +1,98 @@
+"""Plotting helpers for sensor modeling."""
+
+from __future__ import annotations
+
+import logging
+
+import matplotlib.pyplot as plt
+import numpy as np
+import pandas as pd
+from matplotlib.figure import Figure
+
+logger = logging.getLogger(__name__)
+
+
+def plot_quantile_intervals(
+    actual_counts: np.ndarray,
+    quantile_info: dict,
+    title: str = "Model Validation",
+    sensor_name: str = "",
+    show: bool = True,
+) -> Figure:
+    """Plot observed counts against predicted quantile intervals."""
+    time_of_day = np.arange(len(actual_counts)) * 15 / 60.0
+
+    fig, ax = plt.subplots(figsize=(12, 6))
+    ax.fill_between(
+        time_of_day,
+        quantile_info["lower_quantiles"],
+        quantile_info["upper_quantiles"],
+        alpha=0.3,
+        color="blue",
+        label="95% Prediction Interval",
+    )
+    ax.plot(time_of_day, actual_counts, "r-", linewidth=2, label="Observed Events")
+    ax.plot(time_of_day, quantile_info["means"], "k--", label="Predicted Mean")
+    ax.set_xlabel("Hour of Day")
+    ax.set_ylabel("Event Count")
+    ax.set_title(title or f"Model Validation for {sensor_name}")
+    ax.legend()
+    ax.grid(alpha=0.3)
+    fig.tight_layout()
+    logger.info("Displayed validation plot for sensor %s", sensor_name)
+    if show:
+        plt.show()
+    return fig
+
+
+def plot_sensor_activity_patterns(data: pd.DataFrame, show: bool = True) -> Figure:
+    """Plot average sensor activation patterns across the day."""
+    df = data.copy()
+    df["hour"] = df.index.hour
+    means = df.groupby("hour")[data.columns].mean()
+    ax = means.plot(figsize=(10, 6))
+    fig = ax.figure
+    ax.set_xlabel("Hour of Day")
+    ax.set_ylabel("Activation Probability")
+    ax.set_title("Sensor Daily Activity Patterns")
+    ax.grid(alpha=0.3)
+    fig.tight_layout()
+    logger.info("Plotted sensor activity patterns")
+    if show:
+        plt.show()
+    return fig
+
+
+def plot_change_points(
+    series: np.ndarray,
+    change_points: np.ndarray,
+    title: str = "Change Points",
+    show: bool = True,
+) -> Figure:
+    """Plot time series with vertical lines at detected change points."""
+    fig, ax = plt.subplots(figsize=(10, 4))
+    ax.plot(series, label="series")
+    for cp in change_points:
+        ax.axvline(cp, color="red", linestyle="--", alpha=0.7)
+    ax.set_title(title)
+    ax.legend()
+    fig.tight_layout()
+    logger.info("Plotted %d change points", len(change_points))
+    if show:
+        plt.show()
+    return fig
+
+
+def plot_benchmark_results(results: dict[str, float], show: bool = True) -> Figure:
+    """Bar chart of benchmark times for different algorithms."""
+    names = list(results.keys())
+    times = list(results.values())
+    fig, ax = plt.subplots(figsize=(8, 4))
+    ax.bar(names, times, color="skyblue")
+    ax.set_ylabel("Seconds")
+    ax.set_title("CPD Benchmark Runtime")
+    fig.tight_layout()
+    logger.info("Plotted benchmark results for %s", names)
+    if show:
+        plt.show()
+    return fig

sensor_modeling/utils/validation.py

@@ -0,0 +1,117 @@
+"""Validation utilities for sensor models."""
+
+from __future__ import annotations
+
+import logging
+from collections.abc import Mapping
+from typing import Protocol
+
+import numpy as np
+import pandas as pd
+
+from .data_io import SensorDataset
+from .plotting import plot_quantile_intervals
+
+logger = logging.getLogger(__name__)
+
+
+class SupportsPredictionIntervals(Protocol):
+    """Model interface required by prediction validation."""
+
+    def predict_probabilities(self, data: pd.DataFrame) -> np.ndarray:
+        """Return predicted event probabilities."""
+        ...
+
+    def compute_quantile_intervals(
+        self, probabilities: np.ndarray, confidence: float
+    ) -> Mapping[str, np.ndarray]:
+        """Return lower, upper, and mean prediction intervals."""
+        ...
+
+
+def validate_model_predictions(
+    model: SupportsPredictionIntervals,
+    test_data: pd.DataFrame | SensorDataset,
+    sensor_name: str,
+    confidence: float = 0.95,
+    *,
+    plot: bool = True,
+) -> dict[str, object]:
+    """Validate model predictions using quantile coverage metrics."""
+    if not 0 < confidence < 1:
+        raise ValueError("confidence must be between 0 and 1")
+
+    df = test_data.to_dataframe() if isinstance(test_data, SensorDataset) else test_data
+    if sensor_name not in df.columns:
+        raise KeyError(f"Sensor '{sensor_name}' not found in test data")
+
+    probabilities = model.predict_probabilities(df)
+    quantile_info = model.compute_quantile_intervals(probabilities, confidence)
+    if not quantile_info:
+        return {
+            "validation_successful": False,
+            "reason": "Insufficient data for validation",
+        }
+
+    test_array = df[sensor_name].values
+    n_days = len(test_array) // 96
+    if n_days == 0:
+        return {"validation_successful": False, "reason": "Insufficient test data"}
+    test_reshaped = test_array[: n_days * 96].reshape(n_days, 96)
+    actual_counts = np.sum(test_reshaped, axis=0)
+
+    outside_count = np.sum(
+        (actual_counts < quantile_info["lower_quantiles"])
+        | (actual_counts > quantile_info["upper_quantiles"])
+    )
+    outside_percentage = (outside_count / 96) * 100
+    expected_outside = (1 - confidence) * 100
+    mse = np.mean((actual_counts - quantile_info["means"]) ** 2)
+    coverage_accuracy = 100 - abs(outside_percentage - expected_outside)
+
+    if plot:
+        plot_quantile_intervals(actual_counts, quantile_info, sensor_name=sensor_name)
+
+    return {
+        "validation_successful": True,
+        "outside_interval_count": int(outside_count),
+        "outside_interval_percentage": outside_percentage,
+        "expected_outside_percentage": expected_outside,
+        "coverage_accuracy": coverage_accuracy,
+        "mean_squared_error": mse,
+        "actual_counts": actual_counts,
+        "quantile_info": quantile_info,
+        "is_well_calibrated": bool(abs(outside_percentage - expected_outside) < 2.5),
+    }
+
+
+def create_model_comparison_report(results: Mapping[str, object]) -> str:
+    """Create a formatted comparison report for multiple modeling approaches."""
+    report = ["MODEL COMPARISON REPORT", "=" * 40, "\nBIC COMPARISON:"]
+    bic_scores: dict[str, float] = {}
+    for approach, data in results.items():
+        if isinstance(data, Mapping) and isinstance(data.get("total_bic"), int | float):
+            total_bic = float(data["total_bic"])
+            report.append(f"{approach.replace('_', ' ').title()}: {total_bic:.2f}")
+            bic_scores[approach] = total_bic
+
+    improvements = results.get("improvements")
+    if isinstance(improvements, Mapping):
+        report.append("\nIMPROVEMENTS (Lower BIC is better):")
+        for comparison, improvement in improvements.items():
+            if not isinstance(improvement, int | float):
+                continue
+            improvement_value = float(improvement)
+            direction = "better" if improvement_value > 0 else "worse"
+            report.append(
+                f"{comparison.replace('_', ' ').title()}: {improvement_value:.2f} ({direction})"
+            )
+
+    if bic_scores:
+        best_approach = min(bic_scores, key=bic_scores.get)
+        report.append(
+            f"\nBest performing approach: {best_approach.replace('_', ' ').title()}"
+        )
+        report.append(f"BIC score: {bic_scores[best_approach]:.2f}")
+
+    return "\n".join(report)

sensor_modeling/visualization/__init__.py

@@ -0,0 +1,3 @@
+"""Visualization utilities for sensor modeling."""
+
+__all__ = ["interactive", "clinical", "research", "web_app"]

sensor_modeling/visualization/clinical.py

@@ -0,0 +1,67 @@
+"""Healthcare focused visualizations and summaries."""
+
+from __future__ import annotations
+
+from collections.abc import Collection
+from typing import Dict
+
+import pandas as pd
+import plotly.express as px
+
+
+def _require_columns(
+    data: pd.DataFrame, required: Collection[str], context: str
+) -> None:
+    """Raise a clear error when required visualization columns are missing."""
+    missing = sorted(set(required) - set(data.columns))
+    if missing:
+        names = ", ".join(missing)
+        raise ValueError(f"{context} requires columns: {names}")
+
+
+def activity_summary(data: pd.DataFrame) -> px.bar:
+    """Return a patient-friendly bar chart summarizing activity levels."""
+    _require_columns(data, {"activity"}, "activity_summary")
+    summary = data.groupby("activity").size().reset_index(name="count")
+    return px.bar(summary, x="activity", y="count")
+
+
+def clinical_alerts(
+    data: pd.DataFrame, thresholds: Dict[str, float]
+) -> Dict[str, bool]:
+    """Flag sensors that exceed clinical thresholds."""
+    _require_columns(data, {"sensor", "value"}, "clinical_alerts")
+    alerts = {}
+    for sensor, thresh in thresholds.items():
+        alerts[sensor] = bool((data[data["sensor"] == sensor]["value"] > thresh).any())
+    return alerts
+
+
+def trend_monitor(data: pd.DataFrame, window: int = 7) -> px.line:
+    """Plot rolling averages over weeks or months."""
+    _require_columns(data, {"sensor", "timestamp", "value"}, "trend_monitor")
+    if window < 1:
+        raise ValueError("window must be at least 1")
+
+    rolled = (
+        data.set_index("timestamp")
+        .groupby("sensor")["value"]
+        .rolling(window)
+        .mean()
+        .reset_index()
+    )
+    return px.line(rolled, x="timestamp", y="value", color="sensor")
+
+
+def compare_norms(data: pd.DataFrame, normative: pd.DataFrame) -> px.line:
+    """Compare patient data against normative statistics."""
+    required = {"sensor", "timestamp", "value"}
+    _require_columns(data, required, "compare_norms patient data")
+    _require_columns(normative, required, "compare_norms normative data")
+    merged = pd.merge(
+        data, normative, on=["timestamp", "sensor"], suffixes=("_patient", "_norm")
+    )
+    fig = px.line(
+        merged, x="timestamp", y=["value_patient", "value_norm"], color="sensor"
+    )
+    return fig

sensor_modeling/visualization/interactive.py

@@ -0,0 +1,208 @@
+"""Interactive dashboards using Plotly and Bokeh.
+
+This module exposes helpers that make it easy to explore sensor data in
+real time, tune model parameters, drill into detected changes, and export
+figures for presentations. The functions are intentionally lightweight so
+that they can run in environments without a full web server.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Collection, Iterable, Sequence
+from pathlib import Path
+from typing import Any
+
+import numpy as np
+import pandas as pd
+import plotly.express as px
+from bokeh.embed import file_html
+from bokeh.layouts import column
+from bokeh.models import ColumnDataSource, CustomJS, Div, HoverTool, Slider
+from bokeh.plotting import figure
+from bokeh.resources import CDN
+
+
+def _require_columns(
+    data: pd.DataFrame, required: Collection[str], context: str
+) -> None:
+    """Raise a clear error when required visualization columns are missing."""
+    missing = sorted(set(required) - set(data.columns))
+    if missing:
+        names = ", ".join(missing)
+        raise ValueError(f"{context} requires columns: {names}")
+
+
+def real_time_display(data: pd.DataFrame) -> px.line:
+    """Create a Plotly line figure for streaming sensor data.
+
+    Parameters
+    ----------
+    data:
+        DataFrame with at least ``timestamp``, ``sensor`` and ``value`` columns.
+    """
+    _require_columns(data, {"sensor", "timestamp", "value"}, "real_time_display")
+    return px.line(data, x="timestamp", y="value", color="sensor")
+
+
+def _score_parameter(model: Any, param: str, value: float) -> float:
+    """Return a diagnostic score for ``param=value`` using model hooks."""
+    if hasattr(model, "score_parameter"):
+        scorer = getattr(model, "score_parameter")
+        try:
+            return float(scorer(param, value))
+        except TypeError:
+            return float(scorer(value))
+    if hasattr(model, "evaluate_parameter"):
+        evaluator = getattr(model, "evaluate_parameter")
+        try:
+            return float(evaluator(param, value))
+        except TypeError:
+            return float(evaluator(value))
+    raise ValueError(
+        "parameter_tuning requires model.score_parameter(...) or "
+        "model.evaluate_parameter(...) to compute diagnostics"
+    )
+
+
+def _build_parameter_sweep(
+    model: Any, param: str, values: Sequence[float]
+) -> tuple[list[float], list[float]]:
+    """Evaluate the model over candidate parameter values."""
+    ordered_values = sorted({float(v) for v in values})
+    if not ordered_values:
+        raise ValueError("values must contain at least one candidate")
+    if not np.isfinite(ordered_values).all():
+        raise ValueError("values must contain only finite candidates")
+
+    scores = [_score_parameter(model, param, value) for value in ordered_values]
+    if not np.isfinite(scores).all():
+        raise ValueError("parameter scores must be finite")
+    return ordered_values, scores
+
+
+def parameter_tuning(model: Any, param: str, values: Iterable[float]):
+    """Return a Bokeh layout for parameter diagnostics.
+
+    The model must expose either ``score_parameter(param, value)`` or
+    ``evaluate_parameter(param, value)``. The returned view shows a score curve
+    across candidate values and updates the highlighted selection plus summary
+    text when the slider moves. In Python-backed Bokeh sessions, the callback
+    also writes the chosen value back to ``model.<param>``.
+    """
+
+    param_values, scores = _build_parameter_sweep(model, param, list(values))
+    current_value = float(getattr(model, param, param_values[0]))
+    if current_value not in param_values:
+        current_value = param_values[0]
+    current_idx = param_values.index(current_value)
+    best_idx = max(range(len(scores)), key=lambda idx: scores[idx])
+
+    curve_source = ColumnDataSource(
+        {
+            "x": param_values,
+            "y": scores,
+            "label": [f"{value:.4g}" for value in param_values],
+        }
+    )
+    selected_source = ColumnDataSource(
+        {"x": [param_values[current_idx]], "y": [scores[current_idx]]}
+    )
+
+    plot = figure(
+        height=320,
+        sizing_mode="stretch_width",
+        title=f"{param} diagnostic sweep",
+        x_axis_label=param,
+        y_axis_label="score",
+        tools="pan,wheel_zoom,box_zoom,reset,save",
+    )
+    plot.line("x", "y", source=curve_source, line_width=2, color="#1f77b4")
+    plot.scatter("x", "y", source=curve_source, size=8, color="#1f77b4", alpha=0.85)
+    plot.scatter(
+        "x",
+        "y",
+        source=selected_source,
+        size=14,
+        color="#d62728",
+        line_color="white",
+        line_width=2,
+    )
+    plot.add_tools(
+        HoverTool(
+            tooltips=[(param, "@label"), ("score", "@y{0.000}")],
+            renderers=plot.renderers[:2],
+        )
+    )
+
+    summary = Div(
+        text=(
+            f"<b>{param}</b>: {param_values[current_idx]:.4g} | "
+            f"<b>score</b>: {scores[current_idx]:.4f} | "
+            f"<b>best</b>: {param_values[best_idx]:.4g} "
+            f"({scores[best_idx]:.4f})"
+        ),
+        sizing_mode="stretch_width",
+    )
+
+    slider = Slider(
+        start=0,
+        end=len(param_values) - 1,
+        step=1,
+        value=current_idx,
+        title=f"{param}: {param_values[current_idx]:.4g}",
+    )
+
+    def _update(attr: str, old: int, new: int) -> None:  # pragma: no cover
+        value = param_values[int(new)]
+        setattr(model, param, value)
+
+    slider.on_change("value", _update)
+    slider.js_on_change(
+        "value",
+        CustomJS(
+            args=dict(
+                slider=slider,
+                selected=selected_source,
+                summary=summary,
+                values=param_values,
+                scores=scores,
+                param=param,
+                best_value=param_values[best_idx],
+                best_score=scores[best_idx],
+            ),
+            code="""
+const idx = slider.value;
+const value = values[idx];
+const score = scores[idx];
+selected.data = {x: [value], y: [score]};
+selected.change.emit();
+slider.title = `${param}: ${value.toPrecision(4)}`;
+summary.text =
+  `<b>${param}</b>: ${value.toPrecision(4)} | <b>score</b>: ${score.toFixed(4)} | ` +
+  `<b>best</b>: ${best_value.toPrecision(4)} (${best_score.toFixed(4)})`;
+""",
+        ),
+    )
+    return column(slider, summary, plot, sizing_mode="stretch_width")
+
+
+def drill_down(changes: pd.DataFrame) -> px.scatter:
+    """Plot detected changes with interactive hover information."""
+    _require_columns(changes, {"score", "time"}, "drill_down")
+    return px.scatter(changes, x="time", y="score")
+
+
+def export(fig: Any, path: str) -> None:
+    """Export a Plotly or Bokeh figure to an HTML file."""
+    output_path = Path(path)
+    output_path.parent.mkdir(parents=True, exist_ok=True)
+    if hasattr(fig, "write_html"):
+        fig.write_html(output_path)
+    elif fig.__class__.__module__.startswith("bokeh"):
+        html = file_html(fig, CDN, "export")
+        with open(
+            output_path, "w", encoding="utf-8"
+        ) as fh:  # pragma: no cover - simple file write
+            fh.write(html)
+    else:  # pragma: no cover - defensive fallback
+        raise TypeError("Unsupported figure type")