sensor-modeling 0.2.0__py3-none-any.whl

sensor_modeling/models/nhpp_pelt/plotting.py

@@ -0,0 +1,218 @@
+from __future__ import annotations
+
+from collections.abc import Sequence
+from dataclasses import dataclass
+from typing import Any, List, Optional, Tuple, Union
+
+import matplotlib.pyplot as plt
+import numpy as np
+
+from .utils import type_check
+
+Array1D = np.ndarray
+
+
+@dataclass(frozen=True)
+class PlotConfig:
+    """Styling for the raster (top) and λ̂ curves (middle)."""
+
+    figsize: Tuple[float, float] = (12.0, 8.5)
+    dpi: int = 110
+    raster_markersize: float = 4.0
+    raster_alpha: float = 0.9
+    band_alpha: float = 0.08
+    grid_points: int = 400
+    title_top: str = "Daily event raster with segment bands"
+    title_bottom: str = "Estimated segment intensities λ̂(t)"
+
+
+@dataclass(frozen=True)
+class HistConfig:
+    """Config for per-segment average histograms (bottom row)."""
+
+    bins: Union[int, str] = "fd"  # 'fd' | 'sqrt' | 'sturges' | int
+    density: bool = True  # True → average rate (events/day/unit-time)
+    alpha: float = 0.5
+    title: str = "Per-segment average histograms of event times"
+
+
+def _common_bin_edges(
+    all_events: Array1D, delta: float, bins: Union[int, str]
+) -> Array1D:
+    type_check(delta > 0, "delta must be positive.")
+    all_events = np.asarray(all_events, float)
+    all_events = all_events[(all_events >= 0.0) & (all_events <= delta)]
+
+    if isinstance(bins, int):
+        type_check(bins >= 1, "bins must be >= 1.")
+        return np.linspace(0.0, delta, bins + 1)
+
+    if all_events.size == 0:
+        return np.linspace(0.0, delta, 21)
+
+    if bins == "fd":
+        q25, q75 = np.quantile(all_events, [0.25, 0.75])
+        iqr = max(q75 - q25, 1e-12)
+        h = 2.0 * iqr / np.cbrt(all_events.size)
+        k = 20 if (not np.isfinite(h) or h <= 0) else max(1, int(np.ceil(delta / h)))
+        return np.linspace(0.0, delta, k + 1)
+    if bins == "sqrt":
+        k = max(1, int(np.ceil(np.sqrt(all_events.size))))
+        return np.linspace(0.0, delta, k + 1)
+    if bins == "sturges":
+        k = max(1, int(np.ceil(np.log2(all_events.size) + 1.0)))
+        return np.linspace(0.0, delta, k + 1)
+    raise ValueError("bins must be int or one of {'fd','sqrt','sturges'}.")
+
+
+def _avg_hist_per_segment(
+    days: Sequence[Array1D],
+    segments: Sequence[Tuple[int, int]],
+    edges: Array1D,
+    density: bool,
+) -> List[Array1D]:
+    """
+    Average per-day histogram for each segment, using common bin edges.
+    If `density` is True, convert counts/bin/day to rate by dividing by bin width.
+    """
+    edges = np.asarray(edges, float)
+    type_check(np.all(np.diff(edges) > 0), "edges must be strictly increasing.")
+
+    widths = np.diff(edges)
+    avg_counts: List[Array1D] = []
+
+    for i_start, i_end in segments:
+        type_check(1 <= i_start <= i_end <= len(days), "segment indices out of range.")
+        L = i_end - i_start + 1
+
+        acc = np.zeros(edges.size - 1, dtype=float)
+        for d in range(i_start - 1, i_end):
+            ev = np.asarray(days[d], float)
+            if ev.size == 0:
+                continue
+            ev = ev[(ev >= edges[0]) & (ev <= edges[-1])]
+            counts, _ = np.histogram(ev, bins=edges)
+            acc += counts.astype(float)
+
+        mean_per_day = acc / max(L, 1)
+        if density:
+            mean_per_day = mean_per_day / widths
+        avg_counts.append(mean_per_day)
+
+    return avg_counts
+
+
+def plot_segments_and_intensities_with_histograms(
+    days: Sequence[Array1D],
+    model: Any,
+    config: PlotConfig = PlotConfig(),
+    hist: HistConfig = HistConfig(),
+    *,
+    save_path: Optional[str] = None,
+) -> plt.Figure:
+    """
+    Three-row figure:
+      (1) event raster with segment bands,
+      (2) segment λ̂(t) curves (optionally add CI bands outside),
+      (3) per-segment average histograms (small multiples).
+    """
+    type_check(len(days) >= 1, "days must contain at least one array.")
+    need = ["segments_", "knots_", "degree_", "delta_", "P_", "weights_"]
+    missing = [a for a in need if not hasattr(model, a)]
+    type_check(len(missing) == 0, f"Model is missing attributes: {missing}.")
+    type_check(
+        hasattr(model, "intensity_on_grid"), "Model must implement intensity_on_grid()."
+    )
+
+    n_days = len(days)
+    segments: List[Tuple[int, int]] = list(model.segments_)
+    delta: float = float(model.delta_)
+    n_segments = len(segments)
+    type_check(n_segments >= 1, "Model has no segments.")
+
+    all_events = (
+        np.concatenate([np.asarray(d, float) for d in days])
+        if n_days
+        else np.array([], float)
+    )
+    edges = _common_bin_edges(all_events, delta, hist.bins)
+    widths = np.diff(edges)
+    centers = edges[:-1] + 0.5 * widths
+    avg_counts = _avg_hist_per_segment(days, segments, edges, density=hist.density)
+
+    fig = plt.figure(figsize=config.figsize, dpi=config.dpi)
+    gs = fig.add_gridspec(nrows=3, ncols=1, height_ratios=[1.4, 1.0, 1.0], hspace=0.35)
+    ax_top = fig.add_subplot(gs[0, 0])
+    ax_mid = fig.add_subplot(gs[1, 0])
+
+    # -------- Top: raster with segment bands --------
+    for k, (i_start, i_end) in enumerate(segments):
+        y0, y1 = i_start - 0.5, i_end + 0.5
+        if k % 2 == 0:
+            ax_top.axhspan(y0, y1, alpha=config.band_alpha)
+        ax_top.text(
+            delta * 1.002,
+            (y0 + y1) / 2.0,
+            f"Seg {k+1}\n({i_start}–{i_end})",
+            va="center",
+            ha="left",
+            fontsize=9,
+        )
+
+    for i, ev in enumerate(days, start=1):
+        ev = np.asarray(ev, float)
+        if ev.size:
+            ax_top.plot(
+                ev,
+                np.full_like(ev, i, float),
+                linestyle="None",
+                marker="|",
+                markersize=config.raster_markersize,
+                alpha=config.raster_alpha,
+            )
+
+    ax_top.set_xlim(0.0, delta)
+    ax_top.set_ylim(0.5, n_days + 0.5)
+    ax_top.set_xlabel("Time within day")
+    ax_top.set_ylabel("Day index")
+    ax_top.set_title(config.title_top)
+    ax_top.grid(True, linestyle="--", linewidth=0.5, alpha=0.5)
+
+    # -------- Middle: intensity curves (no explicit colors) --------
+    grid = np.linspace(0.0, delta, config.grid_points)
+    for k, (i_start, i_end) in enumerate(segments):
+        lam = np.asarray(model.intensity_on_grid(k, grid), float)
+        type_check(lam.shape == grid.shape, "intensity_on_grid must match grid shape.")
+        label = f"Seg {k+1} (days {i_start}–{i_end})"
+        ax_mid.plot(grid, lam, label=label)
+
+    ax_mid.set_xlim(0.0, delta)
+    ax_mid.set_xlabel("Time within day")
+    ax_mid.set_ylabel("Intensity λ(t)")
+    ax_mid.set_title(config.title_bottom)
+    ax_mid.grid(True, linestyle="--", linewidth=0.5, alpha=0.5)
+    ax_mid.legend(loc="best", fontsize=9, frameon=False)
+
+    # -------- Bottom: per-segment average histograms --------
+    subgs = gs[2, 0].subgridspec(1, n_segments, wspace=0.15)
+    for k in range(n_segments):
+        ax = fig.add_subplot(subgs[0, k])
+        ax.bar(centers, avg_counts[k], width=widths, align="center", alpha=hist.alpha)
+        ax.set_xlim(0.0, delta)
+        if k == 0:
+            ax.set_ylabel(
+                "Avg rate\n(events/day/unit-time)"
+                if hist.density
+                else "Avg count/day/bin"
+            )
+        ax.set_xlabel("Time within day")
+        ax.set_title(f"Seg {k+1}: days {segments[k][0]}–{segments[k][1]}", fontsize=10)
+        ax.grid(True, linestyle="--", linewidth=0.5, alpha=0.5)
+
+    fig.text(0.5, 0.04 + 0.03, hist.title, ha="center", va="center", fontsize=11)
+
+    fig.tight_layout()
+    if save_path:
+        fig.savefig(save_path, dpi=config.dpi, bbox_inches="tight")
+
+    return fig

sensor_modeling/models/nhpp_pelt/quad.py

@@ -0,0 +1,72 @@
+# nhpp/quad.py
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import Tuple
+
+import numpy as np
+
+from .utils import type_check
+
+Array1D = np.ndarray
+
+
+@dataclass(frozen=True)
+class QuadratureConfig:
+    """
+    Settings for Gauss–Legendre quadrature used to integrate terms like
+    ∫ exp(wᵀψ(t)) {1, ψ, ψψᵀ} dt.
+
+    Attributes
+    ----------
+    n_points : int
+        Order of the Gauss–Legendre rule (≥ 2). Larger → more accurate, slower.
+    ridge : float
+        Small diagonal ridge added to the Hessian to ensure positive definiteness.
+    """
+
+    n_points: int = 256
+    ridge: float = 1e-8
+
+
+def leggauss_on_interval(a: float, b: float, n: int) -> Tuple[Array1D, Array1D]:
+    """
+    Compute Gauss–Legendre nodes and weights on a closed interval [a, b].
+
+    Parameters
+    ----------
+    a : float
+        Interval start (can equal b for a degenerate interval; not useful here).
+    b : float
+        Interval end; must satisfy b > a in practice.
+    n : int
+        Number of quadrature points (order). Must be ≥ 2.
+
+    Returns
+    -------
+    nodes : np.ndarray, shape (n,)
+        Quadrature abscissae in [a, b].
+    weights : np.ndarray, shape (n,)
+        Corresponding quadrature weights that integrate polynomials of degree 2n-1 exactly.
+
+    Notes
+    -----
+    We obtain nodes/weights on [-1, 1] via `numpy.polynomial.legendre.leggauss`
+    and map them affinely to [a, b].
+    """
+    type_check(n >= 2, "Quadrature order n must be >= 2.")
+    type_check(np.isfinite(a) and np.isfinite(b), "Interval bounds must be finite.")
+    type_check(b >= a, "Require b >= a.")
+
+    # Nodes/weights on [-1, 1]
+    x, w = np.polynomial.legendre.leggauss(n)
+
+    # Affine map to [a, b]
+    xm = 0.5 * (b + a)
+    xr = 0.5 * (b - a)
+    nodes = xm + xr * x
+    weights = xr * w
+    return nodes.astype(float, copy=False), weights.astype(float, copy=False)
+
+
+__all__ = ["QuadratureConfig", "leggauss_on_interval"]

sensor_modeling/models/nhpp_pelt/regularization.py

@@ -0,0 +1,121 @@
+from __future__ import annotations
+
+from collections.abc import Iterable, Sequence
+from typing import TYPE_CHECKING, List, Tuple
+
+import numpy as np
+
+if TYPE_CHECKING:
+    from .model import NHPPConfig
+
+Array1D = np.ndarray
+Array2D = np.ndarray
+
+
+def difference_matrix(p: int, order: int = 2) -> Array2D:
+    """
+    Finite-difference operator D^(order) for P-splines.
+
+    For p parameters and order=2, returns a (p-2)×p matrix whose rows each
+    realize [1, -2, 1] at consecutive positions. More generally:
+        D^(k) = diff(I_p, n=k, axis=0).
+
+    Returns
+    -------
+    D : np.ndarray, shape (p - order, p)
+    """
+    if p <= order:
+        raise ValueError("p must be > order.")
+    D = np.eye(p, dtype=float)
+    for _ in range(order):
+        D = np.diff(D, n=1, axis=0)
+    return D
+
+
+def p_spline_RtR(p: int, order: int = 2, gamma: float = 0.0) -> Array2D:
+    """
+    Build γ * (D^T D), where D is the 'order'-th difference operator.
+
+    Parameters
+    ----------
+    p : int
+        Number of spline coefficients (P).
+    order : int
+        Difference order (2 for the classic curvature penalty).
+    gamma : float
+        Penalty strength γ ≥ 0.
+
+    Returns
+    -------
+    RtR : np.ndarray, shape (p, p)
+        γ * (Dᵀ D). If gamma==0 returns a zero matrix for convenience.
+    """
+    if gamma <= 0.0:
+        return np.zeros((p, p), dtype=float)
+    D = difference_matrix(p, order=order)
+    return float(gamma) * (D.T @ D)
+
+
+def sweep_gamma(
+    days: Sequence[np.ndarray],
+    base_cfg: NHPPConfig,
+    gammas: Iterable[float],
+    *,
+    order: int = 2,
+) -> List[Tuple[float, int, float]]:
+    """
+    Fit across a grid of γ values (P-spline penalty strength) and return:
+        (gamma, n_changepoints, total_penalized_cost)
+
+    Notes
+    -----
+    - Uses the model's existing penalty_beta (SIC if None).
+    - The reported total cost matches the objective: sum segment costs (incl. γ term)
+      + β per segment.
+    """
+    # local imports to avoid circular deps
+    from .bspline import bspline_design_matrix
+    from .model import NHPPPELT, NHPPConfig
+    from .optimizer import SegmentOptimizer
+    from .quad import QuadratureConfig
+
+    out: List[Tuple[float, int, float]] = []
+
+    for g in gammas:
+        cfg = NHPPConfig(
+            **{
+                **base_cfg.__dict__,
+                "pspline_gamma": float(g),
+                "pspline_order": int(order),
+            }
+        )
+        model = NHPPPELT(cfg).fit(days)
+
+        # re-accumulate the exact objective with the fitted weights
+        total = 0.0
+        quad = QuadratureConfig(n_points=cfg.quad.n_points, ridge=cfg.hessian_ridge)
+        opt = SegmentOptimizer(
+            delta=model.delta_,
+            degree=model.degree_,
+            knots=model.knots_,
+            quad=quad,
+            newton=cfg.newton,
+            pspline_gamma=cfg.pspline_gamma,
+            pspline_order=cfg.pspline_order,
+        )
+
+        for (i, j), w in zip(model.segments_, model.weights_):
+            # sufficient stats s for [i..j]
+            s = np.zeros(cfg.n_basis, dtype=float)
+            for d in range(i - 1, j):
+                ev = np.asarray(days[d], float)
+                if ev.size:
+                    s += bspline_design_matrix(ev, cfg.degree, model.knots_).sum(axis=0)
+            # minimize with warm-start = fitted w to recover cost term
+            _, c = opt.minimize(L=j - i + 1, s=s, w0=w)
+            total += float(c)
+
+        total += len(model.segments_) * float(model.beta_)
+        out.append((float(g), len(model.changepoints_), total))
+
+    return out

sensor_modeling/models/nhpp_pelt/utils.py

@@ -0,0 +1,174 @@
+from __future__ import annotations
+
+import json
+from collections.abc import Iterable, Sequence
+from typing import TYPE_CHECKING, List, Tuple
+
+import numpy as np
+
+if TYPE_CHECKING:
+    from .model import NHPPPELT, NHPPConfig
+
+# ------------------------------- Types -------------------------------
+
+Array1D = np.ndarray
+Array2D = np.ndarray
+
+
+# ---------------------------- Validators ----------------------------
+
+
+def type_check(condition: bool, msg: str) -> None:
+    """Runtime guard with a concise error message."""
+    if not condition:
+        raise ValueError(msg)
+
+
+# -------------------------- Normal PPF (Φ⁻¹) -------------------------
+
+
+def normal_ppf(p: float) -> float:
+    """
+    Approximate the standard-normal inverse CDF Φ⁻¹(p).
+    Accuracy ~ 4e-4 absolute in typical ranges. SciPy-free.
+
+    Parameters
+    ----------
+    p : float
+        Probability in (0, 1).
+
+    Returns
+    -------
+    float
+        Approximate quantile.
+
+    Notes
+    -----
+    Combines a Beasley–Springer/Wichura-style central approximation
+    with a simple tail refinement. Good enough for CI bands.
+    """
+    type_check(0.0 < p < 1.0, "p must be in (0,1).")
+    # exploit symmetry
+    if p > 0.5:
+        return -normal_ppf(1.0 - p)
+
+    # coefficients (central region)
+    a = [2.50662823884, -18.61500062529, 41.39119773534, -25.44106049637]
+    b = [-8.47351093090, 23.08336743743, -21.06224101826, 3.13082909833]
+
+    # auxiliary for mild tails
+    c = [
+        0.3374754822726147,
+        0.9761690190917186,
+        0.1607979714918209,
+        0.0276438810333863,
+        0.0038405729373609,
+        0.0003951896511919,
+        0.0000321767881768,
+        0.0000002888167364,
+        0.0000003960315187,
+    ]
+
+    # central transform
+    t = np.sqrt(-2.0 * np.log(p))
+    x = t - (((a[3] * t + a[2]) * t + a[1]) * t + a[0]) / (
+        (((b[3] * t + b[2]) * t + b[1]) * t + b[0]) * t + 1.0
+    )
+
+    # refine for not-too-small p
+    if p >= 0.02425:
+        q = p - 0.5
+        r = q * q
+        poly = c[8]
+        for k in range(7, -1, -1):
+            poly = poly * r + c[k]
+        return float(q * poly)
+    return float(x)
+
+
+# --------------------------- JSON exporters --------------------------
+
+
+def to_jsonable(model: NHPPPELT) -> dict:
+    """
+    Convert a fitted model into a JSON-serializable dictionary.
+    """
+    return {
+        "delta": float(model.delta_),
+        "degree": int(model.degree_),
+        "n_basis": int(model.P_),
+        "beta": float(model.beta_),
+        "knots": model.knots_.tolist(),
+        "changepoints": [int(c) for c in model.changepoints_],
+        "segments": [(int(i), int(j)) for (i, j) in model.segments_],
+        "weights": [w.astype(float).tolist() for w in model.weights_],
+    }
+
+
+def save_results_json(model: NHPPPELT, path: str) -> None:
+    """
+    Save fitted model parameters/results to a JSON file.
+
+    Parameters
+    ----------
+    model : NHPPPELT
+        Fitted model.
+    path : str
+        Destination path.
+    """
+    with open(path, "w", encoding="utf-8") as f:
+        json.dump(to_jsonable(model), f, ensure_ascii=False, indent=2)
+
+
+# ------------------------------ Sweep β ------------------------------
+
+
+def sweep_beta(
+    days: Sequence[Array1D],
+    base_cfg: NHPPConfig,
+    betas: Iterable[float],
+) -> List[Tuple[float, int, float]]:
+    """
+    Fit across candidate penalties and return tuples of:
+        (beta, n_changepoints, total_penalized_cost)
+
+    Useful to visualize an elbow/stability curve before fixing β.
+
+    Notes
+    -----
+    - Reuses each solution’s segment weights as warm starts to
+      recompute exact segment costs for the reported total.
+    """
+    from .bspline import bspline_design_matrix
+    from .model import NHPPPELT, NHPPConfig
+    from .optimizer import QuadratureConfig, SegmentOptimizer
+
+    results: List[Tuple[float, int, float]] = []
+
+    for b in betas:
+        cfg = NHPPConfig(**{**base_cfg.__dict__, "penalty_beta": float(b)})
+        model = NHPPPELT(cfg).fit(days)
+
+        total_cost = 0.0
+        for (i, j), w in zip(model.segments_, model.weights_):
+            quad = QuadratureConfig(n_points=cfg.quad.n_points, ridge=cfg.hessian_ridge)
+            opt = SegmentOptimizer(
+                delta=model.delta_,
+                degree=model.degree_,
+                knots=model.knots_,
+                quad=quad,
+                newton=cfg.newton,
+            )
+            # build sufficient statistics s for days i..j
+            s = np.zeros(cfg.n_basis, dtype=float)
+            for d in range(i - 1, j):
+                ev = np.asarray(days[d], dtype=float)
+                if ev.size:
+                    s += bspline_design_matrix(ev, cfg.degree, model.knots_).sum(axis=0)
+            _, c = opt.minimize(L=j - i + 1, s=s, w0=w)  # warm start
+            total_cost += float(c)
+
+        total_cost += len(model.segments_) * float(b)
+        results.append((float(b), len(model.changepoints_), total_cost))
+
+    return results

sensor_modeling/observations/__init__.py

@@ -0,0 +1,59 @@
+"""Canonical, hardware-neutral representation of sensor observations.
+
+This package defines the single data model every other stage of the pipeline
+consumes. The central claim it encodes is that a sensor record is *evidence*,
+not behaviour: an :class:`Observation` carries the value, the unit it was
+measured in, the reliability attached to it, and the provenance of any repair
+applied during ingestion, so that later stages can weight it honestly.
+"""
+
+from .adapters import (
+    LegacyConversionError,
+    naive_utc,
+    observations_from_dataset,
+    observations_from_frame,
+    observations_from_records,
+)
+from .ingest import (
+    ClockOffsetEstimator,
+    IngestionReport,
+    ObservationIngestor,
+    RejectedObservation,
+)
+from .observation import Observation, default_kind, require_aware
+from .registry import (
+    SensorContractError,
+    SensorRegistry,
+    SensorSpec,
+    UnknownSensorError,
+)
+from .stream import Gap, ObservationStream
+from .types import Modality, ObservationFlag, ObservationKind
+from .units import Unit, convert, to_canonical
+
+__all__ = [
+    "ClockOffsetEstimator",
+    "Gap",
+    "IngestionReport",
+    "LegacyConversionError",
+    "Modality",
+    "Observation",
+    "ObservationFlag",
+    "ObservationIngestor",
+    "ObservationKind",
+    "ObservationStream",
+    "RejectedObservation",
+    "SensorContractError",
+    "SensorRegistry",
+    "SensorSpec",
+    "Unit",
+    "UnknownSensorError",
+    "convert",
+    "default_kind",
+    "naive_utc",
+    "observations_from_dataset",
+    "observations_from_frame",
+    "observations_from_records",
+    "require_aware",
+    "to_canonical",
+]