tsanomaly 0.1.0__py3-none-any.whl

tsanomaly/__init__.py

@@ -0,0 +1,37 @@
+"""A Python library for autonomous, explainable, real-time anomaly detection on time-series metrics.
+
+Usage guide: docs/usage.md. Architecture: docs/architecture.md.
+"""
+
+from tsanomaly.core.config import DetectorConfig
+from tsanomaly.core.result import DetectionResult
+from tsanomaly.core.types import (
+    Anomaly,
+    Envelope,
+    Episode,
+    EpisodeKind,
+    MetricId,
+    MetricType,
+    Profile,
+    ScoreBreakdown,
+    SeasonalComponent,
+)
+from tsanomaly.detector import Detector
+
+__version__ = "0.1.0"
+
+__all__ = [
+    "Detector",
+    "DetectorConfig",
+    "DetectionResult",
+    "MetricId",
+    "MetricType",
+    "Profile",
+    "SeasonalComponent",
+    "Envelope",
+    "Episode",
+    "EpisodeKind",
+    "ScoreBreakdown",
+    "Anomaly",
+    "__version__",
+]

tsanomaly/calibrate/conformal.py

@@ -0,0 +1,104 @@
+"""Adaptive Conformal Inference calibrator.
+
+Wraps any BaselineModel's (expected, scale) into an interval whose realized
+coverage tracks the target regardless of model misspecification:
+``alpha_{t+1} = alpha_t + gamma * (alpha - err_t)`` (Gibbs & Candes 2021).
+Distribution-free: the interval half-width is a quantile of recent
+standardized nonconformity scores, not a Gaussian z-value.
+
+Freezes during open episodes so an ongoing anomaly cannot widen its own envelope.
+"""
+
+from __future__ import annotations
+
+from collections import deque
+from typing import Any
+
+import numpy as np
+
+
+class ACICalibrator:
+    def __init__(
+        self,
+        target_coverage: float = 0.995,
+        gamma: float = 0.005,
+        reservoir: int = 512,
+    ) -> None:
+        self.alpha_target = 1.0 - target_coverage
+        self.gamma = gamma
+        self.alpha_t = self.alpha_target
+        self.scores: deque[float] = deque(maxlen=reservoir)
+        self.frozen = False
+        self.n_observed = 0
+        self.n_outside = 0
+
+    def _quantile(self) -> float:
+        n = len(self.scores)
+        if n < 30:
+            return 4.0  # conservative cold-start half-width (in scale units)
+        level = min(0.9999, (1.0 - self.alpha_t) * (n + 1) / n)
+        return float(np.quantile(np.array(self.scores), level))
+
+    def halfwidth(self, scale: float) -> float:
+        return self._quantile() * scale
+
+    def observe(self, nonconformity: float, outside: bool, learn_score: bool = True) -> None:
+        """Feed one realized coverage outcome; no-op while frozen.
+
+        The alpha_t coverage-feedback update runs on EVERY sample -- including
+        ones inside an open episode. Freezing it there silences the very misses
+        ACI must react to (an episode's opening points are exactly its coverage
+        errors) and alpha_t drifts upward on pure successes, silently narrowing
+        the envelope. Only the nonconformity reservoir is gated by
+        ``learn_score`` so anomalous magnitudes never widen future envelopes.
+        """
+        if self.frozen:
+            return
+        err = 1.0 if outside else 0.0
+        self.alpha_t = float(
+            np.clip(
+                self.alpha_t + self.gamma * (self.alpha_target - err),
+                self.alpha_target / 20,
+                0.5,
+            )
+        )
+        if learn_score:
+            # Reservoir AND the coverage diagnostic count only non-episode
+            # points: the diagnostic answers "is the envelope honest on normal
+            # data" -- a true incident's points are anomalies, not calibration
+            # misses. (alpha_t above still reacts to every sample.)
+            self.scores.append(float(nonconformity))
+            self.n_observed += 1
+            self.n_outside += int(outside)
+
+    def warm(self, nonconformities: np.ndarray) -> None:
+        for s in nonconformities[np.isfinite(nonconformities)]:
+            self.scores.append(float(s))
+        self.alpha_t = self.alpha_target
+
+    def coverage_error(self) -> float | None:
+        """Realized-minus-target coverage over everything observed (diagnostics)."""
+        if self.n_observed < 50:
+            return None
+        realized = 1.0 - self.n_outside / self.n_observed
+        return realized - (1.0 - self.alpha_target)
+
+    def state_dict(self) -> dict[str, Any]:
+        return {
+            "alpha_target": self.alpha_target,
+            "gamma": self.gamma,
+            "alpha_t": self.alpha_t,
+            "scores": [round(s, 6) for s in self.scores],
+            "reservoir": self.scores.maxlen,
+            "n_observed": self.n_observed,
+            "n_outside": self.n_outside,
+        }
+
+    @classmethod
+    def from_state(cls, d: dict[str, Any]) -> "ACICalibrator":
+        c = cls(1.0 - d["alpha_target"], d["gamma"], d["reservoir"])
+        c.alpha_t = d["alpha_t"]
+        c.scores.extend(d["scores"])
+        c.n_observed = d["n_observed"]
+        c.n_outside = d["n_outside"]
+        return c

tsanomaly/calibrate/evt.py

@@ -0,0 +1,119 @@
+"""Streaming extreme-value tail model.
+
+Generalized Pareto fit over exceedances of a high quantile of standardized
+residual magnitudes -> calibrated tail probabilities for "how far outside the
+envelope" (heavy tails honest, unlike Gaussian). Empirical fallback below the
+threshold or before enough exceedances accumulate, so p_mag works from the
+first episode.
+"""
+
+from __future__ import annotations
+
+from collections import deque
+from typing import Any
+
+import numpy as np
+from scipy import stats
+
+_REFIT_EVERY = 100
+_MIN_EXCEEDANCES = 30
+_THRESHOLD_Q = 0.95
+
+
+class TailModel:
+    def __init__(self, buf_size: int = 2048, exc_size: int = 256) -> None:
+        self.all_s: deque[float] = deque(maxlen=buf_size)
+        self.exceedances: deque[float] = deque(maxlen=exc_size)
+        self.u: float = float("inf")  # POT threshold
+        self.n_total = 0
+        self.n_exc = 0
+        self.gpd: tuple[float, float] | None = None  # (shape c, scale)
+        self._since_refit = 0
+        self.frozen = False
+
+    def warm(self, magnitudes: np.ndarray) -> None:
+        for s in magnitudes[np.isfinite(magnitudes)]:
+            self.all_s.append(float(s))
+        self.n_total = len(self.all_s)
+        self._refit_threshold()
+        for s in self.all_s:
+            if s > self.u:
+                self.exceedances.append(s)
+        self.n_exc = len(self.exceedances)
+        self._fit_gpd()
+
+    def observe(self, magnitude: float) -> None:
+        if self.frozen or not np.isfinite(magnitude):
+            return
+        self.all_s.append(float(magnitude))
+        self.n_total += 1
+        if magnitude > self.u:
+            self.exceedances.append(float(magnitude))
+            self.n_exc += 1
+        self._since_refit += 1
+        if self._since_refit >= _REFIT_EVERY:
+            self._refit_threshold()
+            self._fit_gpd()
+            self._since_refit = 0
+
+    def _refit_threshold(self) -> None:
+        if len(self.all_s) >= 50:
+            self.u = float(np.quantile(np.array(self.all_s), _THRESHOLD_Q))
+
+    def _fit_gpd(self) -> None:
+        exc = np.array([e - self.u for e in self.exceedances if e > self.u])
+        if len(exc) < _MIN_EXCEEDANCES:
+            # too few exceedances to estimate a shape: exponential tail (shape 0)
+            # still decays beyond the observed record
+            if len(exc) >= 5:
+                self.gpd = (0.0, float(max(np.mean(exc), 1e-9)))
+            else:
+                self.gpd = None
+            return
+        try:
+            c, loc, scale = stats.genpareto.fit(exc, floc=0.0)
+            # shape capped at 0.5: heavier tails are infinite-variance
+            self.gpd = (float(np.clip(c, -0.5, 0.5)), float(max(scale, 1e-9)))
+        except Exception:
+            self.gpd = None
+
+    def tail_p(self, magnitude: float) -> float:
+        """P(|z| >= magnitude) under the learned residual-magnitude distribution."""
+        if not np.isfinite(magnitude) or magnitude <= 0:
+            return 1.0
+        n = max(1, self.n_total)
+        p_u = self.n_exc / n if self.n_exc else 1.0 / (n + 1)
+        if magnitude <= self.u or not np.isfinite(self.u):
+            if len(self.all_s) >= 30:
+                arr = np.array(self.all_s)
+                return float(
+                    min(1.0, (np.sum(arr >= magnitude) + 1) / (len(arr) + 1))
+                )
+            # cold start: normal tail as a stopgap
+            return float(min(1.0, 2 * stats.norm.sf(magnitude)))
+        if self.gpd is not None:
+            c, scale = self.gpd
+            return float(max(1e-12, p_u * stats.genpareto.sf(magnitude - self.u, c, scale=scale)))
+        exc = np.array(self.exceedances)
+        return float(max(1e-12, p_u * (np.sum(exc >= magnitude) + 1) / (len(exc) + 1)))
+
+    def state_dict(self) -> dict[str, Any]:
+        return {
+            "all_s": [round(s, 5) for s in self.all_s],
+            "exceedances": [round(s, 5) for s in self.exceedances],
+            "u": self.u if np.isfinite(self.u) else None,
+            "n_total": self.n_total,
+            "n_exc": self.n_exc,
+            "gpd": list(self.gpd) if self.gpd else None,
+        }
+
+    @classmethod
+    def from_state(cls, d: dict[str, Any]) -> "TailModel":
+        t = cls()
+        t.all_s.extend(d["all_s"])
+        t.exceedances.extend(d["exceedances"])
+        t.u = d["u"] if d["u"] is not None else float("inf")
+        t.n_total = d["n_total"]
+        t.n_exc = d["n_exc"]
+        t.gpd = tuple(d["gpd"]) if d["gpd"] else None
+        return t

tsanomaly/core/clock.py

@@ -0,0 +1,36 @@
+"""Injectable clock.
+
+Statistical code never reads the wall clock; time comes from data. The clock
+exists only for operational concerns (stall detection, schedules) and for
+deterministic tests.
+"""
+
+from __future__ import annotations
+
+import time
+from typing import Protocol
+
+
+class Clock(Protocol):
+    def now_ns(self) -> int: ...
+
+
+class SystemClock:
+    def now_ns(self) -> int:
+        return time.time_ns()
+
+
+class ManualClock:
+    """Test clock advanced explicitly."""
+
+    def __init__(self, start_ns: int = 0) -> None:
+        self._now = start_ns
+
+    def now_ns(self) -> int:
+        return self._now
+
+    def advance(self, delta_ns: int) -> None:
+        self._now += delta_ns
+
+    def set(self, ts_ns: int) -> None:
+        self._now = ts_ns

tsanomaly/core/config.py

@@ -0,0 +1,127 @@
+"""Configuration tree.
+
+Three layers: built-in defaults -> tsanomaly.toml -> per-call overrides.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import json
+from pathlib import Path
+from typing import Any
+
+from pydantic import BaseModel, ConfigDict, Field
+
+from tsanomaly.core.errors import ConfigError
+
+
+class _Section(BaseModel):
+    model_config = ConfigDict(extra="forbid", frozen=True)
+
+
+class EnvelopeConfig(_Section):
+    target_coverage: float = Field(0.995, gt=0.5, lt=1.0)  # fraction of normal points inside the envelope
+    aci_gamma: float = Field(0.02, gt=0.0, lt=0.5)  # ACI step size
+    reservoir: int = Field(512, ge=32)  # conformal residual reservoir
+
+
+class EpisodeConfig(_Section):
+    close_after: int = Field(3, ge=1)  # consecutive in-envelope samples to close
+    merge_gap: int = Field(3, ge=0)  # samples between episodes that merge
+    max_points: int = Field(5000, ge=10)  # bounded episode point buffer
+
+
+class SeasonConfig(_Section):
+    max_components: int = Field(2, ge=0)  # max simultaneous seasonal components
+    strength_floor: float = Field(0.3, ge=0.0, le=1.0)  # Hyndman F_s floor
+    n_candidates: int = Field(8, ge=1)  # top-K periodogram peaks
+    null_shuffles: int = Field(100, ge=10)  # block-permutation null size
+    null_quantile: float = Field(0.99, gt=0.5, lt=1.0)
+    min_cycles: float = Field(2.0, ge=1.5)  # data must span >= this many periods
+
+
+class PreprocessConfig(_Section):
+    hampel_k: float = Field(6.0, gt=0.0)  # rolling median +/- k*MAD outlier threshold
+    hampel_window: int = Field(25, ge=5)
+    large_gap_intervals: int = Field(10, ge=2)  # gap > this*interval splits segments
+
+
+class ScoreConfig(_Section):
+    min_alert_score: float = Field(70.0, ge=0.0, le=100.0)
+    log10_full_scale: float = Field(6.0, gt=0.0)  # p=1e-6 -> score 100
+    noise_halfsat: float = Field(5.0, gt=0.0)  # episodes/week at which shrink lambda=1
+    # Episodes below this score are the envelope's *expected* exceedances (a
+    # calibrated 99.5% envelope emits ~0.5% of points by design); they are
+    # tallied in diagnostics, never emitted as anomalies. Not a silent cap:
+    # the count is always reported.
+    report_floor: float = Field(25.0, ge=0.0, le=100.0)
+
+
+class RegimeConfig(_Section):
+    min_samples: int = Field(24, ge=4)  # stationary samples before a shift becomes a re-anchor
+    cooldown_samples: int = Field(64, ge=0)  # post-re-anchor samples with warm-up-damped scores
+
+
+class WarmupConfig(_Section):
+    min_points: int = Field(64, ge=8)  # minimum points before scores are undamped
+    damp: float = Field(0.5, gt=0.0, le=1.0)
+
+
+class StreamConfig(_Section):
+    grace_intervals: int = Field(2, ge=0)  # out-of-order tolerance, in sampling intervals
+
+
+class DetectorConfig(_Section):
+    budget: str = "cheap"  # cheap | balanced | thorough (gates model registry)
+    envelope: EnvelopeConfig = EnvelopeConfig()
+    episodes: EpisodeConfig = EpisodeConfig()
+    season: SeasonConfig = SeasonConfig()
+    preprocess: PreprocessConfig = PreprocessConfig()
+    score: ScoreConfig = ScoreConfig()
+    regime: RegimeConfig = RegimeConfig()
+    warmup: WarmupConfig = WarmupConfig()
+    stream: StreamConfig = StreamConfig()
+
+    def config_hash(self) -> str:
+        blob = json.dumps(self.model_dump(), sort_keys=True, separators=(",", ":"))
+        return hashlib.sha256(blob.encode()).hexdigest()[:16]
+
+    @classmethod
+    def load(
+        cls,
+        toml_path: str | Path | None = None,
+        overrides: dict[str, Any] | None = None,
+    ) -> "DetectorConfig":
+        """Resolve defaults -> toml file -> overrides."""
+        data: dict[str, Any] = {}
+        if toml_path is not None:
+            try:
+                import tomllib  # Python 3.11+
+            except ImportError:  # pragma: no cover - Python 3.10
+                try:
+                    import tomli as tomllib  # type: ignore[no-redef]
+                except ImportError as exc:
+                    raise ConfigError(
+                        "reading TOML config on Python 3.10 requires `pip install tomli`"
+                    ) from exc
+
+            p = Path(toml_path)
+            if not p.exists():
+                raise ConfigError(f"config file not found: {p}")
+            data = tomllib.loads(p.read_text())
+        if overrides:
+            data = _deep_merge(data, overrides)
+        try:
+            return cls.model_validate(data)
+        except Exception as exc:  # pydantic ValidationError -> our error type
+            raise ConfigError(str(exc)) from exc
+
+
+def _deep_merge(base: dict[str, Any], extra: dict[str, Any]) -> dict[str, Any]:
+    out = dict(base)
+    for k, v in extra.items():
+        if isinstance(v, dict) and isinstance(out.get(k), dict):
+            out[k] = _deep_merge(out[k], v)
+        else:
+            out[k] = v
+    return out

tsanomaly/core/determinism.py

@@ -0,0 +1,22 @@
+"""Deterministic seed derivation.
+
+Same inputs -> same seed across processes and platforms. All stochastic steps
+(bootstrap nulls, backtest splits) derive their RNG from here; nothing seeds
+from wall clock or global state.
+"""
+
+from __future__ import annotations
+
+import hashlib
+
+import numpy as np
+
+
+def derive_seed(*parts: str | int) -> int:
+    """Stable 64-bit seed from string/int parts."""
+    h = hashlib.sha256("\x1f".join(str(p) for p in parts).encode("utf-8")).digest()
+    return int.from_bytes(h[:8], "big")
+
+
+def rng_for(*parts: str | int) -> np.random.Generator:
+    return np.random.default_rng(derive_seed(*parts))

tsanomaly/core/errors.py

@@ -0,0 +1,28 @@
+"""Exception hierarchy."""
+
+
+class TSAnomalyError(Exception):
+    """Root of all TSAnomaly exceptions."""
+
+
+class ConfigError(TSAnomalyError):
+    """Invalid or inconsistent configuration."""
+
+
+class IngestionError(TSAnomalyError):
+    """Input data could not be interpreted (schema, timestamps, dimensions)."""
+
+
+class StateError(TSAnomalyError):
+    """Metric state is missing, corrupt, or from an incompatible schema version."""
+
+
+class MissingExtraError(TSAnomalyError):
+    """A plugin needs an optional dependency that is not installed."""
+
+    def __init__(self, plugin: str, extra: str) -> None:
+        super().__init__(
+            f"Plugin {plugin!r} requires the {extra!r} extra: pip install 'tsanomaly[{extra}]'"
+        )
+        self.plugin = plugin
+        self.extra = extra

tsanomaly/core/result.py

@@ -0,0 +1,91 @@
+"""DetectionResult container."""
+
+from __future__ import annotations
+
+import json
+from dataclasses import dataclass, field
+from typing import Any, Iterable
+
+import pandas as pd
+
+from tsanomaly.core.types import Anomaly, Incident
+
+
+@dataclass(slots=True)
+class DetectionResult:
+    anomalies: list[Anomaly] = field(default_factory=list)
+    incidents: list[Incident] = field(default_factory=list)
+    diagnostics: dict[str, Any] = field(default_factory=dict)
+    config_hash: str = ""
+    # Opt-in per-point envelope series: metric key ->
+    # list of {ts, observed, expected, lo, hi, z, outside} dicts.
+    envelopes: dict[str, list[dict[str, Any]]] | None = None
+
+    def __post_init__(self) -> None:
+        self.anomalies.sort(key=lambda a: (-a.score, a.episode.start_ts, a.metric.key()))
+
+    def alerts(self, min_score: float = 70.0) -> list[Anomaly]:
+        return [a for a in self.anomalies if a.score >= min_score]
+
+    def summary(self) -> str:
+        n = len(self.anomalies)
+        n_alert = len(self.alerts())
+        metrics = {a.metric.key() for a in self.anomalies}
+        inc = f", {len(self.incidents)} incidents" if self.incidents else ""
+        return (
+            f"{n} anomalies ({n_alert} above alert threshold) "
+            f"across {len(metrics)} metrics{inc}"
+        )
+
+    def to_frame(self) -> pd.DataFrame:
+        rows = []
+        for a in self.anomalies:
+            e = a.episode
+            rows.append(
+                {
+                    "metric": e.metric.key(),
+                    "start_ts": pd.Timestamp(e.start_ts, unit="ns", tz="UTC"),
+                    "end_ts": (
+                        pd.Timestamp(e.end_ts, unit="ns", tz="UTC") if e.end_ts else pd.NaT
+                    ),
+                    "direction": e.direction,
+                    "kind": e.kind.value,
+                    "score": a.score,
+                    "peak_z": e.peak_z,
+                    "p_mag": a.breakdown.p_mag,
+                    "p_dur": a.breakdown.p_dur,
+                    "p_combined": a.breakdown.p_combined,
+                }
+            )
+        return pd.DataFrame(rows)
+
+    def to_json(self) -> str:
+        """Deterministic JSON: identical data and config always serialize identically."""
+        payload = {
+            "config_hash": self.config_hash,
+            "anomalies": [a.to_dict() for a in self.anomalies],
+            "incidents": [
+                {
+                    "id": i.id,
+                    "narration": i.narration,
+                    "started_ts": i.started_ts,
+                    "ended_ts": i.ended_ts,
+                    "aggregate": i.aggregate.to_dict(),
+                }
+                for i in self.incidents
+            ],
+            "diagnostics": self.diagnostics,
+        }
+        return json.dumps(payload, sort_keys=True, separators=(",", ":"), default=_json_default)
+
+    def extend(self, other: Iterable[Anomaly]) -> None:
+        self.anomalies.extend(other)
+        self.__post_init__()
+
+
+def _json_default(o: Any) -> Any:
+    if hasattr(o, "to_dict"):
+        return o.to_dict()
+    if hasattr(o, "item"):  # numpy scalars
+        return o.item()
+    return str(o)