canary-ml 1.0.0__tar.gz

canary_ml-1.0.0/PKG-INFO

@@ -0,0 +1,72 @@
+Metadata-Version: 2.4
+Name: canary-ml
+Version: 1.0.0
+Summary: Lightweight drift and anomaly monitoring for production ML models.
+License: MIT
+Project-URL: Homepage, https://github.com/aitor1717/canary-ml
+Project-URL: Repository, https://github.com/aitor1717/canary-ml
+Requires-Python: >=3.9
+Description-Content-Type: text/markdown
+Requires-Dist: numpy>=1.24
+Requires-Dist: scipy>=1.10
+Requires-Dist: scikit-learn>=1.3
+Requires-Dist: rich>=13.0
+Provides-Extra: keras
+Requires-Dist: tensorflow>=2.13; extra == "keras"
+Provides-Extra: dev
+Requires-Dist: pytest>=7.0; extra == "dev"
+Requires-Dist: pytest-cov>=4.0; extra == "dev"
+
+# canary-ml
+
+Lightweight drift and anomaly monitoring for production ML models.
+
+## Install
+
+```bash
+pip install canary-ml
+```
+
+## Quickstart
+
+```python
+from canary_ml import ModelMonitor
+
+monitor = ModelMonitor(
+    model=your_model,
+    reference_data=X_train,
+    alert_threshold=0.2,
+    log_path="./canary_logs"
+)
+
+# drop-in replacement — monitoring is a side effect
+predictions = monitor.predict(X_new)
+
+report = monitor.get_report()
+print(report.summary())
+# DriftReport | psi=0.41 | features_drifted=3 | anomaly_rate=3.2% | ALERT
+
+monitor.serve_dashboard(port=8501)
+```
+
+## Features
+
+- Data Drift Detection — KS test, PSI, chi-square per feature with configurable thresholds
+- Anomaly Detection — Isolation Forest and z-score ensemble on inputs and outputs
+- Zero Latency — monitoring is a side effect; inference path stays unchanged
+- Live Dashboard — zero-dep HTML/JS, ships with the package, no cloud account needed
+
+## Why
+
+Most ML monitoring tools require a database, a cloud account, or a separate deployment pipeline. canary-ml wraps your model with a single line of code and starts logging drift metrics immediately — to a local JSON-lines file, with no external dependencies.
+
+The dashboard (`monitor.serve_dashboard()`) reads from that file and auto-refreshes every 5 seconds. You can run it on a laptop, in a Docker container, or on any machine with the package installed.
+
+## Roadmap
+
+### v1.1
+- Label-free performance estimation: estimate model accuracy/F1 from confidence score distributions without ground truth labels, using confidence-based performance estimation (CBPE). Alerts when estimated performance degrades, not just when inputs shift.
+
+## License
+
+MIT

canary_ml-1.0.0/README.md

@@ -0,0 +1,53 @@
+# canary-ml
+
+Lightweight drift and anomaly monitoring for production ML models.
+
+## Install
+
+```bash
+pip install canary-ml
+```
+
+## Quickstart
+
+```python
+from canary_ml import ModelMonitor
+
+monitor = ModelMonitor(
+    model=your_model,
+    reference_data=X_train,
+    alert_threshold=0.2,
+    log_path="./canary_logs"
+)
+
+# drop-in replacement — monitoring is a side effect
+predictions = monitor.predict(X_new)
+
+report = monitor.get_report()
+print(report.summary())
+# DriftReport | psi=0.41 | features_drifted=3 | anomaly_rate=3.2% | ALERT
+
+monitor.serve_dashboard(port=8501)
+```
+
+## Features
+
+- Data Drift Detection — KS test, PSI, chi-square per feature with configurable thresholds
+- Anomaly Detection — Isolation Forest and z-score ensemble on inputs and outputs
+- Zero Latency — monitoring is a side effect; inference path stays unchanged
+- Live Dashboard — zero-dep HTML/JS, ships with the package, no cloud account needed
+
+## Why
+
+Most ML monitoring tools require a database, a cloud account, or a separate deployment pipeline. canary-ml wraps your model with a single line of code and starts logging drift metrics immediately — to a local JSON-lines file, with no external dependencies.
+
+The dashboard (`monitor.serve_dashboard()`) reads from that file and auto-refreshes every 5 seconds. You can run it on a laptop, in a Docker container, or on any machine with the package installed.
+
+## Roadmap
+
+### v1.1
+- Label-free performance estimation: estimate model accuracy/F1 from confidence score distributions without ground truth labels, using confidence-based performance estimation (CBPE). Alerts when estimated performance degrades, not just when inputs shift.
+
+## License
+
+MIT

canary_ml-1.0.0/canary_ml/__init__.py

@@ -0,0 +1,5 @@
+from canary_ml.monitor import ModelMonitor
+from canary_ml.report import DriftReport
+
+__all__ = ["ModelMonitor", "DriftReport"]
+__version__ = "1.0.0"

canary_ml-1.0.0/canary_ml/alerts.py

@@ -0,0 +1,40 @@
+from __future__ import annotations
+
+from rich.console import Console
+from rich.panel import Panel
+from rich.text import Text
+
+from canary_ml.report import DriftReport
+
+_console = Console()
+
+
+def check_alert(report: DriftReport, threshold: float) -> bool:
+    """Return True when PSI exceeds *threshold*."""
+    return report.psi_score > threshold
+
+
+def format_alert(report: DriftReport) -> None:
+    """Print a rich-formatted alert panel to the terminal."""
+    drifted = sum(1 for v in report.ks_results.values() if v.get("drifted"))
+
+    if report.alert_triggered:
+        style, title = "bold red", "[bold red]DRIFT ALERT[/bold red]"
+    elif report.drift_detected:
+        style, title = "bold yellow", "[bold yellow]DRIFT WARNING[/bold yellow]"
+    else:
+        style, title = "bold green", "[bold green]STABLE[/bold green]"
+
+    body = Text()
+    body.append(f"  timestamp      ", style="dim")
+    body.append(f"{report.timestamp}\n")
+    body.append(f"  samples        ", style="dim")
+    body.append(f"{report.n_samples}\n")
+    body.append(f"  PSI score      ", style="dim")
+    body.append(f"{report.psi_score:.3f}\n", style=style if report.psi_score > 0.1 else "")
+    body.append(f"  features drifted ", style="dim")
+    body.append(f"{drifted}\n", style=style if drifted > 0 else "")
+    body.append(f"  anomaly rate   ", style="dim")
+    body.append(f"{report.anomaly_rate * 100:.1f}%\n", style="yellow" if report.anomaly_rate > 0.02 else "")
+
+    _console.print(Panel(body, title=title, border_style=style.split()[1]))

canary_ml-1.0.0/canary_ml/anomaly.py

@@ -0,0 +1,77 @@
+from __future__ import annotations
+
+import numpy as np
+from sklearn.ensemble import IsolationForest
+
+
+class AnomalyDetector:
+    """Ensemble anomaly detector: IsolationForest OR z-score (|z| > 3)."""
+
+    def __init__(
+        self,
+        contamination: float = 0.05,
+        method: str = "ensemble",
+    ) -> None:
+        """
+        Args:
+            contamination: Expected fraction of outliers (passed to IsolationForest).
+            method: 'isolation_forest', 'zscore', or 'ensemble'.
+        """
+        self.contamination = contamination
+        self.method = method
+        self._isoforest: IsolationForest | None = None
+        self._ref_mean: np.ndarray | None = None
+        self._ref_std: np.ndarray | None = None
+
+    def fit(self, X: np.ndarray) -> "AnomalyDetector":
+        """Fit on reference (baseline) data."""
+        X = np.asarray(X, dtype=float)
+        if X.ndim == 1:
+            X = X.reshape(-1, 1)
+
+        self._ref_mean = X.mean(axis=0)
+        self._ref_std = X.std(axis=0)
+        # Avoid division by zero in score()
+        self._ref_std = np.where(self._ref_std == 0, 1.0, self._ref_std)
+
+        if self.method in ("isolation_forest", "ensemble"):
+            self._isoforest = IsolationForest(
+                contamination=self.contamination,
+                random_state=42,
+            )
+            self._isoforest.fit(X)
+
+        return self
+
+    def score(self, X: np.ndarray) -> dict:
+        """Detect anomalies in *X*.
+
+        Returns:
+            anomaly_rate: fraction of samples flagged
+            anomaly_mask: bool array, True = anomalous
+            scores: raw scores (lower = more anomalous for IsolationForest)
+        """
+        X = np.asarray(X, dtype=float)
+        if X.ndim == 1:
+            X = X.reshape(-1, 1)
+
+        n = len(X)
+        mask = np.zeros(n, dtype=bool)
+        raw_scores = np.zeros(n, dtype=float)
+
+        if self.method in ("isolation_forest", "ensemble") and self._isoforest is not None:
+            preds = self._isoforest.predict(X)            # -1 = anomaly
+            iso_mask = preds == -1
+            raw_scores = self._isoforest.decision_function(X)
+            mask |= iso_mask
+
+        if self.method in ("zscore", "ensemble") and self._ref_mean is not None:
+            z = np.abs((X - self._ref_mean) / self._ref_std)
+            zscore_mask = z.max(axis=1) > 3.0
+            mask |= zscore_mask
+
+        return {
+            "anomaly_rate": float(mask.sum() / max(n, 1)),
+            "anomaly_mask": mask,
+            "scores": raw_scores,
+        }