modelshift 0.1.0__py3-none-any.whl

modelshift/__init__.py

@@ -0,0 +1,3 @@
+"""
+ModelShift-Lite package initialization.
+"""

modelshift/baseline.py

@@ -0,0 +1,37 @@
+import pandas as pd
+
+
+class BaselineWindow:
+    """
+    Stores and manages reference baseline data
+    representing normal model behavior.
+    """
+
+    def __init__(self, data: pd.DataFrame):
+        self._validate(data)
+        self.data = data.copy()
+        self.feature_names = list(data.columns)
+        self.num_samples = len(data)
+
+    def _validate(self, data):
+        if not isinstance(data, pd.DataFrame):
+            raise TypeError("Baseline data must be a pandas DataFrame")
+
+        if data.empty:
+            raise ValueError("Baseline data cannot be empty")
+
+    def get_data(self) -> pd.DataFrame:
+        """
+        Returns a copy of baseline data.
+        """
+        return self.data.copy()
+
+    def summary(self) -> dict:
+        """
+        Returns basic metadata about the baseline window.
+        """
+        return {
+            "num_samples": self.num_samples,
+            "num_features": len(self.feature_names),
+            "feature_names": self.feature_names,
+        }

modelshift/drift/__init__.py

@@ -0,0 +1,3 @@
+"""
+Drift detection modules.
+"""

modelshift/drift/feature_drift.py

@@ -0,0 +1,50 @@
+import pandas as pd
+from scipy.stats import ks_2samp
+
+
+def compute_feature_drift(
+    baseline_data: pd.DataFrame,
+    live_data: pd.DataFrame
+) -> dict:
+    """
+    Compute feature-level drift using the Kolmogorov–Smirnov test.
+
+    Returns a dictionary:
+    {
+        feature_name: {
+            "ks_statistic": float,
+            "p_value": float
+        }
+    }
+    """
+
+    _validate_inputs(baseline_data, live_data)
+
+    drift_results = {}
+
+    for feature in baseline_data.columns:
+        baseline_values = baseline_data[feature].dropna()
+        live_values = live_data[feature].dropna()
+
+        ks_stat, p_value = ks_2samp(baseline_values, live_values)
+
+        drift_results[feature] = {
+            "ks_statistic": float(ks_stat),
+            "p_value": float(p_value),
+        }
+
+    return drift_results
+
+
+def _validate_inputs(baseline_data, live_data):
+    if not isinstance(baseline_data, pd.DataFrame):
+        raise TypeError("Baseline data must be a pandas DataFrame")
+
+    if not isinstance(live_data, pd.DataFrame):
+        raise TypeError("Live data must be a pandas DataFrame")
+
+    if baseline_data.empty or live_data.empty:
+        raise ValueError("Baseline and live data cannot be empty")
+
+    if list(baseline_data.columns) != list(live_data.columns):
+        raise ValueError("Baseline and live data must have identical features")

modelshift/drift/prediction_drift.py

@@ -0,0 +1,111 @@
+from __future__ import annotations
+
+import numpy as np
+from scipy.stats import ks_2samp
+
+
+def compute_prediction_drift(
+    baseline_predictions: np.ndarray,
+    live_predictions: np.ndarray
+) -> dict:
+    """
+    Compute prediction behavior drift using:
+      1) KS-test on prediction probability distributions
+      2) Binary entropy change (mean entropy of predicted probabilities)
+
+    Notes:
+    - Expects 1D probability arrays (values in [0, 1]).
+    - Uses full binary entropy: -(p*log(p) + (1-p)*log(1-p))
+    """
+
+    baseline = _prepare_predictions("baseline", baseline_predictions)
+    live = _prepare_predictions("live", live_predictions)
+
+    # KS-test on prediction distributions
+    ks_stat, p_value = ks_2samp(baseline, live)
+
+    # Entropy analysis (full binary entropy)
+    baseline_entropy = _binary_entropy_mean(baseline)
+    live_entropy = _binary_entropy_mean(live)
+    entropy_change = live_entropy - baseline_entropy
+
+    # Lightweight shape/center diagnostics (useful for dashboards/reports)
+    baseline_mean = float(np.mean(baseline))
+    live_mean = float(np.mean(live))
+    baseline_std = float(np.std(baseline))
+    live_std = float(np.std(live))
+    baseline_median = float(np.median(baseline))
+    live_median = float(np.median(live))
+
+    return {
+        "ks_statistic": float(ks_stat),
+        "p_value": float(p_value),
+
+        "baseline_entropy": round(float(baseline_entropy), 6),
+        "live_entropy": round(float(live_entropy), 6),
+        "entropy_change": round(float(entropy_change), 6),
+        "abs_entropy_change": round(float(abs(entropy_change)), 6),
+
+        "baseline_mean_prob": round(baseline_mean, 6),
+        "live_mean_prob": round(live_mean, 6),
+        "mean_prob_shift": round(float(live_mean - baseline_mean), 6),
+
+        "baseline_median_prob": round(baseline_median, 6),
+        "live_median_prob": round(live_median, 6),
+        "median_prob_shift": round(float(live_median - baseline_median), 6),
+
+        "baseline_std_prob": round(baseline_std, 6),
+        "live_std_prob": round(live_std, 6),
+        "std_prob_shift": round(float(live_std - baseline_std), 6),
+
+        "n_baseline": int(baseline.size),
+        "n_live": int(live.size),
+    }
+
+
+def _binary_entropy_mean(preds: np.ndarray) -> float:
+    """
+    Mean binary entropy for probability predictions:
+      H(p) = -(p*log(p) + (1-p)*log(1-p))
+
+    Uses natural log (nats).
+    """
+    eps = 1e-9
+    p = np.clip(preds.astype(float), eps, 1.0 - eps)
+    entropy = -(p * np.log(p) + (1.0 - p) * np.log(1.0 - p))
+    return float(np.mean(entropy))
+
+
+def _prepare_predictions(name: str, arr) -> np.ndarray:
+    """
+    Validate and normalize prediction arrays to a clean 1D float numpy array.
+    """
+    if arr is None:
+        raise ValueError(f"{name.capitalize()} predictions cannot be None")
+
+    if not isinstance(arr, np.ndarray):
+        # Allow lists/Series while staying user-friendly
+        try:
+            arr = np.asarray(arr, dtype=float)
+        except Exception as exc:
+            raise TypeError(
+                f"{name.capitalize()} predictions must be a numpy array or array-like of numeric values"
+            ) from exc
+    else:
+        arr = arr.astype(float, copy=False)
+
+    arr = np.ravel(arr)
+
+    if arr.size == 0:
+        raise ValueError(f"{name.capitalize()} prediction array cannot be empty")
+
+    if not np.all(np.isfinite(arr)):
+        raise ValueError(f"{name.capitalize()} predictions contain NaN/Inf values")
+
+    # We treat these as probability predictions for entropy-based drift
+    if np.min(arr) < 0.0 or np.max(arr) > 1.0:
+        raise ValueError(
+            f"{name.capitalize()} predictions must be probability values in [0, 1]"
+        )
+
+    return arr

modelshift/drift/severity.py

@@ -0,0 +1,239 @@
+from __future__ import annotations
+
+from typing import Any, Dict, List, Optional
+
+
+# ----------------------------
+# Basic per-signal thresholds
+# ----------------------------
+FEATURE_KS_LOW = 0.10
+FEATURE_KS_MEDIUM = 0.20
+FEATURE_KS_HIGH = 0.35
+
+PRED_KS_WARNING = 0.10
+PRED_KS_CRITICAL = 0.15
+
+ENTROPY_DELTA_WARNING = 0.01
+ENTROPY_DELTA_CRITICAL = 0.02
+
+
+def classify_severity(ks_statistic: float) -> str:
+    """
+    Classify severity from a KS-like drift signal.
+
+    Returns one of:
+      LOW / MEDIUM / HIGH / CRITICAL
+    """
+    ks = _safe_float(ks_statistic, default=0.0)
+    if ks < FEATURE_KS_LOW:
+        return "LOW"
+    if ks < FEATURE_KS_MEDIUM:
+        return "MEDIUM"
+    if ks < FEATURE_KS_HIGH:
+        return "HIGH"
+    return "CRITICAL"
+
+
+def compute_health_score(feature_drift_results: dict) -> float:
+    """
+    Compute overall model health score (0–100) from feature drift.
+    Higher score = healthier model.
+
+    Uses average feature KS:
+      health = max(0, 100 * (1 - avg_ks))
+    """
+    summary = summarize_feature_drift(feature_drift_results)
+    if summary["feature_count"] == 0:
+        raise ValueError("Feature drift results cannot be empty")
+
+    avg_ks = summary["avg_ks"]
+    health_score = max(0.0, 100.0 * (1.0 - avg_ks))
+    return round(float(health_score), 2)
+
+
+def summarize_feature_drift(feature_drift_results: Optional[dict]) -> Dict[str, Any]:
+    """
+    Extract feature drift summary stats from feature_drift_results.
+    Safe against missing/malformed values.
+    """
+    if not isinstance(feature_drift_results, dict):
+        return {
+            "feature_count": 0,
+            "avg_ks": 0.0,
+            "max_ks": 0.0,
+            "max_feature": None,
+            "ks_values": [],
+        }
+
+    ks_pairs: List[tuple[str, float]] = []
+    for feature, values in feature_drift_results.items():
+        if not isinstance(values, dict):
+            continue
+        ks = _safe_float(values.get("ks_statistic"), default=None)
+        if ks is None:
+            continue
+        ks_pairs.append((str(feature), ks))
+
+    if not ks_pairs:
+        return {
+            "feature_count": 0,
+            "avg_ks": 0.0,
+            "max_ks": 0.0,
+            "max_feature": None,
+            "ks_values": [],
+        }
+
+    ks_values = [ks for _, ks in ks_pairs]
+    max_feature, max_ks = max(ks_pairs, key=lambda x: x[1])
+
+    return {
+        "feature_count": len(ks_values),
+        "avg_ks": round(sum(ks_values) / len(ks_values), 6),
+        "max_ks": round(float(max_ks), 6),
+        "max_feature": max_feature,
+        "ks_values": [round(float(v), 6) for v in ks_values],
+    }
+
+
+def classify_drift_taxonomy(
+    feature_drift_results: Optional[dict] = None,
+    prediction_drift_results: Optional[dict] = None,
+) -> str:
+    """
+    Agreement/disagreement taxonomy between feature drift and prediction drift.
+
+    Returns one of:
+      STABLE
+      ROBUST_SHIFT              (feature drift high, prediction drift low)
+      SILENT_BEHAVIOR_DRIFT     (feature drift low, prediction drift high)
+      DEGRADING_DRIFT           (both high)
+    """
+    f_summary = summarize_feature_drift(feature_drift_results)
+    max_feature_ks = _safe_float(f_summary.get("max_ks"), default=0.0)
+
+    pred_ks = 0.0
+    if isinstance(prediction_drift_results, dict):
+        pred_ks = _safe_float(prediction_drift_results.get("ks_statistic"), default=0.0)
+
+    feature_high = max_feature_ks >= FEATURE_KS_MEDIUM
+    pred_high = pred_ks >= PRED_KS_WARNING
+
+    if not feature_high and not pred_high:
+        return "STABLE"
+    if feature_high and not pred_high:
+        return "ROBUST_SHIFT"
+    if (not feature_high) and pred_high:
+        return "SILENT_BEHAVIOR_DRIFT"
+    return "DEGRADING_DRIFT"
+
+
+def evaluate_drift_state(
+    feature_drift_results: Optional[dict] = None,
+    prediction_drift_results: Optional[dict] = None,
+) -> Dict[str, Any]:
+    """
+    Composite severity + status decision engine.
+
+    Combines:
+    - avg feature KS
+    - max feature KS
+    - prediction KS
+    - entropy delta
+
+    Returns a normalized decision payload with:
+      severity, status, taxonomy, health_score, signals, thresholds
+    """
+    f_summary = summarize_feature_drift(feature_drift_results)
+
+    avg_feature_ks = _safe_float(f_summary.get("avg_ks"), default=0.0)
+    max_feature_ks = _safe_float(f_summary.get("max_ks"), default=0.0)
+    pred_ks = 0.0
+    entropy_change = 0.0
+
+    if isinstance(prediction_drift_results, dict):
+        pred_ks = _safe_float(prediction_drift_results.get("ks_statistic"), default=0.0)
+        entropy_change = _safe_float(prediction_drift_results.get("entropy_change"), default=0.0)
+
+    # Normalize signals into 0..1 severity components
+    # (Threshold denominators chosen to align with your current observed ranges.)
+    avg_comp = min(1.0, avg_feature_ks / FEATURE_KS_MEDIUM)           # avg drift
+    max_comp = min(1.0, max_feature_ks / FEATURE_KS_HIGH)             # worst feature
+    pred_comp = min(1.0, pred_ks / PRED_KS_CRITICAL)                  # behavior drift
+    ent_comp = min(1.0, abs(entropy_change) / ENTROPY_DELTA_CRITICAL) # confidence shift
+
+    # Weighted composite score [0,1]
+    composite_score = (
+        0.30 * avg_comp +
+        0.25 * max_comp +
+        0.35 * pred_comp +
+        0.10 * ent_comp
+    )
+
+    severity = _classify_composite_severity(composite_score)
+
+    # Status favors prediction drift a bit more (behavior-centric monitoring)
+    if pred_ks >= PRED_KS_CRITICAL or max_feature_ks >= FEATURE_KS_HIGH:
+        status = "CRITICAL_DRIFT"
+    elif pred_ks >= PRED_KS_WARNING or max_feature_ks >= FEATURE_KS_MEDIUM or avg_feature_ks >= FEATURE_KS_LOW:
+        status = "WARNING_DRIFT"
+    else:
+        status = "STABLE"
+
+    taxonomy = classify_drift_taxonomy(feature_drift_results, prediction_drift_results)
+
+    # Health score is only meaningful if feature drift exists
+    health_score = None
+    if f_summary["feature_count"] > 0:
+        health_score = compute_health_score(feature_drift_results)
+
+    return {
+        "severity": severity,
+        "status": status,
+        "taxonomy": taxonomy,
+        "health_score": health_score,
+        "signals": {
+            "avg_feature_ks": round(avg_feature_ks, 6),
+            "max_feature_ks": round(max_feature_ks, 6),
+            "max_feature_name": f_summary.get("max_feature"),
+            "prediction_ks": round(pred_ks, 6),
+            "entropy_change": round(entropy_change, 6),
+            "composite_score": round(float(composite_score), 6),
+            "feature_count": int(f_summary.get("feature_count", 0)),
+        },
+        "thresholds": {
+            "feature_ks_low": FEATURE_KS_LOW,
+            "feature_ks_medium": FEATURE_KS_MEDIUM,
+            "feature_ks_high": FEATURE_KS_HIGH,
+            "pred_ks_warning": PRED_KS_WARNING,
+            "pred_ks_critical": PRED_KS_CRITICAL,
+            "entropy_delta_warning": ENTROPY_DELTA_WARNING,
+            "entropy_delta_critical": ENTROPY_DELTA_CRITICAL,
+        },
+    }
+
+
+# ----------------------------
+# Internal helpers
+# ----------------------------
+def _classify_composite_severity(score: float) -> str:
+    """
+    Composite score is already normalized 0..1.
+    """
+    s = max(0.0, min(1.0, _safe_float(score, default=0.0)))
+
+    if s < 0.20:
+        return "LOW"
+    if s < 0.45:
+        return "MEDIUM"
+    if s < 0.70:
+        return "HIGH"
+    return "CRITICAL"
+
+
+def _safe_float(value: Any, default: Optional[float] = 0.0) -> Optional[float]:
+    try:
+        if value is None:
+            return default
+        return float(value)
+    except (TypeError, ValueError):
+        return default

modelshift/monitor.py

@@ -0,0 +1,317 @@
+from __future__ import annotations
+
+import uuid
+import requests
+from datetime import datetime, timezone
+from typing import Any, Dict, List, Optional
+
+import numpy as np
+import pandas as pd
+
+from modelshift.baseline import BaselineWindow
+from modelshift.drift.feature_drift import compute_feature_drift
+from modelshift.drift.prediction_drift import compute_prediction_drift
+from modelshift.drift.severity import (
+    classify_severity,
+    compute_health_score,
+    evaluate_drift_state,
+    summarize_feature_drift,
+)
+
+# -------------------------------------------------------------------
+# Phase 2: Cloud SDK Configuration
+# -------------------------------------------------------------------
+_CLOUD_CONFIG = {
+    "api_key": None,
+    "endpoint": "http://127.0.0.1:8000/api/v1/track"
+}
+
+def init(api_key: str, endpoint: str = "http://127.0.0.1:8000/api/v1/track"):
+    """
+    Initialize the ModelShift-Lite SDK with your cloud API Key.
+    This links your local ML models to your cloud dashboard.
+    """
+    _CLOUD_CONFIG["api_key"] = api_key
+    _CLOUD_CONFIG["endpoint"] = endpoint
+    print(f"[✓] ModelShift SDK Authenticated. Cloud sync enabled.")
+
+# -------------------------------------------------------------------
+# Core Engine
+# -------------------------------------------------------------------
+class ModelMonitor:
+    """
+    Main interface for ModelShift-Lite monitoring.
+
+    Handles:
+    - baseline/live feature drift
+    - baseline/live prediction drift
+    - composite status/severity/taxonomy summary
+    """
+
+    def __init__(self, reference_data: pd.DataFrame):
+        """
+        Initialize monitor with reference baseline data.
+        """
+        if not isinstance(reference_data, pd.DataFrame):
+            raise TypeError("Reference data must be a pandas DataFrame")
+        if reference_data.empty:
+            raise ValueError("Reference data cannot be empty")
+
+        self.baseline = BaselineWindow(reference_data.copy())
+
+        # Data containers
+        self.live_data: Optional[pd.DataFrame] = None
+
+        # Feature drift
+        self.feature_drift_results: Optional[Dict[str, Any]] = None
+
+        # Prediction drift
+        self.baseline_predictions: Optional[np.ndarray] = None
+        self.live_predictions: Optional[np.ndarray] = None
+        self.prediction_drift_results: Optional[Dict[str, Any]] = None
+
+    # -----------------------
+    # Data Update
+    # -----------------------
+    def update(self, live_data: pd.DataFrame):
+        """
+        Update monitor with new live data.
+        Enforces same columns as baseline (reordered if needed).
+        """
+        if not isinstance(live_data, pd.DataFrame):
+            raise TypeError("Live data must be a pandas DataFrame")
+        if live_data.empty:
+            raise ValueError("Live data cannot be empty")
+
+        baseline_df = self.baseline.get_data()
+        baseline_cols = list(baseline_df.columns)
+        live_cols = list(live_data.columns)
+
+        if set(live_cols) != set(baseline_cols):
+            missing = [c for c in baseline_cols if c not in live_cols]
+            extra = [c for c in live_cols if c not in baseline_cols]
+            raise ValueError(
+                f"Live data columns must match baseline columns. Missing={missing}, Extra={extra}"
+            )
+
+        # Reorder to baseline column order for deterministic behavior
+        self.live_data = live_data[baseline_cols].copy()
+
+    # -----------------------
+    # Feature Drift
+    # -----------------------
+    def compute_feature_drift(self) -> dict:
+        """
+        Compute feature-level drift between baseline and live data.
+        """
+        if self.live_data is None:
+            raise RuntimeError("Live data not set. Call update() first.")
+
+        self.feature_drift_results = compute_feature_drift(
+            self.baseline.get_data(),
+            self.live_data
+        )
+        return self.feature_drift_results
+
+    def get_latest_feature_drift(self) -> dict:
+        if self.feature_drift_results is None:
+            raise RuntimeError("No feature drift computed yet.")
+        return self.feature_drift_results
+
+    def get_feature_severity(self) -> dict:
+        if self.feature_drift_results is None:
+            raise RuntimeError("No feature drift computed yet.")
+
+        severity = {}
+        for feature, values in self.feature_drift_results.items():
+            if not isinstance(values, dict):
+                continue
+            severity[feature] = classify_severity(values.get("ks_statistic", 0.0))
+
+        return severity
+
+    def get_model_health_score(self) -> float:
+        if self.feature_drift_results is None:
+            raise RuntimeError("No feature drift computed yet.")
+        return compute_health_score(self.feature_drift_results)
+
+    def get_top_drifted_features(self, k: int = 5) -> List[Dict[str, Any]]:
+        if self.feature_drift_results is None:
+            raise RuntimeError("No feature drift computed yet.")
+        if not isinstance(k, int) or k <= 0:
+            raise ValueError("k must be a positive integer")
+
+        rows: List[Dict[str, Any]] = []
+        for feature, values in self.feature_drift_results.items():
+            if not isinstance(values, dict):
+                continue
+            ks = _safe_float(values.get("ks_statistic"), 0.0)
+            pv = _safe_float(values.get("p_value"), None)
+            rows.append({
+                "feature": str(feature),
+                "ks_statistic": round(ks, 6),
+                "p_value": None if pv is None else round(pv, 6),
+                "severity": classify_severity(ks),
+            })
+
+        rows.sort(key=lambda x: x["ks_statistic"], reverse=True)
+        return rows[:k]
+
+    def get_most_drifted_feature(self) -> Optional[Dict[str, Any]]:
+        top = self.get_top_drifted_features(k=1)
+        return top[0] if top else None
+
+    # -----------------------
+    # Prediction Drift
+    # -----------------------
+    def set_baseline_predictions(self, predictions):
+        self.baseline_predictions = _prepare_prediction_array(predictions, "baseline")
+
+    def update_predictions(self, live_predictions):
+        self.live_predictions = _prepare_prediction_array(live_predictions, "live")
+
+    def compute_prediction_drift(self) -> dict:
+        if self.baseline_predictions is None:
+            raise RuntimeError("Baseline predictions not set.")
+        if self.live_predictions is None:
+            raise RuntimeError("Live predictions not set.")
+
+        self.prediction_drift_results = compute_prediction_drift(
+            self.baseline_predictions,
+            self.live_predictions
+        )
+        return self.prediction_drift_results
+
+    def get_latest_prediction_drift(self) -> dict:
+        if self.prediction_drift_results is None:
+            raise RuntimeError("No prediction drift computed yet.")
+        return self.prediction_drift_results
+
+    # -----------------------
+    # Composite Summary
+    # -----------------------
+    def evaluate_health(self) -> Dict[str, Any]:
+        if self.feature_drift_results is None:
+            raise RuntimeError("No feature drift computed yet.")
+        if self.prediction_drift_results is None:
+            raise RuntimeError("No prediction drift computed yet.")
+
+        decision = evaluate_drift_state(
+            feature_drift_results=self.feature_drift_results,
+            prediction_drift_results=self.prediction_drift_results,
+        )
+
+        feature_summary = summarize_feature_drift(self.feature_drift_results)
+        top_features = self.get_top_drifted_features(k=5)
+        most_feature = top_features[0] if top_features else None
+
+        return {
+            "status": decision.get("status"),
+            "severity": decision.get("severity"),
+            "taxonomy": decision.get("taxonomy"),
+            "health_score": decision.get("health_score"),
+            "feature_summary": feature_summary,
+            "prediction_drift": self.prediction_drift_results,
+            "top_drifted_features": top_features,
+            "most_drifted_feature": most_feature,
+            "signals": decision.get("signals", {}),
+            "thresholds": decision.get("thresholds", {}),
+        }
+
+    def build_snapshot(self) -> Dict[str, Any]:
+        snapshot: Dict[str, Any] = {
+            "feature_drift": self.feature_drift_results,
+            "prediction_drift": self.prediction_drift_results,
+        }
+
+        if self.feature_drift_results is not None:
+            snapshot["feature_severity"] = self.get_feature_severity()
+            snapshot["health_score"] = self.get_model_health_score()
+            snapshot["top_drifted_features"] = self.get_top_drifted_features(k=5)
+            snapshot["most_drifted_feature"] = self.get_most_drifted_feature()
+
+        if self.feature_drift_results is not None and self.prediction_drift_results is not None:
+            snapshot["decision"] = self.evaluate_health()
+
+        return snapshot
+
+    # -----------------------
+    # Phase 2: Cloud Sync Method
+    # -----------------------
+    def push(self) -> Optional[Dict[str, Any]]:
+        """
+        Takes the local drift calculation and beams it securely to your FastAPI dashboard.
+        """
+        if not _CLOUD_CONFIG["api_key"]:
+            print("[!] API Key Missing. Please add 'modelshift.init(api_key=\"YOUR_KEY\")' at the top of your script.")
+            return None
+            
+        snapshot = self.build_snapshot()
+        decision = snapshot.get("decision", {})
+        
+        mdf = snapshot.get("most_drifted_feature") or {}
+        pred_drift = snapshot.get("prediction_drift") or {}
+        
+        # Package the data exactly how your dashboard expects it
+        run_id = f"run_{uuid.uuid4().hex[:8]}"
+        payload = {
+            "run_id": run_id,
+            "generated_at": datetime.now(timezone.utc).isoformat(),
+            "status": decision.get("status", "UNKNOWN"),
+            "window_size": len(self.live_data) if self.live_data is not None else 0,
+            
+            # Dashboard Graph Metrics
+            "clean_health": 100.0,
+            "drifted_health": decision.get("health_score", 0.0),
+            "drifted_pred_ks": pred_drift.get("ks_statistic", 0.0),
+            "drifted_entropy_change": pred_drift.get("delta_entropy", 0.0),
+            "drifted_last_window_feature": mdf.get("feature"),
+            "drifted_last_window_ks": mdf.get("ks_statistic"),
+            
+            "evaluation": snapshot
+        }
+
+        # The Security Checkpoint
+        headers = {
+            "Content-Type": "application/json",
+            "X-API-Key": _CLOUD_CONFIG["api_key"]
+        }
+        
+        try:
+            print(f"[~] Beaming data to {_CLOUD_CONFIG['endpoint']}...")
+            response = requests.post(_CLOUD_CONFIG["endpoint"], json=payload, headers=headers)
+            response.raise_for_status()
+            print(f"[✓] Successfully synced run '{run_id}' to ModelShift Cloud.")
+            return response.json()
+        except requests.exceptions.RequestException as e:
+            print(f"[!] ModelShift Cloud Sync Failed: {e}")
+            if hasattr(e, 'response') and e.response is not None:
+                print(f"[!] Server Context: {e.response.text}")
+            return None
+
+
+# -----------------------
+# Internal helpers
+# -----------------------
+def _prepare_prediction_array(values, name: str) -> np.ndarray:
+    if values is None:
+        raise ValueError(f"{name.capitalize()} predictions cannot be None")
+    try:
+        arr = np.asarray(values, dtype=float).reshape(-1)
+    except Exception as exc:
+        raise TypeError(f"{name.capitalize()} predictions must be numeric array-like") from exc
+
+    if arr.size == 0:
+        raise ValueError(f"{name.capitalize()} predictions cannot be empty")
+    if not np.all(np.isfinite(arr)):
+        raise ValueError(f"{name.capitalize()} predictions contain NaN/Inf")
+    return arr
+
+
+def _safe_float(value, default=None):
+    try:
+        if value is None:
+            return default
+        return float(value)
+    except (TypeError, ValueError):
+        return default

modelshift/selftest.py

@@ -0,0 +1,398 @@
+# selftest.py (repo root)
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import Any, Dict, List, Optional, Tuple
+import math
+import time
+
+import numpy as np
+import pandas as pd
+
+from modelshift.drift.feature_drift import compute_feature_drift
+from modelshift.drift.prediction_drift import compute_prediction_drift
+from modelshift.drift.severity import compute_health_score
+
+
+# -----------------------------
+# Helpers (robust schema)
+# -----------------------------
+def _to_float(x: Any, default: float = 0.0) -> float:
+    try:
+        v = float(x)
+        if np.isfinite(v):
+            return float(v)
+    except Exception:
+        pass
+    return float(default)
+
+
+def _extract_pred_map(pd_: Any) -> Dict[str, Any]:
+    if not isinstance(pd_, dict):
+        return {}
+    for k in ("prediction_drift", "prediction_drift_results", "results"):
+        v = pd_.get(k)
+        if isinstance(v, dict):
+            return v
+    return pd_
+
+
+def _extract_fd_map(fd: Any) -> Dict[str, Any]:
+    if not isinstance(fd, dict):
+        return {}
+    for k in ("feature_drift_results", "feature_drift", "results"):
+        v = fd.get(k)
+        if isinstance(v, dict):
+            return v
+    return fd
+
+
+def _adapt_pred(pd_: Any) -> Dict[str, Any]:
+    m = _extract_pred_map(pd_)
+    if not isinstance(m, dict):
+        return {}
+    ks = m.get("ks_statistic", m.get("ks", m.get("ks_stat", m.get("statistic", 0.0))))
+    pv = m.get("p_value", m.get("p", m.get("pvalue", 1.0)))
+    out = dict(m)
+    out["ks_statistic"] = _to_float(ks, 0.0)
+    out["p_value"] = _to_float(pv, 1.0)
+    return out
+
+
+def _adapt_fd(fd: Any) -> Dict[str, Dict[str, float]]:
+    m = _extract_fd_map(fd)
+    out: Dict[str, Dict[str, float]] = {}
+    if not isinstance(m, dict):
+        return out
+    for feat, v in m.items():
+        if not isinstance(v, dict):
+            continue
+        ks = v.get("ks_statistic", v.get("ks", v.get("ks_stat", v.get("statistic", v.get("D", 0.0)))))
+        pv = v.get("p_value", v.get("p", v.get("pvalue", v.get("p_val", 1.0))))
+        out[str(feat)] = {
+            "ks_statistic": _to_float(ks, 0.0),
+            "p_value": _to_float(pv, 1.0),
+        }
+    return out
+
+
+def _call_health(fd: Any, pd_: Any) -> Tuple[float, str]:
+    fd_fixed = _adapt_fd(fd)
+    pd_fixed = _adapt_pred(pd_)
+
+    # Try dict form (newer)
+    try:
+        out = compute_health_score({"feature_drift": fd_fixed, "prediction_drift": pd_fixed})
+        if isinstance(out, dict):
+            sc = out.get("health_score", out.get("score", out.get("health", None)))
+            md = out.get("mode", out.get("health_compute_mode", "severity"))
+            if sc is not None:
+                return float(sc), str(md)
+        if isinstance(out, (int, float)):
+            return float(out), "severity"
+    except Exception:
+        pass
+
+    # Try 2-arg form (older)
+    try:
+        out = compute_health_score(fd_fixed, pd_fixed)
+        if isinstance(out, dict):
+            sc = out.get("health_score", out.get("score", out.get("health", None)))
+            md = out.get("mode", out.get("health_compute_mode", "severity"))
+            if sc is not None:
+                return float(sc), str(md)
+        if isinstance(out, (int, float)):
+            return float(out), "severity"
+    except Exception:
+        pass
+
+    # Fallback (simple)
+    ks_vals = [v["ks_statistic"] for v in fd_fixed.values()] if fd_fixed else []
+    avg_ks = float(np.mean(ks_vals)) if ks_vals else 0.0
+    pred_ks = _to_float(pd_fixed.get("ks_statistic"), 0.0)
+    score = 100.0 * (1.0 - min(max(0.70 * pred_ks + 0.30 * avg_ks, 0.0), 1.0))
+    return float(np.clip(score, 0.0, 100.0)), "fallback"
+
+
+def _entropy(probs: np.ndarray, bins: int = 24) -> float:
+    p = np.clip(np.asarray(probs, dtype=float), 0.0, 1.0)
+    h, _ = np.histogram(p, bins=bins, range=(0.0, 1.0), density=False)
+    h = h.astype(float)
+    if h.sum() <= 0:
+        return 0.0
+    q = h / h.sum()
+    q = q[q > 0]
+    return float(-np.sum(q * np.log2(q)))
+
+
+def _hist(probs: np.ndarray, bins: int = 32) -> Dict[str, Any]:
+    p = np.clip(np.asarray(probs, dtype=float), 0.0, 1.0)
+    h, edges = np.histogram(p, bins=bins, range=(0.0, 1.0), density=False)
+    return {
+        "bins": [float(x) for x in edges.tolist()],
+        "counts": [int(x) for x in h.tolist()],
+    }
+
+
+def _top_features(fd: Any, k: int = 8) -> List[Dict[str, Any]]:
+    m = _adapt_fd(fd)
+    rows = []
+    for feat, v in m.items():
+        ks = _to_float(v.get("ks_statistic"), 0.0)
+        pv = _to_float(v.get("p_value"), 1.0)
+        if ks >= 0.35:
+            sev = "CRITICAL"
+        elif ks >= 0.20:
+            sev = "HIGH"
+        elif ks >= 0.10:
+            sev = "MEDIUM"
+        else:
+            sev = "LOW"
+        rows.append({"feature": feat, "ks_statistic": ks, "p_value": pv, "severity": sev})
+    rows.sort(key=lambda r: r["ks_statistic"], reverse=True)
+    return rows[:k]
+
+
+# -----------------------------
+# Synthetic scenario generator
+# -----------------------------
+def _make_synthetic(seed: int, n: int = 2400, d: int = 14) -> Dict[str, Any]:
+    rng = np.random.default_rng(int(seed))
+
+    # baseline features
+    base = rng.normal(0, 1.0, size=(n, d))
+    base_df = pd.DataFrame(base, columns=[f"f{i}" for i in range(d)])
+
+    # clean ~ baseline (small noise)
+    clean = base + rng.normal(0, 0.08, size=(n, d))
+    clean_df = pd.DataFrame(clean, columns=base_df.columns)
+
+    # drifted (shift subset of features)
+    drift = base.copy()
+    drift[:, 0] += 2.0
+    drift[:, 1] += 1.3
+    drift[:, 2] *= 1.8
+    drift[:, 3] += rng.normal(0, 2.2, size=n)
+    drift_df = pd.DataFrame(drift, columns=base_df.columns)
+
+    # synthetic "prediction probs"
+    base_p = rng.beta(2.2, 2.6, size=n)  # mild center
+    clean_p = np.clip(base_p + rng.normal(0, 0.02, size=n), 0, 1)
+    drift_p = np.clip(1.0 - base_p + rng.normal(0, 0.03, size=n), 0, 1)  # strong invert shift
+
+    # FIX: Cleaned up the dictionary return to prevent syntax errors
+    return {
+        "base_X": base_df,
+        "clean_X": clean_df,
+        "drift_X": drift_df,
+        "base_p": base_p,
+        "clean_p": clean_p,
+        "drift_p": drift_p,
+    }
+
+
+# -----------------------------
+# Public API
+# -----------------------------
+def run_selftest(seed: int = 7, test: str = "suite") -> Dict[str, Any]:
+    """
+    Returns a payload designed for BOTH:
+      - readable JSON
+      - rich UI animations (histograms, gauges, feature bars)
+
+    test options:
+      - "prediction"
+      - "feature"
+      - "pipeline"
+      - "suite" (default, runs 3 scenarios)
+      - "concept"
+    """
+    t0 = time.time()
+    test = (test or "suite").strip().lower()
+
+    try:
+        if test not in {"prediction", "feature", "pipeline", "suite", "concept"}:
+            test = "suite"
+
+        cases: List[Dict[str, Any]] = []
+        checks: List[Dict[str, Any]] = []
+
+        # We run up to 3 scenarios in suite so it looks “real”
+        seeds = [seed] if test != "suite" else [seed, seed + 11, seed + 23]
+
+        for idx, s in enumerate(seeds, start=1):
+            sim = _make_synthetic(seed=s)
+            base_X = sim["base_X"]
+            clean_X = sim["clean_X"]
+            drift_X = sim["drift_X"]
+            base_p = sim["base_p"]
+            clean_p = sim["clean_p"]
+            drift_p = sim["drift_p"]
+
+            # --- CONCEPT DRIFT LOGIC ADDED HERE ---
+            if test == "concept":
+                # In Concept Drift, the relationship flips but features stay the same!
+                drift_X = base_X.copy()
+                drift_p = np.clip(base_p + 0.35, 0, 1)
+            # --------------------------------------
+
+            # compute drifts
+            fd_clean = compute_feature_drift(base_X, clean_X)
+            fd_drift = compute_feature_drift(base_X, drift_X)
+
+            pd_clean = compute_prediction_drift(base_p, clean_p)
+            pd_drift = compute_prediction_drift(base_p, drift_p)
+
+            pd_clean_m = _adapt_pred(pd_clean)
+            pd_drift_m = _adapt_pred(pd_drift)
+
+            pred_ks_clean = _to_float(pd_clean_m.get("ks_statistic"), 0.0)
+            pred_ks_drift = _to_float(pd_drift_m.get("ks_statistic"), 0.0)
+
+            ent_base = _entropy(base_p)
+            ent_clean = _entropy(clean_p)
+            ent_drift = _entropy(drift_p)
+            delta_ent_clean = float(ent_clean - ent_base)
+            delta_ent_drift = float(ent_drift - ent_base)
+
+            # health score (use pipeline if available)
+            health_clean, mode_clean = _call_health(fd_clean, pd_clean)
+            health_drift, mode_drift = _call_health(fd_drift, pd_drift)
+
+            # histograms for visuals
+            h_base = _hist(base_p, bins=40)
+            h_clean = _hist(clean_p, bins=40)
+            h_drift = _hist(drift_p, bins=40)
+
+            top_feat = _top_features(fd_drift, k=8)
+
+            cases.append(
+                {
+                    "case_id": f"C{idx}",
+                    "seed": int(s),
+                    "name": (
+                        "Prediction Drift Test" if test == "prediction"
+                        else "Feature Drift Test" if test == "feature"
+                        else "Pipeline Health Test" if test == "pipeline"
+                        else "Concept Drift Test" if test == "concept"
+                        else f"Suite Scenario {idx}"
+                    ),
+                    "metrics": {
+                        "pred_ks_clean": float(pred_ks_clean),
+                        "pred_ks_drifted": float(pred_ks_drift),
+                        "delta_entropy_clean": float(delta_ent_clean),
+                        "delta_entropy_drifted": float(delta_ent_drift),
+                        "health_clean": float(health_clean),
+                        "health_drifted": float(health_drift),
+                        "health_mode_clean": str(mode_clean),
+                        "health_mode_drifted": str(mode_drift),
+                    },
+                    "viz": {
+                        "pred_hist": {
+                            "bins": h_base["bins"],
+                            "baseline": h_base["counts"],
+                            "clean": h_clean["counts"],
+                            "drifted": h_drift["counts"],
+                        },
+                        "top_drifted_features": top_feat,
+                    },
+                }
+            )
+
+        # Decide which checks to enforce (based on selected test)
+        # (We validate the FIRST case for “pass/fail”)
+        c0 = cases[0]
+        m0 = c0["metrics"]
+        pred_clean = float(m0["pred_ks_clean"])
+        pred_drift = float(m0["pred_ks_drifted"])
+        h_clean = float(m0["health_clean"])
+        h_drift = float(m0["health_drifted"])
+
+        if test in {"prediction", "pipeline", "suite"}:
+            checks.append(
+                {
+                    "name": "Prediction drift should be low for clean",
+                    "pass": bool(pred_clean < 0.08),
+                    "value": pred_clean,
+                    "threshold": "< 0.08",
+                }
+            )
+            checks.append(
+                {
+                    "name": "Prediction drift should be high for drifted",
+                    "pass": bool(pred_drift > 0.10),
+                    "value": pred_drift,
+                    "threshold": "> 0.10",
+                }
+            )
+
+        if test in {"feature", "pipeline", "suite"}:
+            top = c0["viz"]["top_drifted_features"]
+            mx = float(top[0]["ks_statistic"]) if top else 0.0
+            checks.append(
+                {
+                    "name": "At least one feature should show strong shift",
+                    "pass": bool(mx > 0.20),
+                    "value": mx,
+                    "threshold": "> 0.20",
+                }
+            )
+
+        if test in {"pipeline", "suite"}:
+            checks.append(
+                {
+                    "name": "Health should degrade under drift",
+                    "pass": bool(h_drift < h_clean),
+                    "value": {"clean": h_clean, "drifted": h_drift},
+                    "threshold": "drifted < clean",
+                }
+            )
+
+        # --- CONCEPT DRIFT CHECKS ADDED HERE ---
+        if test == "concept":
+            top = c0["viz"]["top_drifted_features"]
+            mx = float(top[0]["ks_statistic"]) if top else 0.0
+            checks.append(
+                {
+                    "name": "Feature drift should be ZERO (Inputs didn't change)",
+                    "pass": bool(mx < 0.05),
+                    "value": mx,
+                    "threshold": "< 0.05",
+                }
+            )
+            checks.append(
+                {
+                    "name": "Prediction drift should be MASSIVE (Concept flipped)",
+                    "pass": bool(pred_drift > 0.30),
+                    "value": pred_drift,
+                    "threshold": "> 0.30",
+                }
+            )
+        # ---------------------------------------
+
+        ok = all(bool(x.get("pass")) for x in checks) if checks else True
+
+        payload = {
+            "ok": ok,
+            "test": test,
+            "seed": int(seed),
+            "started_at": time.strftime("%Y-%m-%d %H:%M:%S"),
+            "elapsed_ms": int((time.time() - t0) * 1000),
+            "cases": len(cases),
+            "passed": int(sum(1 for x in checks if x.get("pass"))),
+            "failed": int(sum(1 for x in checks if not x.get("pass"))),
+            "summary": cases[0]["metrics"] if cases else {},
+            "checks": checks,
+            "case_results": cases,
+        }
+        return payload
+
+    except Exception as e:
+        import traceback
+        return {
+            "ok": False,
+            "test": test,
+            "seed": int(seed),
+            "error": str(e),
+            "trace": traceback.format_exc(),
+        }

modelshift/storage/__init__.py

@@ -0,0 +1,3 @@
+"""
+Storage modules for ModelShift-Lite.
+"""

modelshift/storage/sqlite_store.py

@@ -0,0 +1,13 @@
+class SQLiteStore:
+    """
+    Simple SQLite storage interface for drift metrics.
+    """
+
+    def __init__(self, db_path="modelshift.db"):
+        self.db_path = db_path
+
+    def connect(self):
+        pass
+
+    def save_metrics(self, metrics):
+        pass

modelshift/utils/__init__.py

@@ -0,0 +1,3 @@
+"""
+Utility functions.
+"""

modelshift/utils/helpers.py

@@ -0,0 +1,5 @@
+def validate_inputs(data):
+    """
+    Validate input data format.
+    """
+    pass

modelshift-0.1.0.dist-info/METADATA

@@ -0,0 +1,129 @@
+Metadata-Version: 2.4
+Name: modelshift
+Version: 0.1.0
+Summary: A lightweight machine learning drift monitoring and alerting engine.
+Author: Krishna
+Author-email: ryomensukuna2530@gmail.com
+Classifier: Programming Language :: Python :: 3
+Classifier: Operating System :: OS Independent
+Requires-Python: >=3.8
+Description-Content-Type: text/markdown
+Requires-Dist: pandas
+Requires-Dist: numpy
+Requires-Dist: scipy
+Requires-Dist: requests
+Dynamic: author
+Dynamic: author-email
+Dynamic: classifier
+Dynamic: description
+Dynamic: description-content-type
+Dynamic: requires-dist
+Dynamic: requires-python
+Dynamic: summary
+
+# 🚦 ModelShift-Lite  
+### Label-Free Monitoring for Deployed Machine Learning Models
+
+> A lightweight, behavior-centric system to detect **silent reliability degradation** in deployed machine learning models — without requiring ground-truth labels.
+
+---
+
+## 📌 Why ModelShift-Lite?
+
+Machine learning models rarely fail loudly after deployment.  
+Instead, they **silently degrade** as real-world data changes — while true labels are unavailable for continuous evaluation.
+
+**ModelShift-Lite addresses this blind spot.**
+
+---
+
+## 🧩 Problem Statement
+
+Deployed machine learning models often degrade silently over time due to changing data distributions, while ground-truth labels are unavailable for continuous performance evaluation.
+
+---
+
+## 🎯 Project Objective
+
+Design a **label-free, post-deployment monitoring system** that tracks:
+
+- Data distribution shifts  
+- Prediction behavior instability  
+- Model reliability trends  
+
+to provide **early warning signals** of degradation **without modifying the deployed model**.
+
+---
+
+## 🚫 What This Project Does *Not* Do
+
+To maintain clarity of scope, ModelShift-Lite explicitly does **not**:
+
+- ❌ Retrain models  
+- ❌ Correct predictions  
+- ❌ Compute accuracy on production data  
+
+It focuses solely on **monitoring and interpretability**.
+
+---
+
+## 🧠 Core Idea (In Simple Terms)
+
+> *If we cannot measure correctness, we can still monitor behavior.*
+
+ModelShift-Lite observes how a model **reacts** to changing data and identifies signs of instability before failures become obvious.
+
+---
+
+## 🛠️ Key Components
+
+- **Reference Baseline Handling**  
+  Captures normal model behavior from historical or validation data
+
+- **Live Inference Monitoring**  
+  Tracks incoming production data and predictions
+
+- **Feature Drift Detection**  
+  Identifies changes in input distributions
+
+- **Prediction Behavior Analysis**  
+  Monitors confidence, stability, and output distribution shifts
+
+- **Model Health Scoring**  
+  Aggregates drift signals into an interpretable reliability indicator
+
+- **Visualization Dashboard**  
+  Displays trends, drift severity, and degradation warnings
+
+---
+Reference Data →
+→ Drift Detection → Health Scoring → Monitoring Dashboard
+Live Inference →
+
+
+*(Detailed architecture diagrams are provided in `/docs`)*
+
+---
+
+## 💻 Technology Stack
+
+- **Language:** Python  
+- **Data Processing:** NumPy, Pandas  
+- **Statistical Analysis:** SciPy  
+- **Visualization:** Streamlit, Matplotlib  
+- **Storage:** SQLite (local, replaceable)  
+
+---
+
+## 📂 Repository Structure
+
+```text
+modelshift-lite/
+├── modelshift/        # Core monitoring logic
+├── dashboard/         # Streamlit visualization app
+├── experiments/       # Drift simulation & analysis
+├── data/              # Reference & live data
+├── docs/              # Architecture and design docs
+└── README.md
+## 🏗️ High-Level Architecture
+

modelshift-0.1.0.dist-info/RECORD

@@ -0,0 +1,16 @@
+modelshift/__init__.py,sha256=_I2AIeZgUPSI1iAoeqE0BW3Lvdx1HNp4kFUt59OnWTs,51
+modelshift/baseline.py,sha256=gdk_dMUc3bq7AigxBOIEW2PGSLpZw8twzMdeQcv65qg,1038
+modelshift/monitor.py,sha256=Kt7c7jv48auJ2cGnGdSo01G5y8iI-ve0bgaSKXdAO1w,12421
+modelshift/selftest.py,sha256=EdgQd-IROhYagll4QTJ83uRBTvBhtrahrxLa_nuJl9M,14274
+modelshift/drift/__init__.py,sha256=TEkGw_vO--HzOCqu4i014te5Vru7yghgeJh2cJjle_Y,36
+modelshift/drift/feature_drift.py,sha256=nU6auZWatpdjgdleU1ZIms2tGE-4iUfs4skv1VdOnME,1413
+modelshift/drift/prediction_drift.py,sha256=v_vNA3JLPmUU4_f2vi05FVzpNCEWu1we8pXG5BGL1qA,3842
+modelshift/drift/severity.py,sha256=OOEzoisoPLjvFTvU0O6wwK1-GTA-KuUyAWxmP4ihTD0,7878
+modelshift/storage/__init__.py,sha256=N-Ydih1vXsRhj6pkXgkaFesGM_4CrnPlEDWyteRAxoA,48
+modelshift/storage/sqlite_store.py,sha256=TD0aMI5IEBI9zQr8OpJOY-H9i_urURTR5UkCzSd-dJE,272
+modelshift/utils/__init__.py,sha256=NBECA9wFkkwn4-O2NJsDwuZB_SciP_pkl5AhfWGe6tE,30
+modelshift/utils/helpers.py,sha256=NGLv7R82B7xnquKLZw-RhChQVQnfrVCuNund4f5tle8,89
+modelshift-0.1.0.dist-info/METADATA,sha256=obDrdvG6gYhRI0Vdx45WKNSIZXFMpLbhXFyUcZi4XoQ,3688
+modelshift-0.1.0.dist-info/WHEEL,sha256=aeYiig01lYGDzBgS8HxWXOg3uV61G9ijOsup-k9o1sk,91
+modelshift-0.1.0.dist-info/top_level.txt,sha256=3d2NcfPXrOeovneJake07097D7ZHHjKHkykxCDbFoe4,11
+modelshift-0.1.0.dist-info/RECORD,,

modelshift-0.1.0.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (82.0.1)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

modelshift-0.1.0.dist-info/top_level.txt

@@ -0,0 +1 @@
+modelshift