invarlock 0.3.4__py3-none-any.whl → 0.3.6__py3-none-any.whl

invarlock/eval/primary_metric.py

@@ -623,6 +623,9 @@ def compute_primary_metric_from_report(
             "preview": float("nan"),
             "final": float("nan"),
             "ratio_vs_baseline": float("nan"),
+            "invalid": True,
+            "degraded": True,
+            "degraded_reason": "non_finite_pm",
         }
     # For accuracy kinds, derive counts from input_ids if aggregates are missing
     if kind in {"accuracy", "vqa_accuracy"}:
@@ -661,6 +664,11 @@ def compute_primary_metric_from_report(
     final_point = metric.point_from_windows(windows=final_win)
 
     ratio_vs_baseline = float("nan")
+    baseline_has_reference = False
+
+    def _is_finite(value: Any) -> bool:
+        return isinstance(value, (int, float)) and math.isfinite(float(value))
+
     if isinstance(baseline, dict):
         try:
             base_metrics = (
@@ -686,14 +694,25 @@ def compute_primary_metric_from_report(
                     is_ppl_like = str(kind).lower().startswith("ppl")
                     if is_ppl_like and base_ref > 0:
                         ratio_vs_baseline = float(final_point) / float(base_ref)
+                        baseline_has_reference = True
                     elif (
                         str(kind).lower() in {"accuracy", "vqa_accuracy"}
                         and 0 <= base_ref <= 1
                     ):
                         ratio_vs_baseline = float(final_point) - float(base_ref)
+                        baseline_has_reference = True
         except Exception:
             ratio_vs_baseline = float("nan")
 
+    invalid = not (_is_finite(preview_point) and _is_finite(final_point))
+    degraded_reason = None
+    if invalid:
+        degraded_reason = "non_finite_pm"
+    elif baseline_has_reference and not _is_finite(ratio_vs_baseline):
+        degraded_reason = "non_finite_delta"
+
+    degraded = bool(degraded_reason or invalid)
+
     payload = {
         "kind": metric.kind,
         "unit": metric.unit,
@@ -705,7 +724,11 @@ def compute_primary_metric_from_report(
         "preview": preview_point,
         "final": final_point,
         "ratio_vs_baseline": ratio_vs_baseline,
+        "invalid": invalid,
+        "degraded": degraded,
     }
+    if degraded and degraded_reason:
+        payload["degraded_reason"] = degraded_reason
     # Carry counts for accuracy to aid gating
     if kind in {"accuracy", "vqa_accuracy"}:
         if "n_prev" in locals() and n_prev is not None:

invarlock/eval/tail_stats.py

@@ -0,0 +1,230 @@
+from __future__ import annotations
+
+import math
+from collections.abc import Mapping, Sequence
+from typing import Any
+
+__all__ = [
+    "compute_tail_summary",
+    "evaluate_metric_tail",
+]
+
+
+def _as_finite_float(value: Any) -> float | None:
+    try:
+        out = float(value)
+    except Exception:
+        return None
+    return out if math.isfinite(out) else None
+
+
+def _linear_quantile(sorted_values: Sequence[float], q: float) -> float:
+    """Deterministic linear-interpolated quantile on sorted values (q in [0, 1])."""
+    n = len(sorted_values)
+    if n == 0:
+        return float("nan")
+    if n == 1:
+        return float(sorted_values[0])
+    if q <= 0.0:
+        return float(sorted_values[0])
+    if q >= 1.0:
+        return float(sorted_values[-1])
+    pos = float(q) * float(n - 1)
+    lo = int(math.floor(pos))
+    hi = int(math.ceil(pos))
+    if lo == hi:
+        return float(sorted_values[lo])
+    frac = pos - float(lo)
+    a = float(sorted_values[lo])
+    b = float(sorted_values[hi])
+    return a + frac * (b - a)
+
+
+def compute_tail_summary(
+    deltas: Sequence[float] | Sequence[Any],
+    *,
+    quantiles: Sequence[float] = (0.5, 0.9, 0.95, 0.99),
+    epsilon: float = 1e-4,
+    weights: Sequence[float] | Sequence[Any] | None = None,
+) -> dict[str, Any]:
+    """Compute deterministic tail summaries for Δlog-loss samples.
+
+    - Quantiles are computed unweighted using linear interpolation on sorted values.
+    - tail_mass is Pr[delta > epsilon] (unweighted).
+    - tail_mass_weighted is included when weights are provided and finite.
+    """
+    eps = _as_finite_float(epsilon)
+    if eps is None or eps < 0.0:
+        eps = 0.0
+
+    values: list[float] = []
+    paired_weights: list[float] | None = [] if weights is not None else None
+
+    if weights is None:
+        for d in deltas:
+            dv = _as_finite_float(d)
+            if dv is None:
+                continue
+            values.append(float(dv))
+    else:
+        for d, w in zip(deltas, weights, strict=False):
+            dv = _as_finite_float(d)
+            if dv is None:
+                continue
+            wv = _as_finite_float(w)
+            if wv is None or wv < 0.0:
+                wv = 0.0
+            values.append(float(dv))
+            if paired_weights is not None:
+                paired_weights.append(float(wv))
+
+    n = int(len(values))
+    values_sorted = sorted(values)
+
+    summary: dict[str, Any] = {
+        "n": n,
+        "epsilon": float(eps),
+    }
+    if n == 0:
+        summary.update({"max": float("nan"), "tail_mass": 0.0})
+        for q in quantiles:
+            try:
+                qf = float(q)
+            except Exception:
+                continue
+            label = f"q{int(round(100.0 * max(0.0, min(1.0, qf))))}"
+            summary[label] = float("nan")
+        return summary
+
+    summary["max"] = float(values_sorted[-1])
+    tail_ct = sum(1 for v in values if v > eps)
+    summary["tail_mass"] = float(tail_ct / n)
+
+    if paired_weights is not None:
+        total_w = 0.0
+        tail_w = 0.0
+        for v, w in zip(values, paired_weights, strict=False):
+            total_w += float(w)
+            if v > eps:
+                tail_w += float(w)
+        if total_w > 0.0:
+            summary["tail_mass_weighted"] = float(tail_w / total_w)
+            summary["tail_mass_weighted_by"] = "weights"
+
+    for q in quantiles:
+        try:
+            qf = float(q)
+        except Exception:
+            continue
+        qf = max(0.0, min(1.0, qf))
+        label = f"q{int(round(100.0 * qf))}"
+        summary[label] = float(_linear_quantile(values_sorted, qf))
+
+    return summary
+
+
+def evaluate_metric_tail(
+    *,
+    deltas: Sequence[float] | Sequence[Any],
+    policy: Mapping[str, Any] | None = None,
+    weights: Sequence[float] | Sequence[Any] | None = None,
+) -> dict[str, Any]:
+    """Evaluate a tail policy against Δlog-loss samples.
+
+    Policy keys:
+      - mode: "off" | "warn" | "fail" (default: "warn")
+      - min_windows: int (default: 1)
+      - quantile: float in [0, 1] (default: 0.95)
+      - quantile_max: float threshold in Δlog-loss (optional)
+      - epsilon: float deadband for tail_mass (default: 1e-4)
+      - mass_max: float in [0, 1] (optional)
+    """
+    pol = dict(policy or {})
+    mode = str(pol.get("mode", "warn") or "warn").strip().lower()
+    if mode not in {"off", "warn", "fail"}:
+        mode = "warn"
+
+    min_windows = pol.get("min_windows", 1)
+    try:
+        min_windows_i = int(min_windows)
+    except Exception:
+        min_windows_i = 1
+    min_windows_i = max(1, min_windows_i)
+
+    q = _as_finite_float(pol.get("quantile", 0.95))
+    if q is None:
+        q = 0.95
+    q = max(0.0, min(1.0, float(q)))
+
+    eps = _as_finite_float(pol.get("epsilon", 1e-4))
+    if eps is None or eps < 0.0:
+        eps = 0.0
+
+    qmax = _as_finite_float(pol.get("quantile_max"))
+    mmax = _as_finite_float(pol.get("mass_max"))
+    if mmax is not None:
+        mmax = max(0.0, min(1.0, float(mmax)))
+
+    quantiles = sorted({0.5, 0.9, 0.95, 0.99, float(q)})
+    stats = compute_tail_summary(
+        deltas, quantiles=tuple(quantiles), epsilon=float(eps), weights=weights
+    )
+    n = int(stats.get("n", 0) or 0)
+
+    thresholds_present = (qmax is not None) or (mmax is not None)
+    evaluated = bool(mode != "off" and thresholds_present and n >= min_windows_i)
+
+    violations: list[dict[str, Any]] = []
+    passed = True
+    if evaluated:
+        passed = True
+        q_label = f"q{int(round(100.0 * q))}"
+        q_obs = stats.get(q_label)
+        if not (isinstance(q_obs, int | float) and math.isfinite(float(q_obs))):
+            q_obs = float("nan")
+        if qmax is not None and math.isfinite(q_obs) and q_obs > float(qmax):
+            passed = False
+            violations.append(
+                {
+                    "type": "quantile_max_exceeded",
+                    "quantile": float(q),
+                    "observed": float(q_obs),
+                    "threshold": float(qmax),
+                }
+            )
+
+        tail_mass = stats.get("tail_mass")
+        if (
+            mmax is not None
+            and isinstance(tail_mass, int | float)
+            and math.isfinite(float(tail_mass))
+            and float(tail_mass) > float(mmax)
+        ):
+            passed = False
+            violations.append(
+                {
+                    "type": "tail_mass_exceeded",
+                    "epsilon": float(eps),
+                    "observed": float(tail_mass),
+                    "threshold": float(mmax),
+                }
+            )
+
+    warned = bool(evaluated and (not passed) and mode == "warn")
+
+    return {
+        "mode": mode,
+        "evaluated": evaluated,
+        "passed": bool(passed),
+        "warned": warned,
+        "violations": violations,
+        "policy": {
+            "mode": mode,
+            "min_windows": int(min_windows_i),
+            "quantile": float(q),
+            "quantile_max": float(qmax) if qmax is not None else None,
+            "epsilon": float(eps),
+            "mass_max": float(mmax) if mmax is not None else None,
+        },
+        "stats": stats,
+    }

invarlock/guards/_estimators.py

@@ -0,0 +1,154 @@
+from __future__ import annotations
+
+import math
+from typing import Any
+
+import torch
+
+__all__ = [
+    "power_iter_sigma_max",
+    "frobenius_norm_sq",
+    "row_col_norm_extrema",
+    "stable_rank_estimate",
+]
+
+
+def _as_matrix(tensor: torch.Tensor) -> torch.Tensor:
+    if tensor.ndim == 2:
+        return tensor
+    return tensor.view(tensor.shape[0], -1)
+
+
+def power_iter_sigma_max(
+    matrix: Any,
+    *,
+    iters: int,
+    init: str = "ones",
+    eps: float = 1e-12,
+) -> float:
+    """Estimate the largest singular value (spectral norm) via fixed-iter power iteration.
+
+    Contract properties (vNext):
+    - fixed iteration budget (no convergence stopping)
+    - deterministic initialization (`init`)
+    - device-resident matvecs (no `.cpu()` transfers)
+    """
+    try:
+        iters_i = int(iters)
+    except Exception:
+        iters_i = 4
+    if iters_i < 1:
+        iters_i = 1
+
+    if not isinstance(matrix, torch.Tensor):
+        return 0.0
+    if matrix.numel() == 0:
+        return 0.0
+    if matrix.dtype in {torch.int8, torch.uint8}:
+        return 0.0
+
+    W = _as_matrix(matrix.detach())
+    if W.numel() == 0 or W.shape[0] == 0 or W.shape[1] == 0:
+        return 0.0
+
+    device = W.device
+    dtype = W.dtype
+    n = int(W.shape[1])
+
+    with torch.no_grad():
+        if init == "ones":
+            v = torch.ones((n,), device=device, dtype=dtype)
+        else:
+            # Deterministic fallback: unit vector e0.
+            v = torch.zeros((n,), device=device, dtype=dtype)
+            v[0] = 1
+
+        v_norm = torch.linalg.vector_norm(v.float()).clamp_min(eps)
+        v = v / v_norm.to(dtype)
+
+        sigma = 0.0
+        for _ in range(iters_i):
+            u = W @ v
+            u_norm = torch.linalg.vector_norm(u.float()).clamp_min(eps)
+            sigma_val = float(u_norm.item())
+            if not math.isfinite(sigma_val):
+                return 0.0
+            u = u / u_norm.to(dtype)
+            v = W.T @ u
+            v_norm = torch.linalg.vector_norm(v.float()).clamp_min(eps)
+            v = v / v_norm.to(dtype)
+            sigma = sigma_val
+        return float(sigma)
+
+
+def frobenius_norm_sq(matrix: torch.Tensor) -> float:
+    """Return ||matrix||_F^2 with float32 accumulation (device-resident)."""
+    W = _as_matrix(matrix.detach())
+    if W.numel() == 0:
+        return 0.0
+    with torch.no_grad():
+        # Use a fused reduction to avoid materializing a W*W intermediate.
+        norm = torch.linalg.vector_norm(W.reshape(-1), ord=2, dtype=torch.float32)
+        out = float((norm * norm).item())
+        return out if math.isfinite(out) else 0.0
+
+
+def row_col_norm_extrema(
+    matrix: torch.Tensor, *, eps: float = 1e-12
+) -> dict[str, float]:
+    """Compute min/median/max of row/col L2 norms with float32 accumulation."""
+    W = _as_matrix(matrix.detach())
+    if W.numel() == 0 or W.shape[0] == 0 or W.shape[1] == 0:
+        return {
+            "row_min": 0.0,
+            "row_median": 0.0,
+            "row_max": 0.0,
+            "col_min": 0.0,
+            "col_median": 0.0,
+            "col_max": 0.0,
+        }
+    with torch.no_grad():
+        # Avoid materializing W*W: use fused reductions.
+        row = torch.linalg.vector_norm(W, ord=2, dim=1, dtype=torch.float32).clamp_min(
+            eps
+        )
+        col = torch.linalg.vector_norm(W, ord=2, dim=0, dtype=torch.float32).clamp_min(
+            eps
+        )
+
+        row_sorted, _ = torch.sort(row)
+        col_sorted, _ = torch.sort(col)
+
+        def _median(sorted_vec: torch.Tensor) -> float:
+            n = int(sorted_vec.numel())
+            if n <= 0:
+                return 0.0
+            mid = n // 2
+            if n % 2 == 1:
+                return float(sorted_vec[mid].item())
+            return float((sorted_vec[mid - 1] + sorted_vec[mid]).mul(0.5).item())
+
+        return {
+            "row_min": float(row_sorted[0].item()),
+            "row_median": _median(row_sorted),
+            "row_max": float(row_sorted[-1].item()),
+            "col_min": float(col_sorted[0].item()),
+            "col_median": _median(col_sorted),
+            "col_max": float(col_sorted[-1].item()),
+        }
+
+
+def stable_rank_estimate(
+    matrix: torch.Tensor, *, sigma_max: float, eps: float = 1e-12
+) -> float:
+    """Estimate stable rank: ||W||_F^2 / ||W||_2^2, using a provided σ̂max."""
+    try:
+        denom = float(sigma_max) ** 2
+    except Exception:
+        return 0.0
+    if not math.isfinite(denom) or denom <= 0.0:
+        return 0.0
+    denom = max(denom, eps)
+    num = frobenius_norm_sq(matrix)
+    out = float(num) / denom if denom > 0 else 0.0
+    return out if math.isfinite(out) else 0.0

invarlock/guards/policies.py

@@ -15,7 +15,7 @@ from typing import Any, Literal
 
 try:  # Python 3.12+
     from typing import NotRequired, TypedDict
-except ImportError:  # Legacy fallback
+except ImportError:  # Python <3.12 fallback
     from typing import NotRequired
 
     from typing_extensions import TypedDict
@@ -40,6 +40,7 @@ SPECTRAL_CONSERVATIVE: SpectralPolicy = {
     "scope": "ffn",  # FFN layers only (safest)
     "correction_enabled": True,
     "max_caps": 3,
+    "max_spectral_norm": None,
     "multiple_testing": {"method": "bonferroni", "alpha": 0.02, "m": 4},
 }
 
@@ -50,6 +51,7 @@ SPECTRAL_BALANCED: SpectralPolicy = {
     "scope": "ffn",  # FFN layers only
     "correction_enabled": False,
     "max_caps": 5,
+    "max_spectral_norm": None,
     "multiple_testing": {"method": "bh", "alpha": 0.05, "m": 4},
 }
 
@@ -60,6 +62,7 @@ SPECTRAL_AGGRESSIVE: SpectralPolicy = {
     "scope": "all",  # All layers including attention
     "correction_enabled": True,
     "max_caps": 8,
+    "max_spectral_norm": None,
     "multiple_testing": {"method": "bh", "alpha": 0.1, "m": 4},
 }
 
@@ -70,6 +73,7 @@ SPECTRAL_ATTN_AWARE: SpectralPolicy = {
     "scope": "attn",  # Attention layers only
     "correction_enabled": False,
     "max_caps": 5,
+    "max_spectral_norm": None,
     "multiple_testing": {"method": "bh", "alpha": 0.05, "m": 4},
 }
 
@@ -81,7 +85,8 @@ RMT_CONSERVATIVE: RMTPolicyDict = {
     "deadband": 0.05,  # 5% deadband - strict threshold
     "margin": 1.3,  # Lower margin for conservative detection
     "correct": True,  # Enable automatic correction
-    "epsilon": {"attn": 0.05, "ffn": 0.06, "embed": 0.07, "other": 0.07},
+    "epsilon_default": 0.06,
+    "epsilon_by_family": {"attn": 0.05, "ffn": 0.06, "embed": 0.07, "other": 0.07},
 }
 
 # Balanced RMT policy - good for most use cases
@@ -90,7 +95,8 @@ RMT_BALANCED: RMTPolicyDict = {
     "deadband": 0.10,  # 10% deadband - reasonable tolerance
     "margin": 1.5,  # Standard margin for outlier detection
     "correct": False,  # Monitor-only by default
-    "epsilon": {"attn": 0.08, "ffn": 0.10, "embed": 0.12, "other": 0.12},
+    "epsilon_default": 0.10,
+    "epsilon_by_family": {"attn": 0.08, "ffn": 0.10, "embed": 0.12, "other": 0.12},
 }
 
 # Aggressive RMT policy - for research/experimental use
@@ -99,7 +105,8 @@ RMT_AGGRESSIVE: RMTPolicyDict = {
     "deadband": 0.15,  # 15% deadband - more permissive
     "margin": 1.8,  # Higher margin allows more deviation
     "correct": True,  # Enable automatic correction
-    "epsilon": {"attn": 0.15, "ffn": 0.15, "embed": 0.15, "other": 0.15},
+    "epsilon_default": 0.15,
+    "epsilon_by_family": {"attn": 0.15, "ffn": 0.15, "embed": 0.15, "other": 0.15},
 }
 
 # === Variance Guard Policies ===
@@ -276,6 +283,8 @@ def get_spectral_policy(
                     policy["scope"] = tier_config["scope"]
                 if "max_caps" in tier_config:
                     policy["max_caps"] = tier_config["max_caps"]
+                if "max_spectral_norm" in tier_config:
+                    policy["max_spectral_norm"] = tier_config["max_spectral_norm"]
                 if "family_caps" in tier_config:
                     policy["family_caps"] = tier_config["family_caps"]
                 if "multiple_testing" in tier_config:
@@ -390,9 +399,10 @@ def get_rmt_policy(name: str = "balanced", *, use_yaml: bool = True) -> RMTPolic
                     policy["deadband"] = tier_config["deadband"]
                 if "margin" in tier_config:
                     policy["margin"] = tier_config["margin"]
-                # Use epsilon_by_family as the epsilon dict
+                if "epsilon_default" in tier_config:
+                    policy["epsilon_default"] = tier_config["epsilon_default"]
                 if "epsilon_by_family" in tier_config:
-                    policy["epsilon"] = tier_config["epsilon_by_family"]
+                    policy["epsilon_by_family"] = tier_config["epsilon_by_family"]
         except Exception:
             # Fallback to hardcoded values on any error
             pass