invarlock 0.3.5__py3-none-any.whl → 0.3.7__py3-none-any.whl

invarlock/eval/primary_metric.py

@@ -623,6 +623,9 @@ def compute_primary_metric_from_report(
             "preview": float("nan"),
             "final": float("nan"),
             "ratio_vs_baseline": float("nan"),
+            "invalid": True,
+            "degraded": True,
+            "degraded_reason": "non_finite_pm",
         }
     # For accuracy kinds, derive counts from input_ids if aggregates are missing
     if kind in {"accuracy", "vqa_accuracy"}:
@@ -661,6 +664,11 @@ def compute_primary_metric_from_report(
     final_point = metric.point_from_windows(windows=final_win)
 
     ratio_vs_baseline = float("nan")
+    baseline_has_reference = False
+
+    def _is_finite(value: Any) -> bool:
+        return isinstance(value, (int, float)) and math.isfinite(float(value))
+
     if isinstance(baseline, dict):
         try:
             base_metrics = (
@@ -686,14 +694,25 @@ def compute_primary_metric_from_report(
                     is_ppl_like = str(kind).lower().startswith("ppl")
                     if is_ppl_like and base_ref > 0:
                         ratio_vs_baseline = float(final_point) / float(base_ref)
+                        baseline_has_reference = True
                     elif (
                         str(kind).lower() in {"accuracy", "vqa_accuracy"}
                         and 0 <= base_ref <= 1
                     ):
                         ratio_vs_baseline = float(final_point) - float(base_ref)
+                        baseline_has_reference = True
         except Exception:
             ratio_vs_baseline = float("nan")
 
+    invalid = not (_is_finite(preview_point) and _is_finite(final_point))
+    degraded_reason = None
+    if invalid:
+        degraded_reason = "non_finite_pm"
+    elif baseline_has_reference and not _is_finite(ratio_vs_baseline):
+        degraded_reason = "non_finite_delta"
+
+    degraded = bool(degraded_reason or invalid)
+
     payload = {
         "kind": metric.kind,
         "unit": metric.unit,
@@ -705,7 +724,11 @@ def compute_primary_metric_from_report(
         "preview": preview_point,
         "final": final_point,
         "ratio_vs_baseline": ratio_vs_baseline,
+        "invalid": invalid,
+        "degraded": degraded,
     }
+    if degraded and degraded_reason:
+        payload["degraded_reason"] = degraded_reason
     # Carry counts for accuracy to aid gating
     if kind in {"accuracy", "vqa_accuracy"}:
         if "n_prev" in locals() and n_prev is not None:

invarlock/eval/tail_stats.py

@@ -0,0 +1,230 @@
+from __future__ import annotations
+
+import math
+from collections.abc import Mapping, Sequence
+from typing import Any
+
+__all__ = [
+    "compute_tail_summary",
+    "evaluate_metric_tail",
+]
+
+
+def _as_finite_float(value: Any) -> float | None:
+    try:
+        out = float(value)
+    except Exception:
+        return None
+    return out if math.isfinite(out) else None
+
+
+def _linear_quantile(sorted_values: Sequence[float], q: float) -> float:
+    """Deterministic linear-interpolated quantile on sorted values (q in [0, 1])."""
+    n = len(sorted_values)
+    if n == 0:
+        return float("nan")
+    if n == 1:
+        return float(sorted_values[0])
+    if q <= 0.0:
+        return float(sorted_values[0])
+    if q >= 1.0:
+        return float(sorted_values[-1])
+    pos = float(q) * float(n - 1)
+    lo = int(math.floor(pos))
+    hi = int(math.ceil(pos))
+    if lo == hi:
+        return float(sorted_values[lo])
+    frac = pos - float(lo)
+    a = float(sorted_values[lo])
+    b = float(sorted_values[hi])
+    return a + frac * (b - a)
+
+
+def compute_tail_summary(
+    deltas: Sequence[float] | Sequence[Any],
+    *,
+    quantiles: Sequence[float] = (0.5, 0.9, 0.95, 0.99),
+    epsilon: float = 1e-4,
+    weights: Sequence[float] | Sequence[Any] | None = None,
+) -> dict[str, Any]:
+    """Compute deterministic tail summaries for Δlog-loss samples.
+
+    - Quantiles are computed unweighted using linear interpolation on sorted values.
+    - tail_mass is Pr[delta > epsilon] (unweighted).
+    - tail_mass_weighted is included when weights are provided and finite.
+    """
+    eps = _as_finite_float(epsilon)
+    if eps is None or eps < 0.0:
+        eps = 0.0
+
+    values: list[float] = []
+    paired_weights: list[float] | None = [] if weights is not None else None
+
+    if weights is None:
+        for d in deltas:
+            dv = _as_finite_float(d)
+            if dv is None:
+                continue
+            values.append(float(dv))
+    else:
+        for d, w in zip(deltas, weights, strict=False):
+            dv = _as_finite_float(d)
+            if dv is None:
+                continue
+            wv = _as_finite_float(w)
+            if wv is None or wv < 0.0:
+                wv = 0.0
+            values.append(float(dv))
+            if paired_weights is not None:
+                paired_weights.append(float(wv))
+
+    n = int(len(values))
+    values_sorted = sorted(values)
+
+    summary: dict[str, Any] = {
+        "n": n,
+        "epsilon": float(eps),
+    }
+    if n == 0:
+        summary.update({"max": float("nan"), "tail_mass": 0.0})
+        for q in quantiles:
+            try:
+                qf = float(q)
+            except Exception:
+                continue
+            label = f"q{int(round(100.0 * max(0.0, min(1.0, qf))))}"
+            summary[label] = float("nan")
+        return summary
+
+    summary["max"] = float(values_sorted[-1])
+    tail_ct = sum(1 for v in values if v > eps)
+    summary["tail_mass"] = float(tail_ct / n)
+
+    if paired_weights is not None:
+        total_w = 0.0
+        tail_w = 0.0
+        for v, w in zip(values, paired_weights, strict=False):
+            total_w += float(w)
+            if v > eps:
+                tail_w += float(w)
+        if total_w > 0.0:
+            summary["tail_mass_weighted"] = float(tail_w / total_w)
+            summary["tail_mass_weighted_by"] = "weights"
+
+    for q in quantiles:
+        try:
+            qf = float(q)
+        except Exception:
+            continue
+        qf = max(0.0, min(1.0, qf))
+        label = f"q{int(round(100.0 * qf))}"
+        summary[label] = float(_linear_quantile(values_sorted, qf))
+
+    return summary
+
+
+def evaluate_metric_tail(
+    *,
+    deltas: Sequence[float] | Sequence[Any],
+    policy: Mapping[str, Any] | None = None,
+    weights: Sequence[float] | Sequence[Any] | None = None,
+) -> dict[str, Any]:
+    """Evaluate a tail policy against Δlog-loss samples.
+
+    Policy keys:
+      - mode: "off" | "warn" | "fail" (default: "warn")
+      - min_windows: int (default: 1)
+      - quantile: float in [0, 1] (default: 0.95)
+      - quantile_max: float threshold in Δlog-loss (optional)
+      - epsilon: float deadband for tail_mass (default: 1e-4)
+      - mass_max: float in [0, 1] (optional)
+    """
+    pol = dict(policy or {})
+    mode = str(pol.get("mode", "warn") or "warn").strip().lower()
+    if mode not in {"off", "warn", "fail"}:
+        mode = "warn"
+
+    min_windows = pol.get("min_windows", 1)
+    try:
+        min_windows_i = int(min_windows)
+    except Exception:
+        min_windows_i = 1
+    min_windows_i = max(1, min_windows_i)
+
+    q = _as_finite_float(pol.get("quantile", 0.95))
+    if q is None:
+        q = 0.95
+    q = max(0.0, min(1.0, float(q)))
+
+    eps = _as_finite_float(pol.get("epsilon", 1e-4))
+    if eps is None or eps < 0.0:
+        eps = 0.0
+
+    qmax = _as_finite_float(pol.get("quantile_max"))
+    mmax = _as_finite_float(pol.get("mass_max"))
+    if mmax is not None:
+        mmax = max(0.0, min(1.0, float(mmax)))
+
+    quantiles = sorted({0.5, 0.9, 0.95, 0.99, float(q)})
+    stats = compute_tail_summary(
+        deltas, quantiles=tuple(quantiles), epsilon=float(eps), weights=weights
+    )
+    n = int(stats.get("n", 0) or 0)
+
+    thresholds_present = (qmax is not None) or (mmax is not None)
+    evaluated = bool(mode != "off" and thresholds_present and n >= min_windows_i)
+
+    violations: list[dict[str, Any]] = []
+    passed = True
+    if evaluated:
+        passed = True
+        q_label = f"q{int(round(100.0 * q))}"
+        q_obs = stats.get(q_label)
+        if not (isinstance(q_obs, int | float) and math.isfinite(float(q_obs))):
+            q_obs = float("nan")
+        if qmax is not None and math.isfinite(q_obs) and q_obs > float(qmax):
+            passed = False
+            violations.append(
+                {
+                    "type": "quantile_max_exceeded",
+                    "quantile": float(q),
+                    "observed": float(q_obs),
+                    "threshold": float(qmax),
+                }
+            )
+
+        tail_mass = stats.get("tail_mass")
+        if (
+            mmax is not None
+            and isinstance(tail_mass, int | float)
+            and math.isfinite(float(tail_mass))
+            and float(tail_mass) > float(mmax)
+        ):
+            passed = False
+            violations.append(
+                {
+                    "type": "tail_mass_exceeded",
+                    "epsilon": float(eps),
+                    "observed": float(tail_mass),
+                    "threshold": float(mmax),
+                }
+            )
+
+    warned = bool(evaluated and (not passed) and mode == "warn")
+
+    return {
+        "mode": mode,
+        "evaluated": evaluated,
+        "passed": bool(passed),
+        "warned": warned,
+        "violations": violations,
+        "policy": {
+            "mode": mode,
+            "min_windows": int(min_windows_i),
+            "quantile": float(q),
+            "quantile_max": float(qmax) if qmax is not None else None,
+            "epsilon": float(eps),
+            "mass_max": float(mmax) if mmax is not None else None,
+        },
+        "stats": stats,
+    }

invarlock/eval/tasks/__init__.py

@@ -0,0 +1,12 @@
+from __future__ import annotations
+
+from .classification import accuracy_from_records
+from .qa import exact_match_from_records
+from .text_generation import bleu1_from_records, rouge_l_from_records
+
+__all__ = [
+    "accuracy_from_records",
+    "exact_match_from_records",
+    "bleu1_from_records",
+    "rouge_l_from_records",
+]

invarlock/eval/tasks/classification.py

@@ -0,0 +1,48 @@
+from __future__ import annotations
+
+from collections.abc import Iterable
+from typing import Any
+
+
+def _iter_pairs(record: dict[str, Any]) -> list[tuple[Any, Any]]:
+    if "correct" in record:
+        return [(bool(record.get("correct")), True)]
+
+    label = record.get("label")
+    pred = record.get("prediction")
+    if label is None:
+        label = record.get("labels")
+    if pred is None:
+        pred = record.get("pred")
+    if pred is None:
+        pred = record.get("predictions")
+
+    if isinstance(label, list) and isinstance(pred, list):
+        return list(zip(label, pred, strict=False))
+    if label is None or pred is None:
+        return []
+    return [(label, pred)]
+
+
+def accuracy_from_records(records: Iterable[dict[str, Any]]) -> float:
+    """Compute accuracy from records with labels/predictions.
+
+    Accepted record shapes:
+    - {"label": <label>, "prediction": <label>}
+    - {"labels": [...], "predictions": [...]}
+    - {"correct": <bool>}
+    """
+    total = 0
+    correct = 0
+    for record in records:
+        if not isinstance(record, dict):
+            continue
+        for label, pred in _iter_pairs(record):
+            total += 1
+            if isinstance(label, bool):
+                correct += int(label is pred)
+            else:
+                correct += int(label == pred)
+    if total == 0:
+        return float("nan")
+    return float(correct / total)

invarlock/eval/tasks/qa.py

@@ -0,0 +1,36 @@
+from __future__ import annotations
+
+from collections.abc import Iterable
+from typing import Any
+
+
+def _normalize(text: str) -> str:
+    return " ".join(str(text).strip().lower().split())
+
+
+def exact_match_from_records(records: Iterable[dict[str, Any]]) -> float:
+    """Compute exact-match accuracy for QA-style records.
+
+    Accepted record shapes:
+    - {"prediction": "...", "answer": "..."}
+    - {"prediction": "...", "answers": ["...", ...]}
+    """
+    total = 0
+    correct = 0
+    for record in records:
+        if not isinstance(record, dict):
+            continue
+        pred = record.get("prediction")
+        answers = record.get("answers")
+        if answers is None and "answer" in record:
+            answers = [record.get("answer")]
+        if pred is None or answers is None:
+            continue
+        pred_norm = _normalize(pred)
+        answer_list = answers if isinstance(answers, list) else [answers]
+        total += 1
+        if any(_normalize(a) == pred_norm for a in answer_list if a is not None):
+            correct += 1
+    if total == 0:
+        return float("nan")
+    return float(correct / total)

invarlock/eval/tasks/text_generation.py

@@ -0,0 +1,102 @@
+from __future__ import annotations
+
+from collections import Counter
+from collections.abc import Iterable
+from typing import Any
+
+
+def _tokenize(text: str) -> list[str]:
+    return [tok for tok in str(text).strip().lower().split() if tok]
+
+
+def _bleu1(pred: str, ref: str) -> float:
+    pred_tokens = _tokenize(pred)
+    ref_tokens = _tokenize(ref)
+    if not pred_tokens or not ref_tokens:
+        return 0.0
+    pred_counts = Counter(pred_tokens)
+    ref_counts = Counter(ref_tokens)
+    overlap = sum(min(pred_counts[tok], ref_counts.get(tok, 0)) for tok in pred_counts)
+    precision = overlap / float(len(pred_tokens))
+    bp = 1.0
+    if len(pred_tokens) < len(ref_tokens):
+        bp = pow(2.718281828, 1.0 - (len(ref_tokens) / float(len(pred_tokens))))
+    return float(precision * bp)
+
+
+def bleu1_from_records(records: Iterable[dict[str, Any]]) -> float:
+    """Compute BLEU-1 from records with predictions and references."""
+    scores: list[float] = []
+    for record in records:
+        if not isinstance(record, dict):
+            continue
+        pred = record.get("prediction")
+        refs = record.get("references")
+        if pred is None:
+            continue
+        if refs is None and "reference" in record:
+            refs = [record.get("reference")]
+        if refs is None:
+            continue
+        ref_list = refs if isinstance(refs, list) else [refs]
+        best = 0.0
+        for ref in ref_list:
+            if ref is None:
+                continue
+            best = max(best, _bleu1(str(pred), str(ref)))
+        scores.append(best)
+    if not scores:
+        return float("nan")
+    return float(sum(scores) / float(len(scores)))
+
+
+def _lcs_len(a: list[str], b: list[str]) -> int:
+    if not a or not b:
+        return 0
+    dp = [[0] * (len(b) + 1) for _ in range(len(a) + 1)]
+    for i, tok_a in enumerate(a, start=1):
+        for j, tok_b in enumerate(b, start=1):
+            if tok_a == tok_b:
+                dp[i][j] = dp[i - 1][j - 1] + 1
+            else:
+                dp[i][j] = max(dp[i - 1][j], dp[i][j - 1])
+    return dp[-1][-1]
+
+
+def _rouge_l(pred: str, ref: str) -> float:
+    pred_tokens = _tokenize(pred)
+    ref_tokens = _tokenize(ref)
+    if not pred_tokens or not ref_tokens:
+        return 0.0
+    lcs = _lcs_len(pred_tokens, ref_tokens)
+    prec = lcs / float(len(pred_tokens))
+    rec = lcs / float(len(ref_tokens))
+    if prec + rec == 0:
+        return 0.0
+    return float(2 * prec * rec / (prec + rec))
+
+
+def rouge_l_from_records(records: Iterable[dict[str, Any]]) -> float:
+    """Compute ROUGE-L (F1) from records with predictions and references."""
+    scores: list[float] = []
+    for record in records:
+        if not isinstance(record, dict):
+            continue
+        pred = record.get("prediction")
+        refs = record.get("references")
+        if pred is None:
+            continue
+        if refs is None and "reference" in record:
+            refs = [record.get("reference")]
+        if refs is None:
+            continue
+        ref_list = refs if isinstance(refs, list) else [refs]
+        best = 0.0
+        for ref in ref_list:
+            if ref is None:
+                continue
+            best = max(best, _rouge_l(str(pred), str(ref)))
+        scores.append(best)
+    if not scores:
+        return float("nan")
+    return float(sum(scores) / float(len(scores)))

invarlock/guards/_estimators.py

@@ -0,0 +1,154 @@
+from __future__ import annotations
+
+import math
+from typing import Any
+
+import torch
+
+__all__ = [
+    "power_iter_sigma_max",
+    "frobenius_norm_sq",
+    "row_col_norm_extrema",
+    "stable_rank_estimate",
+]
+
+
+def _as_matrix(tensor: torch.Tensor) -> torch.Tensor:
+    if tensor.ndim == 2:
+        return tensor
+    return tensor.view(tensor.shape[0], -1)
+
+
+def power_iter_sigma_max(
+    matrix: Any,
+    *,
+    iters: int,
+    init: str = "ones",
+    eps: float = 1e-12,
+) -> float:
+    """Estimate the largest singular value (spectral norm) via fixed-iter power iteration.
+
+    Contract properties (vNext):
+    - fixed iteration budget (no convergence stopping)
+    - deterministic initialization (`init`)
+    - device-resident matvecs (no `.cpu()` transfers)
+    """
+    try:
+        iters_i = int(iters)
+    except Exception:
+        iters_i = 4
+    if iters_i < 1:
+        iters_i = 1
+
+    if not isinstance(matrix, torch.Tensor):
+        return 0.0
+    if matrix.numel() == 0:
+        return 0.0
+    if matrix.dtype in {torch.int8, torch.uint8}:
+        return 0.0
+
+    W = _as_matrix(matrix.detach())
+    if W.numel() == 0 or W.shape[0] == 0 or W.shape[1] == 0:
+        return 0.0
+
+    device = W.device
+    dtype = W.dtype
+    n = int(W.shape[1])
+
+    with torch.no_grad():
+        if init == "ones":
+            v = torch.ones((n,), device=device, dtype=dtype)
+        else:
+            # Deterministic fallback: unit vector e0.
+            v = torch.zeros((n,), device=device, dtype=dtype)
+            v[0] = 1
+
+        v_norm = torch.linalg.vector_norm(v.float()).clamp_min(eps)
+        v = v / v_norm.to(dtype)
+
+        sigma = 0.0
+        for _ in range(iters_i):
+            u = W @ v
+            u_norm = torch.linalg.vector_norm(u.float()).clamp_min(eps)
+            sigma_val = float(u_norm.item())
+            if not math.isfinite(sigma_val):
+                return 0.0
+            u = u / u_norm.to(dtype)
+            v = W.T @ u
+            v_norm = torch.linalg.vector_norm(v.float()).clamp_min(eps)
+            v = v / v_norm.to(dtype)
+            sigma = sigma_val
+        return float(sigma)
+
+
+def frobenius_norm_sq(matrix: torch.Tensor) -> float:
+    """Return ||matrix||_F^2 with float32 accumulation (device-resident)."""
+    W = _as_matrix(matrix.detach())
+    if W.numel() == 0:
+        return 0.0
+    with torch.no_grad():
+        # Use a fused reduction to avoid materializing a W*W intermediate.
+        norm = torch.linalg.vector_norm(W.reshape(-1), ord=2, dtype=torch.float32)
+        out = float((norm * norm).item())
+        return out if math.isfinite(out) else 0.0
+
+
+def row_col_norm_extrema(
+    matrix: torch.Tensor, *, eps: float = 1e-12
+) -> dict[str, float]:
+    """Compute min/median/max of row/col L2 norms with float32 accumulation."""
+    W = _as_matrix(matrix.detach())
+    if W.numel() == 0 or W.shape[0] == 0 or W.shape[1] == 0:
+        return {
+            "row_min": 0.0,
+            "row_median": 0.0,
+            "row_max": 0.0,
+            "col_min": 0.0,
+            "col_median": 0.0,
+            "col_max": 0.0,
+        }
+    with torch.no_grad():
+        # Avoid materializing W*W: use fused reductions.
+        row = torch.linalg.vector_norm(W, ord=2, dim=1, dtype=torch.float32).clamp_min(
+            eps
+        )
+        col = torch.linalg.vector_norm(W, ord=2, dim=0, dtype=torch.float32).clamp_min(
+            eps
+        )
+
+        row_sorted, _ = torch.sort(row)
+        col_sorted, _ = torch.sort(col)
+
+        def _median(sorted_vec: torch.Tensor) -> float:
+            n = int(sorted_vec.numel())
+            if n <= 0:
+                return 0.0
+            mid = n // 2
+            if n % 2 == 1:
+                return float(sorted_vec[mid].item())
+            return float((sorted_vec[mid - 1] + sorted_vec[mid]).mul(0.5).item())
+
+        return {
+            "row_min": float(row_sorted[0].item()),
+            "row_median": _median(row_sorted),
+            "row_max": float(row_sorted[-1].item()),
+            "col_min": float(col_sorted[0].item()),
+            "col_median": _median(col_sorted),
+            "col_max": float(col_sorted[-1].item()),
+        }
+
+
+def stable_rank_estimate(
+    matrix: torch.Tensor, *, sigma_max: float, eps: float = 1e-12
+) -> float:
+    """Estimate stable rank: ||W||_F^2 / ||W||_2^2, using a provided σ̂max."""
+    try:
+        denom = float(sigma_max) ** 2
+    except Exception:
+        return 0.0
+    if not math.isfinite(denom) or denom <= 0.0:
+        return 0.0
+    denom = max(denom, eps)
+    num = frobenius_norm_sq(matrix)
+    out = float(num) / denom if denom > 0 else 0.0
+    return out if math.isfinite(out) else 0.0