invarlock 0.3.0__py3-none-any.whl → 0.3.2__py3-none-any.whl

invarlock/__init__.py

@@ -12,7 +12,7 @@ For torch-dependent functionality, see subpackages under `invarlock.*`:
 - `invarlock.eval`: Metrics, guard-overhead checks, and certification
 """
 
-__version__ = "0.3.0"
+__version__ = "0.3.2"
 
 # Core exports - torch-independent
 from .config import CFG, Defaults, get_default_config

invarlock/_data/runtime/profiles/ci_cpu.yaml

@@ -11,5 +11,10 @@ dataset:
   final_n: 120
   stride: 512
 
+primary_metric:
+  acceptance_range:
+    min: 0.95
+    max: 1.15
+
 context:
   telemetry_profile: "ci_cpu"

invarlock/_data/runtime/tiers.yaml

@@ -5,6 +5,17 @@
 # embedded in certificates and referenced by automation documentation.
 
 balanced:
+  metrics:
+    pm_ratio:
+      ratio_limit_base: 1.10
+      min_tokens: 50000
+      hysteresis_ratio: 0.002
+      min_token_fraction: 0.01
+    accuracy:
+      delta_min_pp: -1.0
+      min_examples: 200
+      hysteresis_delta_pp: 0.1
+      min_examples_fraction: 0.01
   variance_guard:
     deadband: 0.02
     min_abs_adjust: 0.012
@@ -41,6 +52,17 @@ balanced:
       other: 0.12
 
 conservative:
+  metrics:
+    pm_ratio:
+      ratio_limit_base: 1.05
+      min_tokens: 20000
+      hysteresis_ratio: 0.002
+      min_token_fraction: 0.01
+    accuracy:
+      delta_min_pp: -0.5
+      min_examples: 200
+      hysteresis_delta_pp: 0.1
+      min_examples_fraction: 0.01
   variance_guard:
     deadband: 0.03
     min_abs_adjust: 0.02
@@ -74,3 +96,42 @@ conservative:
       attn: 0.05
       embed: 0.07
       other: 0.07
+
+aggressive:
+  metrics:
+    pm_ratio:
+      ratio_limit_base: 1.20
+      min_tokens: 50000
+      hysteresis_ratio: 0.002
+      min_token_fraction: 0.01
+    accuracy:
+      delta_min_pp: -2.0
+      min_examples: 200
+      hysteresis_delta_pp: 0.1
+      min_examples_fraction: 0.01
+  variance_guard:
+    deadband: 0.12
+    min_effect_lognll: 0.0005
+  spectral_guard:
+    sigma_quantile: 0.98
+    deadband: 0.15
+    scope: ffn
+    max_caps: 8
+    family_caps:
+      ffn: 3.0
+      attn: 3.5
+      embed: 2.5
+      other: 3.5
+    multiple_testing:
+      method: bh
+      alpha: 0.1
+      m: 4
+  rmt_guard:
+    deadband: 0.15
+    margin: 1.8
+    epsilon_default: 0.15
+    epsilon_by_family:
+      ffn: 0.15
+      attn: 0.15
+      embed: 0.15
+      other: 0.15

invarlock/adapters/hf_loading.py

@@ -0,0 +1,97 @@
+"""Helpers for Hugging Face model loading.
+
+Centralizes security- and performance-sensitive defaults used by HF adapters.
+"""
+
+from __future__ import annotations
+
+import os
+from typing import Any
+
+import torch
+
+_TRUE = {"1", "true", "yes", "on"}
+_FALSE = {"0", "false", "no", "off"}
+
+
+def _coerce_bool(val: Any) -> bool | None:
+    if isinstance(val, bool):
+        return val
+    if isinstance(val, int):
+        return bool(val)
+    if isinstance(val, str):
+        s = val.strip().lower()
+        if s in _TRUE:
+            return True
+        if s in _FALSE:
+            return False
+    return None
+
+
+def resolve_trust_remote_code(
+    kwargs: dict[str, Any] | None = None, *, default: bool = False
+) -> bool:
+    """Resolve trust_remote_code with config override and env opt-in."""
+    if kwargs and "trust_remote_code" in kwargs:
+        coerced = _coerce_bool(kwargs.get("trust_remote_code"))
+        if coerced is not None:
+            return coerced
+
+    for env_name in (
+        "INVARLOCK_TRUST_REMOTE_CODE",
+        "TRUST_REMOTE_CODE_BOOL",
+        "ALLOW_REMOTE_CODE",
+    ):
+        env_val = os.environ.get(env_name)
+        coerced = _coerce_bool(env_val)
+        if coerced is not None:
+            return coerced
+
+    return default
+
+
+def default_torch_dtype() -> torch.dtype:
+    """Pick a safe default dtype for HF loads based on hardware."""
+    if torch.cuda.is_available():
+        try:
+            if (
+                hasattr(torch.cuda, "is_bf16_supported")
+                and torch.cuda.is_bf16_supported()
+            ):
+                return torch.bfloat16
+        except Exception:
+            pass
+        return torch.float16
+
+    if hasattr(torch.backends, "mps") and torch.backends.mps.is_available():
+        return torch.float16
+
+    return torch.float32
+
+
+def resolve_torch_dtype(kwargs: dict[str, Any] | None = None) -> torch.dtype | str:
+    """Resolve torch_dtype from kwargs or choose a hardware-aware default."""
+    if kwargs and "torch_dtype" in kwargs:
+        val = kwargs.get("torch_dtype")
+        if isinstance(val, torch.dtype):
+            return val
+        if isinstance(val, str):
+            s = val.strip().lower()
+            if s == "auto":
+                return "auto"
+            mapping = {
+                "float16": torch.float16,
+                "fp16": torch.float16,
+                "half": torch.float16,
+                "bfloat16": torch.bfloat16,
+                "bf16": torch.bfloat16,
+                "float32": torch.float32,
+                "fp32": torch.float32,
+            }
+            if s in mapping:
+                return mapping[s]
+
+    return default_torch_dtype()
+
+
+__all__ = ["resolve_trust_remote_code", "default_torch_dtype", "resolve_torch_dtype"]

invarlock/calibration/__init__.py

@@ -0,0 +1,6 @@
+"""Calibration helpers and sweep harness utilities."""
+
+__all__ = [
+    "spectral_null",
+    "variance_ve",
+]

invarlock/calibration/spectral_null.py

@@ -0,0 +1,301 @@
+from __future__ import annotations
+
+import math
+from collections import Counter, defaultdict
+from typing import Any
+
+
+def _finite01(value: Any) -> bool:
+    try:
+        f = float(value)
+        return math.isfinite(f) and 0.0 <= f <= 1.0
+    except Exception:
+        return False
+
+
+def _bh_reject_families(
+    family_pvals: dict[str, float],
+    *,
+    alpha: float,
+    m: int,
+) -> set[str]:
+    if not family_pvals:
+        return set()
+    try:
+        alpha_f = float(alpha)
+    except Exception:
+        return set()
+    if not (0.0 < alpha_f <= 1.0):
+        return set()
+
+    names = list(family_pvals.keys())
+    pvals = [family_pvals[name] for name in names]
+    n = len(pvals)
+    m_eff = max(int(m) if isinstance(m, int) else 0, n, 1)
+
+    order = sorted(
+        range(n),
+        key=lambda idx: (float("inf") if not _finite01(pvals[idx]) else pvals[idx]),
+    )
+    max_k = 0
+    for rank, idx in enumerate(order, start=1):
+        p = pvals[idx]
+        if not _finite01(p):
+            continue
+        if p <= (alpha_f * rank) / m_eff:
+            max_k = rank
+    if max_k <= 0:
+        return set()
+    cutoff = (alpha_f * max_k) / m_eff
+    selected: set[str] = set()
+    for idx in order:
+        p = pvals[idx]
+        if _finite01(p) and p <= cutoff:
+            selected.add(names[idx])
+    return selected
+
+
+def _bonferroni_reject_families(
+    family_pvals: dict[str, float],
+    *,
+    alpha: float,
+    m: int,
+) -> set[str]:
+    if not family_pvals:
+        return set()
+    try:
+        alpha_f = float(alpha)
+    except Exception:
+        return set()
+    if not (0.0 < alpha_f <= 1.0):
+        return set()
+    m_eff = max(int(m) if isinstance(m, int) else 0, len(family_pvals), 1)
+    cutoff = alpha_f / m_eff
+    return {fam for fam, p in family_pvals.items() if _finite01(p) and p <= cutoff}
+
+
+def _extract_guard(report: dict[str, Any], name: str) -> dict[str, Any] | None:
+    guards = report.get("guards")
+    if isinstance(guards, list):
+        for item in guards:
+            if isinstance(item, dict) and item.get("name") == name:
+                return item
+    return None
+
+
+def _extract_family_max_z(metrics: dict[str, Any]) -> dict[str, float]:
+    out: dict[str, float] = {}
+    summary = metrics.get("family_z_summary")
+    if isinstance(summary, dict):
+        for fam, vals in summary.items():
+            if not isinstance(vals, dict):
+                continue
+            z = vals.get("max")
+            try:
+                if z is not None and math.isfinite(float(z)):
+                    out[str(fam)] = float(z)
+            except Exception:
+                continue
+    q = metrics.get("family_z_quantiles")
+    if isinstance(q, dict):
+        for fam, vals in q.items():
+            if not isinstance(vals, dict):
+                continue
+            z = vals.get("max")
+            try:
+                if z is not None and math.isfinite(float(z)):
+                    out[str(fam)] = max(out.get(str(fam), float("-inf")), float(z))
+            except Exception:
+                continue
+    return out
+
+
+def _extract_multiple_testing(metrics: dict[str, Any]) -> dict[str, Any]:
+    mt = metrics.get("multiple_testing")
+    if not isinstance(mt, dict):
+        return {}
+    out: dict[str, Any] = {}
+    method = mt.get("method")
+    if isinstance(method, str) and method.strip():
+        out["method"] = method.strip().lower()
+    try:
+        alpha = mt.get("alpha")
+        if alpha is not None:
+            out["alpha"] = float(alpha)
+    except Exception:
+        pass
+    try:
+        m_val = mt.get("m")
+        if m_val is not None:
+            out["m"] = int(m_val)
+    except Exception:
+        pass
+    return out
+
+
+def _selected_families_for_alpha(
+    pvals: dict[str, float],
+    *,
+    method: str,
+    alpha: float,
+    m: int,
+) -> set[str]:
+    meth = (method or "").strip().lower()
+    if meth == "bonferroni":
+        return _bonferroni_reject_families(pvals, alpha=alpha, m=m)
+    # Default: BH
+    return _bh_reject_families(pvals, alpha=alpha, m=m)
+
+
+def summarize_null_sweep_reports(
+    reports: list[dict[str, Any]],
+    *,
+    tier: str,
+    safety_margin: float = 0.05,
+    target_any_warning_rate: float = 0.01,
+) -> dict[str, Any]:
+    """Summarize spectral null-sweep results and recommend κ/alpha.
+
+    Inputs are run report dicts produced by `invarlock run` (or equivalent).
+    """
+
+    tier_norm = (tier or "").strip().lower() or "balanced"
+    margin = float(safety_margin or 0.0)
+    if not (0.0 <= margin <= 1.0):
+        margin = 0.05
+    target = float(target_any_warning_rate or 0.0)
+    if not (0.0 <= target <= 1.0):
+        target = 0.01
+
+    family_max_z: dict[str, float] = defaultdict(lambda: float("-inf"))
+    has_warning_default: list[bool] = []
+    run_pvals: list[dict[str, float]] = []
+
+    mt_method = "bh"
+    mt_alpha = 0.05
+    mt_m = 4
+
+    selected_by_family: Counter[str] = Counter()
+    candidate_by_family: Counter[str] = Counter()
+
+    for report in reports:
+        if not isinstance(report, dict):
+            continue
+        g = _extract_guard(report, "spectral") or {}
+        metrics = g.get("metrics", {}) if isinstance(g.get("metrics"), dict) else {}
+        mt = _extract_multiple_testing(metrics)
+        if mt:
+            mt_method = str(mt.get("method", mt_method))
+            if mt.get("alpha") is not None:
+                mt_alpha = float(mt.get("alpha"))
+            if mt.get("m") is not None:
+                mt_m = int(mt.get("m"))
+
+        fam_z = _extract_family_max_z(metrics)
+        for fam, z in fam_z.items():
+            family_max_z[fam] = max(family_max_z[fam], float(z))
+
+        selection = (
+            metrics.get("multiple_testing_selection")
+            if isinstance(metrics.get("multiple_testing_selection"), dict)
+            else {}
+        )
+        pvals = selection.get("family_pvalues")
+        if not isinstance(pvals, dict):
+            pvals = {}
+        parsed_pvals: dict[str, float] = {}
+        for fam, p in pvals.items():
+            try:
+                pf = float(p)
+            except Exception:
+                continue
+            if _finite01(pf):
+                parsed_pvals[str(fam)] = pf
+        run_pvals.append(parsed_pvals)
+
+        families_selected = selection.get("families_selected")
+        if isinstance(families_selected, list):
+            for fam in families_selected:
+                selected_by_family[str(fam)] += 1
+
+        fam_counts = selection.get("family_violation_counts")
+        if isinstance(fam_counts, dict):
+            for fam, count in fam_counts.items():
+                try:
+                    candidate_by_family[str(fam)] += int(count)
+                except Exception:
+                    continue
+
+        caps_applied = metrics.get("caps_applied")
+        try:
+            caps_applied_int = int(caps_applied) if caps_applied is not None else 0
+        except Exception:
+            caps_applied_int = 0
+        violations = g.get("violations", [])
+        has_warning_default.append(bool(caps_applied_int) or bool(violations))
+
+    n = max(len(has_warning_default), 1)
+    observed_any_rate = sum(1 for v in has_warning_default if v) / float(n)
+
+    # κ recommendation: max observed z per family (+ margin), rounded for stable tiers.yaml diffs.
+    rec_caps: dict[str, float] = {}
+    for fam, z in sorted(family_max_z.items()):
+        if not math.isfinite(z):
+            continue
+        kappa = z * (1.0 + margin)
+        rec_caps[fam] = float(round(kappa, 3))
+
+    # α calibration: choose the largest alpha that meets target_any_warning_rate.
+    # This uses per-run family p-values (from spectral.multiple_testing_selection).
+    def _rate_for_alpha(alpha: float) -> float:
+        any_sel = 0
+        for pvals in run_pvals:
+            selected = _selected_families_for_alpha(
+                pvals, method=mt_method, alpha=alpha, m=mt_m
+            )
+            any_sel += 1 if selected else 0
+        return any_sel / float(max(len(run_pvals), 1))
+
+    recommended_alpha = float(mt_alpha)
+    if run_pvals and observed_any_rate > target:
+        # Halving search is stable/deterministic and avoids dependency-heavy optimizers.
+        alpha_grid: list[float] = []
+        a = float(mt_alpha)
+        for _ in range(20):
+            if a <= 1e-6:
+                break
+            alpha_grid.append(a)
+            a *= 0.5
+        alpha_grid.append(1e-6)
+        best = None
+        for candidate in alpha_grid:
+            rate = _rate_for_alpha(candidate)
+            if rate <= target:
+                best = candidate
+                break
+        if best is not None:
+            recommended_alpha = float(best)
+
+    return {
+        "tier": tier_norm,
+        "n_runs": int(len(has_warning_default)),
+        "observed": {
+            "any_warning_rate": float(observed_any_rate),
+            "selected_by_family_runs": dict(selected_by_family),
+            "candidate_violations_by_family_total": dict(candidate_by_family),
+            "family_max_z": {
+                k: float(v) for k, v in sorted(family_max_z.items()) if math.isfinite(v)
+            },
+        },
+        "recommendations": {
+            "family_caps": rec_caps,
+            "multiple_testing": {
+                "method": str(mt_method),
+                "alpha": float(recommended_alpha),
+                "m": int(mt_m),
+            },
+        },
+    }
+
+
+__all__ = ["summarize_null_sweep_reports"]

invarlock/calibration/variance_ve.py

@@ -0,0 +1,154 @@
+from __future__ import annotations
+
+import math
+from typing import Any
+
+
+def _extract_guard(report: dict[str, Any], name: str) -> dict[str, Any] | None:
+    guards = report.get("guards")
+    if isinstance(guards, list):
+        for item in guards:
+            if isinstance(item, dict) and item.get("name") == name:
+                return item
+    return None
+
+
+def _coerce_delta_ci(value: Any) -> tuple[float, float] | None:
+    if not (isinstance(value, tuple | list) and len(value) == 2):
+        return None
+    try:
+        lo = float(value[0])
+        hi = float(value[1])
+    except Exception:
+        return None
+    if not (math.isfinite(lo) and math.isfinite(hi)):
+        return None
+    return (lo, hi)
+
+
+def _gain_lower_bound(
+    *, mean_delta: float | None, delta_ci: tuple[float, float] | None, one_sided: bool
+) -> float:
+    if delta_ci is None:
+        return 0.0
+    lo, hi = delta_ci
+    if hi >= 0.0:
+        return 0.0
+    if one_sided and (mean_delta is None or not (mean_delta < 0.0)):
+        return 0.0
+    # Gain CI lower bound is -upper (worst-case gain).
+    return max(0.0, -hi)
+
+
+def _recommend_threshold_for_target_rate(
+    gains: list[float],
+    *,
+    target_rate: float,
+    safety_margin: float,
+) -> tuple[float, float]:
+    n = len(gains)
+    if n <= 0:
+        return 0.0, 0.0
+    target = float(target_rate)
+    if not (0.0 <= target <= 1.0):
+        target = 0.05
+    desired_passes = int(math.floor(target * n))
+    gains_desc = sorted((max(0.0, float(g)) for g in gains), reverse=True)
+
+    def pass_count(thr: float) -> int:
+        return sum(1 for g in gains_desc if g >= thr)
+
+    if desired_passes <= 0:
+        thr = (
+            (gains_desc[0] * (1.0 + max(0.0, safety_margin)))
+            if gains_desc[0] > 0
+            else 0.0
+        )
+        return float(round(thr, 3)), 0.0
+
+    unique_vals = sorted(set(gains_desc), reverse=True)
+    chosen = None
+    chosen_rate = 0.0
+    for val in unique_vals:
+        cnt = pass_count(val)
+        rate = cnt / float(n)
+        if cnt <= desired_passes:
+            chosen = float(val)
+            chosen_rate = float(rate)
+            break
+
+    if chosen is None:
+        # Ties at max prevent meeting desired_passes; force zero enable rate.
+        thr = gains_desc[0] * (1.0 + max(0.0, safety_margin))
+        return float(round(thr, 3)), 0.0
+
+    thr = chosen * (1.0 + max(0.0, safety_margin))
+    return float(round(thr, 3)), float(chosen_rate)
+
+
+def summarize_ve_sweep_reports(
+    reports: list[dict[str, Any]],
+    *,
+    tier: str,
+    target_enable_rate: float = 0.05,
+    safety_margin: float = 0.0,
+    predictive_one_sided: bool = True,
+) -> dict[str, Any]:
+    """Summarize VE predictive-gate sweeps and recommend min_effect_lognll."""
+
+    tier_norm = (tier or "").strip().lower() or "balanced"
+    one_sided = bool(predictive_one_sided)
+    margin = float(safety_margin or 0.0)
+    if not (0.0 <= margin <= 1.0):
+        margin = 0.0
+
+    gains: list[float] = []
+    widths: list[float] = []
+    evaluated = 0
+
+    for report in reports:
+        if not isinstance(report, dict):
+            continue
+        g = _extract_guard(report, "variance") or {}
+        metrics = g.get("metrics", {}) if isinstance(g.get("metrics"), dict) else {}
+        pg = metrics.get("predictive_gate")
+        if not isinstance(pg, dict):
+            continue
+        evaluated += 1 if pg.get("evaluated") is True else 0
+
+        delta_ci = _coerce_delta_ci(pg.get("delta_ci"))
+        mean_delta = pg.get("mean_delta")
+        try:
+            mean_delta_f = float(mean_delta) if mean_delta is not None else None
+        except Exception:
+            mean_delta_f = None
+
+        gains.append(
+            _gain_lower_bound(
+                mean_delta=mean_delta_f, delta_ci=delta_ci, one_sided=one_sided
+            )
+        )
+        if delta_ci is not None:
+            widths.append(float(abs(delta_ci[1] - delta_ci[0])))
+
+    recommended, expected_rate = _recommend_threshold_for_target_rate(
+        gains, target_rate=target_enable_rate, safety_margin=margin
+    )
+
+    mean_width = float(sum(widths) / max(len(widths), 1)) if widths else None
+
+    return {
+        "tier": tier_norm,
+        "n_runs": int(len(gains)),
+        "observed": {
+            "evaluated_runs": int(evaluated),
+            "mean_ci_width": mean_width,
+        },
+        "recommendations": {
+            "min_effect_lognll": float(recommended),
+            "expected_enable_rate": float(expected_rate),
+        },
+    }
+
+
+__all__ = ["summarize_ve_sweep_reports"]

invarlock/cli/app.py

@@ -34,6 +34,7 @@ class OrderedGroup(TyperGroup):
     def list_commands(self, ctx):  # type: ignore[override]
         return [
             "certify",
+            "calibrate",
             "report",
             "verify",
             "run",
@@ -161,10 +162,24 @@ def _register_subapps() -> None:
     from .commands.plugins import plugins_app as _plugins_app
     from .commands.report import report_app as _report_app
 
+    # Always-available subapps (lightweight imports)
     app.add_typer(_report_app, name="report")
     app.add_typer(_plugins_app, name="plugins")
     app.command(name="doctor")(_doctor_cmd)
 
+    # Optional: calibration subapp. This transitively imports guards, which may
+    # depend on torch/transformers. In minimal environments (no heavy deps),
+    # skip registration so `python -m invarlock --help` stays import-safe.
+    try:
+        from .commands.calibrate import calibrate_app as _calibrate_app
+    except ModuleNotFoundError as exc:  # pragma: no cover - exercised in venv test
+        missing = getattr(exc, "name", "") or ""
+        if missing in {"torch", "transformers"}:
+            return
+        raise
+    else:
+        app.add_typer(_calibrate_app, name="calibrate")
+
 
 @app.command(
     name="verify",