invarlock 0.3.1__py3-none-any.whl → 0.3.2__py3-none-any.whl

invarlock/cli/commands/plugins.py

@@ -897,11 +897,21 @@ def _check_plugin_extras(plugin_name: str, plugin_type: str) -> str:
     if not plugin_info or not plugin_info["packages"]:
         return ""  # No extra dependencies needed
 
-    # Check each required package using import to play nice with tests that mock __import__
+    # Check each required package. For most packages we use a light import so
+    # tests can monkeypatch __import__; for GPU-only stacks like bitsandbytes
+    # we only probe presence via importlib.util.find_spec to avoid crashing on
+    # CPU-only builds during simple listing.
     missing_packages: list[str] = []
     for pkg in plugin_info["packages"]:
         try:
-            __import__(pkg)
+            if pkg == "bitsandbytes":
+                import importlib.util as _util
+
+                spec = _util.find_spec(pkg)
+                if spec is None:
+                    raise ImportError("bitsandbytes not importable")
+            else:
+                __import__(pkg)
         except Exception:
             missing_packages.append(pkg)
 

invarlock/cli/commands/run.py

@@ -9,6 +9,7 @@ prefer Compare & Certify via `invarlock certify --baseline ... --subject ...`.
 
 import copy
 import hashlib
+import inspect
 import json
 import math
 import os
@@ -818,6 +819,51 @@ def _resolve_provider_and_split(
     return data_provider, resolved_split, used_fallback_split
 
 
+def _extract_model_load_kwargs(cfg: InvarLockConfig) -> dict[str, Any]:
+    """Return adapter.load_model kwargs from config (excluding core fields)."""
+    try:
+        data = cfg.model_dump()
+    except Exception:
+        data = {}
+    model = data.get("model") if isinstance(data, dict) else None
+    if not isinstance(model, dict):
+        return {}
+    return {
+        key: value
+        for key, value in model.items()
+        if key not in {"id", "adapter", "device"} and value is not None
+    }
+
+
+def _load_model_with_cfg(adapter: Any, cfg: InvarLockConfig, device: str) -> Any:
+    """Load a model with config-provided kwargs, filtering for strict adapters."""
+    try:
+        model_id = cfg.model.id
+    except Exception:
+        try:
+            model_id = (cfg.model_dump().get("model") or {}).get("id")
+        except Exception:
+            model_id = None
+    if not isinstance(model_id, str) or not model_id:
+        raise ValueError("Missing model.id in config")
+
+    extra = _extract_model_load_kwargs(cfg)
+    try:
+        sig = inspect.signature(adapter.load_model)
+        accepts_var_kw = any(
+            p.kind == inspect.Parameter.VAR_KEYWORD for p in sig.parameters.values()
+        )
+        if accepts_var_kw:
+            return adapter.load_model(model_id, device=device, **extra)
+        allowed = {k: v for k, v in extra.items() if k in sig.parameters}
+        if allowed:
+            return adapter.load_model(model_id, device=device, **allowed)
+    except Exception:
+        # Fall back to the strictest call shape.
+        pass
+    return adapter.load_model(model_id, device=device)
+
+
 def _run_bare_control(
     *,
     adapter: Any,
@@ -899,6 +945,7 @@ def _run_bare_control(
         "errors": [],
         "checks": {},
         "source": f"{profile_normalized or 'ci'}_profile",
+        "mode": "bare",
     }
 
     if getattr(bare_report, "status", "").lower() not in {"success", "completed", "ok"}:
@@ -977,7 +1024,7 @@ def _postprocess_and_summarize(
     match_fraction: float | None,
     overlap_fraction: float | None,
     console: Console,
-) -> None:
+) -> dict[str, str]:
     """Finalize report windows stats and print/save summary artifacts."""
     try:
         ds = report.setdefault("dataset", {}).setdefault("windows", {})
@@ -1001,6 +1048,7 @@ def _postprocess_and_summarize(
     console.print(f"📄 Report: {saved_files['json']}")
     if run_config.event_path:
         console.print(f"📝 Events: {run_config.event_path}")
+    return saved_files
 
 
 def _compute_provider_digest(report: dict[str, Any]) -> dict[str, str] | None:
@@ -1537,6 +1585,7 @@ def run_command(
     no_cleanup = bool(_coerce_option(no_cleanup, False))
 
     # Use shared CLI coercers from invarlock.cli.utils
+    report_path_out: str | None = None
 
     def _fail_run(message: str) -> None:
         console.print(f"[red]❌ {message}[/red]")
@@ -1673,6 +1722,26 @@ def run_command(
             cfg, device=device, out=out, console=console
         )
 
+        determinism_meta: dict[str, Any] | None = None
+        try:
+            from invarlock.cli.determinism import apply_determinism_preset
+
+            preset = apply_determinism_preset(
+                profile=profile_label,
+                device=resolved_device,
+                seed=int(seed_bundle.get("python") or seed_value),
+                threads=int(os.environ.get("INVARLOCK_OMP_THREADS", 1) or 1),
+            )
+            if isinstance(preset, dict) and preset:
+                determinism_meta = preset
+                preset_seeds = preset.get("seeds")
+                if isinstance(preset_seeds, dict) and preset_seeds:
+                    for key in ("python", "numpy", "torch"):
+                        if key in preset_seeds:
+                            seed_bundle[key] = preset_seeds.get(key)
+        except Exception:
+            determinism_meta = None
+
         # Create run directory with timestamp
         timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
         run_dir = output_dir / timestamp
@@ -2934,7 +3003,23 @@ def run_command(
                 )
 
             guard_overhead_payload: dict[str, Any] | None = None
-            if measure_guard_overhead:
+            if skip_overhead and profile_normalized in {"ci", "release"}:
+                guard_overhead_payload = {
+                    "overhead_threshold": GUARD_OVERHEAD_THRESHOLD,
+                    "evaluated": False,
+                    "passed": True,
+                    "skipped": True,
+                    "skip_reason": "INVARLOCK_SKIP_OVERHEAD_CHECK",
+                    "mode": "skipped",
+                    "source": "env:INVARLOCK_SKIP_OVERHEAD_CHECK",
+                    "messages": [
+                        "Overhead check skipped via INVARLOCK_SKIP_OVERHEAD_CHECK"
+                    ],
+                    "warnings": [],
+                    "errors": [],
+                    "checks": {},
+                }
+            elif measure_guard_overhead:
                 guard_overhead_payload = _run_bare_control(
                     adapter=adapter,
                     edit_op=edit_op,
@@ -3076,6 +3161,8 @@ def run_command(
                 meta_payload["invarlock_version"] = invarlock_version
             if env_flags:
                 meta_payload["env_flags"] = env_flags
+            if determinism_meta:
+                meta_payload["determinism"] = determinism_meta
             report["meta"].update(meta_payload)
             if pm_acceptance_range:
                 report["meta"]["pm_acceptance_range"] = pm_acceptance_range
@@ -3235,87 +3322,90 @@ def run_command(
                 report["metrics"].update(metrics_payload)
 
             if guard_overhead_payload is not None:
-                # Compute guarded primary-metric snapshot; pass structured reports into validator
-                try:
-                    # Map loss type to ppl family kind
-                    lk = str(resolved_loss_type or "causal").lower()
-                    if lk == "mlm":
-                        pm_kind_for_overhead = "ppl_mlm"
-                    elif lk in {"seq2seq", "s2s", "t5"}:
-                        pm_kind_for_overhead = "ppl_seq2seq"
-                    else:
-                        pm_kind_for_overhead = "ppl_causal"
+                if bool(guard_overhead_payload.get("skipped", False)):
+                    report["guard_overhead"] = guard_overhead_payload
+                else:
+                    # Compute guarded primary-metric snapshot; pass structured reports into validator
+                    try:
+                        # Map loss type to ppl family kind
+                        lk = str(resolved_loss_type or "causal").lower()
+                        if lk == "mlm":
+                            pm_kind_for_overhead = "ppl_mlm"
+                        elif lk in {"seq2seq", "s2s", "t5"}:
+                            pm_kind_for_overhead = "ppl_seq2seq"
+                        else:
+                            pm_kind_for_overhead = "ppl_causal"
 
-                    # Prefer computing from the in-memory core_report windows to avoid ordering issues
-                    pm_guarded = _extract_pm_snapshot_for_overhead(
-                        core_report, kind=pm_kind_for_overhead
-                    )
-                    if not isinstance(pm_guarded, dict) or not pm_guarded:
+                        # Prefer computing from the in-memory core_report windows to avoid ordering issues
                         pm_guarded = _extract_pm_snapshot_for_overhead(
-                            report, kind=pm_kind_for_overhead
+                            core_report, kind=pm_kind_for_overhead
                         )
+                        if not isinstance(pm_guarded, dict) or not pm_guarded:
+                            pm_guarded = _extract_pm_snapshot_for_overhead(
+                                report, kind=pm_kind_for_overhead
+                            )
 
-                    guard_overhead_payload["guarded_report"] = (
-                        {"metrics": {"primary_metric": pm_guarded}}
-                        if isinstance(pm_guarded, dict) and pm_guarded
-                        else None
+                        guard_overhead_payload["guarded_report"] = (
+                            {"metrics": {"primary_metric": pm_guarded}}
+                            if isinstance(pm_guarded, dict) and pm_guarded
+                            else None
+                        )
+                    except Exception:
+                        guard_overhead_payload["guarded_report"] = None
+                    bare_struct = guard_overhead_payload.get("bare_report") or {}
+                    guarded_struct = guard_overhead_payload.get("guarded_report") or {}
+                    # Be robust to mocks or minimal objects returned by validators
+                    result = validate_guard_overhead(
+                        bare_struct,
+                        guarded_struct,
+                        overhead_threshold=guard_overhead_payload.get(
+                            "overhead_threshold", GUARD_OVERHEAD_THRESHOLD
+                        ),
                     )
-                except Exception:
-                    guard_overhead_payload["guarded_report"] = None
-                bare_struct = guard_overhead_payload.get("bare_report") or {}
-                guarded_struct = guard_overhead_payload.get("guarded_report") or {}
-                # Be robust to mocks or minimal objects returned by validators
-                result = validate_guard_overhead(
-                    bare_struct,
-                    guarded_struct,
-                    overhead_threshold=guard_overhead_payload.get(
-                        "overhead_threshold", GUARD_OVERHEAD_THRESHOLD
-                    ),
-                )
-                try:
-                    messages = list(getattr(result, "messages", []))
-                except Exception:  # pragma: no cover - defensive
-                    messages = []
-                try:
-                    warnings = list(getattr(result, "warnings", []))
-                except Exception:  # pragma: no cover - defensive
-                    warnings = []
-                try:
-                    errors = list(getattr(result, "errors", []))
-                except Exception:  # pragma: no cover - defensive
-                    errors = []
-                try:
-                    checks = dict(getattr(result, "checks", {}))
-                except Exception:  # pragma: no cover - defensive
-                    checks = {}
-                metrics_obj = getattr(result, "metrics", {})
-                if not isinstance(metrics_obj, dict):
-                    metrics_obj = {}
-                overhead_ratio = metrics_obj.get("overhead_ratio")
-                if overhead_ratio is None:
-                    overhead_ratio = getattr(result, "overhead_ratio", None)
-                overhead_percent = metrics_obj.get("overhead_percent")
-                if overhead_percent is None:
-                    overhead_percent = getattr(result, "overhead_percent", None)
-                passed_flag = bool(getattr(result, "passed", False))
-
-                guard_overhead_payload.update(
-                    {
-                        "messages": messages,
-                        "warnings": warnings,
-                        "errors": errors,
-                        "checks": checks,
-                        "overhead_ratio": overhead_ratio,
-                        "overhead_percent": overhead_percent,
-                        "passed": passed_flag,
-                        "evaluated": True,
-                    }
-                )
-                # Normalize for non-finite/degenerate cases
-                guard_overhead_payload = _normalize_overhead_result(
-                    guard_overhead_payload, profile=profile_normalized
-                )
-                report["guard_overhead"] = guard_overhead_payload
+                    try:
+                        messages = list(getattr(result, "messages", []))
+                    except Exception:  # pragma: no cover - defensive
+                        messages = []
+                    try:
+                        warnings = list(getattr(result, "warnings", []))
+                    except Exception:  # pragma: no cover - defensive
+                        warnings = []
+                    try:
+                        errors = list(getattr(result, "errors", []))
+                    except Exception:  # pragma: no cover - defensive
+                        errors = []
+                    try:
+                        checks = dict(getattr(result, "checks", {}))
+                    except Exception:  # pragma: no cover - defensive
+                        checks = {}
+                    metrics_obj = getattr(result, "metrics", {})
+                    if not isinstance(metrics_obj, dict):
+                        metrics_obj = {}
+                    overhead_ratio = metrics_obj.get("overhead_ratio")
+                    if overhead_ratio is None:
+                        overhead_ratio = getattr(result, "overhead_ratio", None)
+                    overhead_percent = metrics_obj.get("overhead_percent")
+                    if overhead_percent is None:
+                        overhead_percent = getattr(result, "overhead_percent", None)
+                    passed_flag = bool(getattr(result, "passed", False))
+
+                    guard_overhead_payload.update(
+                        {
+                            "messages": messages,
+                            "warnings": warnings,
+                            "errors": errors,
+                            "checks": checks,
+                            "overhead_ratio": overhead_ratio,
+                            "overhead_percent": overhead_percent,
+                            "passed": passed_flag,
+                            "evaluated": True,
+                        }
+                    )
+                    # Normalize for non-finite/degenerate cases
+                    guard_overhead_payload = _normalize_overhead_result(
+                        guard_overhead_payload, profile=profile_normalized
+                    )
+                    report["guard_overhead"] = guard_overhead_payload
 
             had_baseline = bool(baseline and Path(baseline).exists())
             if (
@@ -3860,7 +3950,7 @@ def run_command(
             except Exception:
                 pass
 
-            _postprocess_and_summarize(
+            saved_files = _postprocess_and_summarize(
                 report=report,
                 run_dir=run_dir,
                 run_config=run_config,
@@ -3870,6 +3960,11 @@ def run_command(
                 overlap_fraction=overlap_fraction,
                 console=console,
             )
+            try:
+                if isinstance(saved_files, dict) and saved_files.get("json"):
+                    report_path_out = str(saved_files["json"])
+            except Exception:
+                pass
 
             # Metrics display
             pm_obj = None
@@ -4060,6 +4155,7 @@ def run_command(
             pass
 
         # Normal path falls through; cleanup handled below in finally
+        return report_path_out
 
     except FileNotFoundError as e:
         console.print(f"[red]❌ Configuration file not found: {e}[/red]")

invarlock/cli/commands/verify.py

@@ -35,6 +35,22 @@ from .run import _enforce_provider_parity, _resolve_exit_code
 console = Console()
 
 
+def _coerce_float(value: Any) -> float | None:
+    try:
+        out = float(value)
+    except (TypeError, ValueError):
+        return None
+    return out if math.isfinite(out) else None
+
+
+def _coerce_int(value: Any) -> int | None:
+    try:
+        out = int(value)
+    except (TypeError, ValueError):
+        return None
+    return out if out >= 0 else None
+
+
 def _load_certificate(path: Path) -> dict[str, Any]:
     """Load certificate JSON from disk."""
     with path.open("r", encoding="utf-8") as handle:
@@ -315,6 +331,30 @@ def _validate_certificate_payload(
         errors.extend(_validate_drift_band(certificate))
     errors.extend(_apply_profile_lints(certificate))
     errors.extend(_validate_tokenizer_hash(certificate))
+    # Release-only enforcement: guard overhead must be measured or explicitly skipped.
+    if prof == "release":
+        go = certificate.get("guard_overhead")
+        if not isinstance(go, dict) or not go:
+            errors.append(
+                "Release verification requires guard_overhead (missing). "
+                "Set INVARLOCK_SKIP_OVERHEAD_CHECK=1 to explicitly skip during certification."
+            )
+        else:
+            skipped = bool(go.get("skipped", False)) or (
+                str(go.get("mode", "")).strip().lower() == "skipped"
+            )
+            if not skipped:
+                evaluated = go.get("evaluated")
+                if evaluated is not True:
+                    errors.append(
+                        "Release verification requires evaluated guard_overhead (not evaluated). "
+                        "Set INVARLOCK_SKIP_OVERHEAD_CHECK=1 to explicitly skip during certification."
+                    )
+                ratio = go.get("overhead_ratio")
+                if ratio is None:
+                    errors.append(
+                        "Release verification requires guard_overhead.overhead_ratio (missing)."
+                    )
     # Legacy cross-checks removed; primary_metric is canonical
 
     return errors

invarlock/cli/determinism.py

@@ -0,0 +1,237 @@
+"""Determinism presets for CI/release runs.
+
+Centralizes:
+- Seeds (python/numpy/torch)
+- Thread caps (OMP/MKL/etc + torch threads)
+- TF32 policy
+- torch deterministic algorithms
+- A structured "determinism level" for certificate provenance
+"""
+
+from __future__ import annotations
+
+import os
+import random
+from typing import Any
+
+import numpy as np
+
+from invarlock.model_utils import set_seed
+
+try:  # optional torch
+    import torch
+except Exception:  # pragma: no cover
+    torch = None  # type: ignore[assignment]
+
+
+_THREAD_ENV_VARS: tuple[str, ...] = (
+    "OMP_NUM_THREADS",
+    "MKL_NUM_THREADS",
+    "OPENBLAS_NUM_THREADS",
+    "NUMEXPR_NUM_THREADS",
+    "VECLIB_MAXIMUM_THREADS",
+)
+
+
+def _coerce_int(value: Any, default: int) -> int:
+    try:
+        return int(value)
+    except Exception:
+        return int(default)
+
+
+def _coerce_profile(profile: str | None) -> str:
+    try:
+        return (profile or "").strip().lower()
+    except Exception:
+        return ""
+
+
+def _coerce_device(device: str | None) -> str:
+    try:
+        return (device or "").strip().lower()
+    except Exception:
+        return "cpu"
+
+
+def apply_determinism_preset(
+    *,
+    profile: str | None,
+    device: str | None,
+    seed: int,
+    threads: int = 1,
+) -> dict[str, Any]:
+    """Apply a determinism preset and return a provenance payload."""
+
+    prof = _coerce_profile(profile)
+    dev = _coerce_device(device)
+    threads_i = max(1, _coerce_int(threads, 1))
+
+    requested = "off"
+    if prof in {"ci", "release"}:
+        requested = "strict"
+
+    env_set: dict[str, Any] = {}
+    torch_flags: dict[str, Any] = {}
+    notes: list[str] = []
+
+    # Thread caps (best-effort): make CPU determinism explicit and reduce drift.
+    if requested == "strict":
+        for var in _THREAD_ENV_VARS:
+            os.environ[var] = str(threads_i)
+            env_set[var] = os.environ.get(var)
+
+    # CUDA determinism: cuBLAS workspace config.
+    if requested == "strict" and dev.startswith("cuda"):
+        os.environ.setdefault("CUBLAS_WORKSPACE_CONFIG", ":16:8")
+        env_set["CUBLAS_WORKSPACE_CONFIG"] = os.environ.get("CUBLAS_WORKSPACE_CONFIG")
+
+    # Seed all RNGs (python/numpy/torch) using the existing helper for parity.
+    set_seed(int(seed))
+
+    # Derive a stable seed bundle for provenance.
+    seed_bundle = {
+        "python": int(seed),
+        "numpy": int(seed),
+        "torch": None,
+    }
+    try:
+        numpy_seed = int(np.random.get_state()[1][0])
+        seed_bundle["numpy"] = int(numpy_seed)
+    except Exception:
+        pass
+    if torch is not None:
+        try:
+            seed_bundle["torch"] = int(torch.initial_seed())
+        except Exception:
+            seed_bundle["torch"] = int(seed)
+
+    # Torch-specific controls.
+    level = "off" if requested == "off" else "strict"
+    if requested == "strict":
+        if torch is None:
+            level = "tolerance"
+            notes.append("torch_unavailable")
+        else:
+            # Thread caps.
+            try:
+                if hasattr(torch, "set_num_threads"):
+                    torch.set_num_threads(threads_i)
+                if hasattr(torch, "set_num_interop_threads"):
+                    torch.set_num_interop_threads(threads_i)
+                torch_flags["torch_threads"] = threads_i
+            except Exception:
+                level = "tolerance"
+                notes.append("torch_thread_caps_failed")
+
+            # Disable TF32 for determinism.
+            try:
+                matmul = getattr(
+                    getattr(torch.backends, "cuda", object()), "matmul", None
+                )
+                if matmul is not None and hasattr(matmul, "allow_tf32"):
+                    matmul.allow_tf32 = False
+                cudnn_mod = getattr(torch.backends, "cudnn", None)
+                if cudnn_mod is not None and hasattr(cudnn_mod, "allow_tf32"):
+                    cudnn_mod.allow_tf32 = False
+            except Exception:
+                level = "tolerance"
+                notes.append("tf32_policy_failed")
+
+            # Deterministic algorithms.
+            try:
+                if hasattr(torch, "use_deterministic_algorithms"):
+                    torch.use_deterministic_algorithms(True, warn_only=False)
+            except Exception:
+                # Downgrade to tolerance-based determinism rather than crashing.
+                level = "tolerance"
+                notes.append("deterministic_algorithms_unavailable")
+                try:
+                    if hasattr(torch, "use_deterministic_algorithms"):
+                        torch.use_deterministic_algorithms(True, warn_only=True)
+                except Exception:
+                    pass
+
+            # cuDNN knobs.
+            try:
+                cudnn_mod = getattr(torch.backends, "cudnn", None)
+                if cudnn_mod is not None:
+                    cudnn_mod.benchmark = False
+                    if hasattr(cudnn_mod, "deterministic"):
+                        cudnn_mod.deterministic = True
+            except Exception:
+                level = "tolerance"
+                notes.append("cudnn_determinism_failed")
+
+            # Snapshot applied flags for provenance.
+            try:
+                det_enabled = getattr(
+                    torch, "are_deterministic_algorithms_enabled", None
+                )
+                if callable(det_enabled):
+                    torch_flags["deterministic_algorithms"] = bool(det_enabled())
+            except Exception:
+                pass
+            try:
+                cudnn_mod = getattr(torch.backends, "cudnn", None)
+                if cudnn_mod is not None:
+                    torch_flags["cudnn_deterministic"] = bool(
+                        getattr(cudnn_mod, "deterministic", False)
+                    )
+                    torch_flags["cudnn_benchmark"] = bool(
+                        getattr(cudnn_mod, "benchmark", False)
+                    )
+                    if hasattr(cudnn_mod, "allow_tf32"):
+                        torch_flags["cudnn_allow_tf32"] = bool(
+                            getattr(cudnn_mod, "allow_tf32", False)
+                        )
+            except Exception:
+                pass
+            try:
+                matmul = getattr(
+                    getattr(torch.backends, "cuda", object()), "matmul", None
+                )
+                if matmul is not None and hasattr(matmul, "allow_tf32"):
+                    torch_flags["cuda_matmul_allow_tf32"] = bool(matmul.allow_tf32)
+            except Exception:
+                pass
+
+    # Normalized level is always one of these.
+    if level not in {"off", "strict", "tolerance"}:
+        level = "tolerance" if requested == "strict" else "off"
+
+    # Extra breadcrumb: random module state is not easily serializable; include a coarse marker.
+    try:
+        torch_flags["python_random"] = isinstance(random.random(), float)
+    except Exception:
+        pass
+
+    payload = {
+        "requested": requested,
+        "level": level,
+        "profile": prof or None,
+        "device": dev,
+        "threads": threads_i if requested == "strict" else None,
+        "seed": int(seed),
+        "seeds": seed_bundle,
+        "env": env_set,
+        "torch": torch_flags,
+        "notes": notes,
+    }
+
+    # Remove empty sections for stable artifacts.
+    if not payload["env"]:
+        payload.pop("env", None)
+    if not payload["torch"]:
+        payload.pop("torch", None)
+    if not payload["notes"]:
+        payload.pop("notes", None)
+    if payload.get("threads") is None:
+        payload.pop("threads", None)
+    if payload.get("profile") is None:
+        payload.pop("profile", None)
+
+    return payload
+
+
+__all__ = ["apply_determinism_preset"]