invarlock 0.3.6__py3-none-any.whl → 0.3.8__py3-none-any.whl

invarlock/plugins/hf_bnb_adapter.py

@@ -7,7 +7,7 @@ through Transformers. Requires GPU for practical use.
 Install with the `gpu` extra on supported platforms.
 
 This adapter handles both:
-1. Fresh quantization of FP16 models (load_in_8bit/load_in_4bit)
+1. Fresh quantization of FP16 models (via quantization_config)
 2. Loading pre-quantized BNB checkpoints (auto-detected via quantization_config)
 """
 
@@ -55,16 +55,17 @@ def _detect_pre_quantized_bnb(model_id: str) -> tuple[bool, int]:
         if not quant_cfg:
             return False, 0
 
-        # Check for BNB quantization
-        quant_method = quant_cfg.get("quant_method", "").lower()
-        if quant_method == "bitsandbytes" or "load_in_8bit" in quant_cfg:
+        # Check for BNB quantization. Prefer explicit bits, then legacy flags.
+        quant_method = str(quant_cfg.get("quant_method", "")).lower()
+        if "bitsandbytes" in quant_method or "bnb" in quant_method:
+            bits = quant_cfg.get("bits")
+            if isinstance(bits, int) and bits in {4, 8}:
+                return True, bits
             if quant_cfg.get("load_in_8bit"):
                 return True, 8
             if quant_cfg.get("load_in_4bit"):
                 return True, 4
-            # Fallback to bits field
-            bits = quant_cfg.get("bits", 8)
-            return True, bits
+            return True, 8
 
     except Exception:
         pass
@@ -82,11 +83,17 @@ class HF_BNB_Adapter(HFAdapterMixin, ModelAdapter):
             "DEPENDENCY-MISSING: transformers",
             lambda e: {"dependency": "transformers"},
         ):
-            from transformers import AutoModelForCausalLM
+            from transformers import AutoModelForCausalLM, BitsAndBytesConfig
 
         # Check if this is a pre-quantized checkpoint
         is_pre_quantized, pre_quant_bits = _detect_pre_quantized_bnb(model_id)
 
+        if "load_in_8bit" in kwargs or "load_in_4bit" in kwargs:
+            raise ValueError(
+                "hf_bnb adapter: load_in_8bit/load_in_4bit are not supported. "
+                "Use model.quantization_config instead."
+            )
+
         if is_pre_quantized:
             # Load pre-quantized checkpoint WITHOUT re-applying quantization
             with wrap_errors(
@@ -99,20 +106,25 @@ class HF_BNB_Adapter(HFAdapterMixin, ModelAdapter):
                     model_id,
                     device_map="auto",
                     trust_remote_code=True,
-                    # Do NOT pass load_in_8bit/load_in_4bit for pre-quantized
-                    **{
-                        k: v
-                        for k, v in kwargs.items()
-                        if k not in ("load_in_8bit", "load_in_4bit")
-                    },
+                    **kwargs,
                 )
         else:
             # Fresh quantization of FP16 model
-            load_in_8bit = bool(kwargs.pop("load_in_8bit", True))
-            load_in_4bit = bool(kwargs.pop("load_in_4bit", False))
-
-            if load_in_4bit:
-                load_in_8bit = False
+            quantization_config = kwargs.pop("quantization_config", None)
+            if quantization_config is None:
+                quantization_config = BitsAndBytesConfig(load_in_8bit=True)
+            elif isinstance(quantization_config, dict):
+                qdict = dict(quantization_config)
+                bits = qdict.pop("bits", None)
+                qdict.pop("quant_method", None)
+                if isinstance(bits, int):
+                    if bits == 4:
+                        qdict.setdefault("load_in_4bit", True)
+                        qdict.setdefault("load_in_8bit", False)
+                    elif bits == 8:
+                        qdict.setdefault("load_in_8bit", True)
+                        qdict.setdefault("load_in_4bit", False)
+                quantization_config = BitsAndBytesConfig(**qdict)
 
             with wrap_errors(
                 ModelLoadError,
@@ -123,9 +135,8 @@ class HF_BNB_Adapter(HFAdapterMixin, ModelAdapter):
                 model = AutoModelForCausalLM.from_pretrained(
                     model_id,
                     device_map="auto",
-                    load_in_8bit=load_in_8bit,
-                    load_in_4bit=load_in_4bit,
                     trust_remote_code=True,
+                    quantization_config=quantization_config,
                     **kwargs,
                 )
 

invarlock/reporting/__init__.py

@@ -1,7 +1,21 @@
 """
-Reporting package (aliasing evaluation reporting modules).
+Evaluation report tooling (`invarlock.reporting`).
 
-Provides a stable unified import path for report schema, builders,
-and certificate generation while keeping backward compatibility with
-`invarlock.eval.*` imports.
+Provides the evaluation report schema, builder, and renderers.
 """
+
+from __future__ import annotations
+
+from .html import render_report_html
+from .render import render_report_markdown
+from .report_builder import make_report, validate_report
+from .report_schema import REPORT_JSON_SCHEMA, REPORT_SCHEMA_VERSION
+
+__all__ = [
+    "REPORT_SCHEMA_VERSION",
+    "REPORT_JSON_SCHEMA",
+    "make_report",
+    "render_report_markdown",
+    "render_report_html",
+    "validate_report",
+]

invarlock/reporting/guards_analysis.py

@@ -23,7 +23,9 @@ def _measurement_contract_digest(contract: Any) -> str | None:
 
 
 @no_type_check
-def _extract_invariants(report: RunReport) -> dict[str, Any]:
+def _extract_invariants(
+    report: RunReport, baseline: RunReport | None = None
+) -> dict[str, Any]:
     """Extract invariant check results (matches the shape used in tests)."""
     invariants_data = (report.get("metrics", {}) or {}).get("invariants", {})
     failures: list[dict[str, Any]] = []
@@ -81,6 +83,108 @@ def _extract_invariants(report: RunReport) -> dict[str, Any]:
             guard_entry = guard
             break
 
+    baseline_guard_entry = None
+    if baseline is not None:
+        for guard in baseline.get("guards", []) or []:
+            if str(guard.get("name", "")).lower() == "invariants":
+                baseline_guard_entry = guard
+                break
+
+    def _coerce_checks(value: Any) -> dict[str, Any] | None:
+        return value if isinstance(value, dict) else None
+
+    def _extract_guard_checks(
+        entry: Any,
+    ) -> tuple[dict[str, Any] | None, dict[str, Any] | None]:
+        if not isinstance(entry, dict):
+            return None, None
+        details = entry.get("details")
+        if not isinstance(details, dict):
+            return None, None
+        return _coerce_checks(details.get("baseline_checks")), _coerce_checks(
+            details.get("current_checks")
+        )
+
+    def _compare_invariants(
+        baseline_checks: dict[str, Any],
+        current_checks: dict[str, Any],
+    ) -> tuple[list[dict[str, Any]], int, int]:
+        violations: list[dict[str, Any]] = []
+
+        # LayerNorm coverage check
+        baseline_layer_norms = set(baseline_checks.get("layer_norm_paths", ()))
+        current_layer_norms = set(current_checks.get("layer_norm_paths", ()))
+        missing_layer_norms = sorted(baseline_layer_norms - current_layer_norms)
+        if missing_layer_norms:
+            violations.append(
+                {
+                    "type": "layer_norm_missing",
+                    "missing": missing_layer_norms,
+                    "message": "Expected LayerNorm modules are missing vs baseline",
+                }
+            )
+
+        # Tokenizer / vocab alignment
+        baseline_vocab_sizes = baseline_checks.get("embedding_vocab_sizes")
+        current_vocab_sizes = current_checks.get("embedding_vocab_sizes")
+        if isinstance(baseline_vocab_sizes, dict):
+            for module_name, baseline_size in baseline_vocab_sizes.items():
+                current_size = None
+                if isinstance(current_vocab_sizes, dict):
+                    current_size = current_vocab_sizes.get(module_name)
+                if current_size is None or int(current_size) != int(baseline_size):
+                    mismatch = {
+                        "module": module_name,
+                        "baseline": int(baseline_size),
+                        "current": None if current_size is None else int(current_size),
+                    }
+                    violations.append(
+                        {
+                            "type": "tokenizer_mismatch",
+                            "message": "Embedding vocabulary size changed vs baseline",
+                            **mismatch,
+                        }
+                    )
+
+        handled_keys = {
+            "layer_norm_paths",
+            "embedding_vocab_sizes",
+            "config_vocab_size",
+        }
+        for check_name, baseline_value in baseline_checks.items():
+            if check_name in handled_keys:
+                continue
+            current_value = current_checks.get(check_name)
+            if current_value != baseline_value:
+                violations.append(
+                    {
+                        "type": "invariant_violation",
+                        "check": check_name,
+                        "baseline": baseline_value,
+                        "current": current_value,
+                        "message": (
+                            f"Invariant {check_name} changed from {baseline_value} to {current_value}"
+                        ),
+                    }
+                )
+
+        fatal_violation_types = {"tokenizer_mismatch"}
+        fatal_count = 0
+        warning_count = 0
+        annotated: list[dict[str, Any]] = []
+        for violation in violations:
+            violation_type = str(violation.get("type") or "")
+            severity = "fatal" if violation_type in fatal_violation_types else "warning"
+            annotated_violation = dict(violation)
+            annotated_violation.setdefault("severity", severity)
+            annotated.append(annotated_violation)
+            if severity == "fatal":
+                fatal_count += 1
+            else:
+                warning_count += 1
+
+        return annotated, fatal_count, warning_count
+
     severity_status = "pass"
     if guard_entry:
         gm = guard_entry.get("metrics", {}) or {}
@@ -108,9 +212,51 @@ def _extract_invariants(report: RunReport) -> dict[str, Any]:
                 if detail:
                     row["detail"] = detail
                 failures.append(row)
-        if fatal_count > 0:
+        base_fatal = 0
+        base_warn = 0
+        baseline_failures: list[dict[str, Any]] = []
+        if baseline_guard_entry is not None:
+            baseline_pre, baseline_post = _extract_guard_checks(baseline_guard_entry)
+            current_pre, current_post = _extract_guard_checks(guard_entry)
+            baseline_snapshot = baseline_pre or baseline_post
+            current_snapshot = current_post or current_pre
+            if isinstance(baseline_snapshot, dict) and isinstance(
+                current_snapshot, dict
+            ):
+                baseline_failures, base_fatal, base_warn = _compare_invariants(
+                    baseline_snapshot, current_snapshot
+                )
+                for violation in baseline_failures:
+                    check_name = violation.get("check")
+                    if not check_name:
+                        check_name = (
+                            violation.get("module")
+                            or violation.get("type")
+                            or "invariant"
+                        )
+                    row = {
+                        "check": str(check_name),
+                        "type": str(violation.get("type") or "violation"),
+                        "severity": str(violation.get("severity") or "warning"),
+                    }
+                    detail = {k: v for k, v in violation.items() if k not in row}
+                    if detail:
+                        detail.setdefault("source", "baseline_compare")
+                        row["detail"] = detail
+                    failures.append(row)
+
+        fatal_total = fatal_count + base_fatal
+        warn_total = warning_count + base_warn
+        try:
+            summary["fatal_violations"] = fatal_total
+            summary["warning_violations"] = warn_total
+            summary["violations_found"] = fatal_total + warn_total
+        except Exception:
+            pass
+
+        if fatal_total > 0:
             severity_status = "fail"
-        elif warning_count > 0 or violations:
+        elif warn_total > 0 or violations:
             severity_status = "warn"
 
     # If any error-severity entry exists among failures, escalate to fail
@@ -130,12 +276,16 @@ def _extract_invariants(report: RunReport) -> dict[str, Any]:
             "warning_violations": len(failures),
         }
 
+    details_out = invariants_data
+    if not details_out and guard_entry and isinstance(guard_entry.get("details"), dict):
+        details_out = guard_entry.get("details", {})
+
     return {
         "pre": "pass",
         "post": status,
         "status": status,
         "summary": summary,
-        "details": invariants_data,
+        "details": details_out,
         "failures": failures,
     }
 

invarlock/reporting/html.py

@@ -1,5 +1,5 @@
 """
-Minimal HTML exporter for certificates.
+Minimal HTML exporter for reports.
 
 This implementation wraps the Markdown rendering in a simple HTML template so
 that the numbers and core content remain identical across formats.
@@ -10,23 +10,73 @@ from __future__ import annotations
 from html import escape
 from typing import Any
 
-from .render import render_certificate_markdown
+from .render import render_report_markdown
 
+markdown_module: Any | None = None
+try:
+    import markdown as _markdown  # type: ignore[import-untyped]
+except Exception:  # pragma: no cover - optional dependency
+    _markdown = None
+else:
+    markdown_module = _markdown
 
-def render_certificate_html(certificate: dict[str, Any]) -> str:
-    """Render a certificate as a simple HTML document.
 
-    Uses the Markdown renderer and embeds the content in a <pre> block to ensure
-    stable parity for snapshot tests without extra dependencies.
+_STATUS_BADGES = {
+    "\u2705 PASS": '<span class="badge pass">PASS</span>',
+    "\u2705 OK": '<span class="badge pass">OK</span>',
+    "\u274c FAIL": '<span class="badge fail">FAIL</span>',
+    "\u26a0\ufe0f WARN": '<span class="badge warn">WARN</span>',
+    "\u26a0 WARN": '<span class="badge warn">WARN</span>',
+}
+
+
+def _apply_status_badges(html_body: str) -> str:
+    updated = html_body
+    for token, replacement in _STATUS_BADGES.items():
+        updated = updated.replace(token, replacement)
+    return updated
+
+
+def render_report_html(evaluation_report: dict[str, Any]) -> str:
+    """Render an evaluation report as a simple HTML document.
+
+    Uses the Markdown renderer and converts to HTML when available, falling back
+    to a <pre> block when the markdown dependency is missing.
     """
-    md = render_certificate_markdown(certificate)
-    body = f'<pre class="invarlock-md">{escape(md)}</pre>'
+    md = render_report_markdown(evaluation_report)
+    if markdown_module is None:
+        body = f'<pre class="invarlock-md">{escape(md)}</pre>'
+    else:
+        html_body = markdown_module.markdown(md, extensions=["tables", "fenced_code"])
+        html_body = _apply_status_badges(html_body)
+        body = f'<div class="invarlock-md">{html_body}</div>'
     return (
         '<!DOCTYPE html><html><head><meta charset="utf-8">'
-        "<title>InvarLock Safety Certificate</title>"
-        "<style>body{font-family:ui-monospace,Menlo,monospace;white-space:pre-wrap}</style>"
+        "<title>InvarLock Evaluation Report</title>"
+        "<style>"
+        ":root{--pass:#2da44e;--fail:#cf222e;--warn:#bf8700;--ink:#1f2328;"
+        "--muted:#57606a;--panel:#f6f8fa;--border:#d0d7de}"
+        "body{font-family:ui-sans-serif,system-ui,-apple-system,Segoe UI,sans-serif;"
+        "color:var(--ink);background:linear-gradient(180deg,#fff, #f6f8fa);"
+        "margin:0;padding:32px}"
+        ".invarlock-md{max-width:960px;margin:0 auto;padding:24px;background:#fff;"
+        "border:1px solid var(--border);border-radius:16px;box-shadow:0 10px 30px rgba(0,0,0,0.05)}"
+        "h1,h2,h3{margin-top:1.4em}h1{margin-top:0}"
+        "table{border-collapse:collapse;width:100%;margin:12px 0}"
+        "th,td{border:1px solid var(--border);padding:6px 8px;text-align:left}"
+        "code,pre{background:var(--panel);border-radius:8px}"
+        "pre{padding:12px;overflow:auto}"
+        ".badge{display:inline-block;padding:2px 8px;border-radius:999px;"
+        "font-size:0.75rem;font-weight:700;letter-spacing:0.02em;color:#fff}"
+        ".badge.pass{background:var(--pass)}"
+        ".badge.fail{background:var(--fail)}"
+        ".badge.warn{background:var(--warn)}"
+        "@media print{body{background:#fff;padding:0}.invarlock-md{box-shadow:none;"
+        "border:0}a{color:inherit;text-decoration:none}.badge{color:#000;"
+        "border:1px solid #000;background:transparent}}"
+        "</style>"
         "</head><body>" + body + "</body></html>"
     )
 
 
-__all__ = ["render_certificate_html"]
+__all__ = ["render_report_html"]

invarlock/reporting/normalizer.py

@@ -33,7 +33,7 @@ def normalize_run_report(report: Mapping[str, Any] | RunReport) -> RunReport:
     """Coerce an arbitrary report-like mapping into a canonical RunReport.
 
     This is the single entry point for converting pre-canonical or loosely-typed
-    data into the strict PM-only RunReport shape used by certificate/report.
+    data into the strict PM-only RunReport shape used by evaluation report building.
     """
     src = _as_mapping(report)
 
@@ -53,8 +53,10 @@ def normalize_run_report(report: Mapping[str, Any] | RunReport) -> RunReport:
         "ts": ts,
         "auto": meta_in.get("auto") if isinstance(meta_in.get("auto"), dict) else None,
     }
-    # Preserve additional provenance knobs used by certificate/digests.
+    # Preserve additional provenance knobs used by evaluation report digests.
     for key in (
+        "pm_acceptance_range",
+        "pm_drift_band",
         "policy_overrides",
         "overrides",
         "plugins",
@@ -179,6 +181,11 @@ def normalize_run_report(report: Mapping[str, Any] | RunReport) -> RunReport:
         "latency_ms_p50",
         "latency_ms_p95",
         "memory_mb_peak",
+        "gpu_memory_mb_peak",
+        "gpu_memory_reserved_mb_peak",
+        "timings",
+        "guard_timings",
+        "memory_snapshots",
         "throughput_sps",
         "spectral",
         "rmt",

invarlock/reporting/policy_utils.py

@@ -35,7 +35,7 @@ def _compute_thresholds_payload(
     tier: str, resolved_policy: dict[str, Any]
 ) -> dict[str, Any]:
     """Build canonical thresholds payload for digest stability."""
-    from .certificate import TIER_RATIO_LIMITS  # local to avoid cycles
+    from .report_builder import TIER_RATIO_LIMITS  # local to avoid cycles
 
     tier_lc = (tier or "balanced").lower()
     metrics_policy = (

invarlock/reporting/primary_metric_utils.py

@@ -8,21 +8,21 @@ from .utils import _coerce_interval, _weighted_mean
 
 
 def attach_primary_metric(
-    certificate: dict[str, Any],
+    evaluation_report: dict[str, Any],
     report: dict[str, Any],
     baseline_raw: dict[str, Any] | None,
     baseline_ref: dict[str, Any] | None,
     ppl_analysis: dict[str, Any] | None,
 ) -> None:
-    """Attach/normalize the primary_metric block on the certificate.
+    """Attach/normalize the primary_metric block on the evaluation report.
 
-    Behavior mirrors historical logic in certificate.py and preserves structure:
+    Behavior mirrors historical logic in report_builder.py and preserves structure:
     - Prefer explicit metrics.primary_metric if present
     - Compute missing ratio_vs_baseline, degenerate display_ci
     - ppl window-based analysis info (mean logloss) added when available
     - Fallbacks for classification metrics and eval-window-derived ppl
     - Ensure display_ci always present for schema invariants
-    Mutates the certificate in-place.
+    Mutates the evaluation report in-place.
     """
     # Attach primary metric snapshot when provided in report
     try:
@@ -180,12 +180,12 @@ def attach_primary_metric(
                     ]
             except Exception:
                 pass
-            certificate["primary_metric"] = pm_copy
+            evaluation_report["primary_metric"] = pm_copy
     except Exception:
         pass
 
     def _attach_from_windows() -> None:
-        if isinstance(certificate.get("primary_metric"), dict):
+        if isinstance(evaluation_report.get("primary_metric"), dict):
             return
         try:
             m = (
@@ -212,7 +212,7 @@ def attach_primary_metric(
                 baseline=baseline_raw if isinstance(baseline_raw, dict) else None,
             )
             if isinstance(pm_block, dict) and pm_block:
-                certificate["primary_metric"] = pm_block
+                evaluation_report["primary_metric"] = pm_block
         except Exception:
             pass
 
@@ -220,7 +220,7 @@ def attach_primary_metric(
     _attach_from_windows()
 
     # Minimal fallback for classification-only reports without explicit primary_metric
-    if not isinstance(certificate.get("primary_metric"), dict):
+    if not isinstance(evaluation_report.get("primary_metric"), dict):
         try:
             metrics_map = report.get("metrics", {}) if isinstance(report, dict) else {}
             clf = (
@@ -298,7 +298,7 @@ def attach_primary_metric(
                         acc_pm["ratio_vs_baseline"] = delta_pp
                 except Exception:
                     pass
-                certificate["primary_metric"] = acc_pm
+                evaluation_report["primary_metric"] = acc_pm
         except Exception:
             pass
 
@@ -308,8 +308,8 @@ def attach_primary_metric(
     # Ensure primary_metric has display_ci populated for schema invariants
     try:
         pm = (
-            certificate.get("primary_metric", {})
-            if isinstance(certificate.get("primary_metric"), dict)
+            evaluation_report.get("primary_metric", {})
+            if isinstance(evaluation_report.get("primary_metric"), dict)
             else None
         )
         if isinstance(pm, dict) and pm: