invarlock 0.3.6__py3-none-any.whl → 0.3.7__py3-none-any.whl

invarlock/reporting/certificate.py

@@ -1,8 +1,9 @@
 """
-InvarLock Safety Certificate Generation
-==================================
+InvarLock Evaluation Certificate Generation
+==========================================
 
-Generate standardized safety certificates from RunReport and baseline comparison.
+Generate standardized evaluation certificates from RunReport and baseline
+comparison.
 Certificates are standalone, portable verification artifacts that can be used
 for CI/CD gates and regulatory compliance.
 """
@@ -743,7 +744,7 @@ def make_certificate(
     baseline: RunReport | dict[str, Any],
 ) -> dict[str, Any]:
     """
-    Generate a safety certificate from a RunReport and baseline comparison.
+    Generate an evaluation certificate from a RunReport and baseline comparison.
 
     The certificate is a standalone, portable artifact that contains all
     essential metrics and comparisons needed for safety verification.
@@ -764,6 +765,17 @@ def make_certificate(
     # Normalize baseline input
     baseline_raw = baseline
     baseline_normalized = _normalize_baseline(baseline_raw)
+    baseline_report: RunReport | None = None
+    try:
+        if (
+            isinstance(baseline_raw, dict)
+            and "meta" in baseline_raw
+            and "metrics" in baseline_raw
+            and "edit" in baseline_raw
+        ):
+            baseline_report = _normalize_and_validate_report(baseline_raw)
+    except Exception:  # pragma: no cover - baseline compare is best-effort
+        baseline_report = None
 
     # Extract core metadata with full seed bundle
     meta = _extract_certificate_meta(report)
@@ -1440,7 +1452,7 @@ def make_certificate(
         ppl_analysis["window_plan"] = window_plan_ctx
 
     # Extract invariant status
-    invariants = _extract_invariants(report)
+    invariants = _extract_invariants(report, baseline=baseline_report)
 
     # Extract spectral analysis
     spectral = _extract_spectral_analysis(report, baseline_normalized)
@@ -1558,7 +1570,13 @@ def make_certificate(
     telemetry: dict[str, Any] = {}
     metrics_section = report.get("metrics", {})
     if isinstance(metrics_section, dict):
-        for key in ("latency_ms_per_tok", "memory_mb_peak", "throughput_tok_per_s"):
+        for key in (
+            "latency_ms_per_tok",
+            "memory_mb_peak",
+            "gpu_memory_mb_peak",
+            "gpu_memory_reserved_mb_peak",
+            "throughput_tok_per_s",
+        ):
             value = metrics_section.get(key)
             if isinstance(value, int | float) and math.isfinite(value):
                 telemetry[key] = float(value)
@@ -1755,6 +1773,7 @@ def make_certificate(
         capacity_examples = None
 
     pm_acceptance_range = _resolve_pm_acceptance_range_from_report(report)
+    pm_drift_band = _resolve_pm_drift_band_from_report(report)
 
     # Primary metric tail evidence and gate evaluation (ΔlogNLL vs baseline, per-window).
     pm_tail_result: dict[str, Any] = {}
@@ -1881,6 +1900,12 @@ def make_certificate(
     except Exception:  # pragma: no cover - defensive against patched functions
         validation_kwargs["pm_acceptance_range"] = pm_acceptance_range
 
+    try:
+        if "pm_drift_band" in inspect.signature(_compute_validation_flags).parameters:
+            validation_kwargs["pm_drift_band"] = pm_drift_band
+    except Exception:  # pragma: no cover - defensive against patched functions
+        validation_kwargs["pm_drift_band"] = pm_drift_band
+
     try:
         if "pm_tail" in inspect.signature(_compute_validation_flags).parameters:
             validation_kwargs["pm_tail"] = pm_tail_result
@@ -2176,6 +2201,13 @@ def make_certificate(
     from .primary_metric_utils import attach_primary_metric as _attach_pm
 
     _attach_pm(certificate, report, baseline_raw, baseline_ref, ppl_analysis)
+    try:
+        if isinstance(pm_drift_band, dict) and pm_drift_band:
+            pm_block = certificate.get("primary_metric")
+            if isinstance(pm_block, dict):
+                pm_block.setdefault("drift_band", dict(pm_drift_band))
+    except Exception:  # pragma: no cover
+        pass
     _enforce_display_ci_alignment(
         ratio_ci_source,
         certificate.get("primary_metric"),
@@ -2614,7 +2646,7 @@ def _extract_edit_metadata(
         alg_lower = str(algorithm).strip().lower()
     except Exception:  # pragma: no cover
         alg_lower = ""
-    allowed_algorithms = {"quant_rtn", "noop"}
+    allowed_algorithms = {"quant_rtn", "noop", "custom"}
     if alg_lower not in allowed_algorithms:
         algorithm = ""
 
@@ -3225,6 +3257,105 @@ def _resolve_pm_acceptance_range_from_report(
     return {"min": float(min_val), "max": float(max_val)}
 
 
+def _resolve_pm_drift_band_from_report(
+    report: dict[str, Any] | None,
+) -> dict[str, float]:
+    """Resolve preview→final drift band from report context/meta/env."""
+
+    base_min = 0.95
+    base_max = 1.05
+
+    def _safe_float(val: Any) -> float | None:
+        try:
+            if val is None:
+                return None
+            out = float(val)
+        except Exception:
+            return None
+        return out if math.isfinite(out) else None
+
+    cfg_min = None
+    cfg_max = None
+
+    ctx = report.get("context") if isinstance(report, dict) else None
+    if isinstance(ctx, dict):
+        pm_ctx = ctx.get("primary_metric")
+        if isinstance(pm_ctx, dict):
+            band = pm_ctx.get("drift_band")
+            if isinstance(band, dict):
+                cfg_min = _safe_float(band.get("min"))
+                cfg_max = _safe_float(band.get("max"))
+            elif isinstance(band, list | tuple) and len(band) == 2:
+                cfg_min = _safe_float(band[0])
+                cfg_max = _safe_float(band[1])
+        if cfg_min is None or cfg_max is None:
+            alt = ctx.get("pm_drift_band")
+            if isinstance(alt, dict):
+                cfg_min = (
+                    cfg_min if cfg_min is not None else _safe_float(alt.get("min"))
+                )
+                cfg_max = (
+                    cfg_max if cfg_max is not None else _safe_float(alt.get("max"))
+                )
+
+    if (cfg_min is None or cfg_max is None) and isinstance(report, dict):
+        meta = report.get("meta")
+        if isinstance(meta, dict):
+            meta_band = meta.get("pm_drift_band")
+            if isinstance(meta_band, dict):
+                cfg_min = (
+                    cfg_min
+                    if cfg_min is not None
+                    else _safe_float(meta_band.get("min"))
+                )
+                cfg_max = (
+                    cfg_max
+                    if cfg_max is not None
+                    else _safe_float(meta_band.get("max"))
+                )
+
+    def _parse_env(name: str) -> float | None:
+        try:
+            raw = os.environ.get(name, "")
+            if raw is None or str(raw).strip() == "":
+                return None
+            return float(raw)
+        except Exception:
+            return None
+
+    env_min = _parse_env("INVARLOCK_PM_DRIFT_MIN")
+    env_max = _parse_env("INVARLOCK_PM_DRIFT_MAX")
+
+    has_explicit = any(v is not None for v in (cfg_min, cfg_max, env_min, env_max))
+    if not has_explicit:
+        return {}
+
+    min_val = (
+        env_min if env_min is not None else cfg_min if cfg_min is not None else base_min
+    )
+    max_val = (
+        env_max if env_max is not None else cfg_max if cfg_max is not None else base_max
+    )
+
+    try:
+        if min_val is not None and min_val <= 0:
+            min_val = base_min
+    except Exception:
+        min_val = base_min
+    try:
+        if max_val is not None and max_val <= 0:
+            max_val = base_max
+    except Exception:
+        max_val = base_max
+    try:
+        if min_val is not None and max_val is not None and min_val >= max_val:
+            min_val, max_val = base_min, base_max
+    except Exception:
+        min_val, max_val = base_min, base_max
+
+    return {"min": float(min_val), "max": float(max_val)}
+
+
 def _compute_validation_flags(
     ppl: dict[str, Any],
     spectral: dict[str, Any],
@@ -3238,6 +3369,7 @@ def _compute_validation_flags(
     moe: dict[str, Any] | None = None,
     dataset_capacity: dict[str, Any] | None = None,
     pm_acceptance_range: dict[str, float] | None = None,
+    pm_drift_band: dict[str, float] | None = None,
     pm_tail: dict[str, Any] | None = None,
 ) -> dict[str, bool]:
     """Compute validation flags for the certificate including canonical gates."""
@@ -3301,9 +3433,27 @@ def _compute_validation_flags(
         ratio_limit = min(ratio_limit, float(target_ratio))
 
     # Canonical Gates
-    # 1. Drift gate: 0.95 ≤ final/preview ≤ 1.05
+    # 1. Drift gate: by default 0.95 ≤ final/preview ≤ 1.05 (configurable)
     drift_ratio = ppl.get("preview_final_ratio", 1.0)
-    preview_final_drift_acceptable = 0.95 <= drift_ratio <= 1.05
+    drift_min = 0.95
+    drift_max = 1.05
+    if isinstance(pm_drift_band, dict):
+        try:
+            cand_min = pm_drift_band.get("min")
+            cand_max = pm_drift_band.get("max")
+            if isinstance(cand_min, int | float) and isinstance(cand_max, int | float):
+                cand_min_f = float(cand_min)
+                cand_max_f = float(cand_max)
+                if (
+                    math.isfinite(cand_min_f)
+                    and math.isfinite(cand_max_f)
+                    and 0 < cand_min_f < cand_max_f
+                ):
+                    drift_min = cand_min_f
+                    drift_max = cand_max_f
+        except Exception:  # pragma: no cover
+            pass
+    preview_final_drift_acceptable = drift_min <= drift_ratio <= drift_max
     if _tiny_relax:
         # Treat drift identity as informational in tiny dev demos
         preview_final_drift_acceptable = True

invarlock/reporting/certificate_schema.py

@@ -20,7 +20,7 @@ CERTIFICATE_SCHEMA_VERSION = "v1"
 # separately in metric-specific logic.
 CERTIFICATE_JSON_SCHEMA: dict[str, Any] = {
     "$schema": "https://json-schema.org/draft/2020-12/schema",
-    "title": "InvarLock Safety Certificate",
+    "title": "InvarLock Evaluation Certificate",
     "type": "object",
     "required": [
         "schema_version",

invarlock/reporting/guards_analysis.py

@@ -23,7 +23,9 @@ def _measurement_contract_digest(contract: Any) -> str | None:
 
 
 @no_type_check
-def _extract_invariants(report: RunReport) -> dict[str, Any]:
+def _extract_invariants(
+    report: RunReport, baseline: RunReport | None = None
+) -> dict[str, Any]:
     """Extract invariant check results (matches the shape used in tests)."""
     invariants_data = (report.get("metrics", {}) or {}).get("invariants", {})
     failures: list[dict[str, Any]] = []
@@ -81,6 +83,108 @@ def _extract_invariants(report: RunReport) -> dict[str, Any]:
             guard_entry = guard
             break
 
+    baseline_guard_entry = None
+    if baseline is not None:
+        for guard in baseline.get("guards", []) or []:
+            if str(guard.get("name", "")).lower() == "invariants":
+                baseline_guard_entry = guard
+                break
+
+    def _coerce_checks(value: Any) -> dict[str, Any] | None:
+        return value if isinstance(value, dict) else None
+
+    def _extract_guard_checks(
+        entry: Any,
+    ) -> tuple[dict[str, Any] | None, dict[str, Any] | None]:
+        if not isinstance(entry, dict):
+            return None, None
+        details = entry.get("details")
+        if not isinstance(details, dict):
+            return None, None
+        return _coerce_checks(details.get("baseline_checks")), _coerce_checks(
+            details.get("current_checks")
+        )
+
+    def _compare_invariants(
+        baseline_checks: dict[str, Any],
+        current_checks: dict[str, Any],
+    ) -> tuple[list[dict[str, Any]], int, int]:
+        violations: list[dict[str, Any]] = []
+
+        # LayerNorm coverage check
+        baseline_layer_norms = set(baseline_checks.get("layer_norm_paths", ()))
+        current_layer_norms = set(current_checks.get("layer_norm_paths", ()))
+        missing_layer_norms = sorted(baseline_layer_norms - current_layer_norms)
+        if missing_layer_norms:
+            violations.append(
+                {
+                    "type": "layer_norm_missing",
+                    "missing": missing_layer_norms,
+                    "message": "Expected LayerNorm modules are missing vs baseline",
+                }
+            )
+
+        # Tokenizer / vocab alignment
+        baseline_vocab_sizes = baseline_checks.get("embedding_vocab_sizes")
+        current_vocab_sizes = current_checks.get("embedding_vocab_sizes")
+        if isinstance(baseline_vocab_sizes, dict):
+            for module_name, baseline_size in baseline_vocab_sizes.items():
+                current_size = None
+                if isinstance(current_vocab_sizes, dict):
+                    current_size = current_vocab_sizes.get(module_name)
+                if current_size is None or int(current_size) != int(baseline_size):
+                    mismatch = {
+                        "module": module_name,
+                        "baseline": int(baseline_size),
+                        "current": None if current_size is None else int(current_size),
+                    }
+                    violations.append(
+                        {
+                            "type": "tokenizer_mismatch",
+                            "message": "Embedding vocabulary size changed vs baseline",
+                            **mismatch,
+                        }
+                    )
+
+        handled_keys = {
+            "layer_norm_paths",
+            "embedding_vocab_sizes",
+            "config_vocab_size",
+        }
+        for check_name, baseline_value in baseline_checks.items():
+            if check_name in handled_keys:
+                continue
+            current_value = current_checks.get(check_name)
+            if current_value != baseline_value:
+                violations.append(
+                    {
+                        "type": "invariant_violation",
+                        "check": check_name,
+                        "baseline": baseline_value,
+                        "current": current_value,
+                        "message": (
+                            f"Invariant {check_name} changed from {baseline_value} to {current_value}"
+                        ),
+                    }
+                )
+
+        fatal_violation_types = {"tokenizer_mismatch"}
+        fatal_count = 0
+        warning_count = 0
+        annotated: list[dict[str, Any]] = []
+        for violation in violations:
+            violation_type = str(violation.get("type") or "")
+            severity = "fatal" if violation_type in fatal_violation_types else "warning"
+            annotated_violation = dict(violation)
+            annotated_violation.setdefault("severity", severity)
+            annotated.append(annotated_violation)
+            if severity == "fatal":
+                fatal_count += 1
+            else:
+                warning_count += 1
+
+        return annotated, fatal_count, warning_count
+
     severity_status = "pass"
     if guard_entry:
         gm = guard_entry.get("metrics", {}) or {}
@@ -108,9 +212,51 @@ def _extract_invariants(report: RunReport) -> dict[str, Any]:
                 if detail:
                     row["detail"] = detail
                 failures.append(row)
-        if fatal_count > 0:
+        base_fatal = 0
+        base_warn = 0
+        baseline_failures: list[dict[str, Any]] = []
+        if baseline_guard_entry is not None:
+            baseline_pre, baseline_post = _extract_guard_checks(baseline_guard_entry)
+            current_pre, current_post = _extract_guard_checks(guard_entry)
+            baseline_snapshot = baseline_pre or baseline_post
+            current_snapshot = current_post or current_pre
+            if isinstance(baseline_snapshot, dict) and isinstance(
+                current_snapshot, dict
+            ):
+                baseline_failures, base_fatal, base_warn = _compare_invariants(
+                    baseline_snapshot, current_snapshot
+                )
+                for violation in baseline_failures:
+                    check_name = violation.get("check")
+                    if not check_name:
+                        check_name = (
+                            violation.get("module")
+                            or violation.get("type")
+                            or "invariant"
+                        )
+                    row = {
+                        "check": str(check_name),
+                        "type": str(violation.get("type") or "violation"),
+                        "severity": str(violation.get("severity") or "warning"),
+                    }
+                    detail = {k: v for k, v in violation.items() if k not in row}
+                    if detail:
+                        detail.setdefault("source", "baseline_compare")
+                        row["detail"] = detail
+                    failures.append(row)
+
+        fatal_total = fatal_count + base_fatal
+        warn_total = warning_count + base_warn
+        try:
+            summary["fatal_violations"] = fatal_total
+            summary["warning_violations"] = warn_total
+            summary["violations_found"] = fatal_total + warn_total
+        except Exception:
+            pass
+
+        if fatal_total > 0:
             severity_status = "fail"
-        elif warning_count > 0 or violations:
+        elif warn_total > 0 or violations:
             severity_status = "warn"
 
     # If any error-severity entry exists among failures, escalate to fail
@@ -130,12 +276,16 @@ def _extract_invariants(report: RunReport) -> dict[str, Any]:
             "warning_violations": len(failures),
         }
 
+    details_out = invariants_data
+    if not details_out and guard_entry and isinstance(guard_entry.get("details"), dict):
+        details_out = guard_entry.get("details", {})
+
     return {
         "pre": "pass",
         "post": status,
         "status": status,
         "summary": summary,
-        "details": invariants_data,
+        "details": details_out,
         "failures": failures,
     }
 

invarlock/reporting/html.py

@@ -12,19 +12,69 @@ from typing import Any
 
 from .render import render_certificate_markdown
 
+markdown_module: Any | None = None
+try:
+    import markdown as _markdown  # type: ignore[import-untyped]
+except Exception:  # pragma: no cover - optional dependency
+    _markdown = None
+else:
+    markdown_module = _markdown
+
+
+_STATUS_BADGES = {
+    "\u2705 PASS": '<span class="badge pass">PASS</span>',
+    "\u2705 OK": '<span class="badge pass">OK</span>',
+    "\u274c FAIL": '<span class="badge fail">FAIL</span>',
+    "\u26a0\ufe0f WARN": '<span class="badge warn">WARN</span>',
+    "\u26a0 WARN": '<span class="badge warn">WARN</span>',
+}
+
+
+def _apply_status_badges(html_body: str) -> str:
+    updated = html_body
+    for token, replacement in _STATUS_BADGES.items():
+        updated = updated.replace(token, replacement)
+    return updated
+
 
 def render_certificate_html(certificate: dict[str, Any]) -> str:
     """Render a certificate as a simple HTML document.
 
-    Uses the Markdown renderer and embeds the content in a <pre> block to ensure
-    stable parity for snapshot tests without extra dependencies.
+    Uses the Markdown renderer and converts to HTML when available, falling back
+    to a <pre> block when the markdown dependency is missing.
     """
     md = render_certificate_markdown(certificate)
-    body = f'<pre class="invarlock-md">{escape(md)}</pre>'
+    if markdown_module is None:
+        body = f'<pre class="invarlock-md">{escape(md)}</pre>'
+    else:
+        html_body = markdown_module.markdown(md, extensions=["tables", "fenced_code"])
+        html_body = _apply_status_badges(html_body)
+        body = f'<div class="invarlock-md">{html_body}</div>'
     return (
         '<!DOCTYPE html><html><head><meta charset="utf-8">'
-        "<title>InvarLock Safety Certificate</title>"
-        "<style>body{font-family:ui-monospace,Menlo,monospace;white-space:pre-wrap}</style>"
+        "<title>InvarLock Evaluation Certificate</title>"
+        "<style>"
+        ":root{--pass:#2da44e;--fail:#cf222e;--warn:#bf8700;--ink:#1f2328;"
+        "--muted:#57606a;--panel:#f6f8fa;--border:#d0d7de}"
+        "body{font-family:ui-sans-serif,system-ui,-apple-system,Segoe UI,sans-serif;"
+        "color:var(--ink);background:linear-gradient(180deg,#fff, #f6f8fa);"
+        "margin:0;padding:32px}"
+        ".invarlock-md{max-width:960px;margin:0 auto;padding:24px;background:#fff;"
+        "border:1px solid var(--border);border-radius:16px;box-shadow:0 10px 30px rgba(0,0,0,0.05)}"
+        "h1,h2,h3{margin-top:1.4em}h1{margin-top:0}"
+        "table{border-collapse:collapse;width:100%;margin:12px 0}"
+        "th,td{border:1px solid var(--border);padding:6px 8px;text-align:left}"
+        "code,pre{background:var(--panel);border-radius:8px}"
+        "pre{padding:12px;overflow:auto}"
+        ".badge{display:inline-block;padding:2px 8px;border-radius:999px;"
+        "font-size:0.75rem;font-weight:700;letter-spacing:0.02em;color:#fff}"
+        ".badge.pass{background:var(--pass)}"
+        ".badge.fail{background:var(--fail)}"
+        ".badge.warn{background:var(--warn)}"
+        "@media print{body{background:#fff;padding:0}.invarlock-md{box-shadow:none;"
+        "border:0}a{color:inherit;text-decoration:none}.badge{color:#000;"
+        "border:1px solid #000;background:transparent}}"
+        "</style>"
         "</head><body>" + body + "</body></html>"
     )
 

invarlock/reporting/normalizer.py

@@ -55,6 +55,8 @@ def normalize_run_report(report: Mapping[str, Any] | RunReport) -> RunReport:
     }
     # Preserve additional provenance knobs used by certificate/digests.
     for key in (
+        "pm_acceptance_range",
+        "pm_drift_band",
         "policy_overrides",
         "overrides",
         "plugins",
@@ -179,6 +181,11 @@ def normalize_run_report(report: Mapping[str, Any] | RunReport) -> RunReport:
         "latency_ms_p50",
         "latency_ms_p95",
         "memory_mb_peak",
+        "gpu_memory_mb_peak",
+        "gpu_memory_reserved_mb_peak",
+        "timings",
+        "guard_timings",
+        "memory_snapshots",
         "throughput_sps",
         "spectral",
         "rmt",