invarlock 0.2.0__py3-none-any.whl

invarlock/reporting/certificate_schema.py

@@ -0,0 +1,244 @@
+from __future__ import annotations
+
+import json
+from pathlib import Path
+from typing import Any
+
+# Optional JSON Schema validation support (best-effort)
+try:  # pragma: no cover - exercised in integration
+    import jsonschema
+except Exception:  # pragma: no cover
+    jsonschema = None
+
+
+# Certificate schema version (PM-first canonical)
+CERTIFICATE_SCHEMA_VERSION = "v1"
+
+
+# Minimal JSON Schema describing the canonical shape of a certificate.
+# This focuses on structural validity; numerical thresholds are validated
+# separately in metric-specific logic.
+CERTIFICATE_JSON_SCHEMA: dict[str, Any] = {
+    "$schema": "https://json-schema.org/draft/2020-12/schema",
+    "title": "InvarLock Safety Certificate",
+    "type": "object",
+    "required": [
+        "schema_version",
+        "run_id",
+        "artifacts",
+        "plugins",
+        "meta",
+        "dataset",
+    ],
+    "properties": {
+        "schema_version": {"const": CERTIFICATE_SCHEMA_VERSION},
+        "run_id": {"type": "string", "minLength": 4},
+        "edit_name": {"type": "string"},
+        "policy_digest": {
+            "type": "object",
+            "properties": {
+                "policy_version": {"type": "string"},
+                "tier_policy_name": {"type": "string"},
+                "thresholds_hash": {"type": "string"},
+                "hysteresis": {"type": "object"},
+                "min_effective": {"type": "number"},
+                "changed": {"type": "boolean"},
+            },
+            "additionalProperties": True,
+        },
+        "plugins": {
+            "type": "object",
+            "properties": {
+                "adapters": {"type": "array"},
+                "edits": {"type": "array"},
+                "guards": {"type": "array"},
+            },
+            "additionalProperties": True,
+        },
+        "meta": {"type": "object"},
+        "dataset": {
+            "type": "object",
+            "required": ["provider", "seq_len", "windows"],
+            "properties": {
+                "provider": {"type": "string"},
+                "seq_len": {"type": "integer", "minimum": 1},
+                "windows": {
+                    "type": "object",
+                    "required": ["preview", "final"],
+                    "properties": {
+                        "preview": {"type": "integer", "minimum": 0},
+                        "final": {"type": "integer", "minimum": 0},
+                        "seed": {"type": "integer"},
+                    },
+                },
+            },
+            "additionalProperties": True,
+        },
+        # ppl_* block removed from required schema; may appear for ppl-like tasks but is optional
+        "primary_metric": {
+            "type": "object",
+            "properties": {
+                "kind": {"type": "string"},
+                "unit": {"type": "string"},
+                "direction": {"type": "string"},
+                "aggregation_scope": {"type": "string"},
+                "paired": {"type": "boolean"},
+                "gating_basis": {"type": "string"},
+                "preview": {"type": "number"},
+                "final": {"type": "number"},
+                "ratio_vs_baseline": {"type": "number"},
+                "reps": {"type": "number"},
+                "ci_level": {"type": "number"},
+                "counts_source": {"enum": ["measured", "pseudo_config"]},
+                "estimated": {"type": "boolean"},
+                "ci": {
+                    "type": "array",
+                    "minItems": 2,
+                    "maxItems": 2,
+                    "items": {"type": "number"},
+                },
+                "display_ci": {
+                    "type": "array",
+                    "minItems": 2,
+                    "maxItems": 2,
+                    "items": {"type": "number"},
+                },
+            },
+            "additionalProperties": True,
+        },
+        "system_overhead": {
+            "type": "object",
+            # Numeric keys must match these patterns when present; allow flexibility otherwise
+            "patternProperties": {
+                "^latency_ms_(p50|p95)$": {"type": "number"},
+                "^throughput_.*$": {"type": "number"},
+            },
+            "additionalProperties": True,
+        },
+        "validation": {
+            "type": "object",
+            # properties populated at import-time from allow-list; default permissive
+            "properties": {},
+            "additionalProperties": {"type": "boolean"},
+        },
+        "artifacts": {"type": "object"},
+        "provenance": {"type": "object"},
+        "resolved_policy": {"type": "object"},
+        "policy_provenance": {"type": "object"},
+        "structure": {"type": "object"},
+        "confidence": {
+            "type": "object",
+            "properties": {
+                "label": {"enum": ["High", "Medium", "Low"]},
+                "basis": {"type": "string"},
+                "width": {"type": "number"},
+                "threshold": {"type": "number"},
+                "unstable": {"type": "boolean"},
+            },
+            "required": ["label", "basis"],
+            "additionalProperties": True,
+        },
+    },
+    "additionalProperties": True,
+}
+
+
+_VALIDATION_ALLOWLIST_DEFAULT = {
+    "primary_metric_acceptable",
+    "preview_final_drift_acceptable",
+    "guard_overhead_acceptable",
+    "invariants_pass",
+    "spectral_stable",
+    "rmt_stable",
+    # Compatibility keys were removed; PM-only surface
+    "hysteresis_applied",
+    "moe_observed",
+    "moe_identity_ok",
+}
+
+
+def _load_validation_allowlist() -> set[str]:
+    """Load validation key allow-list from contracts/validation_keys.json when available.
+
+    Falls back to a safe built-in default when the contracts directory is not present
+    (e.g., installed wheel) or when parsing fails.
+    """
+    try:
+        root = Path(__file__).resolve().parents[3]
+        path = root / "contracts" / "validation_keys.json"
+        if path.exists():
+            data = json.loads(path.read_text(encoding="utf-8"))
+            if isinstance(data, list):
+                return {str(k) for k in data}
+    except Exception:
+        pass
+    return set(_VALIDATION_ALLOWLIST_DEFAULT)
+
+
+def _validate_with_jsonschema(certificate: dict[str, Any]) -> bool:
+    """Validate certificate with JSON Schema when available."""
+    if jsonschema is None:
+        return True  # Schema library unavailable; fall back to legacy checks
+    try:
+        jsonschema.validate(instance=certificate, schema=CERTIFICATE_JSON_SCHEMA)
+        return True
+    except Exception:
+        return False
+
+
+def validate_certificate(certificate: dict[str, Any]) -> bool:
+    """Validate certificate structure and essential flags."""
+    try:
+        if certificate.get("schema_version") != CERTIFICATE_SCHEMA_VERSION:
+            return False
+
+        # Prefer JSON Schema structural validation; if unavailable or too strict,
+        # fall back to a lenient minimal check used by unit tests.
+        # Tighten JSON Schema: populate validation.properties from allow-list and
+        # disallow unknown validation keys at schema level.
+        try:
+            vkeys = _load_validation_allowlist()
+            if isinstance(CERTIFICATE_JSON_SCHEMA.get("properties"), dict):
+                vspec = CERTIFICATE_JSON_SCHEMA["properties"].get("validation")
+                if isinstance(vspec, dict):
+                    vspec["properties"] = {k: {"type": "boolean"} for k in vkeys}
+                    vspec["additionalProperties"] = False
+        except Exception:
+            pass
+
+        if not _validate_with_jsonschema(certificate):
+            # Minimal fallback: require schema version + run_id + primary_metric
+            run_id_ok = isinstance(certificate.get("run_id"), str) and bool(
+                certificate.get("run_id")
+            )
+            pm = certificate.get("primary_metric")
+            pm_ok = isinstance(pm, dict) and (
+                isinstance(pm.get("final"), int | float)
+                or (isinstance(pm.get("kind"), str) and bool(pm.get("kind")))
+            )
+            if not (run_id_ok and pm_ok):
+                return False
+
+        validation = certificate.get("validation", {})
+        for flag in [
+            "preview_final_drift_acceptable",
+            "primary_metric_acceptable",
+            "invariants_pass",
+            "spectral_stable",
+            "rmt_stable",
+            "guard_overhead_acceptable",
+        ]:
+            # If present, must be boolean; tolerate missing opt-in flags
+            if flag in validation and not isinstance(validation.get(flag), bool):
+                return False
+
+        return True
+    except (KeyError, TypeError, ValueError):
+        return False
+
+
+__all__ = [
+    "CERTIFICATE_SCHEMA_VERSION",
+    "CERTIFICATE_JSON_SCHEMA",
+    "validate_certificate",
+]

invarlock/reporting/dataset_hashing.py

@@ -0,0 +1,215 @@
+from __future__ import annotations
+
+from typing import Any
+
+from ..eval.data import EvaluationWindow, compute_window_hash
+
+
+def compute_window_hashes(
+    preview_window: EvaluationWindow, final_window: EvaluationWindow
+) -> dict[str, str | int]:
+    """Compute SHA256 hashes of evaluation windows for dataset integrity.
+
+    Returns a mapping with preview/final sha256 and total token count.
+    """
+    preview_hash = compute_window_hash(preview_window, include_data=True)
+    final_hash = compute_window_hash(final_window, include_data=True)
+    return {
+        "preview": f"sha256:{preview_hash}",
+        "final": f"sha256:{final_hash}",
+        "total_tokens": sum(
+            len(ids) for ids in preview_window.input_ids + final_window.input_ids
+        ),
+    }
+
+
+def _compute_actual_window_hashes(report: dict[str, Any]) -> dict[str, Any]:
+    """Compute actual window hashes from explicit token IDs when available.
+
+    Expects `report["evaluation_windows"]["preview"|"final"]["input_ids"]` to be present.
+    Returns dict with per-split sha256 hashes and token counts when computable; otherwise
+    returns an empty dict to signal fallback to config-based hashing.
+    """
+    try:
+        # Prefer explicit preview/final hashes included in the report data
+        data_config = report.get("data", {}) if isinstance(report, dict) else {}
+        if (
+            isinstance(data_config, dict)
+            and data_config.get("preview_hash")
+            and data_config.get("final_hash")
+        ):
+            preview_tokens_ct = data_config.get("preview_total_tokens")
+            final_tokens_ct = data_config.get("final_total_tokens")
+            total_tokens = 0
+            if isinstance(preview_tokens_ct, int):
+                total_tokens += preview_tokens_ct
+            if isinstance(final_tokens_ct, int):
+                total_tokens += final_tokens_ct
+            return {
+                "preview": f"blake2s:{data_config.get('preview_hash')}",
+                "final": f"blake2s:{data_config.get('final_hash')}",
+                "dataset": data_config.get("dataset_hash"),
+                "total_tokens": total_tokens,
+                "preview_tokens": preview_tokens_ct,
+                "final_tokens": final_tokens_ct,
+            }
+
+        windows = report.get("evaluation_windows", {})
+        if not isinstance(windows, dict):
+            windows = {}
+        preview_ids = (
+            windows.get("preview", {}).get("input_ids")
+            if windows.get("preview")
+            else None
+        )
+        final_ids = (
+            windows.get("final", {}).get("input_ids") if windows.get("final") else None
+        )
+        if not (isinstance(preview_ids, list) and isinstance(final_ids, list)):
+            # Config-based fallback (stable sha256 of config tuple)
+            import hashlib as _hashlib
+
+            data_config = report.get("data", {}) if isinstance(report, dict) else {}
+            dataset = str(data_config.get("dataset", "unknown"))
+            split = str(
+                data_config.get("split", data_config.get("dataset_split", "val"))
+            )
+            seed = (
+                report.get("meta", {}).get("seed", 0)
+                if isinstance(report.get("meta"), dict)
+                else 0
+            )
+            preview_n = int(data_config.get("preview_n", 0) or 0)
+            final_n = int(data_config.get("final_n", 0) or 0)
+            seq_len = int(data_config.get("seq_len", 0) or 0)
+            config_str = f"{dataset}{split}{seq_len}{preview_n}{final_n}{seed}"
+            digest = _hashlib.sha256(config_str.encode()).hexdigest()
+            preview_tokens = preview_n * seq_len if preview_n and seq_len else 0
+            final_tokens = final_n * seq_len if final_n and seq_len else 0
+            return {
+                "preview": f"sha256:{digest[:32]}",
+                "final": f"sha256:{digest[32:64] if len(digest) >= 64 else digest[:32]}",
+                "dataset": data_config.get("provider_hash")
+                or data_config.get("tokenizer_hash"),
+                "preview_tokens": preview_tokens,
+                "final_tokens": final_tokens,
+                "total_tokens": preview_tokens + final_tokens,
+            }
+        # Compute hashes directly from token ID sequences for robustness
+        import hashlib as _hashlib
+
+        def _hash_sequences(seqs: list[list[int]]) -> str:
+            h = _hashlib.sha256()
+            for seq in seqs:
+                try:
+                    h.update(str(list(seq)).encode("utf-8"))
+                except Exception:
+                    continue
+            return h.hexdigest()
+
+        preview_hash = _hash_sequences(preview_ids)
+        final_hash = _hash_sequences(final_ids)
+        preview_tokens = sum(len(s) for s in preview_ids)
+        final_tokens = sum(len(s) for s in final_ids)
+        return {
+            "preview": f"sha256:{preview_hash}",
+            "final": f"sha256:{final_hash}",
+            "preview_tokens": preview_tokens,
+            "final_tokens": final_tokens,
+            "dataset": None,
+            "total_tokens": preview_tokens + final_tokens,
+        }
+    except Exception:
+        # Signal caller to use config-based fallback
+        return {}
+
+
+def _extract_dataset_info(report: dict[str, Any]) -> dict[str, Any]:
+    """Extract dataset configuration and compute window hashes.
+
+    Tolerates missing `data` by falling back to evaluation_windows lengths and
+    placeholder values for non-essential fields.
+    """
+    data_config = report.get("data", {}) if isinstance(report, dict) else {}
+    eval_windows = (
+        report.get("evaluation_windows", {}) if isinstance(report, dict) else {}
+    )
+    preview_section = (
+        eval_windows.get("preview", {}) if isinstance(eval_windows, dict) else {}
+    )
+    final_section = (
+        eval_windows.get("final", {}) if isinstance(eval_windows, dict) else {}
+    )
+
+    def _len_ids(sec: dict[str, Any]) -> int:
+        ids = sec.get("window_ids")
+        return int(len(ids)) if isinstance(ids, list) else 0
+
+    preview_n = int(data_config.get("preview_n", 0) or 0) or _len_ids(preview_section)
+    final_n = int(data_config.get("final_n", 0) or 0) or _len_ids(final_section)
+    seq_len = int(data_config.get("seq_len", 0) or 0)
+    stride = int(data_config.get("stride", 0) or 0)
+
+    dataset = str(data_config.get("dataset", "unknown"))
+    split = str(data_config.get("split", data_config.get("dataset_split", "val")))
+
+    # Prefer actual window hashes when explicit token IDs are present
+    actual_hashes = _compute_actual_window_hashes(report)
+    if actual_hashes:
+        window_hash = actual_hashes
+    else:
+        # Config-based fallback: produce a stable sha256 of config tuple
+        import hashlib as _hashlib
+
+        seed = (
+            report.get("meta", {}).get("seed", 0)
+            if isinstance(report.get("meta"), dict)
+            else 0
+        )
+        preview_n = int(data_config.get("preview_n", 0) or 0)
+        final_n = int(data_config.get("final_n", 0) or 0)
+        seq_len = int(data_config.get("seq_len", 0) or 0)
+        config_str = f"{dataset}{split}{seq_len}{preview_n}{final_n}{seed}"
+        digest = _hashlib.sha256(config_str.encode()).hexdigest()
+        preview_tokens = preview_n * seq_len if preview_n and seq_len else 0
+        final_tokens = final_n * seq_len if final_n and seq_len else 0
+        window_hash = {
+            "preview": f"sha256:{digest[:32]}",
+            "final": f"sha256:{digest[32:64] if len(digest) >= 64 else digest[:32]}",
+            "dataset": data_config.get("provider_hash")
+            or data_config.get("tokenizer_hash"),
+            "preview_tokens": preview_tokens,
+            "final_tokens": final_tokens,
+            "total_tokens": preview_tokens + final_tokens,
+        }
+
+    tokenizer_info = {
+        "name": data_config.get("tokenizer_name", "unknown"),
+        "hash": data_config.get("tokenizer_hash"),
+        "vocab_size": data_config.get("vocab_size"),
+        "bos_token": data_config.get("bos_token"),
+        "eos_token": data_config.get("eos_token"),
+        "pad_token": data_config.get("pad_token"),
+        "add_prefix_space": data_config.get("add_prefix_space"),
+    }
+
+    return {
+        "provider": dataset,
+        "split": split,
+        "seq_len": seq_len,
+        "stride": stride,
+        "windows": {
+            "preview": preview_n,
+            "final": final_n,
+            "seed": data_config.get("seed"),
+        },
+        "hash": window_hash,
+        "tokenizer": tokenizer_info,
+    }
+
+
+__all__ = [
+    "compute_window_hashes",
+    "_compute_actual_window_hashes",
+    "_extract_dataset_info",
+]