brainpatch 1.2.0__py3-none-any.whl

brainpatch/verify/vectors.py

@@ -0,0 +1,95 @@
+"""Pure-Python vector helpers for corruption generation and static validation.
+
+No torch. A corruption generator transforms an *inert data file* — a manifest and
+a list of floats — and has no business requiring the ML stack. Keeping it
+dependency-free is what lets the corruption suite and every static detection layer
+run in the fast local test suite, where `tests/conftest.py` blocks torch on
+purpose.
+
+Generation and execution tracing genuinely need a model; those live in
+`brainpatch.verify.behavioural` and stay Modal-only.
+
+Arithmetic is done in Python floats (IEEE double), which is *more* precise than
+the float32 path it replaces — an earlier float32 dot product over 1536 dimensions
+reported cosine 0.99999994 for a bit-identical vector.
+"""
+
+from __future__ import annotations
+
+import math
+import random
+import struct
+from collections.abc import Sequence
+
+
+def dot(a: Sequence[float], b: Sequence[float]) -> float:
+    return math.fsum(x * y for x, y in zip(a, b))
+
+
+def norm(a: Sequence[float]) -> float:
+    return math.sqrt(math.fsum(x * x for x in a))
+
+
+def scaled(a: Sequence[float], factor: float) -> list[float]:
+    return [x * factor for x in a]
+
+
+def unit(a: Sequence[float]) -> list[float]:
+    n = norm(a)
+    if n == 0.0:
+        raise ValueError("cannot normalise a zero vector")
+    return [x / n for x in a]
+
+
+def cosine(a: Sequence[float], b: Sequence[float]) -> float:
+    denominator = norm(a) * norm(b)
+    if denominator == 0.0:
+        raise ValueError("cosine of a zero vector is undefined")
+    return dot(a, b) / denominator
+
+
+def gaussian(size: int, *, seed: int) -> list[float]:
+    """Deterministic standard normals.
+
+    `random.Random(seed).gauss` is reproducible across platforms and Python
+    versions, which is what a recorded corruption needs.
+    """
+    rng = random.Random(seed)
+    return [rng.gauss(0.0, 1.0) for _ in range(size)]
+
+
+def orthogonalise(candidate: Sequence[float], against: Sequence[float]) -> list[float]:
+    """Remove the component of `candidate` parallel to `against`."""
+    direction = unit(against)
+    projection = dot(candidate, direction)
+    return [c - projection * d for c, d in zip(candidate, direction)]
+
+
+def to_float16(value: float) -> float:
+    """Round a Python float to IEEE half precision and back.
+
+    `struct` format `'e'` is binary16 with round-to-nearest-even, matching what a
+    float16 store does.
+    """
+    return struct.unpack("<e", struct.pack("<e", value))[0]
+
+
+def to_bfloat16(value: float) -> float:
+    """Round to bfloat16 and back, round-to-nearest-even.
+
+    bfloat16 keeps float32's exponent and truncates the mantissa to 7 bits, so it
+    is coarser than float16 in precision while covering a far wider range. Done by
+    hand because `struct` has no bf16 code; the rounding bias below is the
+    standard RNE construction and matches `torch.Tensor.to(torch.bfloat16)`.
+    """
+    bits = struct.unpack("<I", struct.pack("<f", value))[0]
+    if (bits & 0x7F800000) == 0x7F800000:  # inf or NaN: pass through untouched
+        return value
+    bias = 0x7FFF + ((bits >> 16) & 1)
+    rounded = (bits + bias) & 0xFFFF0000
+    return struct.unpack("<f", struct.pack("<I", rounded))[0]
+
+
+def round_to(values: Sequence[float], dtype: str) -> list[float]:
+    convert = {"float16": to_float16, "bfloat16": to_bfloat16}[dtype]
+    return [convert(v) for v in values]

brainpatch/verify/workflow.py

@@ -0,0 +1,331 @@
+"""The `brainpatch verify` workflow.
+
+One entry point that runs every check the available inputs allow, and is explicit
+about which ones it could not run. A check that was skipped is reported as
+`skipped`, never silently omitted -- an absent check reads as a pass, and that is
+how a bad artifact gets trusted.
+
+The levels degrade gracefully:
+
+* no reference, no model  -> structural only
+* reference, no model     -> structural + numerical
+* model, no reference     -> structural + execution
+* both                    -> structural + numerical + execution
+* plus recorded behaviour -> all four
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Any
+
+from brainpatch.verify import vectors as vec
+from brainpatch.verify.checks import (
+    COSINE_TOLERANCE,
+    NORM_TOLERANCE,
+    Reference,
+    reference_from,
+    run_execution_trace,
+    run_static_layers,
+)
+
+#: Signed cosine below this fails numerical verification. A cosine against an
+#: *unsigned* reference is never used: it cannot distinguish v from -v.
+MIN_SIGNED_COSINE = 1.0 - COSINE_TOLERANCE
+
+LEVELS = ("structural", "numerical", "execution", "behavioural")
+
+PASS = "pass"
+FAIL = "fail"
+SKIPPED = "skipped"
+
+
+@dataclass
+class Check:
+    """One named check and what it found."""
+
+    name: str
+    level: str
+    status: str
+    detail: str = ""
+    measured: dict[str, Any] = field(default_factory=dict)
+
+    @property
+    def ok(self) -> bool:
+        return self.status != FAIL
+
+
+@dataclass
+class VerificationResult:
+    """The outcome of `verify_artifact`.
+
+    ``faithful`` is True only when nothing failed. Skipped checks do not fail a
+    verification, but they do mean the artifact is unverified in that respect,
+    which :attr:`skipped_levels` makes explicit so a caller cannot mistake
+    "not checked" for "checked and fine".
+    """
+
+    artifact: str
+    sha256: str
+    checks: list[Check]
+    levels: dict[str, str]
+
+    @property
+    def failures(self) -> list[Check]:
+        return [c for c in self.checks if c.status == FAIL]
+
+    @property
+    def skipped_levels(self) -> list[str]:
+        return [name for name, state in self.levels.items() if state == SKIPPED]
+
+    @property
+    def faithful(self) -> bool:
+        return not self.failures
+
+    @property
+    def verdict(self) -> str:
+        if self.failures:
+            behavioural_only = all(c.level == "behavioural" for c in self.failures)
+            return "FAILED_BEHAVIORAL_FIDELITY" if behavioural_only else "UNFAITHFUL"
+        return "FAITHFUL"
+
+    def to_dict(self) -> dict[str, Any]:
+        return {
+            "artifact": self.artifact,
+            "sha256": self.sha256,
+            "verdict": self.verdict,
+            "faithful": self.faithful,
+            "levels": self.levels,
+            "skipped_levels": self.skipped_levels,
+            "checks": [
+                {
+                    "name": c.name,
+                    "level": c.level,
+                    "status": c.status,
+                    "detail": c.detail,
+                    "measured": c.measured,
+                }
+                for c in self.checks
+            ],
+        }
+
+
+def _digest(path: Path) -> str:
+    import hashlib
+
+    return hashlib.sha256(path.read_bytes()).hexdigest()
+
+
+def verify_artifact(
+    path: str | Path,
+    *,
+    reference: str | Path | Reference | None = None,
+    backend: Any = None,
+    expect_model: str | None = None,
+) -> VerificationResult:
+    """Verify one artifact as far as the given inputs allow.
+
+    Parameters
+    ----------
+    path:
+        The `.brainpatch` archive under test.
+    reference:
+        A trusted artifact, or a :class:`Reference`, to compare direction and
+        magnitude against. Without it, numerical verification is skipped -- there
+        is nothing to be right or wrong relative to.
+    backend:
+        A loaded backend. Without it, execution verification is skipped and the
+        artifact's declared layer and site remain unconfirmed against what the
+        runtime would actually do.
+    expect_model:
+        Model id the caller intends to load the patch into. Compared against the
+        manifest's declared base model.
+    """
+    from brainpatch.patch.format import PatchFormatError
+    from brainpatch.patch.loader import PatchLoadError, load_patch
+
+    artifact = Path(path)
+    checks: list[Check] = []
+    levels = {level: SKIPPED for level in LEVELS}
+
+    # ---- structural ---------------------------------------------------------
+    try:
+        loaded = load_patch(artifact, verify_checksums=True)
+        checks.append(Check("checksums", "structural", PASS, "archive checksums verify"))
+    except PatchLoadError as exc:
+        checks.append(Check("checksums", "structural", FAIL, str(exc)))
+        levels["structural"] = FAIL
+        return VerificationResult(str(artifact), _digest(artifact), checks, levels)
+
+    try:
+        loaded.manifest.validate()
+        checks.append(Check("schema", "structural", PASS, "manifest is self-consistent"))
+    except PatchFormatError as exc:
+        checks.append(Check("schema", "structural", FAIL, str(exc)))
+
+    manifest = loaded.manifest
+    intervention = manifest.interventions[0]
+    tensor = loaded.vectors[intervention.vector]
+    hidden = manifest.base_model.hidden_size
+    shape_ok = list(tensor.shape) == [hidden]
+    checks.append(
+        Check(
+            "shape_dtype", "structural", PASS if shape_ok else FAIL,
+            f"vector {intervention.vector!r} shape {list(tensor.shape)}, "
+            f"dtype {tensor.dtype}, model hidden size {hidden}",
+            {"shape": list(tensor.shape), "dtype": tensor.dtype},
+        )
+    )
+
+    if expect_model is not None:
+        declared = manifest.base_model.model_id
+        match = declared == expect_model
+        checks.append(
+            Check(
+                "model_compatibility", "structural", PASS if match else FAIL,
+                f"declares {declared!r}, target is {expect_model!r}",
+                {"declared": declared, "expected": expect_model},
+            )
+        )
+    else:
+        checks.append(
+            Check(
+                "model_compatibility", "structural", SKIPPED,
+                f"no target model given; artifact declares "
+                f"{manifest.base_model.model_id!r}",
+            )
+        )
+
+    levels["structural"] = (
+        FAIL if any(c.level == "structural" and c.status == FAIL for c in checks) else PASS
+    )
+
+    # ---- numerical ----------------------------------------------------------
+    if reference is None:
+        checks.append(
+            Check(
+                "signed_direction", "numerical", SKIPPED,
+                "no reference artifact given; direction and magnitude are unverified",
+            )
+        )
+    else:
+        ref = reference if isinstance(reference, Reference) else reference_from(Path(reference))
+        delta = vec.scaled([float(v) for v in tensor.data], intervention.coefficient)
+        cosine = vec.cosine(delta, ref.delta)
+        ratio = vec.norm(delta) / vec.norm(ref.delta)
+
+        direction_ok = cosine >= MIN_SIGNED_COSINE
+        checks.append(
+            Check(
+                "signed_direction", "numerical", PASS if direction_ok else FAIL,
+                f"signed cosine {cosine:+.8f} (threshold {MIN_SIGNED_COSINE:.6f}); "
+                f"unsigned |cos| {abs(cosine):.8f} would have "
+                f"{'passed' if abs(cosine) >= MIN_SIGNED_COSINE else 'failed'}",
+                {"signed_cosine": cosine, "unsigned_cosine": abs(cosine)},
+            )
+        )
+        # Direction and magnitude together fully determine the applied delta, so
+        # the coefficient is reported rather than checked separately: a wrong
+        # coefficient shows up here as a norm ratio, or as a negative cosine if
+        # its sign flipped.
+        norm_ok = abs(ratio - 1.0) <= NORM_TOLERANCE
+        checks.append(
+            Check(
+                "delta_norm", "numerical", PASS if norm_ok else FAIL,
+                f"applied |delta| {vec.norm(delta):.5f} against reference "
+                f"{vec.norm(ref.delta):.5f} (ratio {ratio:.6f}), "
+                f"coefficient {intervention.coefficient:+.6f}",
+                {
+                    "delta_norm": vec.norm(delta),
+                    "norm_ratio": ratio,
+                    "coefficient": intervention.coefficient,
+                },
+            )
+        )
+        levels["numerical"] = (
+            FAIL if any(c.level == "numerical" and c.status == FAIL for c in checks) else PASS
+        )
+
+    # ---- execution ----------------------------------------------------------
+    if backend is None or getattr(backend, "model", None) is None:
+        checks.append(
+            Check(
+                "execution_trace", "execution", SKIPPED,
+                "no loaded backend given; declared layer and site are unconfirmed "
+                "against what the runtime would do",
+            )
+        )
+    else:
+        declared = Reference(
+            delta=[],
+            layer=int(intervention.layer),
+            site=str(intervention.site),
+            model_id=manifest.base_model.model_id,
+            revision=manifest.base_model.revision,
+            shape=list(tensor.shape),
+            schedule=manifest.schedule,
+        )
+        num_layers = len(backend._decoder_layers())
+        trace = run_execution_trace(backend, artifact, declared, num_layers)
+        # `caught` here means "differs from what the manifest declares", which is
+        # a verification failure rather than a detection success.
+        checks.append(
+            Check(
+                "execution_trace", "execution", FAIL if trace["caught"] else PASS,
+                f"runtime edits on prompt pass at layers {trace['prompt_layers']}, "
+                f"on continuation pass at {trace['continuation_layers']}; "
+                f"manifest declares layer {declared.layer} site {declared.site!r}",
+                trace,
+            )
+        )
+        schedule_ok = manifest.schedule == declared.schedule
+        checks.append(
+            Check(
+                "schedule", "execution", PASS if schedule_ok else FAIL,
+                f"schedule {manifest.schedule!r}",
+                {"schedule": manifest.schedule},
+            )
+        )
+        levels["execution"] = (
+            FAIL if any(c.level == "execution" and c.status == FAIL for c in checks) else PASS
+        )
+
+    checks.append(
+        Check(
+            "behavioural_regression", "behavioural", SKIPPED,
+            "run `verify_behavioural_regression` with a recorded result and a "
+            "frozen evaluation set to check this level",
+        )
+    )
+
+    return VerificationResult(str(artifact), _digest(artifact), checks, levels)
+
+
+def verify_behavioural_regression(
+    measured_effect: float,
+    recorded_effect: float,
+    *,
+    tolerance: float,
+) -> Check:
+    """Compare a freshly measured effect against a recorded one.
+
+    ``tolerance`` must be chosen before measuring. Note the empirical caveat from
+    ``experiments/artifact_fidelity_v1``: at a tolerance calibrated to path noise,
+    this check was *less* sensitive to magnitude errors than a delta-norm
+    comparison, so it complements the numerical level rather than replacing it.
+    """
+    delta = abs(measured_effect - recorded_effect)
+    return Check(
+        "behavioural_regression",
+        "behavioural",
+        PASS if delta <= tolerance else FAIL,
+        f"effect {measured_effect:+.4f} against recorded {recorded_effect:+.4f} "
+        f"(delta {delta:.4f}, tolerance {tolerance})",
+        {
+            "measured_effect": measured_effect,
+            "recorded_effect": recorded_effect,
+            "delta": delta,
+            "tolerance": tolerance,
+        },
+    )