brainpatch 1.2.0__py3-none-any.whl

brainpatch/verify/checks.py

@@ -0,0 +1,204 @@
+"""The detection layers, cheapest first.
+
+Each layer answers a different question, and each is blind to something. The
+point of running all of them against every corruption is to find out *what each
+is blind to*, empirically, rather than assuming.
+
+`unsigned_cosine` is included deliberately and is **not** a recommended check.
+It is here because it is the check this project actually ran, which reported
+cosine = 1.0 on an artifact that was behaviourally the sign control. Its column
+in the detection matrix is the argument.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Any
+
+from brainpatch.verify import vectors as vec
+
+#: Ordered cheapest to most expensive. Behavioural reproduction is run
+#: separately -- it needs a GPU and a frozen evaluation set.
+DETECTION_LAYERS: tuple[str, ...] = (
+    "schema",
+    "checksum",
+    "shape_dtype",
+    "model_compatibility",
+    "unsigned_cosine",
+    "signed_cosine",
+    "delta_norm",
+    "execution_trace",
+    "schedule_trace",
+    "runtime_smoke",
+)
+
+#: A layer flags a defect when the quantity it inspects differs from the
+#: reference by more than this. Deliberately loose: the question is whether a
+#: layer can see a defect *at all*, not how finely it can measure.
+COSINE_TOLERANCE = 1e-3
+NORM_TOLERANCE = 1e-3
+
+
+@dataclass
+class Reference:
+    """Everything the static layers compare a candidate against."""
+
+    delta: list[float]
+    layer: int
+    site: str
+    model_id: str
+    revision: str | None
+    shape: list[int]
+    schedule: dict[str, float] | None
+
+
+def reference_from(path: Path) -> Reference:
+    from brainpatch.patch.loader import load_patch
+
+    loaded = load_patch(path)
+    intervention = loaded.manifest.interventions[0]
+    values = [float(v) for v in loaded.vectors[intervention.vector].data]
+    return Reference(
+        delta=vec.scaled(values, intervention.coefficient),
+        layer=int(intervention.layer),
+        site=str(intervention.site),
+        model_id=loaded.manifest.base_model.model_id,
+        revision=loaded.manifest.base_model.revision,
+        shape=list(loaded.vectors[intervention.vector].shape),
+        schedule=loaded.manifest.schedule,
+    )
+
+
+def run_static_layers(path: Path, reference: Reference) -> dict[str, Any]:
+    """Run every layer that does not need a GPU.
+
+    Returns ``{layer: caught}`` plus the measured quantities, so the matrix can
+    be audited rather than trusted.
+    """
+    from brainpatch.patch.format import PatchFormatError
+    from brainpatch.patch.loader import PatchLoadError, load_patch
+
+    caught: dict[str, bool] = {}
+    measured: dict[str, Any] = {}
+
+    # -- checksum: load_patch verifies internally and raises on mismatch -------
+    try:
+        loaded = load_patch(path, verify_checksums=True)
+        caught["checksum"] = False
+    except PatchLoadError as exc:
+        measured["checksum_error"] = str(exc)
+        return {"caught": {layer: layer == "checksum" for layer in DETECTION_LAYERS},
+                "measured": measured, "loadable": False}
+
+    # -- schema ---------------------------------------------------------------
+    try:
+        loaded.manifest.validate()
+        caught["schema"] = False
+    except PatchFormatError as exc:
+        caught["schema"] = True
+        measured["schema_error"] = str(exc)
+
+    intervention = loaded.manifest.interventions[0]
+    tensor = loaded.vectors[intervention.vector]
+    delta = vec.scaled([float(v) for v in tensor.data], intervention.coefficient)
+
+    # -- shape / dtype --------------------------------------------------------
+    caught["shape_dtype"] = list(tensor.shape) != reference.shape
+    measured["shape"] = list(tensor.shape)
+
+    # -- model compatibility --------------------------------------------------
+    caught["model_compatibility"] = (
+        loaded.manifest.base_model.model_id != reference.model_id
+        or loaded.manifest.base_model.revision != reference.revision
+    )
+
+    # -- direction, signed and unsigned --------------------------------------
+    ref_norm = vec.norm(reference.delta)
+    cand_norm = vec.norm(delta)
+    cosine = vec.cosine(delta, reference.delta)
+    measured["signed_cosine"] = cosine
+    measured["unsigned_cosine"] = abs(cosine)
+    caught["signed_cosine"] = abs(cosine - 1.0) > COSINE_TOLERANCE
+    # The blind spot, made explicit: |cos| cannot distinguish v from -v.
+    caught["unsigned_cosine"] = abs(abs(cosine) - 1.0) > COSINE_TOLERANCE
+
+    # -- magnitude ------------------------------------------------------------
+    ratio = cand_norm / ref_norm
+    measured["delta_norm"] = cand_norm
+    measured["norm_ratio"] = ratio
+    caught["delta_norm"] = abs(ratio - 1.0) > NORM_TOLERANCE
+
+    # -- declared structure ---------------------------------------------------
+    measured["layer"] = int(intervention.layer)
+    measured["site"] = str(intervention.site)
+    measured["schedule"] = loaded.manifest.schedule
+    caught["schedule_trace"] = loaded.manifest.schedule != reference.schedule
+
+    return {"caught": caught, "measured": measured, "loadable": True}
+
+
+def run_execution_trace(
+    backend: Any, path: Path, reference: Reference, num_layers: int
+) -> dict[str, Any]:
+    """Ask the *runtime* where and when it would intervene.
+
+    Reads ``resolve_edits``, not the manifest. A manifest declaring
+    ``site: prompt`` and a runtime steering every generated token is exactly the
+    disagreement this layer exists to find, and no amount of manifest inspection
+    would surface it.
+    """
+    from brainpatch.patch.loader import load_patch
+
+    loaded = load_patch(path)
+    active = backend.install_patch(loaded)
+    try:
+        # Which layers carry an edit, on which kind of pass?
+        prompt_layers = [
+            layer
+            for layer in range(num_layers)
+            if backend.resolve_edits(0, layer=layer, is_prompt_pass=True)
+        ]
+        continuation_layers = [
+            layer
+            for layer in range(num_layers)
+            if backend.resolve_edits(0, layer=layer, is_prompt_pass=False)
+        ]
+    finally:
+        backend.remove_patch(active.patch.manifest.name)
+
+    expected_prompt = reference.site in ("prompt", "all")
+    expected_continuation = reference.site in ("continuation", "all")
+
+    caught = (
+        prompt_layers != ([reference.layer] if expected_prompt else [])
+        or continuation_layers != ([reference.layer] if expected_continuation else [])
+    )
+    return {
+        "caught": bool(caught),
+        "prompt_layers": prompt_layers,
+        "continuation_layers": continuation_layers,
+        "expected_layer": reference.layer,
+        "expected_site": reference.site,
+    }
+
+
+def run_runtime_smoke(backend: Any, path: Path, prompt: str) -> dict[str, Any]:
+    """Does it load, install and produce non-degenerate text at all?
+
+    The weakest possible behavioural check, and included precisely to show how
+    little it catches. Almost every corruption here will pass it.
+    """
+    from brainpatch.patch.loader import load_patch
+    from brainpatch.runtime.base import GenerationConfig
+
+    loaded = load_patch(path)
+    active = backend.install_patch(loaded)
+    try:
+        text = backend.generate(prompt, GenerationConfig(max_new_tokens=24, temperature=0.0))
+    finally:
+        backend.remove_patch(active.patch.manifest.name)
+    stripped = text.strip()
+    words = stripped.split()
+    degenerate = not stripped or (len(words) > 6 and len(set(words)) <= 2)
+    return {"caught": bool(degenerate), "sample": text[:160]}

brainpatch/verify/corruptions.py

@@ -0,0 +1,335 @@
+"""Controlled, deterministic corruptions of a compiled `.brainpatch` artifact.
+
+Every corruption starts from a known-correct reference and changes exactly one
+thing. Every product is a **well-formed archive with valid checksums** -- that is
+deliberate and it is the whole design. A variant that fails to load teaches
+nothing about which validation layer catches it, and the class this suite exists
+to expose is precisely the one that survives loading, schema validation and
+checksum verification and is still behaviourally wrong.
+
+No corruption introduces executable content. The `.brainpatch` format is inert
+data by construction; a corruption suite that weakened that would be testing a
+different, worse format.
+
+**No torch.** Everything here transforms a manifest and a list of floats, so it
+runs in the fast local test suite where `tests/conftest.py` blocks the ML stack.
+Vector arithmetic lives in `brainpatch.verify.vectors` and is done in Python doubles, which is
+more precise than the float32 path it replaces. Generation and execution tracing
+genuinely need a model; those stay in `brainpatch.verify.behavioural`, Modal-only.
+
+Corruptions are declarative and recorded: each returns a description of exactly
+what it did, so `corruptions.json` is a reproduction recipe rather than a label.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Any, Callable
+
+from brainpatch.patch.format import Intervention, Manifest
+from brainpatch.patch.loader import LoadedPatch, load_patch, save_patch
+from brainpatch.verify import vectors as vec
+
+
+@dataclass
+class Corruption:
+    """One defect, its rationale, and where we predict it will be caught."""
+
+    key: str
+    #: What real-world mistake this stands in for.
+    defect: str
+    #: Pre-registered prediction, so the detection matrix can contradict it.
+    expected_detection: str
+    apply: Callable[[Manifest, dict[str, Any]], tuple[Manifest, dict[str, Any]]]
+    #: Filled in when the variant is built.
+    detail: dict[str, Any] = field(default_factory=dict)
+
+
+def _clone(manifest: Manifest, **changes: Any) -> Manifest:
+    """A manifest identical to the original except for named fields."""
+    fields = {
+        "name": manifest.name,
+        "base_model": manifest.base_model,
+        "interventions": list(manifest.interventions),
+        "description": manifest.description,
+        "format_version": manifest.format_version,
+        "evidence_level": manifest.evidence_level,
+        "evaluation": dict(manifest.evaluation),
+        "compatibility": dict(manifest.compatibility),
+        "provenance": dict(manifest.provenance),
+        "max_abs_strength": manifest.max_abs_strength,
+        "default_strength": manifest.default_strength,
+        "license": manifest.license,
+        "authors": list(manifest.authors),
+        "schedule": dict(manifest.schedule) if manifest.schedule else None,
+    }
+    fields.update(changes)
+    return Manifest(**fields)
+
+
+def _with_intervention(manifest: Manifest, **changes: Any) -> Manifest:
+    original = manifest.interventions[0]
+    fields = {
+        "layer": original.layer,
+        "vector": original.vector,
+        "coefficient": original.coefficient,
+        "site": original.site,
+    }
+    fields.update(changes)
+    return _clone(manifest, interventions=[Intervention(**fields)])
+
+
+def _values(vectors: dict[str, Any], key: str) -> list[float]:
+    """The stored vector as plain Python floats."""
+    return [float(v) for v in vectors[key].data]
+
+
+def _store(vectors: dict[str, Any], key: str, values: list[float]) -> dict[str, Any]:
+    """Rebuild the vector map with one vector replaced, preserving dtype and shape."""
+    from brainpatch.patch import tensors as ts
+
+    out = dict(vectors)
+    out[key] = ts.Tensor(
+        dtype=vectors[key].dtype,
+        shape=list(vectors[key].shape),
+        data=[float(v) for v in values],
+    )
+    return out
+
+
+# -- the corruptions -----------------------------------------------------------
+
+
+def flip_sign(manifest: Manifest, vectors: dict[str, Any]):
+    """The defect this project actually shipped.
+
+    Discovery selected the negated decoder column; the compiler emits the
+    unsigned column and carries sign in the coefficient; the coefficient was
+    written positive. Behaviourally the sign control.
+    """
+    return _with_intervention(
+        manifest, coefficient=-manifest.interventions[0].coefficient
+    ), vectors
+
+
+def shift_layer(delta: int):
+    """Off-by-one in a layer index. Plausible in any hand-written manifest."""
+
+    def apply(manifest: Manifest, vectors: dict[str, Any]):
+        layer = manifest.interventions[0].layer + delta
+        return _with_intervention(manifest, layer=layer), vectors
+
+    return apply
+
+
+def scale_coefficient(factor: float):
+    """Calibration drift: right direction, wrong dose."""
+
+    def apply(manifest: Manifest, vectors: dict[str, Any]):
+        coefficient = manifest.interventions[0].coefficient * factor
+        return _with_intervention(manifest, coefficient=coefficient), vectors
+
+    return apply
+
+
+def replace_site(site: str):
+    """Injection site changed -- e.g. prompt-only becomes always-on.
+
+    The `site` field exists because v3's validated configuration was prompt-token
+    only and the format could not express it. Losing it silently re-runs the
+    intervention on every generated token, a configuration with no test evidence.
+    """
+
+    def apply(manifest: Manifest, vectors: dict[str, Any]):
+        return _with_intervention(manifest, site=site), vectors
+
+    return apply
+
+
+def precision_roundtrip(dtype: str):
+    """Store the vector through a lower-precision type and back.
+
+    Models the real cost of serialization precision. Round-to-nearest-even in
+    both cases, matching what a real store does.
+    """
+
+    def apply(manifest: Manifest, vectors: dict[str, Any]):
+        key = manifest.interventions[0].vector
+        return manifest, _store(vectors, key, vec.round_to(_values(vectors, key), dtype))
+
+    return apply
+
+
+def renormalize_vector(scale: float):
+    """Vector rescaled while the coefficient is left alone.
+
+    The applied delta is `coefficient x vector`, so rescaling the tensor changes
+    the dose without changing any declared number. A reader of the manifest sees
+    the validated coefficient and is wrong about what gets applied.
+    """
+
+    def apply(manifest: Manifest, vectors: dict[str, Any]):
+        key = manifest.interventions[0].vector
+        return manifest, _store(vectors, key, vec.scaled(_values(vectors, key), scale))
+
+    return apply
+
+
+def stored_vector_wrong(seed: int = 0):
+    """Metadata correct, tensor wrong.
+
+    Everything a reader can inspect in the manifest is right. The archive is
+    consistent and its checksums verify -- they are computed over whatever is
+    stored. Only a comparison against an external reference, or behaviour, sees it.
+    """
+
+    def apply(manifest: Manifest, vectors: dict[str, Any]):
+        key = manifest.interventions[0].vector
+        original = _values(vectors, key)
+        replacement = vec.unit(vec.gaussian(len(original), seed=seed))
+        return manifest, _store(
+            vectors, key, vec.scaled(replacement, vec.norm(original))
+        )
+
+    return apply
+
+
+def manifest_coefficient_wrong(factor: float = 0.5):
+    """Tensor correct, declared coefficient wrong.
+
+    The mirror image of `renormalize_vector`, and the reason numerical fidelity
+    must be computed on the *applied delta* rather than on either half alone.
+    """
+
+    def apply(manifest: Manifest, vectors: dict[str, Any]):
+        coefficient = manifest.interventions[0].coefficient * factor
+        return _with_intervention(manifest, coefficient=coefficient), vectors
+
+    return apply
+
+
+def orthogonal_direction(seed: int = 7):
+    """A direction orthogonal to the validated one, same norm.
+
+    The strongest test of "numerically plausible, semantically absent": shape,
+    dtype, norm and coefficient are all exactly right, and the intervention is
+    unrelated to the one that was validated.
+    """
+
+    def apply(manifest: Manifest, vectors: dict[str, Any]):
+        key = manifest.interventions[0].vector
+        original = _values(vectors, key)
+        candidate = vec.orthogonalise(vec.gaussian(len(original), seed=seed), original)
+        return manifest, _store(
+            vectors, key, vec.scaled(vec.unit(candidate), vec.norm(original))
+        )
+
+    return apply
+
+
+#: The registry. Ordered so the detection matrix reads from crudest to subtlest.
+CORRUPTIONS: tuple[Corruption, ...] = (
+    Corruption(
+        "sign_flip", "coefficient sign flipped (the defect this project shipped)",
+        "signed cosine and behaviour; NOT unsigned cosine", flip_sign,
+    ),
+    Corruption(
+        "layer_plus_1", "layer index off by one, upward",
+        "execution trace and behaviour", shift_layer(+1),
+    ),
+    Corruption(
+        "layer_minus_1", "layer index off by one, downward",
+        "execution trace and behaviour", shift_layer(-1),
+    ),
+    Corruption(
+        "coefficient_half", "coefficient scaled x0.5 (under-dosed)",
+        "delta norm and behaviour", scale_coefficient(0.5),
+    ),
+    Corruption(
+        "coefficient_double", "coefficient scaled x2 (over-dosed)",
+        "delta norm and behaviour", scale_coefficient(2.0),
+    ),
+    Corruption(
+        "site_all", "prompt-only schedule replaced with always-on",
+        "execution trace and behaviour", replace_site("all"),
+    ),
+    Corruption(
+        "site_continuation", "injection moved to generated tokens only",
+        "execution trace and behaviour", replace_site("continuation"),
+    ),
+    # NEGATIVE CONTROL, not a defect. The v1 format already stores vectors as
+    # F16, so an F16 round-trip is bit-exact identity. It is retained on purpose:
+    # a suite of thirteen defects with no null case cannot distinguish "this
+    # layer detects defects" from "this layer fires at everything". Any layer
+    # that flags this one is producing a false positive.
+    Corruption(
+        "f16_roundtrip", "NEGATIVE CONTROL: vector stored through float16 and back "
+        "(identity -- the format already stores F16)",
+        "nothing: this is a no-op and must be caught by no layer",
+        precision_roundtrip("float16"),
+    ),
+    Corruption(
+        "bf16_roundtrip", "vector stored through bfloat16 and back "
+        "(real perturbation: bf16 trades mantissa for exponent)",
+        "nothing cheap; too small for any cosine or norm threshold",
+        precision_roundtrip("bfloat16"),
+    ),
+    Corruption(
+        "vector_rescaled", "vector rescaled x2, coefficient untouched",
+        "delta norm and behaviour; NOT cosine", renormalize_vector(2.0),
+    ),
+    Corruption(
+        "vector_wrong", "manifest correct, stored vector replaced by a random direction",
+        "signed cosine and behaviour", stored_vector_wrong(),
+    ),
+    Corruption(
+        "coefficient_declared_wrong", "vector correct, declared coefficient halved",
+        "delta norm and behaviour", manifest_coefficient_wrong(0.5),
+    ),
+    Corruption(
+        "orthogonal_direction", "same norm and coefficient, direction orthogonal to validated",
+        "signed cosine and behaviour; NOT norm, shape or schema", orthogonal_direction(),
+    ),
+)
+
+
+def build_variants(reference: Path, out_dir: Path) -> list[dict[str, Any]]:
+    """Write every corrupted variant. Returns the `corruptions.json` payload."""
+    import hashlib
+
+    loaded: LoadedPatch = load_patch(reference)
+    out_dir.mkdir(parents=True, exist_ok=True)
+
+    key = loaded.manifest.interventions[0].vector
+    reference_delta = vec.scaled(
+        _values(loaded.vectors, key), loaded.manifest.interventions[0].coefficient
+    )
+
+    records: list[dict[str, Any]] = []
+    for corruption in CORRUPTIONS:
+        manifest, vectors = corruption.apply(loaded.manifest, loaded.vectors)
+        manifest = _clone(manifest, name=f"corrupt-{corruption.key.replace('_', '-')}")
+        path = out_dir / f"{corruption.key}.brainpatch"
+        save_patch(manifest, vectors, path, overwrite=True)
+
+        intervention = manifest.interventions[0]
+        delta = vec.scaled(_values(vectors, intervention.vector), intervention.coefficient)
+        cos = vec.cosine(delta, reference_delta)
+        records.append(
+            {
+                "key": corruption.key,
+                "defect": corruption.defect,
+                "expected_detection": corruption.expected_detection,
+                "artifact": path.name,
+                "sha256": hashlib.sha256(path.read_bytes()).hexdigest(),
+                "bytes": path.stat().st_size,
+                "layer": intervention.layer,
+                "site": intervention.site,
+                "coefficient": intervention.coefficient,
+                "delta_norm": vec.norm(delta),
+                "signed_cosine_to_reference": cos,
+                "unsigned_cosine_to_reference": abs(cos),
+            }
+        )
+    return records

brainpatch/verify/report.py

@@ -0,0 +1,133 @@
+"""Human-readable artifact-fidelity reports and the detection matrix.
+
+The report always prints all four components, including the ones that passed. The
+interesting artifact is the one that passes three and fails the fourth, and that
+is only legible if all four are shown.
+"""
+
+from __future__ import annotations
+
+from typing import Any
+
+COMPONENTS = ("structural", "numerical", "execution", "behavioural")
+
+#: Column labels for the matrix, short enough to fit side by side.
+LAYER_ABBREV = {
+    "schema": "schm",
+    "checksum": "csum",
+    "shape_dtype": "shape",
+    "model_compatibility": "model",
+    "unsigned_cosine": "ucos",
+    "signed_cosine": "scos",
+    "delta_norm": "norm",
+    "execution_trace": "exec",
+    "schedule_trace": "sched",
+    "runtime_smoke": "smoke",
+    "behavioural_reproduction": "BEHAV",
+}
+
+_CHECK_LABEL = {
+    "schema": "Schema",
+    "checksum": "Checksums",
+    "model_compatibility": "Model compatibility",
+    "signed_cosine": "Signed direction",
+    "delta_norm": "Delta norm",
+    "execution_trace": "Layer/site execution",
+    "behavioural_reproduction": "Behavior reproduction",
+}
+
+
+def render_artifact_report(variant: dict[str, Any], *, tolerance: float) -> str:
+    """The per-artifact report from the spec.
+
+    ``caught = True`` means the layer detected a defect, so it renders as FAIL.
+    """
+    caught = dict(variant["caught"])
+    caught["behavioural_reproduction"] = variant.get("caught_behavioural", False)
+
+    lines = [f"Artifact: {variant['key']}", ""]
+    lines.append("Artifact Fidelity")
+    lines.append("")
+    width = max(len(v) for v in _CHECK_LABEL.values())
+    for layer, label in _CHECK_LABEL.items():
+        if layer not in caught:
+            continue
+        state = "FAIL" if caught[layer] else "PASS"
+        lines.append(f"{label.ljust(width)}   {state}")
+
+    delta = variant.get("behavioural_delta")
+    if delta is not None and delta == delta:  # not NaN
+        lines.append("")
+        lines.append(f"behavioural_delta = {delta:.4f}  (tolerance {tolerance})")
+
+    lines += ["", "Components:"]
+    fidelity = variant.get("fidelity", {})
+    for component in COMPONENTS:
+        lines.append(f"  {component.ljust(12)} {fidelity.get(component, 'not measured')}")
+
+    lines += ["", "Overall:", variant.get("verdict", "UNKNOWN")]
+    return "\n".join(lines)
+
+
+def render_detection_matrix(
+    variants: list[dict[str, Any]], layers: list[str]
+) -> str:
+    """Rows = corruptions, columns = layers. ``X`` means the layer caught it."""
+    header = f"{'corruption':<28}{'delta':>9}  " + " ".join(
+        f"{LAYER_ABBREV.get(layer, layer[:5]):>5}" for layer in layers
+    )
+    lines = [header, "-" * len(header)]
+    for variant in variants:
+        caught = dict(variant["caught"])
+        caught["behavioural_reproduction"] = variant.get("caught_behavioural", False)
+        delta = variant.get("behavioural_delta", float("nan"))
+        cells = " ".join(
+            f"{('X' if caught.get(layer) else '.'):>5}" for layer in layers
+        )
+        lines.append(f"{variant['key']:<28}{delta:>9.4f}  {cells}")
+    return "\n".join(lines)
+
+
+def to_csv(variants: list[dict[str, Any]], layers: list[str]) -> str:
+    rows = [",".join(["corruption", "verdict", "behavioural_delta"] + layers)]
+    for variant in variants:
+        caught = dict(variant["caught"])
+        caught["behavioural_reproduction"] = variant.get("caught_behavioural", False)
+        cells = [
+            variant["key"],
+            variant.get("verdict", ""),
+            f"{variant.get('behavioural_delta', float('nan')):.4f}",
+        ]
+        cells += ["caught" if caught.get(layer) else "-" for layer in layers]
+        rows.append(",".join(cells))
+    return "\n".join(rows) + "\n"
+
+
+def summarise_claim(
+    variants: list[dict[str, Any]], layers: list[str]
+) -> dict[str, Any]:
+    """Does the behavioural layer catch anything the cheap layers miss?
+
+    ``unsigned_cosine`` is excluded from "cheap layers" on purpose. It is in the
+    matrix as the check that failed historically, not as one a validator should
+    rely on; counting it as a legitimate catcher would let the very blind spot
+    under study argue that nothing is blind.
+    """
+    cheap = [
+        layer for layer in layers
+        if layer not in ("behavioural_reproduction", "unsigned_cosine")
+    ]
+    behaviour_only, false_positives = [], []
+    for variant in variants:
+        caught = dict(variant["caught"])
+        failed_behaviour = variant.get("caught_behavioural", False)
+        hits = [layer for layer in cheap if caught.get(layer)]
+        if failed_behaviour and not hits:
+            behaviour_only.append(variant["key"])
+        if "NEGATIVE CONTROL" in variant.get("defect", "") and (hits or failed_behaviour):
+            false_positives.append(variant["key"])
+    return {
+        "claim_supported": bool(behaviour_only),
+        "caught_only_by_behaviour": behaviour_only,
+        "negative_control_false_positives": false_positives,
+    }