brainpatch 1.2.0__py3-none-any.whl

brainpatch/steering/schedule.py

@@ -0,0 +1,138 @@
+"""Token-indexed strength schedules for dynamic steering.
+
+A schedule maps *generated token index* to a strength multiplier, using
+step-hold semantics: the value at index ``n`` is the value of the largest
+keyframe ``<= n``. This is what makes "turn the patch on 20 tokens into the
+answer" expressible::
+
+    StrengthSchedule({0: 0.0, 20: 1.0, 40: 2.0})
+
+    index:     0 ... 19   20 ... 39   40 ...
+    strength:  0.0        1.0         2.0
+
+Optional linear interpolation smooths the transitions, which matters because an
+abrupt large change in the residual stream mid-generation can itself derail the
+model -- a confound worth being able to rule out.
+
+Everything here is pure arithmetic so it can be tested without a model.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import Any, Mapping
+
+
+@dataclass(frozen=True)
+class StrengthSchedule:
+    """Piecewise strength multiplier over generated-token index.
+
+    Parameters
+    ----------
+    keyframes:
+        Mapping of token index -> strength multiplier. Indices count *generated*
+        tokens, starting at 0 for the first token the model produces; prompt
+        tokens are not counted.
+    interpolate:
+        When True, values between keyframes are linearly interpolated. When
+        False (default) the schedule is a step function.
+    default:
+        Strength used before the first keyframe when index 0 is not specified.
+    """
+
+    keyframes: Mapping[int, float]
+    interpolate: bool = False
+    default: float = 1.0
+
+    def __post_init__(self) -> None:
+        if not self.keyframes:
+            raise ValueError("a schedule needs at least one keyframe")
+        normalized: dict[int, float] = {}
+        for key, value in self.keyframes.items():
+            try:
+                index = int(key)
+            except (TypeError, ValueError) as exc:
+                raise ValueError(f"schedule keys must be integers, got {key!r}") from exc
+            if index < 0:
+                raise ValueError(f"schedule token index must be >= 0, got {index}")
+            if not isinstance(value, (int, float)) or isinstance(value, bool):
+                raise ValueError(f"schedule value at {index} must be numeric, got {value!r}")
+            normalized[index] = float(value)
+        # frozen dataclass: bypass __setattr__ to install the cleaned mapping
+        object.__setattr__(self, "keyframes", dict(sorted(normalized.items())))
+
+    @property
+    def steps(self) -> list[int]:
+        """Sorted keyframe indices."""
+        return list(self.keyframes)
+
+    def strength_at(self, token_index: int) -> float:
+        """Strength multiplier for the given generated-token index.
+
+        >>> s = StrengthSchedule({0: 0.0, 20: 1.0, 40: 2.0})
+        >>> s.strength_at(0), s.strength_at(19), s.strength_at(20), s.strength_at(100)
+        (0.0, 0.0, 1.0, 2.0)
+        """
+        if token_index < 0:
+            raise ValueError(f"token_index must be >= 0, got {token_index}")
+
+        steps = self.steps
+        # Before the first keyframe: fall back to `default`.
+        if token_index < steps[0]:
+            return self.default
+
+        # Find the last keyframe at or before token_index.
+        lo = 0
+        for i, step in enumerate(steps):
+            if step <= token_index:
+                lo = i
+            else:
+                break
+
+        left = steps[lo]
+        left_value = self.keyframes[left]
+        if not self.interpolate or lo + 1 >= len(steps):
+            return left_value
+
+        right = steps[lo + 1]
+        right_value = self.keyframes[right]
+        span = right - left
+        if span <= 0:  # defensive; sorted unique keys make this unreachable
+            return left_value
+        alpha = (token_index - left) / span
+        return left_value + alpha * (right_value - left_value)
+
+    def is_constant(self) -> bool:
+        """True if the schedule never changes strength."""
+        values = set(self.keyframes.values())
+        if len(values) > 1:
+            return False
+        return self.steps[0] == 0 or values == {self.default}
+
+    def to_dict(self) -> dict[str, Any]:
+        return {
+            "keyframes": {str(k): v for k, v in self.keyframes.items()},
+            "interpolate": self.interpolate,
+            "default": self.default,
+        }
+
+    @classmethod
+    def from_dict(cls, data: Mapping[str, Any]) -> "StrengthSchedule":
+        """Build from a serialized schedule.
+
+        Accepts both the full form ``{"keyframes": {...}, ...}`` and the bare
+        ``{"0": 0.0, "20": 1.0}`` shorthand used inside patch files.
+        """
+        if "keyframes" in data:
+            raw = data["keyframes"]
+            return cls(
+                keyframes={int(k): float(v) for k, v in raw.items()},
+                interpolate=bool(data.get("interpolate", False)),
+                default=float(data.get("default", 1.0)),
+            )
+        return cls(keyframes={int(k): float(v) for k, v in data.items()})
+
+    @classmethod
+    def constant(cls, strength: float = 1.0) -> "StrengthSchedule":
+        """A schedule that holds one strength for the whole generation."""
+        return cls(keyframes={0: float(strength)}, default=float(strength))

brainpatch/ui/__init__.py

@@ -0,0 +1,11 @@
+"""Local web UI. Requires ``pip install 'brainpatch[ui]'``."""
+
+__all__ = ["launch"]
+
+
+def __getattr__(name: str):
+    if name == "launch":
+        from brainpatch.ui.app import launch
+
+        return launch
+    raise AttributeError(f"module {__name__!r} has no attribute {name!r}")

brainpatch/ui/app.py

@@ -0,0 +1,201 @@
+"""Local Gradio UI.
+
+Runs entirely on the user's machine: their backend, their model, their installed
+patches. No hosted service, no Modal, no telemetry.
+
+The UI deliberately shows each patch's ``evidence_level`` next to its slider. If
+a patch has no validated behavioural effect, the interface says so rather than
+implying that moving the slider does something known.
+"""
+
+from __future__ import annotations
+
+from typing import Any
+
+BASELINE_LABEL = "BASELINE (patch off)"
+PATCHED_LABEL = "PATCHED"
+
+
+def launch(
+    *,
+    model: str | None = None,
+    backend: str = "auto",
+    device: str = "auto",
+    host: str = "127.0.0.1",
+    port: int = 7860,
+    share: bool = False,
+) -> None:
+    """Build and serve the UI."""
+    import gradio as gr
+
+    from brainpatch.patch.registry import default_registry
+    from brainpatch.runtime.auto import available_backends
+    from brainpatch.runtime.base import GenerationConfig
+    from brainpatch.runtime.model import BrainPatchedModel
+
+    registry = default_registry()
+    state: dict[str, Any] = {"model": None, "model_id": None}
+
+    def installed_names() -> list[str]:
+        return [p.name for p in registry.list_patches()]
+
+    def ensure_model(model_id: str, backend_name: str) -> BrainPatchedModel:
+        if state["model"] is None or state["model_id"] != (model_id, backend_name):
+            if state["model"] is not None:
+                state["model"].unload()
+            state["model"] = BrainPatchedModel.from_pretrained(
+                model_id, backend=backend_name, device=device
+            )
+            state["model_id"] = (model_id, backend_name)
+        return state["model"]
+
+    def run_compare(
+        model_id: str,
+        backend_name: str,
+        patch_name: str,
+        prompt: str,
+        strength: float,
+        max_tokens: int,
+        temperature: float,
+        use_schedule: bool,
+        turn_on_at: int,
+    ) -> tuple[str, str, str]:
+        if not model_id:
+            return "", "", "Enter a model id first."
+        if not prompt.strip():
+            return "", "", "Enter a prompt."
+        try:
+            patched = ensure_model(model_id, backend_name)
+        except Exception as exc:  # noqa: BLE001
+            return "", "", f"Could not load model: {exc}"
+
+        for name in list(patched.list_patches()):
+            patched.remove_patch(name)
+
+        info_bits = []
+        if patch_name and patch_name != "(none)":
+            try:
+                handle = patched.install(patch_name, strength=strength)
+                if use_schedule:
+                    handle.schedule = {0: 0.0, int(turn_on_at): 1.0}
+                info_bits.append(
+                    f"patch **{handle.name}** · strength {handle.strength} · "
+                    f"evidence `{handle.evidence_level}`"
+                )
+                if handle.evidence_level in {"none", "correlational"}:
+                    info_bits.append(
+                        "⚠️ This patch has **no validated behavioural effect**. "
+                        "Any difference below is an activation perturbation, not a "
+                        "demonstrated capability."
+                    )
+            except Exception as exc:  # noqa: BLE001
+                return "", "", f"Could not install patch: {exc}"
+
+        cfg = GenerationConfig(max_new_tokens=int(max_tokens), temperature=float(temperature))
+        try:
+            result = patched.compare(prompt, cfg)
+        except Exception as exc:  # noqa: BLE001
+            return "", "", f"Generation failed: {exc}"
+
+        if result["baseline"] == result["patched"]:
+            info_bits.append("Outputs are identical at this strength on this prompt.")
+        return result["baseline"], result["patched"], "\n\n".join(info_bits)
+
+    def patch_details(name: str) -> str:
+        if not name or name == "(none)":
+            return "Select a patch."
+        try:
+            loaded = registry.get(name).load()
+        except Exception as exc:  # noqa: BLE001
+            return f"Could not read patch: {exc}"
+        manifest = loaded.manifest
+        lines = [
+            f"# {manifest.name}",
+            "",
+            manifest.description or "_no description_",
+            "",
+            f"- base model: `{manifest.base_model.model_id}`",
+            f"- revision: `{manifest.base_model.revision or 'unpinned'}`",
+            f"- layers: {manifest.layers}",
+            f"- size: {loaded.archive_bytes / 1024:.1f} KB",
+            f"- evidence level: **{manifest.evidence_level}**",
+            "",
+            "## Backend compatibility",
+        ]
+        if manifest.compatibility:
+            for backend_name, entry in sorted(manifest.compatibility.items()):
+                lines.append(f"- `{backend_name}`: **{entry.get('status', 'unsupported')}**")
+        else:
+            lines.append("- none recorded")
+        if manifest.evaluation:
+            lines += ["", "## Recorded evaluation", "```json", _pretty(manifest.evaluation), "```"]
+        else:
+            lines += ["", "_No evaluation recorded: this patch has no measured effect._"]
+        return "\n".join(lines)
+
+    def backend_status() -> str:
+        rows = ["| backend | status | detail |", "|---|---|---|"]
+        for status in available_backends():
+            mark = "available" if status.available else "unavailable"
+            rows.append(f"| `{status.name}` | {mark} | {status.detail} |")
+        return "\n".join(rows)
+
+    with gr.Blocks(title="BrainPatch") as blocks:
+        gr.Markdown(
+            "# BrainPatch\n"
+            "Tiny activation patches for frozen language models. "
+            "Everything here runs locally on your machine.\n\n"
+            "> Feature directions are not concepts. A patch changes activations; "
+            "whether that produces a *specific* behaviour is an empirical question "
+            "answered by each patch's evidence level."
+        )
+
+        with gr.Tab("Compare"):
+            with gr.Row():
+                with gr.Column(scale=2):
+                    model_box = gr.Textbox(label="Model", value=model or "", placeholder="Qwen/Qwen2.5-1.5B-Instruct")
+                    prompt_box = gr.Textbox(label="Prompt", lines=5, value="Evaluate my idea.")
+                    with gr.Row():
+                        max_tokens = gr.Slider(16, 512, value=128, step=16, label="max new tokens")
+                        temperature = gr.Slider(0.0, 1.5, value=0.0, step=0.1, label="temperature")
+                with gr.Column(scale=1):
+                    backend_box = gr.Dropdown(
+                        ["auto", "transformers", "llamacpp", "vllm", "mlx"],
+                        value=backend,
+                        label="backend",
+                    )
+                    patch_box = gr.Dropdown(
+                        ["(none)"] + installed_names(), value="(none)", label="patch"
+                    )
+                    strength_box = gr.Slider(-8, 8, value=1.0, step=0.1, label="strength")
+                    schedule_box = gr.Checkbox(label="dynamic schedule", value=False)
+                    turn_on_box = gr.Slider(0, 128, value=24, step=4, label="turn on at token")
+                    go = gr.Button("Generate", variant="primary")
+            with gr.Row():
+                baseline_out = gr.Textbox(label=BASELINE_LABEL, lines=14)
+                patched_out = gr.Textbox(label=PATCHED_LABEL, lines=14)
+            info_out = gr.Markdown()
+            go.click(
+                run_compare,
+                [model_box, backend_box, patch_box, prompt_box, strength_box,
+                 max_tokens, temperature, schedule_box, turn_on_box],
+                [baseline_out, patched_out, info_out],
+            )
+
+        with gr.Tab("Patch Inspector"):
+            inspect_box = gr.Dropdown(
+                ["(none)"] + installed_names(), value="(none)", label="installed patch"
+            )
+            details = gr.Markdown()
+            inspect_box.change(patch_details, [inspect_box], [details])
+
+        with gr.Tab("Backends"):
+            gr.Markdown(backend_status())
+
+    blocks.launch(server_name=host, server_port=port, share=share)
+
+
+def _pretty(data: Any) -> str:
+    import json
+
+    return json.dumps(data, indent=2, sort_keys=True)

brainpatch/verify/__init__.py

@@ -0,0 +1,66 @@
+"""Verify that a `.brainpatch` artifact encodes the intervention it claims to.
+
+Checksums prove a file was not modified. They do not prove the sign is right, the
+layer is right, or that the runtime applies the intervention where the manifest
+says it does. This package covers the second half.
+
+Four levels, cheapest first:
+
+``structural``
+    The archive is well-formed, its checksums verify, the manifest is
+    self-consistent, tensor shapes match, and the declared base model is the one
+    you are loading. Needs no model and no GPU.
+
+``numerical``
+    The applied delta (``coefficient x vector``) matches a trusted reference in
+    **signed** direction and magnitude. Requires a reference artifact. The
+    reference must be signed: a cosine against an unsigned reference cannot see a
+    sign error, which is how this project once shipped one.
+
+``execution``
+    The runtime actually applies the intervention at the declared layer and on
+    the declared passes. Read from ``resolve_edits``, not from the manifest --
+    a manifest declaring ``site: prompt`` and a runtime steering every generated
+    token is exactly the disagreement this level exists to find. Requires a
+    loaded backend.
+
+``behavioural``
+    The artifact reproduces a recorded behavioural result within a tolerance you
+    set in advance. Optional, and the most expensive.
+
+Empirically (``experiments/artifact_fidelity_v1``), across thirteen controlled
+corruptions of a real artifact: schema, checksum, shape and model-compatibility
+checks caught **none** of eleven real defects, while signed direction, delta norm
+and execution tracing together caught **all** of them. Layer and site defects were
+invisible to every file-level check -- only execution tracing found them.
+"""
+
+from brainpatch.verify.checks import (
+    COSINE_TOLERANCE,
+    DETECTION_LAYERS,
+    NORM_TOLERANCE,
+    Reference,
+    reference_from,
+    run_execution_trace,
+    run_runtime_smoke,
+    run_static_layers,
+)
+from brainpatch.verify.workflow import (
+    VerificationResult,
+    verify_artifact,
+    verify_behavioural_regression,
+)
+
+__all__ = [
+    "COSINE_TOLERANCE",
+    "DETECTION_LAYERS",
+    "NORM_TOLERANCE",
+    "Reference",
+    "VerificationResult",
+    "reference_from",
+    "run_execution_trace",
+    "run_runtime_smoke",
+    "run_static_layers",
+    "verify_artifact",
+    "verify_behavioural_regression",
+]

brainpatch/verify/behavioural.py

@@ -0,0 +1,156 @@
+"""Behavioural fidelity: does the deployed artifact reproduce the validated effect?
+
+The only detection layer that is not blind by construction, and the only one that
+would have caught the sign defect this project shipped.
+
+**The product path is the only path.** Every measurement here runs
+``load_patch`` -> ``validate_patch`` -> ``install_patch`` and then generates
+through the runtime's *own installed hooks*, so ``resolve_edits``, the site
+gating and the vector cache are all exercised. Reconstructing the research
+direction and testing that is not artifact validation -- it is precisely what let
+the sign defect through.
+
+Generation is batched. That is an I/O choice, not a path change: the hooks are
+the backend's, installed by ``install_patch``, and the pass counter the site
+gating depends on advances identically whether one prompt or twenty-four are in
+flight. :func:`unbatched_reference` re-measures through the fully public
+``backend.generate`` path so the two can be compared rather than assumed equal.
+"""
+
+from __future__ import annotations
+
+from typing import Any
+
+#: Fixed for every condition. Batch composition changes left-padding and can flip
+#: argmax on near-ties, so it must never vary between reference and variant.
+BATCH_SIZE = 24
+GENERATION_KWARGS = {"max_new_tokens": 96, "do_sample": False}
+
+
+def generate_through_installed_patch(
+    backend: Any, prompts: list[str], *, batch_size: int = BATCH_SIZE
+) -> list[str]:
+    """Batched generation driven by the backend's installed hooks.
+
+    Hooks are installed once. The pass counter is reset per batch, exactly as
+    ``backend.generate`` resets it per call -- pass 0 is the prompt, pass n>0 is
+    generated token n-1, which is what the ``site`` gating reads.
+    """
+    import torch
+
+    tokenizer = backend.tokenizer
+    pad_id = tokenizer.pad_token_id or tokenizer.eos_token_id
+    original_side = tokenizer.padding_side
+    tokenizer.padding_side = "left"
+
+    outputs: list[str] = []
+    backend._apply_to_prompt = True
+    backend._trace = []
+    backend._install_hooks()
+    try:
+        for start in range(0, len(prompts), batch_size):
+            chunk = prompts[start : start + batch_size]
+            rendered = [
+                tokenizer.apply_chat_template(
+                    [{"role": "user", "content": p}], tokenize=False, add_generation_prompt=True
+                )
+                for p in chunk
+            ]
+            encoded = tokenizer(
+                rendered, return_tensors="pt", padding=True, add_special_tokens=False
+            ).to(backend.device)
+            # Same reset the public generate() performs between calls.
+            backend._pass_counter = 0
+            backend._current_pass = 0
+            with torch.inference_mode():
+                generated = backend.model.generate(
+                    **encoded, pad_token_id=pad_id, **GENERATION_KWARGS
+                )
+            for row in range(len(chunk)):
+                new_tokens = generated[row, encoded["input_ids"].shape[1] :]
+                outputs.append(tokenizer.decode(new_tokens, skip_special_tokens=True))
+    finally:
+        backend._remove_hooks()
+        tokenizer.padding_side = original_side
+    return outputs
+
+
+def unbatched_reference(backend: Any, prompts: list[str]) -> list[str]:
+    """Fully public product path, one prompt at a time.
+
+    Slow (~3.3 s/item against ~0.15 s/item batched), so it is run once on the
+    reference artifact to establish that the batched path agrees, rather than on
+    every variant.
+    """
+    from brainpatch.runtime.base import GenerationConfig
+
+    config = GenerationConfig(
+        max_new_tokens=GENERATION_KWARGS["max_new_tokens"], temperature=0.0
+    )
+    return [backend.generate(prompt, config) for prompt in prompts]
+
+
+def measure(examples: Any, texts: list[str]) -> dict[str, Any]:
+    """Behavioural summary using the frozen evaluator. Nothing is tuned."""
+    from brainpatch.research.generation_eval import summarise
+
+    polarities = [str(e.metadata.get("polarity", "false_claim")) for e in examples]
+    return summarise(polarities, texts)
+
+
+def commitment_effect(behaviour: dict[str, Any], baseline: dict[str, Any]) -> dict[str, float]:
+    """The construct-correct reading of this artifact.
+
+    Reported as **commitment / anti-evasion**, never as anti-sycophancy: the
+    construct-validity analysis found only ~22% of the correction gain came from
+    agreement moving to challenge, and that sycophantic agreement *rose*. The
+    evasion rate is the primary quantity because it is what the artifact was
+    shown to move.
+    """
+    return {
+        "correction_rate": behaviour["correction_rate_false_claims"],
+        "correction_gain": (
+            behaviour["correction_rate_false_claims"]
+            - baseline["correction_rate_false_claims"]
+        ),
+        "evasion_rate": behaviour["other_rate_false_claims"],
+        "evasion_reduction": (
+            baseline["other_rate_false_claims"] - behaviour["other_rate_false_claims"]
+        ),
+        "sycophantic_rate": behaviour["sycophantic_agreement_rate_false_claims"],
+        "sis": behaviour["selective_independence_score"],
+        "degenerate_rate": behaviour["degenerate_rate"],
+        "mean_response_chars": behaviour["mean_response_chars"],
+    }
+
+
+def behavioural_delta(variant: dict[str, float], reference: dict[str, float]) -> float:
+    """Primary fidelity distance: absolute difference in correction gain.
+
+    A single pre-registered scalar, chosen before results. Exact string identity
+    is deliberately not used -- batching and F16 storage both flip argmax on
+    near-ties, and a Q4 quantization agrees with its own reference on only ~57%
+    of greedy outputs, so string agreement measures decoding noise more than it
+    measures fidelity.
+    """
+    return abs(variant["correction_gain"] - reference["correction_gain"])
+
+
+def transition_counts(
+    examples: Any, before: list[str], after: list[str], polarity: str = "false_claim"
+) -> dict[str, int]:
+    """Per-item label transitions, so two artifacts reaching the same rate by
+    different routes are not mistaken for each other."""
+    from collections import Counter
+
+    from brainpatch.research.generation_eval import per_item_labels
+
+    polarities = [str(e.metadata.get("polarity", "false_claim")) for e in examples]
+    rows_b = per_item_labels(polarities, before)
+    rows_a = per_item_labels(polarities, after)
+    pairs = Counter(
+        (b["label"], a["label"])
+        for b, a in zip(rows_b, rows_a)
+        if b["polarity"] == polarity
+    )
+    return {f"{a}->{b}": n for (a, b), n in sorted(pairs.items())}