openjev-sdk 0.2.0__py3-none-any.whl

openjev/__init__.py

@@ -0,0 +1,33 @@
+"""Local option scoring and trainable candidate-decision heads.
+
+Model backends are loaded only when used, so importing openjev does not initialize
+an accelerator or download weights.
+"""
+from .decision import Candidate, ChoiceTask
+from .scorer import DEFAULT_MODEL, OptionScore, OptionScorer
+
+__all__ = ["DEFAULT_MODEL", "OptionScore", "OptionScorer", "Candidate", "ChoiceTask",
+           "DecisionScorer", "extract_decision_features", "train_decision_head",
+           "TemperatureCalibrator", "run_decision_experiment"]
+
+
+def __getattr__(name):
+    if name == 'TemperatureCalibrator':
+        from .decision_probability import TemperatureCalibrator
+        return TemperatureCalibrator
+    if name == 'run_decision_experiment':
+        from .decision_experiment import run_experiment
+        return run_experiment
+    aliases = {"DecisionScorer": "DecisionScorer",
+               "extract_decision_features": "extract_features",
+               "train_decision_head": "train_artifact"}
+    if name in aliases:
+        from . import decision_mlx
+        value = getattr(decision_mlx, aliases[name])
+        globals()[name] = value
+        return value
+    raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
+
+
+def __dir__():
+    return sorted(set(globals()) | set(__all__))

openjev/cli.py

@@ -0,0 +1,286 @@
+"""Command line entry points: score, eval, bench, check, serve, features, train, eval-head."""
+from __future__ import annotations
+
+import argparse
+import json
+import random
+import statistics
+import sys
+import time
+
+from .scorer import DEFAULT_MODEL, NORMS, OptionScorer, iter_jsonl
+
+
+def _add_common(p: argparse.ArgumentParser) -> None:
+    p.add_argument("--backend", choices=("auto", "mlx", "torch"), default="auto")
+    p.add_argument("--device", default="auto", help="auto, cpu, cuda, cuda:N, or mps (torch backend)")
+    p.add_argument("--model", default=DEFAULT_MODEL, help="local dir or HF repo id")
+    p.add_argument("--adapter", default=None, help="LoRA adapter dir to load on top of --model (e.g. adapters/chess-lora)")
+    p.add_argument("--batch-size", type=int, default=8, help="options per forward pass")
+    p.add_argument("--norm", choices=NORMS, default="mean",
+                   help="mean: per-token log-prob; sum: total; pmi: sum minus unconditional")
+    p.add_argument("--chat", action="store_true",
+                   help="wrap context in Gemma's chat template; options score as the reply")
+    p.add_argument("--sep", default="", help="string inserted between context and option")
+
+
+def _scorer(args: argparse.Namespace) -> OptionScorer:
+    t = time.perf_counter()
+    s = OptionScorer(args.model, batch_size=args.batch_size, chat=args.chat, sep=args.sep, adapter_path=args.adapter, backend=args.backend, device=args.device)
+    print(f"loaded {args.model} in {time.perf_counter() - t:.1f}s", file=sys.stderr)
+    return s
+
+
+def _read_options(path: str) -> list[str]:
+    with open(path) as f:
+        return [line.rstrip("\n") for line in f if line.strip()]
+
+
+def cmd_score(args: argparse.Namespace) -> None:
+    if args.options_file:
+        args.option = _read_options(args.options_file) + args.option
+    if len(args.option) < 2:
+        sys.exit("need at least two options (--option ... or --options-file)")
+    scorer = _scorer(args)
+    result = scorer.score(args.context, args.option, norm=args.norm)
+    scorer.score(args.context, args.option, norm=args.norm)  # warm second run for timing
+    payload = {
+        "context": args.context,
+        "norm": args.norm,
+        "timing": scorer.last_timing,
+        "options": [r.to_dict() for r in result],
+    }
+    if args.json:
+        print(json.dumps(payload, indent=2))
+        return
+    ranked = sorted(result, key=lambda r: -r.score)
+    for r in ranked:
+        print(f"{r.probability:7.3%}  score={r.score:8.3f}  sum={r.logprob_sum:8.3f}  "
+              f"n={r.n_tokens:3d}  {r.option!r}")
+    t = scorer.last_timing
+    print(f"[{t['context_tokens']} ctx tok, {t['option_tokens']} opt tok] "
+          f"prefill {t['prefill_s']*1000:.0f} ms, options {t['options_s']*1000:.0f} ms, "
+          f"total {t['total_s']*1000:.0f} ms", file=sys.stderr)
+
+
+def cmd_eval(args: argparse.Namespace) -> None:
+    scorer = _scorer(args)
+    rows = list(iter_jsonl(args.data))
+    if args.limit:
+        rows = rows[: args.limit]
+    fixed = _read_options(args.fixed_options) if args.fixed_options else None
+    top1 = top3 = labelled = 0
+    lat: list[float] = []
+    for i, row in enumerate(rows):
+        options = row.get("options") or fixed
+        if not options:
+            sys.exit(f"row {i} has no 'options' and no --fixed-options given")
+        res = scorer.score(row["context"], options, norm=args.norm)
+        lat.append(scorer.last_timing["total_s"])
+        order = sorted(range(len(res)), key=lambda j: -res[j].score)
+        label = row.get("label")
+        if label is not None:
+            labelled += 1
+            top1 += order[0] == label
+            top3 += label in order[:3]
+        if args.verbose or label is None:
+            print(json.dumps({"i": i, "label": label, "pred": order[0], "pred_option": options[order[0]],
+                              "probs": [round(r.probability, 4) for r in res]}))
+    n = len(rows)
+    print(json.dumps({
+        "examples": n,
+        "labelled": labelled,
+        "top1": top1 / labelled if labelled else None,
+        "top3": top3 / labelled if labelled else None,
+        "norm": args.norm,
+        "median_latency_s": statistics.median(lat),
+        "mean_latency_s": statistics.fmean(lat),
+    }, indent=2))
+
+
+def _synthetic(scorer: OptionScorer, ctx_tokens: int, n_opts: int, opt_tokens: int, seed: int):
+    rng = random.Random(seed)
+    words = ("river stone cloud engine quiet market ledger signal orbit velvet "
+             "harbour lantern cipher meadow granite").split()
+    def text(n):  # roughly n tokens of plain words
+        return " ".join(rng.choice(words) for _ in range(n))
+    context = text(ctx_tokens)
+    options = [text(opt_tokens) for _ in range(n_opts)]
+    return context, options
+
+
+def cmd_bench(args: argparse.Namespace) -> None:
+    scorer = _scorer(args)
+    context, options = _synthetic(scorer, args.context_tokens, args.options, args.option_tokens, 0)
+    ctx_n = len(scorer.context_ids(context))
+    opt_n = sum(len(scorer.option_ids(o)) for o in options)
+    scorer.score(context, options, norm="mean")  # warm-up
+    scorer.score_naive(context, options[:1])
+
+    cached = []
+    for _ in range(args.repeat):
+        scorer.score(context, options, norm="mean")
+        cached.append(scorer.last_timing["total_s"])
+    naive = []
+    for _ in range(args.repeat):
+        t = time.perf_counter()
+        scorer.score_naive(context, options)
+        naive.append(time.perf_counter() - t)
+    print(json.dumps({
+        "context_tokens": ctx_n,
+        "options": len(options),
+        "option_tokens_total": opt_n,
+        "batch_size": scorer.batch_size,
+        "prefix_cached_batched_s": {"median": statistics.median(cached), "min": min(cached)},
+        "naive_per_option_s": {"median": statistics.median(naive), "min": min(naive)},
+        "speedup": statistics.median(naive) / statistics.median(cached),
+    }, indent=2))
+
+
+def cmd_check(args: argparse.Namespace) -> None:
+    """Verify the prefix-shared batched path against naive full re-encoding."""
+    scorer = _scorer(args)
+    cases = [
+        ("The capital of France is", [" Paris", " Berlin", " a city in Europe", " Lyon"]),
+        ("Q: What is 2 + 2?\nA:", [" 4", " 5", " four", " twenty-two"]),
+    ]
+    context, options = _synthetic(scorer, args.context_tokens, args.options, args.option_tokens, 1)
+    cases.append((context, options))
+    worst = 0.0
+    for ctx, opts in cases:
+        fast = [r.logprob_sum for r in scorer.score(ctx, opts, norm="sum")]
+        slow = scorer.score_naive(ctx, opts)
+        diffs = [abs(a - b) for a, b in zip(fast, slow)]
+        worst = max(worst, max(diffs))
+        rel = max(d / max(1e-6, abs(b)) for d, b in zip(diffs, slow))
+        print(f"ctx_tokens={len(scorer.context_ids(ctx)):4d} options={len(opts):2d} "
+              f"max_abs_diff={max(diffs):.4f} max_rel_diff={rel:.4%}")
+        for o, a, b in zip(opts, fast, slow):
+            print(f"    cached={a:9.3f} naive={b:9.3f}  {o[:40]!r}")
+    ok = worst < args.tol
+    print("OK" if ok else f"MISMATCH (worst abs diff {worst:.4f} > tol {args.tol})")
+    sys.exit(0 if ok else 1)
+
+
+def cmd_serve(args: argparse.Namespace) -> None:
+    from .server import serve
+
+    serve(args.host, args.port, args.model, args.batch_size, backend=args.backend, device=args.device)
+
+
+def cmd_features(args: argparse.Namespace) -> None:
+    import platform
+    if platform.system() != "Darwin" or platform.machine() != "arm64" or getattr(args, "backend", "auto") == "torch":
+        raise SystemExit("Feature extraction and head training/evaluation require MLX on Apple silicon. Scoring and serving support Windows/Linux via torch.")
+    from .features import extract_dataset
+
+    scorer = _scorer(args)
+    meta = extract_dataset(scorer, args.data, args.out, limit=args.limit, chat=args.chat, sep=args.sep,
+                           contextual=args.contextual)
+    print(json.dumps(meta))
+
+
+def cmd_train(args: argparse.Namespace) -> None:
+    import platform
+    if platform.system() != "Darwin" or platform.machine() != "arm64" or getattr(args, "backend", "auto") == "torch":
+        raise SystemExit("Feature extraction and head training/evaluation require MLX on Apple silicon. Scoring and serving support Windows/Linux via torch.")
+    from .train import train
+
+    print(json.dumps(train(args.train, args.validation, args.out, rank=args.rank, epochs=args.epochs,
+                           batch_size=args.batch_size, lr=args.learning_rate, seed=args.seed)))
+
+
+def cmd_eval_head(args: argparse.Namespace) -> None:
+    import platform
+    if platform.system() != "Darwin" or platform.machine() != "arm64" or getattr(args, "backend", "auto") == "torch":
+        raise SystemExit("Feature extraction and head training/evaluation require MLX on Apple silicon. Scoring and serving support Windows/Linux via torch.")
+    from .features import FeatureSet
+    from .head import AttentionHead
+    from .train import evaluate
+
+    head, cfg = AttentionHead.load(args.checkpoint)
+    fs = FeatureSet(args.features)
+    print(json.dumps({"model": evaluate(head, fs), "shuffled_context": evaluate(head, fs, shuffle_context=True),
+                      "checkpoint": args.checkpoint, "rank": cfg["rank"]}, indent=2))
+
+
+def main(argv: list[str] | None = None) -> None:
+    p = argparse.ArgumentParser(prog="openjev", description=__doc__)
+    sub = p.add_subparsers(dest="cmd", required=True)
+
+    s = sub.add_parser("score", help="score options for one context")
+    _add_common(s)
+    s.add_argument("--context", required=True)
+    s.add_argument("--option", action="append", default=[], help="repeat for each option")
+    s.add_argument("--options-file", help="text file with one predefined option per line")
+    s.add_argument("--json", action="store_true")
+    s.set_defaults(fn=cmd_score)
+
+    e = sub.add_parser("eval", help="top-k accuracy on jevlike-style JSONL {context, options, label}")
+    _add_common(e)
+    e.add_argument("data")
+    e.add_argument("--limit", type=int, default=0)
+    e.add_argument("--fixed-options", help="text file of options used for every row lacking an 'options' field")
+    e.add_argument("--verbose", action="store_true")
+    e.set_defaults(fn=cmd_eval)
+
+    for name, fn, helptext in (
+        ("bench", cmd_bench, "latency of prefix-cached batched scoring vs naive re-encoding"),
+        ("check", cmd_check, "verify cached batched scores match naive full-sequence scores"),
+    ):
+        b = sub.add_parser(name, help=helptext)
+        _add_common(b)
+        b.add_argument("--context-tokens", type=int, default=200)
+        b.add_argument("--options", type=int, default=8)
+        b.add_argument("--option-tokens", type=int, default=30)
+        b.add_argument("--repeat", type=int, default=5)
+        b.add_argument("--tol", type=float, default=0.5, help="check: max abs log-prob diff allowed")
+        b.set_defaults(fn=fn)
+
+    f = sub.add_parser("features", help="cache frozen-Gemma features for a JSONL file into an .npz")
+    _add_common(f)
+    f.add_argument("data")
+    f.add_argument("--out", required=True)
+    f.add_argument("--limit", type=int, default=0)
+    f.add_argument("--contextual", action="store_true",
+                   help="encode options as continuations of the context (leaks the match into option features)")
+    f.set_defaults(fn=cmd_features)
+
+    tr = sub.add_parser("train", help="train the attention head on cached features")
+    tr.add_argument("train")
+    tr.add_argument("--validation", required=True)
+    tr.add_argument("--out", default="runs/head.safetensors")
+    tr.add_argument("--rank", type=int, default=256)
+    tr.add_argument("--epochs", type=int, default=8)
+    tr.add_argument("--batch-size", type=int, default=64)
+    tr.add_argument("--learning-rate", type=float, default=5e-4)
+    tr.add_argument("--seed", type=int, default=7)
+    tr.set_defaults(fn=cmd_train)
+
+    eh = sub.add_parser("eval-head", help="top-k, ECE and shuffled-context control for a trained head")
+    eh.add_argument("checkpoint")
+    eh.add_argument("features")
+    eh.set_defaults(fn=cmd_eval_head)
+
+    v = sub.add_parser("serve", help="HTTP server with the model loaded once (POST /score, /v1/systemone)")
+    v.add_argument("--model", default=DEFAULT_MODEL)
+    v.add_argument("--batch-size", type=int, default=8)
+    v.add_argument("--backend", choices=("auto", "mlx", "torch"), default="auto")
+    v.add_argument("--device", default="auto", help="auto, cpu, cuda, cuda:N, or mps")
+    v.add_argument("--host", default="127.0.0.1")
+    v.add_argument("--port", type=int, default=8000)
+    v.set_defaults(fn=cmd_serve)
+
+    from .decision_cli import register
+    register(sub)
+    from .tod.cli import register as register_tod
+    register_tod(sub)
+
+    args = p.parse_args(argv)
+    if len(getattr(args, "option", []) or []) == 1:
+        p.error("--option must be given at least twice")
+    args.fn(args)
+
+
+if __name__ == "__main__":
+    main()

openjev/compute.py

@@ -0,0 +1,68 @@
+"""Where decision models run: MLX on Apple silicon, PyTorch everywhere else (CUDA, ROCm, MPS, CPU).
+
+Features are cached as plain arrays and heads are saved as safetensors with the same
+parameter names on both backends, so a head trained on one backend loads on the other.
+
+Selection order: `select(...)` (the CLI's --backend/--device), then the OPENJEV_BACKEND and
+OPENJEV_DEVICE environment variables, then auto: MLX on Apple silicon when it is installed,
+otherwise PyTorch on the first CUDA/ROCm GPU, then MPS, then CPU.
+"""
+from __future__ import annotations
+
+import importlib
+import importlib.metadata
+import importlib.util
+import os
+import platform
+
+BACKENDS = ('auto', 'mlx', 'torch')
+_selected = {'backend': None, 'device': None}
+
+
+def select(backend: str | None = None, device: str | None = None) -> None:
+    """Choose the backend (auto, mlx, torch) and PyTorch device (auto, cuda, cuda:N, mps, cpu)."""
+    if backend is not None and backend not in BACKENDS:
+        raise ValueError(f'backend must be one of {BACKENDS}')
+    _selected.update(backend=backend, device=device)
+
+
+def backend() -> str:
+    name = _selected['backend'] or os.environ.get('OPENJEV_BACKEND') or 'auto'
+    if name not in BACKENDS:
+        raise ValueError(f'OPENJEV_BACKEND must be one of {BACKENDS}')
+    if name != 'auto':
+        return name
+    apple = platform.system() == 'Darwin' and platform.machine() == 'arm64'
+    return 'mlx' if apple and importlib.util.find_spec('mlx') else 'torch'
+
+
+def device() -> str:
+    """The PyTorch device name (ignored by MLX, which always uses Metal)."""
+    name = _selected['device'] or os.environ.get('OPENJEV_DEVICE') or 'auto'
+    if name != 'auto':
+        return name
+    import torch
+    return 'cuda' if torch.cuda.is_available() else 'mps' if torch.backends.mps.is_available() else 'cpu'
+
+
+def ops(name: str | None = None):
+    """The head module for a backend: AttentionHead, new_head, forward, Optimizer, snapshot, restore."""
+    name = name or backend()
+    if name == 'mlx':
+        if not importlib.util.find_spec('mlx'):
+            raise RuntimeError('MLX needs Apple silicon; use the torch backend (--backend torch or OPENJEV_BACKEND=torch)')
+        return importlib.import_module('openjev.head')
+    try:
+        return importlib.import_module('openjev.head_torch')
+    except ImportError as exc:
+        raise RuntimeError('The torch backend needs PyTorch: pip install torch') from exc
+
+
+def versions(name: str | None = None) -> dict:
+    """Libraries whose versions change extracted features."""
+    name = name or backend()
+    packages = ('mlx', 'mlx-lm') if name == 'mlx' else ('torch',)
+    result = {'backend': name}
+    for package in (*packages, 'transformers', 'numpy'):
+        result[package] = importlib.metadata.version(package)
+    return result

openjev/decision.py

@@ -0,0 +1,231 @@
+"""Native choice contract and artifact identities. Safe to import without MLX."""
+from __future__ import annotations
+
+from dataclasses import asdict, dataclass
+import hashlib
+import json
+import math
+from pathlib import Path
+
+RECIPE = {
+    'version': 1,
+    'context_rendering': 'state-question-json-v1',
+    'chat': False,
+    'separator': '',
+    'candidate_encoding': 'independent-text-after-bos',
+    'candidate_pooling': 'masked-mean-float32-excluding-bos',
+    'context_pooling': 'all-final-hidden-states',
+    'storage_dtype': 'float16',
+    'candidate_ids_encoded': False,
+}
+
+
+def canonical(value):
+    return json.dumps(value, sort_keys=True, ensure_ascii=False, allow_nan=False, separators=(',', ':'))
+
+
+def digest(value):
+    return hashlib.sha256(canonical(value).encode()).hexdigest()
+
+
+def file_digest(path):
+    h = hashlib.sha256()
+    with Path(path).open('rb') as stream:
+        for chunk in iter(lambda: stream.read(8 * 1024 * 1024), b''):
+            h.update(chunk)
+    return h.hexdigest()
+
+
+@dataclass(frozen=True)
+class Candidate:
+    id: str
+    text: str
+
+
+@dataclass(frozen=True)
+class ChoiceTask:
+    id: str
+    state: object
+    question: str
+    candidates: tuple[Candidate, ...]
+    choice_id: str | None = None
+    group_id: str | None = None
+
+    @classmethod
+    def parse(cls, value, labelled=False):
+        if not isinstance(value, dict):
+            raise ValueError('task must be an object')
+        allowed = {'schema_version', 'id', 'state', 'question', 'candidates', 'target', 'group_id'}
+        if set(value) - allowed:
+            raise ValueError(f'unknown task fields: {sorted(set(value)-allowed)}')
+        if type(value.get('schema_version')) is not int or value['schema_version'] != 1:
+            raise ValueError('schema_version must be 1')
+        for name in ('id', 'question'):
+            if not isinstance(value.get(name), str) or not value[name].strip():
+                raise ValueError(f'{name} must be a nonempty string')
+        state = value.get('state')
+        if not isinstance(state, (str, dict, list)):
+            raise ValueError('state must be a string, object, or array')
+        canonical(state)  # Reject non-JSON values and NaN/Infinity.
+        candidates = value.get('candidates')
+        if not isinstance(candidates, list) or not 2 <= len(candidates) <= 255:
+            raise ValueError('need 2..255 candidates')
+        parsed = []
+        for c in candidates:
+            if not isinstance(c, dict) or set(c) != {'id', 'text'}:
+                raise ValueError('each candidate must contain only id and text')
+            if any(not isinstance(c[k], str) or not c[k].strip() for k in ('id', 'text')):
+                raise ValueError('candidate id and text must be nonempty strings')
+            parsed.append(Candidate(**c))
+        if len({c.id for c in parsed}) != len(parsed):
+            raise ValueError('duplicate candidate IDs')
+        if len({c.text for c in parsed}) != len(parsed):
+            raise ValueError('duplicate candidate text cannot identify a unique choice')
+        target = value.get('target')
+        choice_id = None
+        if target is not None:
+            if not isinstance(target, dict) or set(target) != {'choice_id'}:
+                raise ValueError('target must contain only choice_id')
+            choice_id = target['choice_id']
+            if not isinstance(choice_id, str) or choice_id not in {c.id for c in parsed}:
+                raise ValueError('target choice_id is not in candidates')
+        if labelled and choice_id is None:
+            raise ValueError('a choice target is required')
+        group_id = value.get('group_id')
+        if group_id is not None and (not isinstance(group_id, str) or not group_id.strip()):
+            raise ValueError('group_id must be a nonempty string')
+        return cls(value['id'], state, value['question'], tuple(parsed), choice_id, group_id)
+
+    @property
+    def label(self):
+        if self.choice_id is None:
+            raise ValueError('task has no target')
+        return next(i for i, c in enumerate(self.candidates) if c.id == self.choice_id)
+
+    def to_dict(self):
+        value = dict(schema_version=1, id=self.id, state=self.state, question=self.question,
+                     candidates=[asdict(c) for c in self.candidates])
+        if self.choice_id is not None:
+            value['target'] = {'choice_id': self.choice_id}
+        if self.group_id is not None:
+            value['group_id'] = self.group_id
+        return value
+
+    @property
+    def input_fingerprint(self):
+        # Order and IDs do not disguise a repeated semantic example across splits.
+        return digest(dict(state=self.state, question=self.question,
+                           candidates=sorted(c.text for c in self.candidates)))
+
+
+def render_context(task):
+    return 'State:\n' + canonical(task.state) + '\n\nQuestion:\n' + task.question
+
+
+def load_tasks(path, labelled=False):
+    tasks, ids = [], set()
+    with Path(path).open() as stream:
+        for line_number, line in enumerate(stream, 1):
+            if not line.strip():
+                continue
+            try:
+                task = ChoiceTask.parse(json.loads(line), labelled)
+                if task.id in ids:
+                    raise ValueError('duplicate task ID')
+                ids.add(task.id)
+                tasks.append(task)
+            except (ValueError, TypeError, KeyError) as exc:
+                raise ValueError(f'{path}:{line_number}: {exc}') from exc
+    if not tasks:
+        raise ValueError(f'{path}: empty dataset')
+    return tasks
+
+
+def provenance(tasks):
+    return dict(ids=[t.id for t in tasks], inputs=[t.input_fingerprint for t in tasks],
+                groups=sorted({t.group_id for t in tasks if t.group_id is not None}))
+
+
+def check_disjoint(a, b):
+    for key in ('ids', 'inputs', 'groups'):
+        if set(a[key]) & set(b[key]):
+            raise ValueError(f'dataset overlap in {key}; use independent splits')
+
+
+def model_identity(model):
+    root = Path(model).resolve()
+    if not root.is_dir() or not (root/'config.json').is_file():
+        raise ValueError('Phase 1 requires a local MLX-compatible model directory with config.json')
+    files = sorted(p for p in root.rglob('*') if p.is_file() and
+                   '.cache' not in p.relative_to(root).parts and
+                   p.suffix in ('.json', '.safetensors', '.model', '.jinja', '.txt'))
+    if not any(p.suffix == '.safetensors' for p in files):
+        raise ValueError('model directory has no safetensors weights')
+    hashes = {p.relative_to(root).as_posix(): file_digest(p) for p in files}
+    return dict(path=str(root), files=hashes, fingerprint=digest(hashes))
+
+
+def read_artifact(directory):
+    root = Path(directory)
+    manifest = json.loads((root/'manifest.json').read_text())
+    if manifest.get('schema_version') != 1 or manifest.get('strategy') != 'candidate_head':
+        raise ValueError('unsupported decision artifact')
+    if manifest.get('recipe') != RECIPE:
+        raise ValueError('unsupported feature/rendering recipe')
+    if manifest.get('head') != 'attention_head_v1':
+        raise ValueError('unsupported decision head')
+    for name in ('head.safetensors', 'head.json'):
+        if file_digest(root/name) != manifest['files'][name]:
+            raise ValueError(f'artifact integrity mismatch: {name}')
+    backbone = manifest['backbone']
+    signature = manifest['feature_signature']
+    config = json.loads((root/'head.json').read_text())
+    if (digest(backbone['files']) != backbone['fingerprint'] or
+            signature['model_fingerprint'] != backbone['fingerprint'] or
+            signature['recipe'] != RECIPE or signature['limits'] != manifest['limits'] or
+            config['hidden'] != signature['hidden'] or
+            config['features_meta']['signature_sha256'] != digest(signature)):
+        raise ValueError('inconsistent artifact feature contract')
+    return manifest
+
+
+def choice_result(task, logits):
+    if len(logits) != len(task.candidates) or not all(math.isfinite(x) for x in logits):
+        raise ValueError('invalid decision utilities')
+    shifted = [v-max(logits) for v in logits]
+    z = sum(math.exp(v) for v in shifted)
+    options = [dict(id=c.id, utility=u, probability=math.exp(v)/z)
+               for c, u, v in zip(task.candidates, logits, shifted)]
+    # Stable ID tie-breaking is output mapping only; IDs never enter the encoder.
+    best = min(options, key=lambda o: (-o['utility'], o['id']))
+    return dict(id=task.id, strategy='candidate_head', choice_id=best['id'], candidates=options)
+
+
+def summarize(tasks, results):
+    if not tasks or len(tasks) != len(results):
+        raise ValueError('expected nonempty aligned results')
+    hits = top3 = 0
+    nll = brier = 0.
+    bins = [[0, 0., 0.] for _ in range(10)]
+    for task, result in zip(tasks, results):
+        options = result['candidates']
+        gold = next(o for o in options if o['id'] == task.choice_id)
+        temperature = result.get('calibration', {}).get('temperature', 1.0)
+        scores = [o['utility']/temperature for o in options]
+        maximum = max(scores)
+        # Stable NLL even when a probability underflows.
+        nll += maximum + math.log(sum(math.exp(s-maximum) for s in scores)) - gold['utility']/temperature
+        hit = result['choice_id'] == task.choice_id
+        hits += hit
+        ordered = sorted(options, key=lambda o: (-o['utility'], o['id']))
+        top3 += task.choice_id in [o['id'] for o in ordered[:3]]
+        brier += sum((o['probability']-int(o['id'] == task.choice_id))**2 for o in options)
+        confidence = ordered[0]['probability']
+        bucket = bins[min(9, int(confidence*10))]
+        bucket[0] += 1
+        bucket[1] += confidence
+        bucket[2] += hit
+    n = len(tasks)
+    return dict(n=n, accuracy=hits/n, top3=top3/n, nll=nll/n, brier=brier/n,
+                ece=sum(abs(sc-sh) for count, sc, sh in bins)/n,
+                chance_accuracy=sum(1/len(t.candidates) for t in tasks)/n)