strands-decider 0.1.0__py3-none-any.whl

strands_decider/__init__.py

@@ -0,0 +1,3 @@
+"""strands-decider: a System One model -- typed, calibrated answers instead of text."""
+
+__version__ = "0.1.0"

strands_decider/cli.py

@@ -0,0 +1,390 @@
+"""The strands-decider CLI: build data, train, calibrate, evaluate, serve, ask.
+
+Only `ask` and `serve` show in `--help`. The training and evaluation commands stay
+registered but hidden: `strands-decider train --help` works, and training/recipe.sh
+calls them through `python -m strands_decider.cli`.
+"""
+
+from __future__ import annotations
+
+import json
+import os
+from collections import Counter
+from itertools import islice
+
+import torch
+import typer
+from rich.console import Console
+from rich.table import Table
+
+from .data import recipes as _recipes
+from .data.format import Example, read_jsonl, write_jsonl
+from .prompting import build_prompt
+from .schema import ChoiceQuestion, NoulQuestion, Question, ScoreQuestion
+
+app = typer.Typer(
+    name="strands-decider",
+    help="A System One model: typed, calibrated answers instead of text.",
+    no_args_is_help=True,
+    add_completion=False,
+)
+data_app = typer.Typer(help="Build and inspect training corpora.", no_args_is_help=True)
+app.add_typer(data_app, name="data", hidden=True)
+
+console = Console()
+
+
+# ---------------------------------------------------------------- data
+
+
+@data_app.command("recipes")
+def data_recipes() -> None:
+    """List the datasets that can be converted into training examples."""
+    table = Table(title="Available recipes")
+    for col in ("name", "dataset", "config", "split", "max examples"):
+        table.add_column(col)
+    for name in sorted(_recipes.SPECS):
+        s = _recipes.SPECS[name]
+        table.add_row(name, s.path, s.config or "-", s.split, str(s.max_examples))
+    console.print(table)
+
+
+@data_app.command("build")
+def data_build(
+    out: str = typer.Option("data/train.jsonl", help="Output JSONL (.gz supported)."),
+    recipes: list[str] | None = typer.Option(
+        None, "--recipe", "-r",
+        help="Recipe name; repeat. Default: all. If a recipe fails, the build writes nothing.",
+    ),
+    max_options: int = typer.Option(16, help="Cap options per example (large label sets)."),
+    max_chars: int = typer.Option(2000, help="Truncate each state to this many characters."),
+    max_examples: int | None = typer.Option(None, help="Per-recipe example cap."),
+    seed: int = typer.Option(0),
+    holdout: list[str] | None = typer.Option(
+        None, "--holdout", help="Task to route to a separate held-out file; repeat."
+    ),
+) -> None:
+    """Download datasets and convert them to (state, question, answer) examples.
+
+    Tasks named with --holdout are written to a sibling *.holdout.jsonl instead, so
+    you can measure generalisation to label sets the model never trained on.
+    """
+    names = list(recipes) if recipes else _recipes.available_recipes()
+    hold = set(holdout or [])
+
+    kept: list[Example] = []
+    held: list[Example] = []
+    for name in names:
+        console.print(f"[cyan]building[/] {name} ...")
+        try:
+            examples = _recipes.build_recipe(
+                name,
+                max_options=max_options,
+                max_chars=max_chars,
+                seed=seed,
+                max_examples=max_examples,
+            )
+        except Exception as exc:
+            # Stop here. A missing recipe removes rows and moves every later row to a new
+            # position. The frozen teacher and replay targets attach to rows by position.
+            hint = (' Install the "train" extra: pip install -e ".[train]".'
+                    if isinstance(exc, ModuleNotFoundError) and exc.name == "datasets" else "")
+            raise SystemExit(f"recipe {name} failed, no corpus written: {exc!r}.{hint}") from exc
+        (held if name in hold else kept).extend(examples)
+        console.print(f"  [green]{len(examples):,}[/] examples")
+
+    if not kept:
+        raise typer.Exit(code=1)
+
+    n = write_jsonl(out, kept)
+    console.print(f"[green]wrote[/] {n:,} examples -> {out}")
+    if held:
+        hpath = out.replace(".jsonl", ".holdout.jsonl")
+        console.print(f"[green]wrote[/] {write_jsonl(hpath, held):,} held-out -> {hpath}")
+
+
+@data_app.command("stats")
+def data_stats(path: str = typer.Argument(..., help="JSONL corpus to summarise.")) -> None:
+    """Show composition of a corpus: tasks, kinds, option-count distribution."""
+    tasks: Counter = Counter()
+    kinds: Counter = Counter()
+    nopts: Counter = Counter()
+    total = 0
+    for ex in read_jsonl(path):
+        tasks[ex.task] += 1
+        kinds[ex.kind] += 1
+        nopts[ex.n_options] += 1
+        total += 1
+
+    console.print(f"[bold]{total:,}[/] examples in {path}")
+    t = Table(title="By task")
+    t.add_column("task")
+    t.add_column("n", justify="right")
+    t.add_column("share", justify="right")
+    for name, c in tasks.most_common():
+        t.add_row(name, f"{c:,}", f"{100 * c / total:.1f}%")
+    console.print(t)
+
+    k = Table(title="By kind / option count")
+    k.add_column("kind")
+    k.add_column("n", justify="right")
+    for name, c in kinds.most_common():
+        k.add_row(name, f"{c:,}")
+    console.print(k)
+    console.print("option counts: " + ", ".join(f"{n}:{c:,}" for n, c in sorted(nopts.items())))
+
+
+@data_app.command("peek")
+def data_peek(
+    path: str = typer.Argument(...),
+    n: int = typer.Option(3, help="How many rendered prompts to print."),
+) -> None:
+    """Render a few examples exactly as the model will see them."""
+    for ex in islice(read_jsonl(path), n):
+        prompt, rq = build_prompt(ex.state, ex.to_question())
+        console.rule(f"{ex.task} / {ex.kind} / gold slot {ex.label}")
+        console.print(prompt)
+        console.print(f"[dim]slots: {rq.slot_labels}[/]")
+
+
+# ---------------------------------------------------------------- train
+
+
+@app.command("train", hidden=True)
+def train_cmd(
+    config: str | None = typer.Option(None, "--config", "-c", help="YAML training config."),
+    train_file: list[str] | None = typer.Option(None, "--train-file", help="Override data."),
+    base_model: str | None = typer.Option(None, help="Override base model id."),
+    output_dir: str | None = typer.Option(None, help="Override checkpoint directory."),
+    epochs: int | None = typer.Option(None),
+    micro_batch_size: int | None = typer.Option(None),
+    max_steps: int | None = typer.Option(None, help="Hard step cap (smoke tests)."),
+) -> None:
+    """Train the slot heads and a LoRA adapter on the torso."""
+    from .train import TrainConfig, train
+
+    cfg = TrainConfig.from_yaml(config) if config else TrainConfig()
+    if train_file:
+        cfg.train_files = list(train_file)
+    for key, val in (
+        ("base_model", base_model),
+        ("output_dir", output_dir),
+        ("epochs", epochs),
+        ("micro_batch_size", micro_batch_size),
+        ("max_steps", max_steps),
+    ):
+        if val is not None:
+            setattr(cfg, key, val)
+    if not cfg.train_files:
+        raise typer.BadParameter("no training files; pass --train-file or set train_files")
+    train(cfg)
+
+
+@app.command("calibrate", hidden=True)
+def calibrate_cmd(
+    checkpoint: str = typer.Argument(...),
+    data: str = typer.Option(..., "--data", help="Held-out JSONL for fitting temperature."),
+    limit: int = typer.Option(4000, help="Cap examples used."),
+    batch_size: int = typer.Option(16),
+    split: str = typer.Option(
+        "calib", help="Half of --data to use: calib|test|all. Disjoint from `eval --split test`."
+    ),
+) -> None:
+    """Fit a temperature on held-out data and write it into the checkpoint.
+
+    Run this before serving. Without it, `confidence` is whatever the raw head
+    happened to produce, and the documented thresholds will not hold.
+
+    Defaults to the `calib` half so that `strands-decider eval --split test` scores rows the
+    temperature was never fitted on.
+    """
+
+    from .evaluate import calibrate_checkpoint, partition_examples, sample_examples
+
+    examples = sample_examples(partition_examples(list(read_jsonl(data)), split), limit)
+    result = calibrate_checkpoint(checkpoint, examples, batch_size=batch_size)
+    console.print(f"[green]global temperature = {result['temperature']:.4f}[/]")
+    by_kind = result.get("temperature_by_kind") or {}
+    if by_kind:
+        console.print(
+            "[green]per-primitive[/] "
+            + ", ".join(f"{k}={v:.4f}" for k, v in sorted(by_kind.items()))
+            + "  (written to checkpoint)"
+        )
+    console.print("before:      ", result["before"])
+    if "after_global" in result:
+        console.print("after global:", result["after_global"])
+    console.print("after:       ", result["after"])
+
+
+@app.command("eval", hidden=True)
+def eval_cmd(
+    checkpoint: str = typer.Argument(...),
+    data: str = typer.Option(..., "--data"),
+    limit: int = typer.Option(4000),
+    batch_size: int = typer.Option(16),
+    out: str | None = typer.Option(None, help="Write the full report as JSON."),
+    raw: bool = typer.Option(False, "--raw", help="Ignore the fitted temperature."),
+    split: str = typer.Option(
+        "test", help="Half of --data to score: test|calib|all. Disjoint from `calibrate`."
+    ),
+) -> None:
+    """Accuracy, ECE, NLL and score MAE, broken down by task and confidence band.
+
+    Defaults to the `test` half, which `strands-decider calibrate` does not fit on.
+    """
+    from .evaluate import evaluate_checkpoint, partition_examples, sample_examples
+
+    examples = sample_examples(partition_examples(list(read_jsonl(data)), split), limit)
+    report = evaluate_checkpoint(
+        checkpoint, examples, batch_size=batch_size, apply_temperature=not raw
+    )
+
+    overall = report["overall"]
+    console.print(
+        f"[bold]overall[/] n={overall['n']:,} acc={overall['accuracy']:.3f} "
+        f"ece={overall['ece']:.3f} nll={overall['nll']:.3f} T={_fmt_temperature(report['temperature'])}"
+    )
+
+    for section in ("by_kind", "by_task", "by_confidence_band"):
+        rows = {k: v for k, v in report[section].items() if v}
+        if not rows:
+            continue
+        t = Table(title=section)
+        for col in ("group", "n", "accuracy", "ece", "mean conf"):
+            t.add_column(col, justify="right" if col != "group" else "left")
+        for name, m in rows.items():
+            t.add_row(
+                name, f"{m['n']:,}", f"{m['accuracy']:.3f}",
+                f"{m['ece']:.3f}", f"{m['mean_confidence']:.3f}",
+            )
+        console.print(t)
+
+    if out:
+        os.makedirs(os.path.dirname(os.path.abspath(out)), exist_ok=True)
+        with open(out, "w", encoding="utf-8") as fh:
+            json.dump(report, fh, indent=2)
+        console.print(f"[green]report -> {out}[/]")
+
+
+def _auto_device() -> str:
+    """Best available torch device: cuda > mps > cpu."""
+    if torch.cuda.is_available():
+        return "cuda"
+    if getattr(torch.backends, "mps", None) is not None and torch.backends.mps.is_available():
+        return "mps"
+    return "cpu"
+
+
+def _fmt_temperature(t: float | dict[str, float]) -> str:
+    """Temperature is a scalar, or a {primitive: value} map once fitted per kind."""
+    if isinstance(t, dict):
+        return "{" + ", ".join(f"{k}:{v:.3f}" for k, v in sorted(t.items())) + "}"
+    return f"{t:.3f}"
+
+
+# ---------------------------------------------------------------- serve / ask
+
+
+@app.command("serve")
+def serve_cmd(
+    checkpoint: str = typer.Argument(...),
+    host: str = typer.Option("127.0.0.1"),
+    port: int = typer.Option(8000),
+    device: str | None = typer.Option(
+        None, "--device",
+        help="Torch device (cuda|mps|cpu). Auto-detected when omitted.",
+    ),
+    no_prefix_cache: bool = typer.Option(False, "--no-prefix-cache"),
+    model_name: str | None = typer.Option(
+        None, "--model-name",
+        help="Value returned as `model` in responses. Defaults to the checkpoint basename.",
+    ),
+) -> None:
+    """Serve POST /v1/systemone. JevBench's typesafe adapter runs against it unchanged."""
+    from .server import serve
+
+    console.print(f"[green]serving[/] {checkpoint} on http://{host}:{port}")
+    serve(
+        checkpoint, host=host, port=port, device=device or _auto_device(),
+        use_prefix_cache=not no_prefix_cache, model_name=model_name,
+    )
+
+
+@app.command("ask")
+def ask_cmd(
+    checkpoint: str = typer.Argument(...),
+    state: str = typer.Option(..., "--state", "-s", help="The content to evaluate."),
+    noul: list[str] | None = typer.Option(None, "--noul", help="Yes/no question; repeat."),
+    choice: list[str] | None = typer.Option(
+        None, "--choice", help="'question?=opt1,opt2,opt3'; repeat."
+    ),
+    score: list[str] | None = typer.Option(
+        None, "--score", help="'question?=low,mid,high' (ascending); repeat."
+    ),
+    device: str | None = typer.Option(
+        None, "--device",
+        help="Torch device (cuda|mps|cpu). Auto-detected when omitted.",
+    ),
+    as_json: bool = typer.Option(False, "--json", help="Print the raw API response."),
+) -> None:
+    """Ask one state a set of typed questions from the command line."""
+    from .infer import load_engine
+
+    questions: dict[str, Question] = {}
+    for i, q in enumerate(noul or []):
+        questions[f"noul_{i}"] = NoulQuestion(instructions=q)
+    for i, spec in enumerate(choice or []):
+        q, _, opts = spec.partition("=")
+        if not opts:
+            raise typer.BadParameter(f"--choice needs 'question=opt1,opt2': {spec!r}")
+        questions[f"choice_{i}"] = ChoiceQuestion(
+            instructions=q, criteria={o.strip(): "" for o in opts.split(",") if o.strip()}
+        )
+    for i, spec in enumerate(score or []):
+        q, _, levels = spec.partition("=")
+        if not levels:
+            raise typer.BadParameter(f"--score needs 'question=low,high': {spec!r}")
+        questions[f"score_{i}"] = ScoreQuestion(
+            instructions=q, criteria=[s.strip() for s in levels.split(",") if s.strip()]
+        )
+    if not questions:
+        raise typer.BadParameter("ask at least one --noul / --choice / --score question")
+
+    engine = load_engine(checkpoint, device=device or _auto_device())
+    response = engine.ask(state, questions)
+
+    if as_json:
+        console.print_json(response.model_dump_json())
+        return
+
+    for name, ans in response.answers.items():
+        if ans.type == "noul":
+            console.print(f"[bold]{name}[/] noul = [cyan]{ans.noul:.3f}[/]")
+        elif ans.type == "choice":
+            console.print(
+                f"[bold]{name}[/] -> [cyan]{ans.choice}[/] "
+                f"(confidence {ans.confidence:.3f})"
+            )
+            for opt, p in sorted(ans.probabilities.items(), key=lambda kv: -kv[1]):
+                console.print(f"    {opt:<24} {p:.3f}")
+        else:
+            console.print(
+                f"[bold]{name}[/] score = [cyan]{ans.score:.2f}[/] "
+                f"(confidence {ans.confidence:.3f})"
+            )
+            for lvl, p in ans.probabilities.items():
+                console.print(f"    {lvl}: {ans.legend.get(lvl, ''):<40} {p:.3f}")
+
+
+@app.command("info", hidden=True)
+def info_cmd(checkpoint: str = typer.Argument(...)) -> None:
+    """Print a checkpoint's configuration."""
+    from .modeling import StrandsDeciderConfig, checkpoint_dir, config_path
+
+    cfg = StrandsDeciderConfig.from_json(config_path(checkpoint_dir(checkpoint)))
+    console.print_json(cfg.to_json())
+
+
+if __name__ == "__main__":  # pragma: no cover
+    app()

strands_decider/data/__init__.py

@@ -0,0 +1,12 @@
+from .collate import CollatorConfig, SystemOneCollator
+from .format import Example, load_examples, read_jsonl, split_examples, write_jsonl
+
+__all__ = [
+    "CollatorConfig",
+    "Example",
+    "SystemOneCollator",
+    "load_examples",
+    "read_jsonl",
+    "split_examples",
+    "write_jsonl",
+]

strands_decider/data/adequacy.py

@@ -0,0 +1,115 @@
+"""Answer adequacy from HelpSteer2 (nvidia/HelpSteer2, CC-BY-4.0): a request, a response,
+and "is the response adequate?".
+
+JevBench's `adequacy` family asks this question, and nothing in the training corpus
+does: v17 answers "adequate" on 11 of that family's 12 public tasks. HelpSteer2 has
+human ratings (Scale AI, 3-5 annotators per response, 0-4 scales) of responses written
+by NVIDIA's in-house models, two responses per prompt.
+
+  adequate    helpfulness >= 3 and correctness >= 3
+  inadequate  helpfulness <= 1 or correctness <= 1
+  (the middle band is dropped as ambiguous)
+
+Single-turn prompts only. Both splits are balanced to equal "yes" and "no": every
+inadequate response is kept, and the adequate ones are drawn first from the same
+prompts (the other response to a prompt whose first is inadequate), so that the
+request alone does not predict the answer. The eval comes from HelpSteer2's
+validation split and training rows only from its train split.
+
+    python -m strands_decider.data.adequacy --raw data/raw/helpsteer2 \
+        --out data/adequacy_hs2.jsonl --eval-out data/adequacy_hs2_eval.jsonl
+"""
+from __future__ import annotations
+
+import gzip
+import json
+import random
+from collections import defaultdict
+
+from .format import Example
+
+TASK = "adequacy_hs2"
+QUESTION = "Does the response adequately answer the request: correct, complete and on point?"
+VARIANTS = [
+    QUESTION,
+    "Is this response good enough to send as the answer to the request?",
+    "Would a careful reviewer accept this response as an adequate answer to the request?",
+    "Does the response do what the request asks, without significant errors or omissions?",
+]
+OPTIONS = [["false", "the response is not adequate"], ["true", "the response is adequate"]]
+
+
+def verdict(r: dict) -> bool | None:
+    if r["helpfulness"] >= 3 and r["correctness"] >= 3:
+        return True
+    if r["helpfulness"] <= 1 or r["correctness"] <= 1:
+        return False
+    return None
+
+
+def load(path: str) -> list[dict]:
+    with gzip.open(path, "rt", encoding="utf-8") as fh:
+        rows = [json.loads(line) for line in fh if line.strip()]
+    return [r for r in rows if "<extra_id_1>" not in r["prompt"] and verdict(r) is not None]
+
+
+def balanced(rows: list[dict], seed: int = 0) -> list[dict]:
+    """Every inadequate response, and as many adequate ones, pair-mates first."""
+    rng = random.Random(seed)
+    bad = [r for r in rows if not verdict(r)]
+    bad_prompts = {r["prompt"] for r in bad}
+    good = [r for r in rows if verdict(r)]
+    mates = [r for r in good if r["prompt"] in bad_prompts]
+    others = [r for r in good if r["prompt"] not in bad_prompts]
+    rng.shuffle(mates)
+    rng.shuffle(others)
+    return bad + (mates + others)[:len(bad)]
+
+
+def to_example(r: dict) -> Example:
+    state = f"Request:\n{r['prompt'].strip()}\n\nResponse:\n{r['response'].strip()}"
+    return Example(kind="noul", state=state, instructions=QUESTION, options=[list(o) for o in OPTIONS],
+                   label=int(bool(verdict(r))), task=TASK, instruction_variants=list(VARIANTS))
+
+
+def main(argv: list[str] | None = None) -> None:
+    import argparse
+
+    from transformers import AutoTokenizer
+
+    from .format import write_jsonl
+    from .multistep import fits
+
+    ap = argparse.ArgumentParser(description="Build the HelpSteer2 answer-adequacy rows.")
+    ap.add_argument("--raw", default="data/raw/helpsteer2")
+    ap.add_argument("--out", default="", help="training rows (train split); omitted if empty")
+    ap.add_argument("--eval-out", default="data/adequacy_hs2_eval.jsonl")
+    ap.add_argument("--tokenizer", default="Qwen/Qwen3.5-2B-Base")
+    ap.add_argument("--max-tokens", type=int, default=3000)
+    ap.add_argument("--seed", type=int, default=0)
+    args = ap.parse_args(argv)
+    tok = AutoTokenizer.from_pretrained(args.tokenizer)
+
+    splits = {"eval": (f"{args.raw}/validation.jsonl.gz", args.eval_out)}
+    if args.out:
+        splits["train"] = (f"{args.raw}/train.jsonl.gz", args.out)
+    prompts = defaultdict(set)
+    for name, (src, dst) in splits.items():
+        rows = balanced(load(src), seed=args.seed)
+        prompts[name] = {r["prompt"] for r in rows}
+        examples = fits([to_example(r) for r in rows], tok, args.max_tokens)
+        # fits() may drop rows over the window unevenly; rebalance what remains
+        yes = [e for e in examples if e.label == 1]
+        no = [e for e in examples if e.label == 0]
+        n = min(len(yes), len(no))
+        examples = yes[:n] + no[:n]
+        random.Random(args.seed).shuffle(examples)
+        print(f"{name}: {len(examples):,} rows ({n:,} adequate, {n:,} not), "
+              f"{len(prompts[name]):,} distinct prompts")
+        write_jsonl(dst, examples)
+    if "train" in prompts:
+        print(f"prompts shared by train and eval: {len(prompts['train'] & prompts['eval'])}")
+
+
+if __name__ == "__main__":
+    main()

strands_decider/data/catchall.py

@@ -0,0 +1,114 @@
+"""Catch-all options (v20): choice rows where "other" / "none of these" is an option, and
+half the time the right one.
+
+The classification corpus names every category it asks about, so the model has never
+had to pick a catch-all because the true category is missing, or to pass one over
+because it is present. JevBench's routing and intent questions carry such options, and
+v19 misses one of each with high confidence. This adds rows built from the corpus's own
+choice tasks with named categories (ag_news, banking77, clinc150, dbpedia,
+yahoo_topics):
+
+  absent   the gold option is removed and a catch-all is added: the catch-all is right
+  present  a catch-all is added beside the gold option: the gold option is right
+
+in equal numbers, so the presence of a catch-all never predicts the answer. Rows whose
+options already include CLINC150's out-of-scope class are skipped. The catch-all goes
+at the end of the canonical order; options are shuffled at collate time.
+
+The eval applies the same transform to the test half of two held-out tasks never
+trained on (`massive_intent`, `emotion` in data/holdout_v5_norule.jsonl).
+
+    python -m strands_decider.data.catchall --out data/catchall_v20.jsonl --eval-out data/catchall_v20_eval.jsonl
+"""
+from __future__ import annotations
+
+import random
+from collections.abc import Iterable
+
+from .format import Example
+
+SOURCES = ("ag_news", "banking77", "clinc150", "dbpedia", "yahoo_topics")
+HELDOUT = ("massive_intent", "emotion")
+NAMES = ["other", "none of these", "something else", "none of the listed categories"]
+DESCRIPTION = "the text fits none of the other options"
+
+
+def _oos(ex: Example) -> bool:
+    return any("oos" in n.lower() or "out of scope" in n.lower() or "out_of_scope" in n.lower()
+               for n, _ in ex.options)
+
+
+def transform(ex: Example, mode: str, rng: random.Random, max_options: int = 24) -> Example | None:
+    """One catch-all row from `ex`; None if the row cannot take one. A row already at
+    `max_options` (the model's num_slots, the width of its training KL reference) gives
+    up one wrong option at random to make room for the catch-all."""
+    if ex.kind != "choice" or _oos(ex) or any(n.lower() in NAMES for n, _ in ex.options):
+        return None
+    catch = [rng.choice(NAMES), DESCRIPTION]
+    if mode == "absent":
+        rest = [o for i, o in enumerate(ex.options) if i != ex.label]
+        if len(rest) < 2:
+            return None
+        options, label = [*rest, catch], len(rest)
+    elif mode == "present":
+        options, label = list(ex.options), ex.label
+        if len(options) >= max_options:
+            drop = rng.choice([i for i in range(len(options)) if i != label])
+            options = [o for i, o in enumerate(options) if i != drop]
+            label = label - (drop < label)
+        options = [*options, catch]
+    else:
+        raise ValueError(mode)
+    return Example(kind="choice", state=ex.state, instructions=ex.instructions,
+                   options=[list(o) for o in options], label=label, task=f"{ex.task}+catchall:{mode}",
+                   weight=ex.weight, instruction_variants=list(ex.instruction_variants))
+
+
+def build(examples: list[Example], tasks: Iterable[str], per_task: int, seed: int = 0) -> list[Example]:
+    """`per_task` rows from each task, half absent and half present."""
+    rng = random.Random(seed)
+    out: list[Example] = []
+    for task in tasks:
+        pool = [e for e in examples if e.task == task and e.kind == "choice"]
+        rng.shuffle(pool)
+        want = {"absent": per_task // 2, "present": per_task - per_task // 2}
+        for e in pool:
+            mode = "absent" if want["absent"] >= want["present"] and want["absent"] > 0 else "present"
+            if want[mode] == 0:
+                break
+            row = transform(e, mode, rng)
+            if row is not None:
+                out.append(row)
+                want[mode] -= 1
+    rng.shuffle(out)
+    return out
+
+
+def main(argv: list[str] | None = None) -> None:
+    import argparse
+
+    from ..evaluate import partition_examples
+    from .format import read_jsonl, write_jsonl
+
+    ap = argparse.ArgumentParser(description="Build the catch-all option rows (v20).")
+    ap.add_argument("--src", default="data/train_v5.jsonl")
+    ap.add_argument("--heldout", default="data/holdout_v5_norule.jsonl")
+    ap.add_argument("--out", default="data/catchall_v20.jsonl")
+    ap.add_argument("--eval-out", default="data/catchall_v20_eval.jsonl")
+    ap.add_argument("--per-task", type=int, default=1000)
+    ap.add_argument("--eval-per-task", type=int, default=400)
+    ap.add_argument("--seed", type=int, default=0)
+    args = ap.parse_args(argv)
+
+    train = build(list(read_jsonl(args.src)), SOURCES, args.per_task, seed=args.seed)
+    test_half = partition_examples(list(read_jsonl(args.heldout)), "test")
+    evals = build(test_half, HELDOUT, args.eval_per_task, seed=args.seed)
+    for name, rows in (("train", train), ("eval", evals)):
+        absent = sum(r.task.endswith(":absent") for r in rows)
+        print(f"{name}: {len(rows):,} rows ({absent:,} catch-all right, {len(rows) - absent:,} catch-all a distractor)")
+    write_jsonl(args.out, train)
+    write_jsonl(args.eval_out, evals)
+
+
+if __name__ == "__main__":
+    main()