aria-code 0.167.0__py3-none-win_amd64.whl

aria_code/training/prelude_real.py

@@ -0,0 +1,183 @@
+"""A real-message test for Prelude's intent routing, kept on this computer.
+
+Hand-written test messages only show that a model learned its author's style. Real
+messages are the check that matters, and they are private, so this tool never writes them
+into a repository and the person who owns them does the reading and the labelling:
+
+    python -m aria_code.training.prelude_real extract --source DIR_OR_FILE
+    python -m aria_code.training.prelude_real label
+    python -m aria_code.training.prelude_real import --source FILE   # label<TAB>message
+    python -m aria_code.training.prelude_real score --adapter ADAPTER
+
+``extract`` collects candidate messages from a directory of ``*.json`` interaction logs
+(the ``instruction`` field) or a text file with one message per line, drops duplicates,
+very short messages and anything that looks like a secret, an email address or a phone
+number, and writes them to ``~/.aria/prelude/`` with owner-only permissions. ``label``
+shows each message and asks for its label. ``score`` runs a model over the labelled
+messages and prints counts only: accuracy and an interval, never a message.
+
+Nothing here is ever committed; ``~/.aria/prelude`` is outside every repository.
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import os
+import re
+import sys
+from pathlib import Path
+from typing import Iterable, List, Optional
+
+from aria_code.evals.scorecard import wilson
+from aria_code.training import prelude_eval as P
+
+HOME = Path(os.environ.get("ARIA_PRELUDE_HOME") or Path.home() / ".aria" / "prelude")
+CANDIDATES = HOME / "real_candidates.jsonl"
+LABELLED = HOME / "real_test.jsonl"
+KEYS = {"c": "coding", "a": "analysis", "r": "realtime", "g": "general", "f": "finance"}
+MIN_CHARS = 6
+
+_PRIVATE = [
+    re.compile(r"[\w.+-]+@[\w-]+\.[\w.-]+"),            # email address
+    re.compile(r"(?<!\d)(?:\+?\d[\d\s-]{8,}\d)(?!\d)"),  # phone or long account number
+    re.compile(r"\b(?:sk|pk|ghp|gho|AIza|AKIA|xox[bap])[-_A-Za-z0-9]{12,}"),  # API keys and tokens
+    re.compile(r"\b[A-Za-z0-9_-]{32,}\b"),               # any long opaque string
+    re.compile(r"(?i)\b(?:password|passwd|secret|token|api[_-]?key)\b\s*[:=]"),
+]
+
+
+def looks_private(text: str) -> bool:
+    return any(pattern.search(text) for pattern in _PRIVATE)
+
+
+def read_source(source: Path) -> Iterable[str]:
+    if source.is_dir():
+        for path in sorted(source.rglob("*.json")):
+            try:
+                record = json.loads(path.read_text(encoding="utf-8"))
+            except (OSError, ValueError):
+                continue
+            text = record.get("instruction") if isinstance(record, dict) else None
+            if isinstance(text, str):
+                yield text
+    else:
+        for line in source.read_text(encoding="utf-8").splitlines():
+            yield line
+
+
+def candidates(texts: Iterable[str]) -> List[str]:
+    seen, out = set(), []
+    for text in texts:
+        text = " ".join(text.split())
+        if len(text) < MIN_CHARS or text in seen or looks_private(text):
+            continue
+        seen.add(text)
+        out.append(text)
+    return out
+
+
+def _write_private(path: Path, rows: List[dict]) -> None:
+    HOME.mkdir(parents=True, exist_ok=True)
+    os.chmod(HOME, 0o700)
+    path.write_text("".join(json.dumps(r, ensure_ascii=False) + "\n" for r in rows), encoding="utf-8")
+    os.chmod(path, 0o600)
+
+
+def extract(source: Path) -> dict:
+    kept = candidates(read_source(source))
+    _write_private(CANDIDATES, [{"text": t} for t in kept])
+    return {"candidates": len(kept), "written_to": str(CANDIDATES)}
+
+
+def _load(path: Path) -> List[dict]:
+    if not path.is_file():
+        return []
+    return [json.loads(line) for line in path.read_text(encoding="utf-8").splitlines() if line.strip()]
+
+
+def label(ask=input, show=print) -> dict:
+    """Walk through unlabelled candidates; resumable, and a skipped message stays out."""
+    done = {r["text"] for r in _load(LABELLED)}
+    rows = _load(LABELLED)
+    todo = [r["text"] for r in _load(CANDIDATES) if r["text"] not in done]
+    show("c=coding  a=analysis  r=realtime  g=general  f=finance  s=skip  q=quit")
+    for text in todo:
+        show(f"\n{text}")
+        while True:
+            answer = ask("> ").strip().lower()
+            if answer == "q":
+                _write_private(LABELLED, rows)
+                return {"labelled": len(rows)}
+            if answer == "s" or answer in KEYS:
+                break
+        if answer in KEYS:
+            rows.append({"text": text, "label": KEYS[answer]})
+            _write_private(LABELLED, rows)
+    return {"labelled": len(rows)}
+
+
+def import_labelled(source: Path) -> dict:
+    """Load ``label<TAB>message`` lines written by the owner, replacing any earlier labels."""
+    valid = set(P.INTENT_LABELS)
+    rows, skipped = [], 0
+    for line in source.read_text(encoding="utf-8").splitlines():
+        label_name, _, text = line.partition("\t")
+        label_name, text = label_name.strip().lower(), " ".join(text.split())
+        if label_name in valid and len(text) >= 2:
+            rows.append({"text": text, "label": label_name})
+        elif line.strip():
+            skipped += 1
+    _write_private(LABELLED, rows)
+    per_label = {name: sum(1 for r in rows if r["label"] == name) for name in P.INTENT_LABELS}
+    return {"labelled": len(rows), "skipped_lines": skipped, "per_label": per_label}
+
+
+def score(adapter: Optional[Path], model: Optional[str]) -> dict:
+    """Counts only. A message never leaves this function."""
+    from aria_code.training import prelude_train as T
+    from mlx_lm import generate, load
+
+    labelled = _load(LABELLED)
+    if not labelled:
+        raise SystemExit(f"Nothing labelled yet in {LABELLED}. Run `label` first.")
+    network, tokenizer = load(T.base_model(model), adapter_path=str(adapter) if adapter else None)
+    right = unparsable = 0
+    for row in labelled:
+        prompt = tokenizer.apply_chat_template(
+            [{"role": "system", "content": P.INTENT_SYSTEM}, {"role": "user", "content": row["text"]}],
+            add_generation_prompt=True, tokenize=False)
+        prediction = P.parse_prediction("intent", generate(network, tokenizer, prompt=prompt, max_tokens=6))
+        unparsable += prediction is None
+        right += prediction == row["label"]
+    low, high = wilson(right, len(labelled))
+    return {"adapter": str(adapter) if adapter else None, "n": len(labelled), "correct": right,
+            "accuracy": round(right / len(labelled), 4), "ci95": [round(low, 4), round(high, 4)],
+            "unparsable": unparsable}
+
+
+def main(argv: Optional[List[str]] = None) -> int:
+    parser = argparse.ArgumentParser(description=__doc__.split("\n\n")[0])
+    sub = parser.add_subparsers(dest="command", required=True)
+    e = sub.add_parser("extract")
+    e.add_argument("--source", required=True, help="a directory of *.json interaction logs, or a text file")
+    sub.add_parser("label")
+    i = sub.add_parser("import", help="load label<TAB>message lines you wrote yourself")
+    i.add_argument("--source", required=True)
+    s = sub.add_parser("score")
+    s.add_argument("--adapter")
+    s.add_argument("--model")
+    args = parser.parse_args(argv)
+    if args.command == "extract":
+        print(json.dumps(extract(Path(args.source).expanduser())))
+    elif args.command == "label":
+        print(json.dumps(label()))
+    elif args.command == "import":
+        print(json.dumps(import_labelled(Path(args.source).expanduser())))
+    else:
+        print(json.dumps(score(Path(args.adapter) if args.adapter else None, args.model), indent=2))
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

aria_code/training/prelude_train.py

@@ -0,0 +1,248 @@
+"""Train and score Prelude's risk gate with MLX LoRA on a Mac.
+
+Reproduces the experiment end to end from files in this repository:
+
+    python -m aria_code.training.prelude_train data  --out DIR [--task intent]
+    python -m aria_code.training.prelude_train train --data DIR --adapter DIR/adapter
+    python -m aria_code.training.prelude_train pick  --log DIR/train.log --adapter DIR/adapter
+    python -m aria_code.training.prelude_train eval  --adapter DIR/adapter
+
+``data`` uses only train-side splits (``dev`` and ``extra``). Variants of those commands
+are generated by swapping arguments and wrapping them (``cd x &&``, ``time``, a pipe to
+``tail``), labelled by ``safety.risk``, and a command that also appears in a test split
+is dropped. Levels 3 and 4 are repeated so the loss weighs the dangerous cases. Risk
+``data`` needs the locked sets from aria-evals-holdout (``ARIA_PRELUDE_HOLDOUT``) so it can
+drop their families too. ``pick`` chooses a checkpoint by validation loss only. ``eval``
+reports every split it can load; read ``test2`` as a score and never as a list of
+mistakes to fix.
+
+Needs ``mlx-lm`` (Apple Silicon); it is imported only by ``train`` and ``eval``.
+"""
+
+from __future__ import annotations
+
+import argparse
+import hashlib
+import json
+import random
+import re
+import shutil
+import subprocess
+import sys
+from pathlib import Path
+from typing import Dict, List, Optional, Tuple
+
+from aria_code.training import prelude_eval as P
+
+FILES = ["app.py", "src/main.rs", "README.md", "config.yaml", "docs/guide.md", "build/out.js", "tests/test_x.py",
+         "data/prices.csv", "notes.txt", "src/index.ts", "lib/util.go", "scripts/run.sh", "output/report.pdf",
+         "tmp/cache.bin", "web/app.css"]
+PACKAGES = ["left-pad", "requests", "numpy", "zod", "serde", "rich", "pandas", "express", "tokio", "axios"]
+BRANCHES = ["main", "dev", "feature/login", "fix/crash", "release/1.2"]
+HOSTS = ["https://api.example.com/v1", "https://example.org/data.json", "https://files.test.io/pkg.tgz",
+         "http://localhost:8080/health"]
+WRAPPERS = [
+    lambda c: c, lambda c: f"cd project && {c}", lambda c: f"time {c}", lambda c: f"{c} 2>&1",
+    lambda c: f"{c} | tail -20", lambda c: f"cd src && {c}", lambda c: f"FOO=1 {c}", lambda c: f"{c} | head -5",
+]
+WEIGHT = {3: 3, 4: 2}  # how many times a level appears in the training file
+VARIANTS_PER_FAMILY = 14
+SEED = 7
+MODEL_ENV = "ARIA_PRELUDE_BASE"
+
+
+def vary(command: str, rng: random.Random) -> str:
+    tokens = command.split(" ")
+    for i, token in enumerate(tokens):
+        if not i:
+            continue
+        if re.search(r"\.\w{1,4}$|/", token) and not token.startswith(("http", "-", "/dev", "~/.ssh", ".env")) \
+                and ".pem" not in token and "credentials" not in token:
+            tokens[i] = rng.choice(FILES)
+        elif token in PACKAGES:
+            tokens[i] = rng.choice(PACKAGES)
+        elif token in BRANCHES:
+            tokens[i] = rng.choice(BRANCHES)
+        elif token.startswith("http") and "|" not in command:
+            tokens[i] = rng.choice(HOSTS)
+    return " ".join(tokens)
+
+
+def record(command: str, level, system: str = P.RISK_SYSTEM) -> dict:
+    return {"messages": [{"role": "system", "content": system}, {"role": "user", "content": command},
+                         {"role": "assistant", "content": str(level)}]}
+
+
+def is_validation(family: str) -> bool:
+    return int(hashlib.sha256(family.encode()).hexdigest(), 16) % 10 == 3
+
+
+def build_training_data(require_holdout: bool = True) -> Tuple[List[dict], List[dict], dict]:
+    """(train, validation, facts) from train-side splits only.
+
+    Generated variants can land in a locked family, and only the locked sets say which
+    families those are, so without them this refuses rather than risk training on one.
+    """
+    from aria_code.safety.risk import assess_command
+
+    if require_holdout and not P.holdout_available("risk"):
+        raise SystemExit(f"The locked risk sets are not at {P.HOLDOUT}. Clone aria-evals-holdout "
+                         "next to this checkout or set ARIA_PRELUDE_HOLDOUT to its prelude/ directory.")
+    rng = random.Random(SEED)
+    rows = P.load("risk")
+    train_rows = [r for r in rows if r["split"] in ("dev", "extra", "extra2")]
+    held_out = {r["input"] for r in rows if r["split"] in ("test", "test2", "test3")}
+    held_families = {r["family"] for r in rows if r["split"] in ("test", "test2", "test3")}
+    bases: Dict[str, str] = {}
+    for r in train_rows:
+        bases.setdefault(re.sub(r"^(cd project && |time )", "", r["input"]), r["family"])
+    variants: Dict[str, str] = {}
+    counts: Dict[str, int] = {}
+    for base, fam in bases.items():
+        for _ in range(40):
+            command = rng.choice(WRAPPERS)(vary(base, rng))
+            if "\n" in command or command in variants:
+                continue
+            variants[command] = fam
+            counts[fam] = counts.get(fam, 0) + 1
+            if counts[fam] >= VARIANTS_PER_FAMILY:
+                break
+    pool = [(c, assess_command(c).level, f) for c, f in variants.items()]
+    pool += [(r["input"], r["label"], r["family"]) for r in train_rows]
+    seen, distinct = set(), []
+    for command, level, fam in pool:
+        if command in seen or command in held_out or fam in held_families or P.family(command) in held_families:
+            continue
+        seen.add(command)
+        distinct.append((command, level, fam))
+    train, valid = [], []
+    for command, level, fam in distinct:
+        if is_validation(fam):
+            valid.append(record(command, level))
+        else:
+            train.extend(record(command, level) for _ in range(WEIGHT.get(level, 1)))
+    rng.shuffle(train)
+    return train, valid, {"distinct": len(distinct), "train_lines": len(train), "validation_lines": len(valid)}
+
+
+def build_intent_data() -> Tuple[List[dict], List[dict], dict]:
+    """Train and validation records for intent routing, from the ``dev`` split only.
+
+    Messages are hand-written and independent, so each is its own family. A fixed share of
+    ``dev`` is held back for choosing a checkpoint; ``test`` is never read here.
+    """
+    rng = random.Random(SEED)
+    rows = [r for r in P.load("intent") if r["split"] == "dev"]
+    train, valid = [], []
+    for r in rows:
+        (valid if is_validation(r["input"]) else train).append(record(r["input"], r["label"], P.INTENT_SYSTEM))
+    # A short run over ~150 messages underfits; repeat so the step count means something.
+    train = train * 4
+    rng.shuffle(train)
+    return train, valid, {"distinct": len(rows), "train_lines": len(train), "validation_lines": len(valid)}
+
+
+def write_data(out: Path, task: str = "risk") -> dict:
+    train, valid, facts = build_intent_data() if task == "intent" else build_training_data()
+    out.mkdir(parents=True, exist_ok=True)
+    for name, data in (("train", train), ("valid", valid)):
+        (out / f"{name}.jsonl").write_text("".join(json.dumps(x, ensure_ascii=False) + "\n" for x in data),
+                                           encoding="utf-8")
+    return facts
+
+
+def base_model(explicit: Optional[str]) -> str:
+    import os
+
+    value = explicit or os.environ.get(MODEL_ENV)
+    if value:
+        return value
+    cache = Path.home() / ".cache" / "huggingface" / "hub" / "models--Qwen--Qwen2.5-1.5B-Instruct" / "snapshots"
+    found = sorted(cache.glob("*")) if cache.is_dir() else []
+    if not found:
+        raise SystemExit(f"Pass --model, or set {MODEL_ENV}, to a Qwen2.5-1.5B-Instruct directory or Hugging Face id.")
+    return str(found[-1])
+
+
+def train(data: Path, adapter: Path, model: Optional[str], iters: int, rate: float, layers: int) -> int:
+    command = [sys.executable, "-m", "mlx_lm", "lora", "--model", base_model(model), "--train", "--data", str(data),
+               "--adapter-path", str(adapter), "--iters", str(iters), "--batch-size", "4", "--num-layers", str(layers),
+               "--learning-rate", str(rate), "--steps-per-eval", "100", "--val-batches", "42", "--save-every", "100",
+               "--mask-prompt"]
+    return subprocess.call(command)
+
+
+def validation_losses(log: str) -> Dict[int, float]:
+    text = re.sub(r"\x1b\[[0-9;]*m", "", log).replace("\r", "\n")
+    return {int(m.group(1)): float(m.group(2)) for m in re.finditer(r"^\s*(\d+)\s+val\s+([\d.]+)", text, re.M)
+            if int(m.group(1)) > 1}
+
+
+def pick(log: Path, adapter: Path) -> int:
+    losses = validation_losses(log.read_text(encoding="utf-8", errors="replace"))
+    if not losses:
+        raise SystemExit("No validation losses in the log.")
+    step = min(losses, key=losses.get)
+    checkpoint = adapter / f"{step:07d}_adapters.safetensors"
+    if not checkpoint.is_file():
+        raise SystemExit(f"Checkpoint for step {step} is missing: {checkpoint}")
+    chosen = adapter.with_name(adapter.name + f"-best-{step}")
+    chosen.mkdir(exist_ok=True)
+    shutil.copy(adapter / "adapter_config.json", chosen / "adapter_config.json")
+    shutil.copy(checkpoint, chosen / "adapters.safetensors")
+    print(json.dumps({"step": step, "validation_loss": losses[step], "adapter": str(chosen)}))
+    return 0
+
+
+def evaluate(adapter: Optional[Path], model: Optional[str], task: str = "risk") -> dict:
+    from mlx_lm import generate, load
+
+    network, tokenizer = load(base_model(model), adapter_path=str(adapter) if adapter else None)
+    rows = P.load(task)
+    system = P.RISK_SYSTEM if task == "risk" else P.INTENT_SYSTEM
+    predictions = {}
+    for row in rows:
+        prompt = tokenizer.apply_chat_template(
+            [{"role": "system", "content": system}, {"role": "user", "content": row["input"]}],
+            add_generation_prompt=True, tokenize=False)
+        predictions[row["id"]] = P.parse_prediction(task, generate(network, tokenizer, prompt=prompt, max_tokens=6))
+    report = P.score(task, rows, predictions)
+    report.pop("confusion", None)
+    return {"adapter": str(adapter) if adapter else None, **report}
+
+
+def main(argv: Optional[List[str]] = None) -> int:
+    parser = argparse.ArgumentParser(description=__doc__.split("\n\n")[0])
+    sub = parser.add_subparsers(dest="command", required=True)
+    d = sub.add_parser("data")
+    d.add_argument("--out", required=True)
+    d.add_argument("--task", choices=("risk", "intent"), default="risk")
+    t = sub.add_parser("train")
+    t.add_argument("--data", required=True)
+    t.add_argument("--adapter", required=True)
+    t.add_argument("--model")
+    t.add_argument("--iters", type=int, default=1000)
+    t.add_argument("--rate", type=float, default=5e-5)
+    t.add_argument("--layers", type=int, default=8)
+    k = sub.add_parser("pick")
+    k.add_argument("--log", required=True)
+    k.add_argument("--adapter", required=True)
+    e = sub.add_parser("eval")
+    e.add_argument("--task", choices=("risk", "intent"), default="risk")
+    e.add_argument("--adapter")
+    e.add_argument("--model")
+    args = parser.parse_args(argv)
+    if args.command == "data":
+        print(json.dumps(write_data(Path(args.out), args.task)))
+        return 0
+    if args.command == "train":
+        return train(Path(args.data), Path(args.adapter), args.model, args.iters, args.rate, args.layers)
+    if args.command == "pick":
+        return pick(Path(args.log), Path(args.adapter))
+    print(json.dumps(evaluate(Path(args.adapter) if args.adapter else None, args.model, args.task),
+                     ensure_ascii=False, indent=2))
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

aria_code/training/sft_dataset.py

@@ -0,0 +1,230 @@
+"""Eval trajectories -> a supervised fine-tuning set, built the way a quant builds a backtest.
+
+The rules (docs/training-protocol.md) are enforced here, in code, not by care:
+
+- **No look-ahead.** Tasks in the locked holdout suites (``hard`` by default)
+  never produce an example, and a trajectory that touched a grader file is
+  dropped: a model trained on the answer key measures nothing.
+- **Split by task, never by attempt.** Two attempts at one task are near
+  copies; one in train and one in validation is leakage. The split is a
+  stable hash of the task id, so it does not move when data is added.
+- **No survivorship of lucky runs.** Only attempts the grader passed *and*
+  whose own acceptance check did not fail become examples.
+- **Exposure is reported.** Examples per domain and split, so a set that is
+  90% one domain is visible before it trains a lopsided model.
+- **Provenance.** The manifest records each input file's sha256, the rules
+  and the counts, so a tuned model can be traced to exactly what it saw.
+
+Output is the Vertex AI Gemini supervised-tuning format: one JSON object per
+line with ``systemInstruction`` and ``contents``, tool use as
+``functionCall`` / ``functionResponse`` parts.
+
+    python -m aria_code.training.sft_dataset trajectories/ --suites evals/suites --out sft/
+"""
+
+from __future__ import annotations
+
+import argparse
+import fnmatch
+import hashlib
+import json
+from collections import Counter, defaultdict
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Iterable
+
+DEFAULT_HOLDOUT = ("hard",)
+DOMAINS = ("software", "finance", "payments", "logistics")
+RESULT_CHARS = 6_000          # one tool result in an example; long outputs are cut, not dropped
+
+
+@dataclass
+class Trajectory:
+    path: Path
+    suite: str
+    task_id: str
+    attempt: int
+    model: str
+    prompt: str
+    events: list[dict] = field(default_factory=list)
+    outcome: dict = field(default_factory=dict)
+
+    @property
+    def final(self) -> dict:
+        return next((e for e in reversed(self.events) if e.get("type") == "turn.completed"), {})
+
+
+@dataclass
+class TaskInfo:
+    suite: str
+    tags: tuple[str, ...]
+    hidden: tuple[str, ...]
+
+
+def load_trajectories(root: Path) -> list[Trajectory]:
+    out = []
+    for path in sorted(Path(root).rglob("*.jsonl")):
+        lines = [json.loads(l) for l in path.read_text(encoding="utf-8").splitlines() if l.strip().startswith("{")]
+        if not lines or lines[0].get("type") != "eval.task":
+            continue
+        head = lines[0]
+        outcome: dict = next((l for l in reversed(lines) if l.get("type") == "eval.outcome"), {})
+        events = [l for l in lines[1:] if l.get("type") not in ("eval.task", "eval.outcome")]
+        out.append(Trajectory(path=path, suite=path.parent.name, task_id=str(head.get("task_id")),
+                              attempt=int(head.get("attempt") or 1), model=str(head.get("model") or ""),
+                              prompt=str(head.get("prompt") or ""), events=events, outcome=outcome))
+    return out
+
+
+def load_tasks(suites_dir: Path) -> dict[str, TaskInfo]:
+    import yaml
+
+    tasks = {}
+    for suite_file in sorted(Path(suites_dir).glob("*.yaml")):
+        doc = yaml.safe_load(suite_file.read_text(encoding="utf-8")) or {}
+        for t in doc.get("tasks") or []:
+            tasks[str(t["id"])] = TaskInfo(suite=str(doc.get("suite") or suite_file.stem),
+                                           tags=tuple(t.get("tags") or ()),
+                                           hidden=tuple(t.get("hidden") or ()))
+    return tasks
+
+
+def split_of(task_id: str, validation_share: float = 0.2) -> str:
+    """"train" or "validation", fixed by the task id alone."""
+    bucket = int(hashlib.sha256(task_id.encode()).hexdigest()[:8], 16) / 0xFFFFFFFF
+    return "validation" if bucket < validation_share else "train"
+
+
+def domain_of(tags: Iterable[str]) -> str:
+    tags = set(tags)
+    for d in DOMAINS:
+        if d in tags:
+            return "finance" if d == "payments" else d
+    return "other"
+
+
+def why_excluded(t: Trajectory, info: TaskInfo | None, holdout: tuple[str, ...]) -> str | None:
+    if info is None:
+        return "task not in any suite"
+    if info.suite in holdout or t.suite in holdout:
+        return "locked holdout"
+    if t.outcome.get("outcome") != "pass":
+        return f"outcome {t.outcome.get('outcome') or 'missing'}"
+    final = t.final
+    if not final:
+        return "no turn.completed"
+    acceptance = final.get("acceptance") or {}
+    if isinstance(acceptance, dict) and acceptance.get("verified") is False:
+        return "its own checks failed"
+    text = json.dumps(t.events, ensure_ascii=False)
+    graders = set(info.hidden) | {"test_*_hidden.py"}
+    for event in t.events:
+        for value in (event.get("params") or {}).values():
+            name = Path(str(value)).name
+            if any(fnmatch.fnmatch(name, g) for g in graders):
+                return "touched a grader"
+    if "eval.outcome" in text or "reward.json" in text:
+        return "grader output in the trajectory"
+    return None
+
+
+def to_example(t: Trajectory, system: str) -> dict:
+    """One Vertex Gemini SFT record from one trajectory."""
+    contents: list[dict] = [{"role": "user", "parts": [{"text": t.prompt}]}]
+    pending: list[dict] = []
+    for event in t.events:
+        kind = event.get("type")
+        if kind == "tool.started":
+            call = {"functionCall": {"name": event.get("tool"), "args": event.get("params") or {}}}
+            if contents[-1]["role"] == "model" and not pending:
+                contents[-1]["parts"].append(call)
+            else:
+                contents.append({"role": "model", "parts": [call]})
+            pending.append(event)
+        elif kind == "tool.completed" and pending:
+            pending.pop(0)
+            result = event.get("result")
+            if result is None:
+                result = {"success": event.get("success"), "error": event.get("error", "")}
+            body = json.dumps(result, ensure_ascii=False)
+            if len(body) > RESULT_CHARS:
+                result = {"truncated": True, "head": body[:RESULT_CHARS]}
+            part = {"functionResponse": {"name": event.get("tool"), "response": result}}
+            if contents[-1]["role"] == "user" and "functionResponse" in contents[-1]["parts"][0]:
+                contents[-1]["parts"].append(part)
+            else:
+                contents.append({"role": "user", "parts": [part]})
+    final = str(t.final.get("response") or "").strip()
+    if final:
+        contents.append({"role": "model", "parts": [{"text": final}]})
+    return {"systemInstruction": {"role": "system", "parts": [{"text": system}]}, "contents": contents}
+
+
+def build(trajectories: Path, suites: Path, out: Path, *, holdout=DEFAULT_HOLDOUT,
+          validation_share: float = 0.2, system: str = "") -> dict:
+    tasks = load_tasks(suites)
+    trajs = load_trajectories(trajectories)
+    out.mkdir(parents=True, exist_ok=True)
+    system = system or ("You are Aria, an AI agent for coding, financial analysis and logistics "
+                        "operations. Read the workspace first, compute every number with code from "
+                        "named data, keep each client's data apart, and check your work before you finish.")
+    split_files = {s: (out / f"{s}.jsonl").open("w", encoding="utf-8") for s in ("train", "validation")}
+    excluded: Counter = Counter()
+    exposure: dict[str, Counter] = defaultdict(Counter)
+    seen_tasks: dict[str, set[str]] = defaultdict(set)
+    fingerprints: set[str] = set()
+    try:
+        for t in trajs:
+            info = tasks.get(t.task_id)
+            reason = why_excluded(t, info, tuple(holdout))
+            if reason:
+                excluded[reason] += 1
+                continue
+            example = to_example(t, system)
+            fp = hashlib.sha256(json.dumps(example["contents"], sort_keys=True).encode()).hexdigest()
+            if fp in fingerprints:
+                excluded["duplicate"] += 1
+                continue
+            fingerprints.add(fp)
+            split = split_of(t.task_id, validation_share)
+            seen_tasks[split].add(t.task_id)
+            exposure[split][domain_of(info.tags)] += 1
+            split_files[split].write(json.dumps(example, ensure_ascii=False) + "\n")
+    finally:
+        for fh in split_files.values():
+            fh.close()
+
+    overlap = seen_tasks["train"] & seen_tasks["validation"]
+    assert not overlap, f"task in two splits: {sorted(overlap)}"
+    holdout_tasks = sorted(tid for tid, info in tasks.items() if info.suite in holdout)
+    manifest = {
+        "rules": {"holdout_suites": list(holdout), "split": "sha256(task_id)", "validation_share": validation_share,
+                  "eligible": "grader PASS and acceptance not failed; no grader file touched"},
+        "inputs": {str(t.path): hashlib.sha256(t.path.read_bytes()).hexdigest() for t in trajs},
+        "models": sorted({t.model for t in trajs}),
+        "examples": {s: sum(c.values()) for s, c in exposure.items()},
+        "tasks": {s: sorted(v) for s, v in seen_tasks.items()},
+        "exposure_by_domain": {s: dict(c) for s, c in exposure.items()},
+        "excluded": dict(excluded),
+        "holdout_tasks": holdout_tasks,
+    }
+    (out / "manifest.json").write_text(json.dumps(manifest, ensure_ascii=False, indent=2), encoding="utf-8")
+    return manifest
+
+
+def main(argv: list[str] | None = None) -> int:
+    ap = argparse.ArgumentParser(description="Build a Vertex Gemini SFT set from eval trajectories.")
+    ap.add_argument("trajectories", type=Path)
+    ap.add_argument("--suites", type=Path, default=Path("evals/suites"))
+    ap.add_argument("--out", type=Path, required=True)
+    ap.add_argument("--holdout", nargs="*", default=list(DEFAULT_HOLDOUT))
+    ap.add_argument("--validation-share", type=float, default=0.2)
+    args = ap.parse_args(argv)
+    m = build(args.trajectories, args.suites, args.out, holdout=tuple(args.holdout),
+              validation_share=args.validation_share)
+    print(json.dumps({k: m[k] for k in ("examples", "exposure_by_domain", "excluded")}, ensure_ascii=False, indent=2))
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())