self-evolve-framework 1.6.0 → 1.6.1

package/template/skills/skillopt-sleep/scripts/python/experiments/report.py

@@ -1,132 +0,0 @@
-"""SkillOpt-Sleep — turn a sweep JSONL into a presented Markdown scorecard.
-
-Usage:
-  python -m skillopt_sleep.experiments.report --in docs/sleep/sweep.jsonl \
-      --out docs/sleep/benchmark_report.md
-"""
-from __future__ import annotations
-
-import argparse
-import json
-import os
-import sys
-from typing import Any, Dict, List
-
-
-def _load(path: str) -> List[Dict[str, Any]]:
-    rows = []
-    if os.path.exists(path):
-        with open(path) as f:
-            for line in f:
-                line = line.strip()
-                if line:
-                    try:
-                        rows.append(json.loads(line))
-                    except Exception:
-                        pass
-    return rows
-
-
-def _fmt_model(backend: str, model: str) -> str:
-    m = model or "default"
-    return f"{backend}:{m}"
-
-
-def render(rows: List[Dict[str, Any]]) -> str:
-    direct = [r for r in rows if r.get("cfg", {}).get("kind") in ("direct", "dual") and "error" not in r]
-    transfer = [r for r in rows if r.get("cfg", {}).get("kind") == "transfer" and "error" not in r]
-    errors = [r for r in rows if "error" in r]
-
-    out: List[str] = []
-    out.append("# SkillOpt-Sleep — benchmark report")
-    out.append("")
-    out.append("Auto-generated from `sweep.jsonl`. Benchmark: "
-               "[gbrain-evals](https://github.com/garrytan/gbrain-evals) `skillopt-v1` "
-               "(deficient skills, train/held-out split, local rule judge — no judge-API).")
-    out.append("Held-out scores are computed by the harness, not the optimizer.")
-    out.append("")
-
-    # ── direct improvement table ──────────────────────────────────────────
-    out.append("## Direct improvement (optimize, then deploy)")
-    out.append("")
-    out.append("| Optimizer → Target | Seed | Held-out before | Held-out after | Nights | Tokens |")
-    out.append("|---|---|---|---|---|---|")
-    for r in direct:
-        c = r["cfg"]
-        if c.get("kind") == "dual":
-            label = (f"{_fmt_model(c['optimizer_backend'], c.get('optimizer_model',''))}"
-                     f" → {_fmt_model(c['target_backend'], c.get('target_model',''))}")
-        else:
-            m = _fmt_model(c["backend"], c.get("model", ""))
-            label = f"{m} → {m}"
-        out.append(f"| {label} | {c['seed']} | "
-                   f"{r['baseline']:.2f} | **{r['after']:.2f}** | {c['nights']} | "
-                   f"{r.get('tokens','?')} |")
-    if direct:
-        n_imp = sum(1 for r in direct if r.get("improved"))
-        out.append("")
-        out.append(f"**{n_imp}/{len(direct)} configurations improved on held-out.**")
-    out.append("")
-
-    # ── transfer table ────────────────────────────────────────────────────
-    if transfer:
-        out.append("## Cross-model transfer (optimize on SOURCE, deploy frozen on TARGET)")
-        out.append("")
-        out.append("The price-difference story: spend cheap tokens optimizing overnight, "
-                   "then deploy the frozen skill on any model with no further optimization.")
-        out.append("")
-        out.append("| Source (optimizer) | Target (deploy) | Seed | Target baseline | Transferred | Gain |")
-        out.append("|---|---|---|---|---|---|")
-        for r in transfer:
-            c = r["cfg"]
-            s = _fmt_model(c["source_backend"], c.get("source_model", ""))
-            t = _fmt_model(c["target_backend"], c.get("target_model", ""))
-            out.append(f"| {s} | {t} | {c['seed']} | {r['baseline_target']:.2f} | "
-                       f"**{r['transferred']:.2f}** | {r['transfer_gain']:+.2f} |")
-        n_pos = sum(1 for r in transfer if r.get("transfer_gain", 0) > 0)
-        out.append("")
-        out.append(f"**{n_pos}/{len(transfer)} transfers were positive** "
-                   "(frozen skill helped a different model than it was optimized on).")
-        out.append("")
-
-    # ── errors (honest reporting) ─────────────────────────────────────────
-    if errors:
-        out.append("## Configs that errored (reported, not hidden)")
-        out.append("")
-        for r in errors:
-            out.append(f"- `{json.dumps(r['cfg'])}` → {r['error']}")
-        out.append("")
-
-    out.append("## How to reproduce")
-    out.append("")
-    out.append("```bash")
-    out.append("git clone https://github.com/garrytan/gbrain-evals /tmp/gbrain-evals")
-    out.append("python -m skillopt_sleep.experiments.sweep --plan full \\")
-    out.append("    --data-root /tmp/gbrain-evals/eval/data/skillopt-v1 --out docs/sleep/sweep.jsonl")
-    out.append("python -m skillopt_sleep.experiments.report \\")
-    out.append("    --in docs/sleep/sweep.jsonl --out docs/sleep/benchmark_report.md")
-    out.append("```")
-    out.append("")
-    return "\n".join(out)
-
-
-def main(argv=None) -> int:
-    ap = argparse.ArgumentParser(description="Render SkillOpt-Sleep sweep report")
-    ap.add_argument("--in", dest="inp", default="docs/sleep/sweep.jsonl")
-    ap.add_argument("--out", default="docs/sleep/benchmark_report.md")
-    args = ap.parse_args(argv)
-
-    rows = _load(args.inp)
-    if not rows:
-        print(f"no rows in {args.inp}", file=sys.stderr)
-        return 1
-    md = render(rows)
-    os.makedirs(os.path.dirname(args.out) or ".", exist_ok=True)
-    with open(args.out, "w") as f:
-        f.write(md)
-    print(f"wrote {args.out} ({len(rows)} rows)")
-    return 0
-
-
-if __name__ == "__main__":
-    sys.exit(main())

package/template/skills/skillopt-sleep/scripts/python/experiments/run_experiment.py

@@ -1,178 +0,0 @@
-"""SkillOpt-Sleep — validation experiment.
-
-Answers the question the user posed: *does nightly offline self-evolution
-actually improve the agent?*  Runs deterministically with the MockBackend
-(no API key, reproducible) and is the acceptance test for the whole idea.
-
-What it proves:
-  1. MONOTONIC LIFT — over N sleep nights, the held-out score rises from a
-     baseline (empty skill/memory) toward 1.0 as the gate accepts the
-     general rules the persona's tasks require.
-  2. GATE SAFETY — an injected harmful edit is REJECTED (held-out score does
-     not improve), so a bad nightly proposal can never be adopted.
-  3. PLUMBING — harvest->mine->replay->consolidate->stage->adopt all run and
-     the adopted artifact, re-scored, retains the lift.
-
-Run:
-    python -m skillopt_sleep.experiments.run_experiment
-    python -m skillopt_sleep.experiments.run_experiment --persona programmer --nights 3
-    python -m skillopt_sleep.experiments.run_experiment --backend anthropic   # real lift
-"""
-from __future__ import annotations
-
-import argparse
-import json
-import os
-import sys
-import tempfile
-from typing import List
-
-from skillopt_sleep.backend import get_backend
-from skillopt_sleep.consolidate import consolidate
-from skillopt_sleep.experiments.personas import (
-    PERSONAS,
-    harmful_edit_task,
-    researcher_persona,
-)
-from skillopt_sleep.memory import ensure_skill_scaffold
-from skillopt_sleep.replay import aggregate_scores, replay_batch
-from skillopt_sleep.types import TaskRecord
-
-
-def _score_holdout(backend, tasks: List[TaskRecord], skill: str, memory: str,
-                   metric: str = "mixed", w: float = 0.5) -> float:
-    from skillopt_sleep.consolidate import select_gate_score
-    # the persona experiment uses a 2-way split (train/val, no test); score on val
-    holdout = [t for t in tasks if t.split in ("val", "holdout")] or tasks
-    pairs = replay_batch(backend, holdout, skill, memory)
-    h, s = aggregate_scores(pairs)
-    return select_gate_score(h, s, metric, w)
-
-
-def run(persona: str = "researcher", nights: int = 4, backend_name: str = "mock",
-        edit_budget: int = 4, seed: int = 42, model: str = "", codex_path: str = "",
-        limit_tasks: int = 0) -> dict:
-    from skillopt_sleep.mine import assign_splits
-
-    make = PERSONAS.get(persona, researcher_persona)
-    items = make()
-    if limit_tasks and limit_tasks < len(items):
-        items = items[:limit_tasks]
-    tasks = assign_splits(items, holdout_fraction=0.34, seed=seed)
-    backend = get_backend(backend_name, model=model, codex_path=codex_path)
-    is_mock = (backend.name == "mock")
-
-    # start from an empty managed skill + empty memory
-    skill = ensure_skill_scaffold("", name="skillopt-sleep-learned",
-                                  description="Learned preferences.")
-    memory = ""
-
-    baseline = _score_holdout(backend, tasks, skill, memory)
-    trace = [{"night": 0, "holdout_score": round(baseline, 4), "action": "baseline",
-              "n_edits": 0}]
-
-    for night in range(1, nights + 1):
-        res = consolidate(
-            backend, tasks, skill, memory,
-            edit_budget=edit_budget, gate_metric="mixed", gate_mixed_weight=0.5,
-            evolve_skill=True, evolve_memory=True, night=night,
-        )
-        if res.accepted:
-            skill, memory = res.new_skill, res.new_memory
-        trace.append({
-            "night": night,
-            "holdout_score": round(res.candidate_score, 4),
-            "action": res.gate_action,
-            "accepted": res.accepted,
-            "n_edits": len(res.applied_edits),
-            "edits": [e.content for e in res.applied_edits],
-            "n_rejected": len(res.rejected_edits),
-        })
-        # converged: stop early if perfect
-        if res.candidate_score >= 0.999:
-            break
-
-    after = _score_holdout(backend, tasks, skill, memory)
-
-    # ── gate-safety probe (mock only; it relies on the mock's known bad rule) ──
-    harmful_rejected = None
-    if is_mock:
-        harmful_tasks = assign_splits([harmful_edit_task()] + make()[:3],
-                                      holdout_fraction=0.5, seed=seed)
-        _ = _score_holdout(backend, harmful_tasks, skill, memory)
-        res_h = consolidate(backend, harmful_tasks, skill, memory,
-                            edit_budget=edit_budget, gate_metric="mixed",
-                            evolve_skill=True, evolve_memory=False, night=nights + 1)
-        harmful_rule_text = get_backend("mock").RULE_TEXT["__harmful__"]  # type: ignore[attr-defined]
-        harmful_rejected = (harmful_rule_text not in res_h.new_skill)
-
-    result = {
-        "persona": persona,
-        "backend": backend.name,
-        "model": model or "(default)",
-        "n_tasks": len(tasks),
-        "nights_run": len(trace) - 1,
-        "baseline_holdout": round(baseline, 4),
-        "after_holdout": round(after, 4),
-        "lift": round(after - baseline, 4),
-        "improved": after > baseline,
-        "gate_blocks_harmful": harmful_rejected,  # None for real backends
-        "tokens_used": backend.tokens_used(),
-        "final_skill_excerpt": skill[-500:],
-        "trace": trace,
-    }
-    return result
-
-
-def _assert(cond: bool, msg: str) -> None:
-    if not cond:
-        print(f"FAIL: {msg}")
-        raise SystemExit(1)
-
-
-def main(argv=None) -> int:
-    ap = argparse.ArgumentParser(description="SkillOpt-Sleep validation experiment")
-    ap.add_argument("--persona", default="researcher", choices=list(PERSONAS.keys()))
-    ap.add_argument("--nights", type=int, default=4)
-    ap.add_argument("--backend", default="mock", choices=["mock", "claude", "codex", "copilot"])
-    ap.add_argument("--model", default="", help="backend model override")
-    ap.add_argument("--codex-path", default="", help="path to the real @openai/codex binary")
-    ap.add_argument("--edit-budget", type=int, default=4)
-    ap.add_argument("--limit-tasks", type=int, default=0, help="cap #tasks (control API cost)")
-    ap.add_argument("--json", action="store_true")
-    ap.add_argument("--assert-improves", action="store_true",
-                    help="exit nonzero unless lift>0 (and, for mock, gate blocks harmful edit)")
-    args = ap.parse_args(argv)
-
-    res = run(args.persona, nights=args.nights, backend_name=args.backend,
-              edit_budget=args.edit_budget, model=args.model,
-              codex_path=args.codex_path, limit_tasks=args.limit_tasks)
-
-    if args.json:
-        print(json.dumps(res, ensure_ascii=False, indent=2))
-    else:
-        print(f"=== SkillOpt-Sleep experiment: persona={res['persona']} "
-              f"backend={res['backend']} model={res['model']} ===")
-        print(f"tasks: {res['n_tasks']}   tokens(approx): {res['tokens_used']}")
-        print(f"baseline held-out : {res['baseline_holdout']}")
-        print(f"after  held-out   : {res['after_holdout']}   (lift {res['lift']:+.4f})")
-        if res["gate_blocks_harmful"] is not None:
-            print(f"gate blocks harmful edit: {res['gate_blocks_harmful']}")
-        print("trace:")
-        for row in res["trace"]:
-            edits = "; ".join(row.get("edits", []))[:80]
-            print(f"  night {row['night']}: holdout={row['holdout_score']} "
-                  f"{row['action']} (+{row['n_edits']} edits) {edits}")
-
-    if args.assert_improves:
-        _assert(res["improved"], "held-out score did not improve")
-        if res["gate_blocks_harmful"] is not None:
-            _assert(res["gate_blocks_harmful"], "gate failed to block harmful edit")
-            print("\nPASS: nightly consolidation improves held-out score AND gate blocks regressions.")
-        else:
-            print("\nPASS: nightly consolidation improves held-out score (real backend).")
-    return 0
-
-
-if __name__ == "__main__":
-    sys.exit(main())

package/template/skills/skillopt-sleep/scripts/python/experiments/run_gbrain.py

@@ -1,209 +0,0 @@
-"""SkillOpt-Sleep — run the gbrain-evals skillopt-v1 benchmark with our engine.
-
-Reproduces gbrain's "Result 1 — skills measurably improve" scorecard
-(docs/benchmarks/2026-06-03-skillopt.md) using SkillOpt-Sleep's
-consolidate() loop and either the claude or codex backend.
-
-For each deficient seed skill:
-  1. score the held-out tasks with the ORIGINAL skill            -> before
-  2. run N consolidation nights on the training tasks (gated)     -> evolve skill
-  3. score the held-out tasks with the EVOLVED skill             -> after
-
-Held-out scoring is done locally by the rule judge (no judge API). Only the
-agent's `attempt` (and the optimizer's `reflect`) spend tokens.
-
-Usage:
-    python -m skillopt_sleep.experiments.run_gbrain --backend mock
-    python -m skillopt_sleep.experiments.run_gbrain --backend claude --seeds brief-writer --nights 2
-    python -m skillopt_sleep.experiments.run_gbrain --backend codex  --data-root /tmp/gbrain-evals/eval/data/skillopt-v1
-"""
-from __future__ import annotations
-
-import argparse
-import json
-import sys
-from typing import Dict, List, Optional
-
-from skillopt_sleep.backend import build_backend, get_backend
-from skillopt_sleep.consolidate import consolidate, select_gate_score
-from skillopt_sleep.experiments.gbrain_bench import (
-    available_seeds,
-    find_data_root,
-    load_seed,
-)
-from skillopt_sleep.replay import aggregate_scores, replay_batch
-
-
-def _score(backend, tasks, skill, memory, split="test", metric="mixed", w=0.5):
-    sub = [t for t in tasks if t.split == split]
-    if not sub:  # fall back to val, then everything, so we never score on nothing
-        sub = [t for t in tasks if t.split == "val"] or tasks
-    pairs = replay_batch(backend, sub, skill, memory)
-    h, s = aggregate_scores(pairs)
-    return h, s, select_gate_score(h, s, metric, w)
-
-
-def run_seed(backend, seed: str, skill: str, tasks: List, *,
-             nights: int = 3, edit_budget: int = 4, gate_mode: str = "on",
-             slow_update: bool = True, rollouts_k: int = 1,
-             limit_replay: int = 0, limit_holdout: int = 0) -> dict:
-    memory = ""
-    # optionally cap each split to control API cost / latency.
-    # limit_replay caps train; limit_holdout caps BOTH val and test.
-    if limit_replay or limit_holdout:
-        train = [t for t in tasks if t.split == "train"]
-        val = [t for t in tasks if t.split == "val"]
-        test = [t for t in tasks if t.split == "test"]
-        if limit_replay:
-            train = train[:limit_replay]
-        if limit_holdout:
-            val = val[:limit_holdout]
-            test = test[:limit_holdout]
-        tasks = train + val + test
-    # final measure is TEST (the gbrain held-out set); val gates internally
-    bh, bs, bscore = _score(backend, tasks, skill, memory, split="test")
-    trace = [{"night": 0, "test_hard": round(bh, 3), "action": "baseline"}]
-    cur = skill
-    first_night_skill = skill
-    for night in range(1, nights + 1):
-        res = consolidate(
-            backend, tasks, cur, memory,
-            edit_budget=edit_budget, gate_metric="mixed", gate_mixed_weight=0.5,
-            gate_mode=gate_mode, rollouts_k=rollouts_k,
-            evolve_skill=True, evolve_memory=False, night=night,
-        )
-        if res.accepted:
-            cur = res.new_skill
-        if night == 1:
-            first_night_skill = cur
-        # report the TEST score each night (independent of the val gate)
-        th, _ts, _ = _score(backend, tasks, cur, memory, split="test")
-        trace.append({
-            "night": night,
-            "val_hard": round(res.holdout_candidate, 3),
-            "test_hard": round(th, 3),
-            "action": res.gate_action,
-            "accepted": res.accepted,
-            "edits": [e.content for e in res.applied_edits],
-        })
-        if th >= 0.999:
-            break
-
-    # ── SLOW UPDATE: consolidate cross-night experience into the protected
-    # long-term field. Runs regardless of gate mode (it is what preserves
-    # long-term memory even when the gate is OFF).
-    slow_text = None
-    if nights >= 2 and slow_update:
-        try:
-            from skillopt_sleep.slow_update import run_slow_update, replace_slow_field
-            val_tasks = [t for t in tasks if t.split == "val"] or tasks
-            prev_pairs = replay_batch(backend, val_tasks, first_night_skill, memory)
-            curr_pairs = replay_batch(backend, val_tasks, cur, memory)
-            slow_text = run_slow_update(
-                backend, prev_skill=first_night_skill, curr_skill=cur,
-                prev_pairs=[(t, r) for t, r in prev_pairs],
-                curr_pairs=[(t, r) for t, r in curr_pairs],
-            )
-            if slow_text:
-                cur = replace_slow_field(cur, slow_text)
-        except Exception:
-            slow_text = None
-
-    ah, as_, ascore = _score(backend, tasks, cur, memory, split="test")
-    return {
-        "seed": seed,
-        "held_out_before": round(bh, 3),
-        "held_out_after": round(ah, 3),
-        "improved": ah > bh,
-        "nights": len(trace) - 1,
-        "trace": trace,
-        "slow_update": slow_text,
-        "final_skill_tail": cur[-400:],
-    }
-
-
-def main(argv=None) -> int:
-    ap = argparse.ArgumentParser(description="Run gbrain-evals skillopt-v1 with SkillOpt-Sleep")
-    ap.add_argument("--backend", default="mock", choices=["mock", "claude", "codex"])
-    ap.add_argument("--model", default="")
-    ap.add_argument("--optimizer-backend", default="", help="route reflect/judge here (dual)")
-    ap.add_argument("--optimizer-model", default="")
-    ap.add_argument("--target-backend", default="", help="route attempt here (dual)")
-    ap.add_argument("--target-model", default="")
-    ap.add_argument("--codex-path", default="")
-    ap.add_argument("--data-root", default="", help="path to eval/data/skillopt-v1")
-    ap.add_argument("--seeds", default="", help="comma list; default = all available")
-    ap.add_argument("--nights", type=int, default=3)
-    ap.add_argument("--edit-budget", type=int, default=4)
-    ap.add_argument("--gate", default="on", choices=["on", "off", "hard", "soft"],
-                    help="on/hard/soft = validation-gated; off = greedy (no hard filter)")
-    ap.add_argument("--rollouts-k", type=int, default=1,
-                    help=">1 = multi-rollout contrastive reflection per task")
-    ap.add_argument("--budget-tokens", type=int, default=0,
-                    help="approx token budget; auto-plans nights x rollouts when set")
-    ap.add_argument("--budget-minutes", type=float, default=0.0)
-    ap.add_argument("--preferences", default="", help="free-text user preferences (prior for reflect)")
-    ap.add_argument("--limit-replay", type=int, default=0, help="cap #train tasks (cost control)")
-    ap.add_argument("--limit-holdout", type=int, default=0, help="cap #val and #test tasks (cost control)")
-    ap.add_argument("--json", action="store_true")
-    args = ap.parse_args(argv)
-
-    data_root = find_data_root(args.data_root)
-    if not data_root:
-        print("ERROR: could not find eval/data/skillopt-v1. Clone gbrain-evals and pass --data-root.",
-              file=sys.stderr)
-        return 2
-
-    seeds = [s.strip() for s in args.seeds.split(",") if s.strip()] or available_seeds(data_root)
-    backend = build_backend(
-        backend=args.backend, model=args.model,
-        optimizer_backend=args.optimizer_backend, optimizer_model=args.optimizer_model,
-        target_backend=args.target_backend, target_model=args.target_model,
-        codex_path=args.codex_path, preferences=args.preferences,
-    )
-
-    results = []
-    for seed in seeds:
-        skill, tasks = load_seed(data_root, seed)
-        if not tasks:
-            continue
-        # budget auto-planning: derive nights x rollouts_k from a token budget
-        nights, rollouts_k = args.nights, args.rollouts_k
-        if args.budget_tokens:
-            from skillopt_sleep.budget import Budget, plan_depth
-            n_train = len([t for t in tasks if t.split == "train"]) or len(tasks)
-            nights, rollouts_k = plan_depth(
-                Budget(max_tokens=args.budget_tokens), n_tasks=n_train,
-                default_nights=args.nights, default_k=args.rollouts_k,
-            )
-            if not args.json:
-                print(f"  [budget] {args.budget_tokens} tok -> nights={nights} rollouts_k={rollouts_k}")
-        r = run_seed(backend, seed, skill, tasks, nights=nights,
-                     edit_budget=args.edit_budget, rollouts_k=rollouts_k,
-                     gate_mode=("off" if args.gate == "off" else "on"),
-                     limit_replay=args.limit_replay, limit_holdout=args.limit_holdout)
-        results.append(r)
-        if not args.json:
-            print(f"  {seed:<18} held-out {r['held_out_before']:.2f} -> {r['held_out_after']:.2f}"
-                  f"  ({'IMPROVED' if r['improved'] else 'no change'}, {r['nights']} nights)")
-
-    n_improved = sum(1 for r in results if r["improved"])
-    summary = {
-        "benchmark": "gbrain-evals/skillopt-v1",
-        "backend": backend.name,
-        "model": args.model or "(default)",
-        "n_seeds": len(results),
-        "n_improved": n_improved,
-        "tokens_used": backend.tokens_used(),
-        "results": results,
-    }
-    if args.json:
-        print(json.dumps(summary, ensure_ascii=False, indent=2))
-    else:
-        print(f"\n=== {n_improved}/{len(results)} seeds improved on held-out "
-              f"(backend={backend.name}, ~{backend.tokens_used()} tokens) ===")
-    return 0
-
-
-if __name__ == "__main__":
-    sys.exit(main())

package/template/skills/skillopt-sleep/scripts/python/experiments/run_transfer.py

@@ -1,155 +0,0 @@
-"""SkillOpt-Sleep — skill-transfer experiment (sleep scenario).
-
-Answers: "if I optimize a skill while the agent sleeps using a CHEAP model,
-does the learned skill still help an EXPENSIVE model at deploy time?" — and the
-reverse. This is the SkillOpt paper's cross-model transfer result, reproduced
-in the sleep setting, and it is the core price-difference value proposition:
-spend cheap tokens overnight, deploy the frozen skill anywhere.
-
-Protocol, per gbrain seed:
-  1. baseline_target = held-out score of the DEFICIENT skill, run on TARGET model
-  2. optimize the skill for N nights using the SOURCE model (attempt+reflect)
-  3. transferred = held-out score of the LEARNED skill, run on TARGET model,
-     with NO further optimization
-  4. (reference) direct = held-out score of a skill optimized AND run on TARGET
-
-Report baseline / direct / transferred, mirroring SkillOpt Table "transfer".
-
-Usage:
-  python -m skillopt_sleep.experiments.run_transfer \
-     --source-backend claude --source-model haiku \
-     --target-backend claude --target-model sonnet \
-     --seeds brief-writer --nights 2
-"""
-from __future__ import annotations
-
-import argparse
-import json
-import sys
-from typing import List, Optional
-
-from skillopt_sleep.backend import get_backend
-from skillopt_sleep.consolidate import consolidate, select_gate_score
-from skillopt_sleep.experiments.gbrain_bench import (
-    available_seeds, find_data_root, load_seed,
-)
-from skillopt_sleep.replay import aggregate_scores, replay_batch
-
-
-def _holdout_hard(backend, tasks, skill, memory="") -> float:
-    # transfer is measured on the true held-out TEST split
-    ho = [t for t in tasks if t.split == "test"]
-    if not ho:
-        ho = [t for t in tasks if t.split in ("val", "holdout")] or tasks
-    pairs = replay_batch(backend, ho, skill, memory)
-    h, _s = aggregate_scores(pairs)
-    return h
-
-
-def _optimize(backend, skill, tasks, *, nights, edit_budget) -> str:
-    cur = skill
-    for night in range(1, nights + 1):
-        res = consolidate(backend, tasks, cur, "",
-                          edit_budget=edit_budget, gate_metric="mixed",
-                          evolve_skill=True, evolve_memory=False, night=night)
-        if res.accepted:
-            cur = res.new_skill
-        if res.holdout_candidate >= 0.999:
-            break
-    return cur
-
-
-def run_seed(seed, skill, tasks, *, source, target, nights, edit_budget,
-             limit_replay, limit_holdout, do_direct=True) -> dict:
-    if limit_replay or limit_holdout:
-        train = [t for t in tasks if t.split == "train"]
-        val = [t for t in tasks if t.split == "val"]
-        test = [t for t in tasks if t.split == "test"]
-        if limit_replay:
-            train = train[:limit_replay]
-        if limit_holdout:
-            val = val[:limit_holdout]
-            test = test[:limit_holdout]
-        tasks = train + val + test
-
-    baseline_target = _holdout_hard(target, tasks, skill)
-
-    # optimize on SOURCE, evaluate frozen skill on TARGET
-    learned_on_source = _optimize(source, skill, tasks, nights=nights, edit_budget=edit_budget)
-    transferred = _holdout_hard(target, tasks, learned_on_source)
-
-    direct = None
-    if do_direct:
-        learned_on_target = _optimize(target, skill, tasks, nights=nights, edit_budget=edit_budget)
-        direct = _holdout_hard(target, tasks, learned_on_target)
-
-    return {
-        "seed": seed,
-        "baseline_target": round(baseline_target, 3),
-        "direct_target": (round(direct, 3) if direct is not None else None),
-        "transferred": round(transferred, 3),
-        "transfer_gain": round(transferred - baseline_target, 3),
-        "learned_skill_tail": learned_on_source[-300:],
-    }
-
-
-def main(argv=None) -> int:
-    ap = argparse.ArgumentParser(description="SkillOpt-Sleep cross-model transfer")
-    ap.add_argument("--source-backend", default="claude")
-    ap.add_argument("--source-model", default="haiku")
-    ap.add_argument("--target-backend", default="claude")
-    ap.add_argument("--target-model", default="sonnet")
-    ap.add_argument("--codex-path", default="")
-    ap.add_argument("--data-root", default="")
-    ap.add_argument("--seeds", default="brief-writer")
-    ap.add_argument("--nights", type=int, default=2)
-    ap.add_argument("--edit-budget", type=int, default=4)
-    ap.add_argument("--limit-replay", type=int, default=3)
-    ap.add_argument("--limit-holdout", type=int, default=3)
-    ap.add_argument("--no-direct", action="store_true", help="skip the direct reference (saves cost)")
-    ap.add_argument("--json", action="store_true")
-    args = ap.parse_args(argv)
-
-    data_root = find_data_root(args.data_root)
-    if not data_root:
-        print("ERROR: gbrain-evals skillopt-v1 data not found; pass --data-root", file=sys.stderr)
-        return 2
-
-    source = get_backend(args.source_backend, model=args.source_model, codex_path=args.codex_path)
-    target = get_backend(args.target_backend, model=args.target_model, codex_path=args.codex_path)
-
-    seeds = [s.strip() for s in args.seeds.split(",") if s.strip()] or available_seeds(data_root)
-    results = []
-    for seed in seeds:
-        skill, tasks = load_seed(data_root, seed)
-        if not tasks:
-            continue
-        r = run_seed(seed, skill, tasks, source=source, target=target,
-                     nights=args.nights, edit_budget=args.edit_budget,
-                     limit_replay=args.limit_replay, limit_holdout=args.limit_holdout,
-                     do_direct=not args.no_direct)
-        results.append(r)
-        if not args.json:
-            d = f" direct={r['direct_target']}" if r['direct_target'] is not None else ""
-            print(f"  {seed:<16} baseline={r['baseline_target']:.2f}"
-                  f" transferred={r['transferred']:.2f}{d}"
-                  f"  (gain {r['transfer_gain']:+.2f})")
-
-    summary = {
-        "experiment": "skillopt-sleep/transfer",
-        "source": f"{args.source_backend}:{args.source_model}",
-        "target": f"{args.target_backend}:{args.target_model}",
-        "tokens_source": source.tokens_used(),
-        "tokens_target": target.tokens_used(),
-        "results": results,
-    }
-    if args.json:
-        print(json.dumps(summary, ensure_ascii=False, indent=2))
-    else:
-        print(f"\n=== transfer {summary['source']} -> {summary['target']}: "
-              f"{sum(1 for r in results if r['transfer_gain'] > 0)}/{len(results)} positive ===")
-    return 0
-
-
-if __name__ == "__main__":
-    sys.exit(main())