lbagents-eval 0.19.0__tar.gz

lbagents_eval-0.19.0/.gitignore

@@ -0,0 +1,118 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+cov.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+cover/
+
+# Translations
+*.mo
+*.pot
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+.pybuilder/
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+# .python-version
+
+# PEP 582
+__pypackages__/
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Pixi
+.pixi/
+
+# Rope project settings
+.ropeproject
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+# pytype static type analyzer
+.pytype/
+
+# Cython debug symbols
+cython_debug/
+
+# Ruff
+.ruff_cache/
+
+# OS
+.DS_Store
+
+# IDE
+*.code-workspace
+.idea/
+.vscode/
+
+# Eval results (runtime artifacts)
+lbagents-eval/results.json
+eval_results.xml
+
+# threads from local script
+.threads/

lbagents_eval-0.19.0/PKG-INFO

@@ -0,0 +1,18 @@
+Metadata-Version: 2.5
+Name: lbagents-eval
+Version: 0.19.0
+Summary: Evaluation of LbAgents workflows on replayed real failures
+Project-URL: Homepage, https://gitlab.cern.ch/lhcb-dpa/analysis-productions/lbagents
+Classifier: Intended Audience :: Science/Research
+Classifier: License :: OSI Approved :: GNU General Public License v3 (GPLv3)
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Topic :: Scientific/Engineering
+Requires-Python: >=3.12
+Requires-Dist: langfuse<6.0,>=5.0.3
+Requires-Dist: lbagents
+Requires-Dist: mcp>=1.0
+Requires-Dist: pyyaml>=6.0
+Requires-Dist: rich>=13.0
+Requires-Dist: scipy>=1.15
+Requires-Dist: typer>=0.12

lbagents_eval-0.19.0/pyproject.toml

@@ -0,0 +1,49 @@
+[build-system]
+requires = ["hatchling", "hatch-vcs"]
+build-backend = "hatchling.build"
+
+[project]
+name = "lbagents-eval"
+description = "Evaluation of LbAgents workflows on replayed real failures"
+requires-python = ">=3.12"
+dynamic = ["version"]
+classifiers = [
+    "Intended Audience :: Science/Research",
+    "License :: OSI Approved :: GNU General Public License v3 (GPLv3)",
+    "Programming Language :: Python :: 3",
+    "Programming Language :: Python :: 3.12",
+    "Topic :: Scientific/Engineering",
+]
+# One-directional dependency: the eval harness depends on the product, never the
+# reverse. In the pixi workspace `lbagents` resolves to the local editable copy.
+dependencies = [
+    "lbagents",
+    # `export` reads production traces with the v2 observations API (SDK 5).
+    "langfuse>=5.0.3,<6.0",
+    # Live MCP tools (live-fallback replay, snapshot, check-tools).
+    "mcp>=1.0",
+    "typer>=0.12",
+    "rich>=13.0",
+    "pyyaml>=6.0",
+    # Exact sign test and the bootstrap confidence interval of the report.
+    "scipy>=1.15",
+]
+
+[project.scripts]
+lbagents-eval = "lbagents_eval.cli:app"
+
+[project.urls]
+Homepage = "https://gitlab.cern.ch/lhcb-dpa/analysis-productions/lbagents"
+
+# Same git-tag-derived version as `lbagents` (one tag → identical versions).
+[tool.hatch.version]
+source = "vcs"
+raw-options = { root = "..", git_describe_command = ["git", "describe", "--dirty", "--tags", "--long", "--match", "*[0-9]*", "--exclude", "v[0-9]r*", "--exclude", "v[0-9][0-9]r*"] }
+
+[tool.hatch.build.targets.wheel]
+packages = ["src/lbagents_eval"]
+
+[tool.pytest.ini_options]
+testpaths = ["tests"]
+addopts = "-v --cov=lbagents_eval --cov-report=term-missing"
+junit_family = "xunit1"

lbagents_eval-0.19.0/scripts/cases_to_scenarios.py

@@ -0,0 +1,417 @@
+###############################################################################
+# (c) Copyright 2026 CERN for the benefit of the LHCb Collaboration           #
+#                                                                             #
+# This software is distributed under the terms of the GNU General Public      #
+# Licence version 3 (GPL Version 3), copied verbatim in the file "COPYING".   #
+#                                                                             #
+# In applying this licence, CERN does not waive the privileges and immunities #
+# granted to it by virtue of its status as an Intergovernmental Organization  #
+# or submit itself to any jurisdiction.                                       #
+###############################################################################
+"""Turn recorded real CI failures into lbagents-eval scenarios.
+
+    python cases_to_scenarios.py CASES.json CACHE_DIR --tag harvest-2026-10
+
+CASES.json: a list of cases ({"id", "mr_iid", "pipeline_id", "ci_run_name", "sha",
+"reference", "evidence", optional "kind", "confidence", "draft", "reviewed_by",
+"expected_response_contains"}), as produced by the benchmark harness. CACHE_DIR/<id>.json:
+the tool outputs recorded when the case was run live ({key: {"tool", "args", "output"}};
+GitLab tools pinned to the failing commit).
+
+For each case it writes scenarios/harness/ap_ci_<id>.yaml (every recorded call as a
+mock response matched on its arguments, a "no output recorded" fallback per tool, and a
+scripted gold path: pipeline -> CI run -> first failed job -> post the expected answer)
+and adds or replaces the case's entry in scenarios/datasets/scenarios.jsonl.
+
+Draft cases (expected answer not yet checked by an operator: every case without a
+"reviewed_by", whatever its "draft" says) get no expected_response_contains: those
+keywords are chosen when the answer is confirmed. Proposed keywords are kept in
+metadata.draft_expected_response_contains, where nothing gates on them.
+
+Each entry carries metadata: draft, confidence, kind, side (request / outside / either,
+the side the expected cause is on; see SIDE) and needs_review.
+
+get_mr_info is recorded live, so it returns the MR as it is now, not as it was when the
+pipeline failed: the fields that describe what happened after the failure (state, merge
+and close data, the current head, labels...) are removed, head_sha is set to the failing
+commit, and sentences of the description that state the fix are cut (REDACT). The title
+and description are still the current ones (GitLab keeps no history of them).
+
+    python cases_to_scenarios.py CASES.json --update-existing
+
+rewrites only the scenarios.jsonl entries of CASES.json (metadata, tags, keywords,
+expected_tool_calls from the gold path of the existing harness) and strips get_mr_info
+in the existing harness YAMLs, without re-reading the recordings.
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import pathlib
+
+import yaml
+
+# Shared with the live-fallback replay, so a recording made later is cleaned the same way.
+from lbagents_eval.scenarios.recordings import FALLBACK, REDACT, at_failure_mr_info, commits_before, trimmed
+
+HERE = pathlib.Path(__file__).resolve().parent
+SCENARIOS = HERE.parent / "src" / "lbagents_eval" / "scenarios"
+# References that no benchmark report matched: possibly too narrow or wrong
+# (dataset-eval-review.md, section 1.2). Low confidence until an operator reviews them.
+TO_REVIEW = {"mr7370_40783", "mr7276_40327", "mr7298_40397", "mr7403_40923", "mr7471_41311"}
+
+# Side of the expected cause: "request" (the author changes the MR), "outside" (site,
+# DIRAC, storage, CI service, permissions: an owner adds the author, so the 2 process
+# cases are outside), "either" (both remedies are valid: the "insufficient disk replicas"
+# cases, where the data can be staged or the request changed).
+EITHER = {"mr7395_40884", "mr7471_41311", "mr6756_40661"}
+
+
+MR_INFO_NOTE = (
+    "# get_mr_info: post-failure fields removed, head_sha = the failing commit; title and description\n"
+    "# are the MR's current text (recorded after the failure), not the text at the failure time.\n"
+)
+REDACT_NOTE = "# The sentence of the current description that states the fix is removed (REDACT).\n"
+
+
+# Accepted / rejected fixes that the single prose reference does not spell out
+# (dataset-eval-review.md, section 1.2), for the goal judge (metadata.grading_notes).
+# Not reviewed by an operator. Kept for reviewers; the bench judges (scoring/verdict.py) do not read them.
+GRADING_NOTES = {
+    "mr7463": "Also accept removing the END_V*/end-vertex variables, as in the similar case mr7466.",
+    "mr7462": "Accept any unpacking that takes the last two returned values.",
+    "mr7407": "Rejected fix: removing the input_pvs argument (create_dtf requires pvs).",
+    "mr7398_41072": "Also accept fixing the import so that it works with the Moore version used.",
+    "mr7258_40226": (
+        "Accept any change that reduces the combinatorics (tighter cuts, fewer combinations). "
+        "Rejected fix: raising the memory limit."
+    ),
+    "mr7276_40327": (
+        "Also accept setting input_process/input_stream/geometry_version in extra_options by hand, "
+        "as the error message suggests. Rejected fix: waiting for a SprucingConfig release."
+    ),
+    "mr7403_40923": "Also accept using other input data or adding a re-merge step.",
+    "mr7440_41131": "Also accept initialising D_data or guarding the tuples.",
+    "mr7343_40747": "Also accept dropping the cut or applying the track-type cut on the pion.",
+    "mr7288_40362": "Accept removing TupleToolTagging, or a tagger list or DaVinci version that has SSKaonTagger.",
+    "mr7404_40924": "Accept removing flavour tagging, or configuring the tagging particles.",
+    "mr7137_39844": "Accept any fix of the input locations (the job reads HLT2 locations on Spruce input).",
+    "mr7395_40884": "Accept reducing n_test_lfns, staging the files to disk, or another sample on disk.",
+    "mr7471_41311": "Accept staging the files to disk or another sample that is on disk.",
+    "mr6756_40661": "Accept staging the files to disk, fewer test LFNs, or another sample on disk.",
+    "mr7025_39809": (
+        "Accept start_run/end_run instead of the enumerated runs, with or without reporting the Bookkeeping limit."
+    ),
+    "mr7125_39716": (
+        "Accept start_run/end_run instead of the enumerated runs, with or without reporting the Bookkeeping limit."
+    ),
+}
+
+
+def side(c):
+    if c["id"] in EITHER:
+        return "either"
+    return "outside" if c.get("kind") in ("infrastructure", "process") else "request"
+
+
+def message(c):
+    return (
+        f'Diagnose the CI failure for pipeline {c["pipeline_id"]}, CI run "{c["ci_run_name"]}" '
+        f"(MR !{c['mr_iid']}). Investigate the logs, identify the root cause, and post a review on the merge request."
+    )
+
+
+def recorded(cache, tool, args):
+    for v in cache.values():
+        if v["tool"] == tool and v["args"] == args:
+            return v["output"]
+    return None
+
+
+def pipeline_created(pipeline_info: str | None) -> str | None:
+    """Creation time of the failing pipeline (ISO 8601, UTC) from its get_pipeline_info output."""
+    try:
+        return json.loads(pipeline_info or "").get("created")
+    except (ValueError, AttributeError):
+        return None
+
+
+def first_failed_job(run_info: str | None):
+    try:
+        jobs = json.loads(run_info or "").get("jobs", [])
+    except (ValueError, AttributeError):
+        return None
+    return next((j["name"] for j in jobs if j.get("status") == "FAILED"), None)
+
+
+def strip_post_failure_commits(mocks: list, sha: str | None, created: str | None) -> bool:
+    """Cut the get_mr_commits recordings at the failure (commits_before) and drop the
+    get_commit_diff recordings of the commits cut: a commit pushed after the failure is
+    often the fix. Edits *mocks* in place; True if anything changed."""
+    removed, changed = set(), False
+    for m in mocks:
+        if m["tool_name"] == "get_mr_commits" and m.get("match_arguments") is not None:
+            new = commits_before(m["response"], sha, created)
+            if new != m["response"]:
+                try:
+                    old_shas = {str(x.get("sha", ""))[:12] for x in json.loads(m["response"])}
+                    removed |= old_shas - {str(x.get("sha", ""))[:12] for x in json.loads(new)}
+                except (ValueError, AttributeError):
+                    pass
+                m["response"], changed = new, True
+    keep = [
+        m
+        for m in mocks
+        if not (
+            m["tool_name"] == "get_commit_diff"
+            and m.get("match_arguments")
+            and str(m["match_arguments"].get("commit_sha", ""))[:12] in removed
+        )
+    ]
+    if len(keep) != len(mocks):
+        mocks[:] = keep
+        changed = True
+    return changed
+
+
+def harness(c, cache):
+    mocks, tools = [], set()
+    for v in cache.values():
+        if v["tool"] == "post_mr_review":
+            continue
+        output = v["output"]
+        if v["tool"] == "get_mr_info":
+            output = at_failure_mr_info(output, c["id"], c.get("sha"))
+        mocks.append(
+            {
+                "tool_name": v["tool"],
+                "arguments": v["args"],
+                "response": trimmed(v["tool"], output),
+                "match_arguments": v["args"],
+            }
+        )
+        tools.add(v["tool"])
+    info = recorded(cache, "get_pipeline_info", {"pipeline_id": c["pipeline_id"]})
+    strip_post_failure_commits(mocks, c.get("sha"), c.get("pipeline_created") or pipeline_created(info))
+    for t in sorted(tools):
+        mocks.append({"tool_name": t, "response": FALLBACK})
+    mocks.append({"tool_name": "post_mr_review", "response": '{"status": "ok"}'})
+
+    pid, run = c["pipeline_id"], c["ci_run_name"]
+    path = [("get_pipeline_info", {"pipeline_id": pid}), ("get_ci_run_info", {"pipeline_id": pid, "ci_run_name": run})]
+    job = first_failed_job(recorded(cache, "get_ci_run_info", path[1][1]))
+    if job:
+        path.append(("get_ci_job_info", {"pipeline_id": pid, "ci_run_name": run, "job_name": job}))
+    scripted = [
+        {"content": "", "tool_calls": [{"tool_name": t, "arguments": a}]}
+        for t, a in path
+        if recorded(cache, t, a) is not None
+    ]
+    body = f"## CI failure: pipeline {pid}, run {run}\n\n{c['reference']}\n"
+    scripted.append({"content": "", "tool_calls": [{"tool_name": "post_mr_review", "arguments": {"body": body}}]})
+    scripted.append({"content": "Review proposed."})
+    return {"mock_responses": mocks, "scripted_llm_responses": scripted}
+
+
+def gold_path_calls(scripted):
+    """expected_tool_calls = the tool calls of the scripted gold path (post_mr_review without its body)."""
+    calls = []
+    for r in scripted:
+        for tc in r.get("tool_calls", []):
+            name = tc.get("tool_name") or tc.get("name")
+            if name == "post_mr_review":
+                calls.append({"tool_name": name})
+            else:
+                calls.append({"tool_name": name, "arguments": tc.get("arguments", {})})
+    return calls
+
+
+def entry(c, tag, scripted):
+    kind = c.get("kind", "request")
+    draft = not c.get("reviewed_by")
+    confidence = "low" if c["id"] in TO_REVIEW else (c.get("confidence") or "unrated")
+    tags = ["ci_failure", "real_case", tag, {"request": "request_bug"}.get(kind, kind), f"side_{side(c)}"]
+    if draft:
+        tags += ["draft", f"confidence_{confidence}"]
+    if c["id"] in TO_REVIEW:
+        tags.append("needs_review")
+    metadata = {
+        "case_id": c["id"],
+        "draft": draft,
+        "confidence": confidence,
+        "kind": kind,
+        "side": side(c),
+        "needs_review": c["id"] in TO_REVIEW,
+    }
+    if c.get("confidence") and confidence != c["confidence"]:
+        metadata["draft_confidence"] = c["confidence"]
+    if c["id"] in TO_REVIEW:
+        metadata["review_note"] = (
+            "No benchmark report (7 setups, Claude included) matched this reference: it may be too narrow or wrong."
+        )
+    if c["id"] in GRADING_NOTES:
+        metadata["grading_notes"] = GRADING_NOTES[c["id"]]
+    if c.get("reviewed_by"):
+        metadata["reviewed_by"] = c["reviewed_by"]
+    # What a live-fallback replay needs to see the failure as it was (lbagents_eval.execution.replay).
+    metadata["sha"] = c["sha"]
+    if c.get("pipeline_created"):
+        metadata["pipeline_created"] = c["pipeline_created"]
+    e = {
+        "name": f"ap_ci_{c['id']}",
+        "description": (
+            f"Real failed Analysis Productions CI run (MR !{c['mr_iid']}, pipeline {c['pipeline_id']}, "
+            f"run {c['ci_run_name']}, commit {c['sha'][:12]}). {c.get('evidence', '')}"
+            + (" Expected answer is a DRAFT, not yet checked by an operator." if draft else "")
+            + " Runs with either CI workflow (run --workflow)."
+        ),
+        "workflow": "ap_debug_ci_failure",
+        "user_input": message(c),
+        "initial_state": {"pipeline_id": c["pipeline_id"], "mr_iid": c["mr_iid"], "ci_run_name": c["ci_run_name"]},
+        "expected_goal": c["reference"],
+        "expected_tool_calls": [
+            tc
+            for tc in gold_path_calls(scripted)
+            if tc["tool_name"] in ("get_pipeline_info", "get_ci_run_info", "get_ci_job_info", "post_mr_review")
+        ],
+        "tags": tags,
+        "metadata": metadata,
+    }
+    keywords = c.get("expected_response_contains")
+    if keywords and not draft:
+        e["expected_response_contains"] = keywords
+    elif keywords:
+        metadata["draft_expected_response_contains"] = keywords
+    return e
+
+
+def split_header(text: str) -> tuple[str, str]:
+    lines = text.splitlines(True)
+    i = 0
+    while i < len(lines) and lines[i].startswith("#"):
+        i += 1
+    return "".join(lines[:i]), "".join(lines[i:])
+
+
+def strip_mr_info_in_harness(path: pathlib.Path, case_id: str, sha: str | None) -> bool:
+    """Apply at_failure_mr_info to the get_mr_info outputs of an existing harness YAML.
+
+    Files written by this script are re-dumped the same way; hand-written ones (the 7
+    pilot cases, with other formatting) get only their response lines replaced, so
+    the rest of the file is unchanged. Returns True if the file changed.
+    """
+    header, body = split_header(path.read_text())
+    data = yaml.safe_load(body)
+    changed = False
+    if yaml.safe_dump(data, sort_keys=False, allow_unicode=True, width=100000) == body:
+        for m in data["mock_responses"]:
+            if m["tool_name"] == "get_mr_info" and m.get("response", "").startswith("{"):
+                new = at_failure_mr_info(m["response"], case_id, sha, strict=False)
+                changed |= new != m["response"]
+                m["response"] = new
+        if changed:
+            path.write_text(header + yaml.safe_dump(data, sort_keys=False, allow_unicode=True, width=100000))
+        return changed
+    lines = body.splitlines(True)
+    for i, line in enumerate(lines):
+        if line.startswith("- tool_name: get_mr_info"):
+            j = next(j for j in range(i + 1, len(lines)) if lines[j].startswith("  response: "))
+            old = yaml.safe_load(lines[j][len("  response: ") :])
+            if not str(old).startswith("{"):
+                continue
+            new = at_failure_mr_info(old, case_id, sha, strict=False)
+            if new != old:
+                dumped = yaml.safe_dump({"response": new}, allow_unicode=True, width=100000)
+                if dumped.count("\n") != 1:
+                    raise ValueError(f"{path}: the new get_mr_info response does not fit on one line")
+                lines[j] = "  " + dumped
+                changed = True
+                for m in data["mock_responses"]:
+                    if m["tool_name"] == "get_mr_info" and m.get("response") == old:
+                        m["response"] = new
+    if changed:
+        new_body = "".join(lines)
+        if yaml.safe_load(new_body) != data:  # only the get_mr_info responses may differ
+            raise ValueError(f"{path}: rewriting the get_mr_info response changed other content")
+        path.write_text(header + new_body)
+    return changed
+
+
+def update_existing(cases, tag_of):
+    """Rewrite the dataset entries and strip get_mr_info in the harnesses, from CASES only."""
+    jsonl = SCENARIOS / "datasets" / "scenarios.jsonl"
+    entries = [json.loads(line) for line in jsonl.read_text().splitlines() if line.strip()]
+    by_name = {e["name"]: e for e in entries}
+    for c in cases:
+        name = f"ap_ci_{c['id']}"
+        path = SCENARIOS / "harness" / f"{name}.yaml"
+        if name not in by_name or not path.exists():
+            print("not in the dataset, skipped:", c["id"])
+            continue
+        old = by_name[name]
+        c = dict(c)
+        c.setdefault("expected_response_contains", old.get("expected_response_contains"))
+        strip_mr_info_in_harness(path, c["id"], c.get("sha"))
+        header, body = split_header(path.read_text())
+        h = yaml.safe_load(body)
+        info = next((m["response"] for m in h["mock_responses"] if m["tool_name"] == "get_pipeline_info"), None)
+        c.setdefault("pipeline_created", pipeline_created(info))
+        if strip_post_failure_commits(h["mock_responses"], c.get("sha"), c["pipeline_created"]):
+            if yaml.safe_dump(yaml.safe_load(body), sort_keys=False, allow_unicode=True, width=100000) != body:
+                raise ValueError(f"{path}: post-failure commits to cut in a hand-written harness: edit it by hand")
+            path.write_text(header + yaml.safe_dump(h, sort_keys=False, allow_unicode=True, width=100000))
+            print("post-failure commits cut:", c["id"])
+        by_name[name] = entry(c, tag_of(old), h.get("scripted_llm_responses", []))
+    jsonl.write_text("".join(json.dumps(e) + "\n" for e in by_name.values()))
+
+
+def main():
+    ap = argparse.ArgumentParser()
+    ap.add_argument("cases")
+    ap.add_argument("cache_dir", nargs="?")
+    ap.add_argument("--tag")
+    ap.add_argument("--update-existing", action="store_true")
+    a = ap.parse_args()
+    cases = json.loads(pathlib.Path(a.cases).read_text())
+    if a.update_existing:
+        known = ("pilot-2026-10", "harvest-2026-10")
+        update_existing(cases, lambda e: a.tag or next((t for t in e.get("tags", []) if t in known), "harvest-2026-10"))
+        return
+    if not (a.cache_dir and a.tag):
+        ap.error("cache_dir and --tag are required unless --update-existing")
+    jsonl = SCENARIOS / "datasets" / "scenarios.jsonl"
+    entries = [json.loads(line) for line in jsonl.read_text().splitlines() if line.strip()]
+    by_name = {e["name"]: e for e in entries}
+    skipped = []
+    for c in cases:
+        cache_file = pathlib.Path(a.cache_dir) / f"{c['id']}.json"
+        if not cache_file.exists():
+            skipped.append(c["id"])
+            continue
+        cache = json.loads(cache_file.read_text())
+        name = f"ap_ci_{c['id']}"
+        header = (
+            f"# Real case {c['id']} (MR !{c['mr_iid']}, pipeline {c['pipeline_id']}): tool outputs recorded\n"
+            f"# live ({a.tag}; LbAPI MCP + GitLab tools pinned to {c['sha'][:12]}).\n"
+            "# scripted_llm_responses = a gold trajectory over those outputs (the tests' scripted model).\n"
+            + MR_INFO_NOTE
+            + (REDACT_NOTE if c["id"] in REDACT else "")
+        )
+        h = harness(c, cache)
+        c = {
+            **c,
+            "pipeline_created": pipeline_created(
+                recorded(cache, "get_pipeline_info", {"pipeline_id": c["pipeline_id"]})
+            ),
+        }
+        (SCENARIOS / "harness" / f"{name}.yaml").write_text(
+            header + yaml.safe_dump(h, sort_keys=False, allow_unicode=True, width=100000)
+        )
+        by_name[name] = entry(c, a.tag, h["scripted_llm_responses"])
+    jsonl.write_text("".join(json.dumps(e) + "\n" for e in by_name.values()))
+    print(f"{len(cases) - len(skipped)} scenarios written; no recording for: {skipped or 'none'}")
+
+
+if __name__ == "__main__":
+    main()