contambench 0.1.0__py3-none-any.whl

contambench/__init__.py

@@ -0,0 +1,5 @@
+"""ContamBench: stressor suites, labeled contaminated workloads, and reference
+measurements for testing open-weight LLM serving."""
+
+__version__ = "0.1.0"
+__data_version__ = "0.1.0"

contambench/cli.py

@@ -0,0 +1,245 @@
+"""contam-bench command-line interface."""
+from __future__ import annotations
+
+import argparse
+import json
+import sys
+from pathlib import Path
+
+from contambench import __version__, data
+
+
+def _data_base(args) -> Path | None:
+    return Path(args.data_dir).expanduser() if getattr(args, "data_dir", None) else None
+
+
+def cmd_data(args) -> int:
+    base = _data_base(args)
+    if args.action == "fetch":
+        parts = [args.subset] if args.subset else None
+        dest = data.fetch(parts, base=base, source=args.source)
+        print(f"data ready at {dest}")
+    elif args.action == "path":
+        print(data.root(base))
+    elif args.action == "status":
+        print(json.dumps({"root": str(data.root(base)), **data.status(base)}, indent=2))
+    return 0
+
+
+def cmd_export(args) -> int:
+    from contambench import export
+
+    pairs_file = Path(args.pairs) if args.pairs else data.suite_file(args.suite, _data_base(args))
+    pairs = export.load_pairs(pairs_file)
+    benign = export.read_benign_csv(Path(args.benign)) if args.benign else export.benign_pool(pairs, args.rows)
+    rows, manifest = export.contaminate(benign, pairs, args.c, args.k)
+    manifest = {"suite": args.suite or str(pairs_file), "seed": args.seed,
+                "seed_note": "recorded for compatibility; the selection rule is deterministic", **manifest}
+    man = export.write(Path(args.out), rows, manifest)
+    print(f"wrote {args.out} ({manifest['n_attack']}/{manifest['n_rows']} stressor rows, "
+          f"{manifest['distinct_attack_prompts']} distinct prompts) and {man}")
+    return 0
+
+
+def cmd_run(args, extra: list[str]) -> int:
+    from contambench.harness import experiment
+
+    experiment.main(extra)
+    return 0
+
+
+def cmd_calibrate(args) -> int:
+    import pandas as pd
+
+    from contambench.harness import experiment
+    from contambench.harness.analysis import calibrate_capacity
+    from contambench.harness.config import config_from_dict
+
+    if args.config:
+        from contambench.harness.config import load_config
+
+        cfg = load_config(args.config)
+        cfg.contamination_ratio = [0.0]  # calibration is benign by definition
+        if args.endpoint:
+            cfg.endpoint = args.endpoint
+        if args.api_key:
+            cfg.api_key = args.api_key
+        if args.simulate:
+            cfg.simulate = True
+        if args.results_dir:
+            cfg.results_dir = args.results_dir
+        run_dir = experiment.run_experiment(cfg)
+        num_clients = max(cfg.client_counts)
+    else:
+        if not args.suite:
+            raise SystemExit("calibrate needs --config or --suite")
+        run_dir, num_clients = None, args.num_clients
+    if run_dir is None:
+        cfg = config_from_dict({
+        "name": "capacity_calibrate",
+        "endpoint": args.endpoint, "api_key": args.api_key, "model": args.model,
+        "attack_model": data.SUITES[data.suite_slug(args.suite)],
+        "attack_file": str(data.suite_file(args.suite, _data_base(args))),
+        "contamination_ratio": [0.0], "num_clients": args.num_clients,
+        "arrival_rate": [float(x) for x in args.arrival_rates.split(",")],
+        "arrival_model": "poisson", "requests_per_client": args.requests_per_client,
+        "num_trials": args.num_trials, "max_tokens": args.max_tokens, "temperature": 0.7,
+        "auto_slo": True, "seed": args.seed, "results_dir": args.results_dir, "simulate": args.simulate,
+        })
+        run_dir = experiment.run_experiment(cfg)
+    df = pd.read_parquet(Path(run_dir) / "requests.parquet")
+    out = Path(run_dir) / "calibrate-capacity"
+    res = calibrate_capacity.run(df, out, num_clients=num_clients)
+    print(json.dumps({k: res.get(k) for k in ("capacity_throughput_req_s", "recommended_arrival_rates")}, indent=2))
+    print(f"calibration written to {out}")
+    return 0
+
+
+def cmd_score(args) -> int:
+    from contambench import score
+
+    df = score.load(args.run)
+    rates = [args.arrival_rate] if args.arrival_rate is not None else sorted(df.arrival_rate.unique())
+    results = [score.sweep(df, r) for r in rates]
+    if args.json:
+        print(json.dumps({"sweeps": results, "stage_b": score.stage_b(df)}, indent=2))
+    else:
+        for s in results:
+            print(score.format_sweep(s), end="\n\n")
+        sb = score.stage_b(df)
+        if sb["stressor_cap_hit"] is not None:
+            print(f"Stage B: {100 * sb['stressor_cap_hit']:.1f}% of {sb['n_stressor']} stressor requests reached the "
+                  f"output cap ({100 * sb['benign_cap_hit']:.2f}% of {sb['n_benign']} benign)")
+    return 0
+
+
+def cmd_examples(args) -> int:
+    import shutil
+    from importlib.resources import files
+
+    dest = Path(args.dest)
+    dest.mkdir(parents=True, exist_ok=True)
+    src = files("contambench") / "examples"
+    for f in sorted(p for p in src.iterdir() if p.name.endswith(".yaml")):
+        target = dest / f.name
+        if target.exists() and not args.force:
+            print(f"kept existing {target}")
+            continue
+        target.write_text(f.read_text(encoding="utf-8"), encoding="utf-8")
+        print(f"wrote {target}")
+    return 0
+
+
+def cmd_report(args) -> int:
+    from contambench import report
+
+    print(report.render(_data_base(args), check=not args.no_check))
+    return 0
+
+
+def cmd_telemetry(args, extra: list[str]) -> int:
+    from contambench.harness import telemetry
+
+    return telemetry.main(extra)
+
+
+def cmd_guidellm(args, extra: list[str]) -> int:
+    import shutil
+    import subprocess
+
+    exe = shutil.which("guidellm")
+    if not exe:
+        print("GuideLLM is not installed: pip install 'contambench[guidellm]'", file=sys.stderr)
+        return 2
+    return subprocess.call([exe, *extra])
+
+
+def build_parser() -> argparse.ArgumentParser:
+    p = argparse.ArgumentParser(prog="contam-bench", description=__doc__)
+    p.add_argument("--version", action="version", version=f"contambench {__version__}")
+    p.add_argument("--data-dir", help="data root (default: $CONTAMBENCH_DATA or ~/.cache/contambench/<version>)")
+    sub = p.add_subparsers(dest="cmd", required=True)
+
+    d = sub.add_parser("data", help="fetch, locate, or inspect the data record")
+    d.add_argument("action", choices=["fetch", "path", "status"])
+    d.add_argument("--subset", choices=list(data.PARTS), help="fetch one product only")
+    d.add_argument("--from", dest="source", help="copy the record from a local directory instead of Zenodo")
+
+    e = sub.add_parser("export", help="write a D2 labeled workload at contamination ratio c")
+    e.add_argument("--suite", help=f"one of: {', '.join(data.SUITES)}")
+    e.add_argument("--pairs", help="use a pairs file instead of a fetched suite")
+    e.add_argument("--benign", help="benign pool CSV (prompt_id, prompt, success_rate); default: tile the suite")
+    e.add_argument("--c", type=float, required=True)
+    e.add_argument("--k", type=int, default=50, help="draw stressors from the top-k prompts (default 50)")
+    e.add_argument("--rows", type=int, default=1000)
+    e.add_argument("--seed", type=int, default=23)
+    e.add_argument("--out", required=True)
+
+    sub.add_parser("run", help="execute a workload against an endpoint (options as in the harness; see run --help)",
+                   add_help=False)
+
+    c = sub.add_parser("calibrate", help="benign arrival-rate sweep -> capacity and operating points")
+    c.add_argument("--config", help="harness YAML; contamination is forced to 0")
+    c.add_argument("--endpoint", required=False)
+    c.add_argument("--api-key")
+    c.add_argument("--model", help="model name sent in the request body")
+    c.add_argument("--suite", help="suite whose benign prompts drive the sweep (when no --config)")
+    c.add_argument("--arrival-rates", default="0.1,0.2,0.3,0.4,0.5,0.6,0.8,1.0")
+    c.add_argument("--num-clients", type=int, default=50)
+    c.add_argument("--requests-per-client", type=int, default=20)
+    c.add_argument("--num-trials", type=int, default=1)
+    c.add_argument("--max-tokens", type=int, default=4096)
+    c.add_argument("--seed", type=int, default=23)
+    c.add_argument("--results-dir")
+    c.add_argument("--simulate", action="store_true", help="synthetic latencies, no endpoint")
+
+    s = sub.add_parser("score", help="apply the SLO oracle and label split to a D3-format run")
+    s.add_argument("run", help="run directory or requests.parquet/csv")
+    s.add_argument("--arrival-rate", type=float)
+    s.add_argument("--json", action="store_true")
+
+    ex = sub.add_parser("examples", help="copy the example YAML campaign configurations")
+    ex.add_argument("dest", nargs="?", default="examples")
+    ex.add_argument("--force", action="store_true", help="overwrite existing files")
+
+    r = sub.add_parser("report", help="reprint the paper's tables from the data record")
+    r.add_argument("--no-check", action="store_true", help="skip the comparison with the paper's values")
+
+    sub.add_parser("telemetry", help="poll a vLLM /metrics endpoint into a JSONL log (see telemetry --help)",
+                   add_help=False)
+    sub.add_parser("guidellm", help="run GuideLLM (pass its arguments after --)", add_help=False)
+    return p
+
+
+def _command_index(argv: list[str]) -> int | None:
+    """Index of the subcommand token, skipping global options."""
+    i = 0
+    while i < len(argv):
+        a = argv[i]
+        if a == "--data-dir":
+            i += 2
+        elif a.startswith("-"):
+            i += 1
+        else:
+            return i
+    return None
+
+
+def main(argv: list[str] | None = None) -> int:
+    argv = list(sys.argv[1:] if argv is None else argv)
+    i = _command_index(argv)
+    # `run`, `telemetry`, and `guidellm` forward everything after the subcommand.
+    passthrough = {"run": cmd_run, "guidellm": cmd_guidellm, "telemetry": cmd_telemetry}
+    if i is not None and argv[i] in passthrough:
+        args = build_parser().parse_args(argv[: i + 1])
+        extra = argv[i + 1:]
+        if extra[:1] == ["--"]:
+            extra = extra[1:]
+        return passthrough[argv[i]](args, extra)
+    args = build_parser().parse_args(argv)
+    return {"data": cmd_data, "export": cmd_export, "calibrate": cmd_calibrate, "score": cmd_score,
+            "report": cmd_report, "examples": cmd_examples}[args.cmd](args)
+
+
+if __name__ == "__main__":
+    sys.exit(main())

contambench/data.py

@@ -0,0 +1,142 @@
+"""Locate, download, and verify the ContamBench data record.
+
+The record holds three zips (D1, D2, D3) plus documentation and a SHA256SUMS file.
+``fetch`` copies them from Zenodo (or from a local directory with ``source=``),
+checks every hash, and extracts the zips into the data root.
+
+Data root, in order of precedence: the ``root`` argument, ``$CONTAMBENCH_DATA``,
+then ``~/.cache/contambench/<version>``.
+"""
+from __future__ import annotations
+
+import hashlib
+import os
+import shutil
+import urllib.request
+import zipfile
+from pathlib import Path
+
+from contambench import __data_version__
+
+# Zenodo record of data version 0.1.0 (DOI 10.5281/zenodo.23482023).
+ZENODO_RECORD = os.environ.get("CONTAMBENCH_ZENODO_RECORD", "23482023")
+ZENODO_FILES = "https://zenodo.org/records/{record}/files/{name}?download=1"
+
+PARTS = {"d1": "d1_stressors", "d2": "d2_workloads", "d3": "d3_execution"}
+DOCS = ["README.md", "DATASET_CARD.md", "LICENSE-DATA.md", "CITATION.cff", "runs_index.json"]
+
+# Short suite names accepted on the command line, mapped to model ids.
+SUITES = {
+    "llama-3.2-3b-instruct": "meta-llama/Llama-3.2-3B-Instruct",
+    "qwen2.5-7b-instruct": "Qwen/Qwen2.5-7B-Instruct",
+    "llama-3.1-8b-instruct": "meta-llama/Meta-Llama-3.1-8B-Instruct",
+    "mistral-7b-instruct-v0.3": "mistralai/Mistral-7B-Instruct-v0.3",
+    "qwen2.5-math-7b": "Qwen/Qwen2.5-Math-7B",
+}
+CAMPAIGNS = {
+    "aws_llama3b": "llama-3.2-3b-instruct",
+    "aws_qwen7b": "qwen2.5-7b-instruct",
+    "runpod_llama8b": "llama-3.1-8b-instruct",
+    "aws_mistral7b": "mistral-7b-instruct-v0.3",
+}
+
+
+class DataNotFound(RuntimeError):
+    pass
+
+
+def root(path: str | os.PathLike | None = None) -> Path:
+    if path:
+        return Path(path).expanduser()
+    env = os.environ.get("CONTAMBENCH_DATA")
+    if env:
+        return Path(env).expanduser()
+    return Path.home() / ".cache" / "contambench" / __data_version__
+
+
+def require(part: str, base: Path | None = None) -> Path:
+    """Path to an extracted product directory, with a helpful error if it is missing."""
+    d = (base or root()) / PARTS[part]
+    if not d.is_dir():
+        raise DataNotFound(
+            f"{PARTS[part]} not found under {d.parent}. Run `contam-bench data fetch` "
+            "(or set CONTAMBENCH_DATA to an extracted record)."
+        )
+    return d
+
+
+def suite_slug(name: str) -> str:
+    """Accept a short suite name or a Hugging Face model id."""
+    if name in SUITES:
+        return name
+    for slug, model in SUITES.items():
+        if model.lower() == name.lower():
+            return slug
+    raise KeyError(f"unknown suite {name!r}; choose one of {', '.join(SUITES)}")
+
+
+def suite_file(name: str, base: Path | None = None) -> Path:
+    return require("d1", base) / "suites" / suite_slug(name) / "pairs.json"
+
+
+def _sha256(path: Path) -> str:
+    h = hashlib.sha256()
+    with path.open("rb") as fh:
+        for chunk in iter(lambda: fh.read(1 << 20), b""):
+            h.update(chunk)
+    return h.hexdigest()
+
+
+def _download(url: str, dst: Path) -> None:
+    tmp = dst.with_suffix(dst.suffix + ".part")
+    with urllib.request.urlopen(url) as r, tmp.open("wb") as fh:
+        shutil.copyfileobj(r, fh, length=1 << 20)
+    tmp.replace(dst)
+
+
+def _get(name: str, dst: Path, source: Path | None) -> None:
+    if source is not None:
+        shutil.copyfile(source / name, dst)
+        return
+    if not ZENODO_RECORD:
+        raise DataNotFound(
+            "The Zenodo record id is not set in this build. Pass --from <dir> with a local copy "
+            "of the record, or set CONTAMBENCH_ZENODO_RECORD."
+        )
+    _download(ZENODO_FILES.format(record=ZENODO_RECORD, name=name), dst)
+
+
+def fetch(parts: list[str] | None = None, *, base: Path | None = None, source: str | None = None,
+          log=print) -> Path:
+    """Download (or copy), verify, and extract the requested products."""
+    dest = base or root()
+    dest.mkdir(parents=True, exist_ok=True)
+    src = Path(source).expanduser() if source else None
+    _get("SHA256SUMS", dest / "SHA256SUMS", src)
+    sums = {}
+    for line in (dest / "SHA256SUMS").read_text().splitlines():
+        digest, name = line.split(maxsplit=1)
+        sums[name.strip()] = digest
+    wanted = [f"{PARTS[p]}.zip" for p in (parts or list(PARTS))]
+    names = wanted + [d for d in DOCS if d in sums] + (["schemas.zip"] if "schemas.zip" in sums else [])
+    for name in names:
+        target = dest / name
+        target.parent.mkdir(parents=True, exist_ok=True)
+        if not (target.exists() and _sha256(target) == sums[name]):
+            log(f"fetching {name}")
+            _get(name, target, src)
+        if _sha256(target) != sums[name]:
+            raise RuntimeError(f"checksum mismatch for {name}")
+    for name in wanted + (["schemas.zip"] if "schemas.zip" in sums else []):
+        out = dest / name[:-4]
+        if out.is_dir():
+            shutil.rmtree(out)
+        with zipfile.ZipFile(dest / name) as z:
+            z.extractall(dest)
+        log(f"extracted {name[:-4]}/")
+    return dest
+
+
+def status(base: Path | None = None) -> dict[str, bool]:
+    b = base or root()
+    return {k: (b / v).is_dir() for k, v in PARTS.items()}

contambench/examples/calibrate.yaml

@@ -0,0 +1,20 @@
+# Step 1: benign capacity calibration. Raises the per-client arrival rate until
+# achieved throughput stops rising, then places operating points at 60/80/90% of it.
+#   export OPENAI_API_KEY=...            # key of YOUR endpoint, if it needs one
+#   contam-bench calibrate --config examples/calibrate.yaml --endpoint http://HOST:8000
+name: capacity_calibrate
+backend: openai
+attack_model: meta-llama/Llama-3.2-3B-Instruct
+workload_size: 100
+contamination_ratio: [0.0]
+num_clients: 50
+arrival_rate: [0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.8, 1.0]
+arrival_model: poisson
+requests_per_client: 15
+num_trials: 1
+max_tokens: 4096
+temperature: 0.7
+timeout_s: 600
+config_cooldown_s: 20
+auto_slo: true
+seed: 23

contambench/examples/control_semantic_suffix.yaml

@@ -0,0 +1,19 @@
+# Control run: a plain-language "keep writing" suffix in place of the optimized one.
+# Swap the file for random_suffix.json or long_output_instruction.json for the other controls.
+#   contam-bench run --config examples/control_semantic_suffix.yaml --endpoint http://HOST:8000
+name: control_semantic_suffix
+backend: openai
+attack_model: meta-llama/Llama-3.2-3B-Instruct
+attack_file: contambench:d1_stressors/controls/semantic_suffix.json
+contamination_ratio: [0.0, 0.02, 0.05, 0.10, 0.15]
+num_clients: 50
+arrival_rate: [0.27]
+arrival_model: poisson
+requests_per_client: 20
+num_trials: 3
+max_tokens: 4096
+temperature: 0.7
+timeout_s: 600
+config_cooldown_s: 20
+auto_slo: true
+seed: 23

contambench/examples/output_caps.yaml

@@ -0,0 +1,19 @@
+# Output-cap run: the same contaminated workload under several output caps.
+# Include 0.0 in contamination_ratio if you want SLO budgets from the same run.
+#   contam-bench run --config examples/output_caps.yaml --endpoint http://HOST:8000
+name: output_caps
+backend: openai
+attack_model: meta-llama/Llama-3.2-3B-Instruct
+workload_size: 50
+contamination_ratio: [0.0, 0.10]
+num_clients: 50
+arrival_rate: [0.23]
+arrival_model: poisson
+requests_per_client: 20
+num_trials: 3
+max_tokens: [512, 1024, 2048, 4096]
+temperature: 0.7
+timeout_s: 600
+config_cooldown_s: 20
+auto_slo: true
+seed: 23

contambench/examples/simulate.yaml

@@ -0,0 +1,18 @@
+# Offline run with synthetic latencies. No endpoint and no GPU are needed.
+# It exercises the harness and the scorer; its numbers are not measurements.
+#   contam-bench run --config examples/simulate.yaml
+#   contam-bench score contambench-results/raw/<run_id>
+name: simulate_demo
+simulate: true
+attack_model: meta-llama/Llama-3.2-3B-Instruct   # resolves the D1 suite in the fetched record
+workload_size: 50
+contamination_ratio: [0.0, 0.10]
+num_clients: 4
+arrival_rate: [1.0]
+arrival_model: poisson
+requests_per_client: 10
+num_trials: 2
+max_tokens: 256  # short outputs keep the simulation to about a minute
+temperature: 0.7
+auto_slo: true
+seed: 23

contambench/examples/sweep.yaml

@@ -0,0 +1,20 @@
+# Step 2: contamination sweep at the calibrated operating points.
+# Replace arrival_rate with the rates printed by `contam-bench calibrate`.
+#   contam-bench run --config examples/sweep.yaml --endpoint http://HOST:8000
+#   contam-bench score contambench-results/raw/<run_id>
+name: contamination_sweep
+backend: openai
+attack_model: meta-llama/Llama-3.2-3B-Instruct
+workload_size: 50
+contamination_ratio: [0.0, 0.02, 0.05, 0.10, 0.15]
+num_clients: 50
+arrival_rate: [0.16, 0.23, 0.27]
+arrival_model: poisson
+requests_per_client: 20
+num_trials: 3
+max_tokens: 4096
+temperature: 0.7
+timeout_s: 600
+config_cooldown_s: 20
+auto_slo: true
+seed: 23

contambench/export.py

@@ -0,0 +1,101 @@
+"""Write a D2 labeled workload: a replayable CSV plus a manifest.
+
+The benign pool is the suite's benign prompts in suite order, cycled up to ``rows``
+rows (or a user-supplied CSV with ``prompt_id, prompt, success_rate``). A fraction
+``c`` of rows is replaced by the matching stressor prompt. Stressor rows are chosen
+deterministically: among rows whose prompt is in the top-``k`` prompts by Stage-A
+success rate, take the ``round(c * rows)`` rows with the highest success rate, with
+ties broken by row index. This reproduces the exports shipped in D2.
+"""
+from __future__ import annotations
+
+import csv
+import json
+from pathlib import Path
+
+
+def load_pairs(path: Path) -> list[dict]:
+    data = json.loads(Path(path).read_text(encoding="utf-8"))
+    if isinstance(data, dict):
+        data = data.get("prompts") or data.get("entries") or []
+        if isinstance(data, dict):
+            data = list(data.values())
+    return [p for p in data if p.get("prompt") is not None and p.get("adv_prompt") is not None]
+
+
+def benign_pool(pairs: list[dict], rows: int) -> list[dict]:
+    n = len(pairs)
+    return [
+        {"prompt_id": str(i % n), "prompt": pairs[i % n]["prompt"], "success_rate": str(float(pairs[i % n]["success_rate"]))}
+        for i in range(rows)
+    ]
+
+
+def read_benign_csv(path: Path) -> list[dict]:
+    with Path(path).open(newline="", encoding="utf-8") as fh:
+        rows = list(csv.DictReader(fh))
+    for col in ("prompt_id", "prompt", "success_rate"):
+        if rows and col not in rows[0]:
+            raise ValueError(f"{path} is missing column {col!r}")
+    return rows
+
+
+def contaminate(benign: list[dict], pairs: list[dict], c: float, k: int) -> tuple[list[dict], dict]:
+    if not 0.0 <= c <= 1.0:
+        raise ValueError("c must be in [0, 1]")
+    adv = {p["prompt"]: p["adv_prompt"] for p in pairs}
+    missing = {r["prompt"] for r in benign} - adv.keys()
+    if missing:
+        raise ValueError(f"{len(missing)} benign prompts have no stressor in the suite")
+    best: dict[str, float] = {}
+    for r in benign:
+        best[r["prompt_id"]] = max(best.get(r["prompt_id"], -1.0), float(r["success_rate"]))
+    ranked = sorted(best.items(), key=lambda kv: (-kv[1], int(kv[0])))
+    eligible = {pid for pid, _ in ranked[:k]}
+    idx = [i for i, r in enumerate(benign) if r["prompt_id"] in eligible]
+    n_attack = int(round(c * len(benign)))
+    if n_attack > len(idx):
+        raise ValueError(f"need {n_attack} stressor rows but only {len(idx)} rows belong to the top-{k} prompts")
+    idx.sort(key=lambda i: (-float(benign[i]["success_rate"]), i))
+    chosen = sorted(idx[:n_attack])
+    cs = set(chosen)
+    out = [
+        {
+            "prompt_id": r["prompt_id"],
+            "prompt": adv[r["prompt"]] if i in cs else r["prompt"],
+            "success_rate": r["success_rate"],
+            "is_attack": int(i in cs),
+            "benign_prompt": r["prompt"],
+            "row_index": i,
+        }
+        for i, r in enumerate(benign)
+    ]
+    manifest = {
+        "n_rows": len(out),
+        "contamination_ratio": c,
+        "n_attack": n_attack,
+        "n_benign": len(out) - n_attack,
+        "realized_contamination_ratio": n_attack / len(out) if out else 0.0,
+        "workload_size": k,
+        "eligible_unique_prompts": len(eligible),
+        "eligible_rows": len(idx),
+        "selection": "rows whose prompt_id is among the top-workload_size prompts by Stage-A success_rate; "
+        "take the n_attack rows with the highest success_rate, ties broken by row index",
+        "attack_row_indices": chosen,
+        "attack_prompt_ids": sorted({benign[i]["prompt_id"] for i in chosen}, key=int),
+        "distinct_attack_prompts": len({benign[i]["prompt_id"] for i in chosen}),
+        "text_column": "prompt",
+    }
+    return out, manifest
+
+
+def write(out_csv: Path, rows: list[dict], manifest: dict) -> Path:
+    out_csv = Path(out_csv)
+    out_csv.parent.mkdir(parents=True, exist_ok=True)
+    with out_csv.open("w", newline="", encoding="utf-8") as fh:
+        w = csv.DictWriter(fh, fieldnames=["prompt_id", "prompt", "success_rate", "is_attack", "benign_prompt", "row_index"])
+        w.writeheader()
+        w.writerows(rows)
+    man = out_csv.with_suffix(".manifest.json")
+    man.write_text(json.dumps(manifest, indent=2) + "\n", encoding="utf-8")
+    return man