benchkit-cli 0.1.0__py3-none-any.whl

benchkit/perf_report.py

@@ -0,0 +1,207 @@
+"""Write portable performance-profile reports."""
+
+from __future__ import annotations
+
+import csv
+import json
+from datetime import datetime
+from importlib.resources import files
+from pathlib import Path
+
+
+def _safe_json(data: object) -> str:
+    return (
+        json.dumps(data, ensure_ascii=False)
+        .replace("&", "\\u0026")
+        .replace("<", "\\u003c")
+        .replace(">", "\\u003e")
+    )
+
+
+def render_perf_html(profile: dict) -> str:
+    template = (
+        files("benchkit").joinpath("templates/perf.html").read_text(encoding="utf-8")
+    )
+    return template.replace("__BENCHKIT_PERF_DATA__", _safe_json(profile))
+
+
+def _number(case: dict, metric: str, field: str = "median") -> float:
+    return float((case.get(metric) or {}).get(field) or 0.0)
+
+
+def _markdown(profile: dict) -> str:
+    environment = profile.get("environment") or {}
+    settings = profile.get("settings") or {}
+    note = environment.get("config_note") or environment.get("hardware") or ""
+    lines = [
+        "# BenchKit Performance Profile",
+        "",
+        f"**Model:** `{profile['model']}`",
+        f"**Provider:** {profile.get('provider') or 'unknown'}",
+        f"**Host:** `{profile.get('host') or ''}`",
+        f"**Generated:** {profile.get('generated_at') or ''}",
+        (
+            f"**Settings:** {settings.get('gen_tokens', 0)} output tokens · "
+            f"{settings.get('reps', 0)} repetitions · "
+            f"{settings.get('warmups', 0)} warmup(s)"
+        ),
+    ]
+    if note:
+        lines.append(f"**Configuration:** {note}")
+    startup = profile.get("startup") or {}
+    if startup.get("ttft_s") is not None:
+        lines.append(
+            f"**First request TTFT:** {startup['ttft_s']:.3f}s "
+            f"({startup.get('label', 'startup probe')})"
+        )
+    drift = profile.get("drift_check") or {}
+    if drift and not drift.get("error"):
+        warning = " ⚠ possible warmup/drift anomaly" if not drift["stable"] else ""
+        lines.append(
+            f"**Drift check:** {drift['depth_label']} TG "
+            f"{drift['initial_tg_tps']:.1f} → {drift['final_tg_tps']:.1f} tok/s "
+            f"({drift['change']:+.1%}){warning}"
+        )
+    sweep = profile.get("sweep_summary") or {}
+    lines.append(
+        f"**Sweep median:** "
+        f"TG {float(sweep.get('tg_tps_median') or 0):.1f} tok/s · "
+        f"PP {float(sweep.get('pp_tps_median') or 0):.1f} tok/s"
+    )
+
+    lines.extend(
+        [
+            "",
+            "| Depth | Actual input | PP tok/s | TG tok/s | TTFT | Wall | Overhead | Output |",
+            "|------:|-------------:|---------:|---------:|-----:|-----:|---------:|-------:|",
+        ]
+    )
+    for case in profile.get("cases", []):
+        actual = case.get("actual_input_tokens") or "—"
+        marker = "!" if not case.get("depth_within_tolerance", True) else ""
+        pp = f"{_number(case, 'pp_tps'):.1f}" if case.get("pp_reliable") else "n/a"
+        overhead = (
+            f"{_number(case, 'overhead_s'):.3f}s"
+            if case.get("overhead_available")
+            else "n/a"
+        )
+        lines.append(
+            f"| {case['depth_label']}{marker} | {actual} "
+            f"| {pp} "
+            f"| {_number(case, 'tg_tps'):.1f} "
+            f"| {_number(case, 'ttft_s'):.3f}s "
+            f"| {_number(case, 'wall_time_s'):.2f}s "
+            f"| {overhead} "
+            f"| {_number(case, 'output_tokens'):.0f} |"
+        )
+
+    calibrated = any(
+        case.get("depth_method") == "calibrated" for case in profile.get("cases", [])
+    )
+    inaccurate = [
+        case["depth_label"]
+        for case in profile.get("cases", [])
+        if not case.get("depth_within_tolerance", True)
+    ]
+    estimated = [
+        case["depth_label"]
+        for case in profile.get("cases", [])
+        if case.get("depth_method") == "estimated"
+    ]
+    lines.extend(
+        [
+            "",
+            "> Values are medians. Min/max and every raw repetition are available "
+            "in `perf.json`.",
+        ]
+    )
+    if calibrated:
+        lines.extend(
+            [
+                "",
+                "Depth method: calibrated. The server does not expose its "
+                "native tokenizer route, so BenchKit calibrated deterministic "
+                "text against the reported input-token count.",
+            ]
+        )
+    if inaccurate:
+        lines.extend(
+            [
+                "",
+                f"⚠ Depth calibration remained outside ±1% for: "
+                f"{', '.join(inaccurate)}. Compare using Actual input.",
+            ]
+        )
+    if estimated:
+        lines.extend(
+            [
+                "",
+                f"⚠ Token-depth calibration was unavailable for: "
+                f"{', '.join(estimated)}. Compare using Actual input.",
+            ]
+        )
+    lines.extend(
+        [
+            "",
+            "PP is server-reported prompt processing throughput; TG is "
+            "server-reported token generation throughput. PP is shown as n/a "
+            "for tiny prompts where fixed overhead dominates the value.",
+            "",
+            "> Tip: Run the `ruler` benchmark to verify how much of the "
+            "configured context window is actually usable.",
+            "",
+        ]
+    )
+    return "\n".join(lines)
+
+
+def save_profile(profile: dict) -> Path:
+    timestamp = datetime.now().strftime("%Y-%m-%d_%H-%M-%S")
+    output = Path("results") / timestamp
+    output.mkdir(parents=True, exist_ok=True)
+
+    with open(output / "perf.json", "w", encoding="utf-8") as handle:
+        json.dump(profile, handle, indent=2, ensure_ascii=False)
+
+    rows = []
+    for case in profile.get("cases", []):
+        rows.append(
+            {
+                "model": profile["model"],
+                "depth": case["depth_label"],
+                "actual_input_tokens": case.get("actual_input_tokens", 0),
+                "depth_method": case.get("depth_method", ""),
+                "depth_deviation": case.get("depth_deviation", 0),
+                "depth_within_tolerance": case.get("depth_within_tolerance", False),
+                "pp_reliable": case.get("pp_reliable", False),
+                "overhead_available": case.get("overhead_available", False),
+                "pp_tps_median": _number(case, "pp_tps"),
+                "pp_tps_min": _number(case, "pp_tps", "min"),
+                "pp_tps_max": _number(case, "pp_tps", "max"),
+                "tg_tps_median": _number(case, "tg_tps"),
+                "tg_tps_min": _number(case, "tg_tps", "min"),
+                "tg_tps_max": _number(case, "tg_tps", "max"),
+                "ttft_s_median": _number(case, "ttft_s"),
+                "ttft_s_min": _number(case, "ttft_s", "min"),
+                "ttft_s_max": _number(case, "ttft_s", "max"),
+                "wall_time_s_median": _number(case, "wall_time_s"),
+                "overhead_s_median": _number(case, "overhead_s"),
+                "overhead_s_min": _number(case, "overhead_s", "min"),
+                "overhead_s_max": _number(case, "overhead_s", "max"),
+                "output_tokens_median": _number(case, "output_tokens"),
+                "successful_reps": case.get("successful_reps", 0),
+            }
+        )
+    if rows:
+        with open(output / "perf.csv", "w", newline="", encoding="utf-8") as handle:
+            writer = csv.DictWriter(handle, fieldnames=rows[0].keys())
+            writer.writeheader()
+            writer.writerows(rows)
+
+    with open(output / "perf.md", "w", encoding="utf-8") as handle:
+        handle.write(_markdown(profile))
+
+    with open(output / "perf.html", "w", encoding="utf-8") as handle:
+        handle.write(render_perf_html(profile))
+
+    return output

benchkit/perturbations.py

@@ -0,0 +1,208 @@
+"""Deterministic, benchmark-aware input perturbations."""
+
+from __future__ import annotations
+
+import hashlib
+import random
+from collections.abc import Iterable
+from dataclasses import dataclass, replace
+
+from benchkit.benchmarks.base import Task
+
+CHOICE_ORDER = "choice-order"
+PERTURBATIONS = (CHOICE_ORDER,)
+
+# Every benchmark here stores visible options in ``metadata["choices"]`` and
+# the correct option letter in ``metadata["answer"]``. TruthfulQA is the one
+# exception to the all-choices-visible rule: its prompt intentionally shows at
+# most A-D even though the source MC1 rows can carry more distractors.
+CHOICE_ORDER_BENCHMARKS = frozenset(
+    {
+        "arc",
+        "gpqa",
+        "hellaswag",
+        "mmlu",
+        "mmlu-pro",
+        "mmvp",
+        "openbookqa",
+        "piqa",
+        "truthfulqa",
+        "vstar",
+        "winogrande",
+    }
+)
+VISIBLE_CHOICE_LIMITS = {"truthfulqa": 4}
+
+
+@dataclass(frozen=True)
+class PerturbedCase:
+    """Prompt and scoring tasks produced by one perturbation."""
+
+    prompt_task: Task
+    evaluation_task: Task
+    details: dict
+
+
+def perturbations_for(benchmark_key: str) -> tuple[str, ...]:
+    """Return perturbations supported by one benchmark."""
+    if benchmark_key in CHOICE_ORDER_BENCHMARKS:
+        return (CHOICE_ORDER,)
+    return ()
+
+
+def supports_perturbation(benchmark_key: str, name: str) -> bool:
+    """Return whether a benchmark opts into a named perturbation."""
+    return name in perturbations_for(benchmark_key)
+
+
+def unsupported_benchmarks(name: str, keys: Iterable[str]) -> list[str]:
+    """Return the benchmarks in ``keys`` that cannot run a perturbation, once each."""
+    return list(
+        dict.fromkeys(key for key in keys if not supports_perturbation(key, name))
+    )
+
+
+def _task_rng(benchmark_key: str, task_id: str, name: str, seed: int) -> random.Random:
+    material = f"{seed}\0{benchmark_key}\0{task_id}\0{name}".encode()
+    digest = hashlib.sha256(material).digest()
+    return random.Random(int.from_bytes(digest[:8], "big"))
+
+
+def _choice_order(benchmark_key: str, task: Task, seed: int) -> PerturbedCase:
+    raw_choices = task.metadata.get("choices")
+    answer = str(task.metadata.get("answer", "")).upper()
+    if not isinstance(raw_choices, (list, tuple)):
+        raise ValueError(f"{benchmark_key}/{task.id} has no choice list")
+
+    choices = list(raw_choices)
+    visible_count = min(
+        len(choices),
+        VISIBLE_CHOICE_LIMITS.get(benchmark_key, len(choices)),
+    )
+    if visible_count < 2:
+        raise ValueError(f"{benchmark_key}/{task.id} needs at least two choices")
+
+    answer_index = ord(answer) - ord("A") if len(answer) == 1 else -1
+    if answer_index < 0 or answer_index >= visible_count:
+        raise ValueError(
+            f"{benchmark_key}/{task.id} answer {answer!r} is not a visible choice"
+        )
+
+    rng = _task_rng(benchmark_key, task.id, CHOICE_ORDER, seed)
+    order = list(range(visible_count))
+    rng.shuffle(order)
+
+    # A choice-order run should exercise answer-position sensitivity on every
+    # task. If the random permutation left the correct answer under its old
+    # letter, swap it with a deterministic different position.
+    new_answer_index = order.index(answer_index)
+    if new_answer_index == answer_index:
+        candidates = [index for index in range(visible_count) if index != answer_index]
+        swap_index = rng.choice(candidates)
+        order[answer_index], order[swap_index] = (
+            order[swap_index],
+            order[answer_index],
+        )
+        new_answer_index = order.index(answer_index)
+
+    visible_choices = [choices[index] for index in order]
+    perturbed_choices = visible_choices + choices[visible_count:]
+    perturbed_answer = chr(ord("A") + new_answer_index)
+    metadata = dict(task.metadata)
+    metadata.update(choices=perturbed_choices, answer=perturbed_answer)
+    perturbed_task = replace(task, metadata=metadata)
+    details = {
+        "name": CHOICE_ORDER,
+        "seed": seed,
+        # Original labels in their new A.. position. For example ["C", "A",
+        # "B"] means original C is now A, original A is now B, and so on.
+        "choice_order": [chr(ord("A") + index) for index in order],
+        "original_answer": answer,
+        "perturbed_answer": perturbed_answer,
+    }
+    return PerturbedCase(perturbed_task, perturbed_task, details)
+
+
+def perturb_task(
+    benchmark_key: str,
+    task: Task,
+    name: str | None,
+    seed: int,
+) -> PerturbedCase:
+    """Copy and perturb a task without mutating the cached source task."""
+    if name is None:
+        return PerturbedCase(task, task, {})
+    if not supports_perturbation(benchmark_key, name):
+        raise ValueError(f"{benchmark_key} does not support perturbation {name!r}")
+    if name == CHOICE_ORDER:
+        return _choice_order(benchmark_key, task, seed)
+    raise ValueError(f"Unknown perturbation: {name}")
+
+
+def _result_key(result: dict) -> tuple[object, ...]:
+    return (
+        result.get("model"),
+        result.get("benchmark"),
+        result.get("variant"),
+        result.get("slice"),
+        result.get("harness", "direct"),
+        result.get("repair_attempts", 0),
+    )
+
+
+def annotate_robustness(results: list[dict]) -> None:
+    """Attach paired clean-versus-perturbed metrics to completed results."""
+    baselines = {
+        _result_key(result): result
+        for result in results
+        if not result.get("perturbation")
+    }
+
+    for result in results:
+        if not result.get("perturbation"):
+            continue
+        baseline = baselines.get(_result_key(result))
+        if baseline is None:
+            continue
+
+        baseline_tasks = {task["task_id"]: task for task in baseline.get("tasks", [])}
+        pairs = [
+            (baseline_tasks[task["task_id"]], task)
+            for task in result.get("tasks", [])
+            if task["task_id"] in baseline_tasks
+        ]
+        if not pairs:
+            continue
+
+        paired_total = len(pairs)
+        baseline_score = sum(float(clean.get("score", 0)) for clean, _ in pairs)
+        perturbed_score = sum(
+            float(perturbed.get("score", 0)) for _, perturbed in pairs
+        )
+        regressions = sum(
+            bool(clean.get("passed")) and not bool(perturbed.get("passed"))
+            for clean, perturbed in pairs
+        )
+        recoveries = sum(
+            not bool(clean.get("passed")) and bool(perturbed.get("passed"))
+            for clean, perturbed in pairs
+        )
+        comparison = {
+            "perturbation": result["perturbation"],
+            "perturbation_seed": result.get("perturbation_seed"),
+            "paired_total": paired_total,
+            "baseline_score": round(baseline_score / paired_total, 1),
+            "perturbed_score": round(perturbed_score / paired_total, 1),
+            "score_delta_pp": round(
+                (perturbed_score - baseline_score) / paired_total,
+                1,
+            ),
+            "regressions": regressions,
+            "regression_rate": round(regressions / paired_total * 100, 1),
+            "recoveries": recoveries,
+            "recovery_rate": round(recoveries / paired_total * 100, 1),
+        }
+        result.update(comparison)
+        result["baseline_benchmark_label"] = baseline.get(
+            "benchmark_label", baseline.get("benchmark")
+        )