pergamon-bench 0.8.0__py3-none-any.whl

pbench_grader/__init__.py

@@ -0,0 +1,6 @@
+"""Pergamon Bench grading: rubric judging, weighted legal scoring, rewardkit criteria, `pbench` CLI.
+
+Does not need Harbor: the verifier container installs only this package.
+"""
+
+__version__ = "0.8.0"

pbench_grader/cache.py

@@ -0,0 +1,69 @@
+"""File cache for judge verdicts.
+
+Key = sha256(answer, criterion text, judge model, prompt hash, reasoning/temperature).
+Re-running a job or a regrade with the same answer and judge then costs nothing.
+Off unless a directory is configured (``--cache-dir`` for ``pbench grade`` or the
+``PBENCH_JUDGE_CACHE_DIR`` environment variable).
+"""
+
+from __future__ import annotations
+
+import hashlib
+import json
+import os
+from pathlib import Path
+from typing import Any
+
+
+def cache_dir_from_env() -> Path | None:
+    value = os.environ.get("PBENCH_JUDGE_CACHE_DIR")
+    return Path(value).expanduser() if value else None
+
+
+class JudgeCache:
+    def __init__(self, directory: Path | None) -> None:
+        self.directory = directory
+        self.hits = 0
+        self.misses = 0
+        if directory is not None:
+            directory.mkdir(parents=True, exist_ok=True)
+
+    @property
+    def enabled(self) -> bool:
+        return self.directory is not None
+
+    @staticmethod
+    def key(*parts: str) -> str:
+        h = hashlib.sha256()
+        for part in parts:
+            h.update(part.encode("utf-8"))
+            h.update(b"\0")
+        return h.hexdigest()
+
+    def _path(self, key: str) -> Path:
+        assert self.directory is not None
+        return self.directory / key[:2] / f"{key}.json"
+
+    def get(self, key: str) -> dict[str, Any] | None:
+        if not self.enabled:
+            return None
+        path = self._path(key)
+        if not path.exists():
+            self.misses += 1
+            return None
+        try:
+            data = json.loads(path.read_text(encoding="utf-8"))
+        except (OSError, ValueError):
+            self.misses += 1
+            return None
+        self.hits += 1
+        return data
+
+    def put(self, key: str, value: dict[str, Any]) -> None:
+        if not self.enabled:
+            return
+        path = self._path(key)
+        path.parent.mkdir(parents=True, exist_ok=True)
+        tmp = path.with_suffix(".tmp")
+        tmp.write_text(json.dumps(value, ensure_ascii=False), encoding="utf-8")
+        tmp.replace(path)

pbench_grader/cli.py

@@ -0,0 +1,456 @@
+"""The grading subcommands of the ``pergamon-bench`` command (no Harbor needed).
+
+    pergamon-bench grade    --tasks ../pergamon-bench/tasks --answers answers.jsonl -o graded/
+    pergamon-bench summary  jobs/<job-dir> [--markdown]
+    pergamon-bench manifest --tasks ../pergamon-bench/tasks
+
+``add_commands`` registers them on a parser; ``run`` executes the parsed arguments.
+"""
+
+from __future__ import annotations
+
+import argparse
+import asyncio
+import json
+import sys
+from pathlib import Path
+from typing import Any
+
+from . import versions
+from .grading import GradingSettings, grade_answer
+from .rk import (
+    DEFAULT_ADJUDICATOR_MODEL,
+    DEFAULT_JUDGE_REASONING_EFFORT,
+    DEFAULT_JUDGE_TEMPERATURE,
+)
+from .rk import DEFAULT_JUDGE_MODELS as DEFAULT_JUDGE
+from .rubric import load_rubric
+
+# ── pbench grade ────────────────────────────────────────────────────────
+
+
+def _read_answers(path: Path) -> list[dict[str, Any]]:
+    """JSONL rows: {"task": "<task dir name>", "answer": "..."} or {"task", "answer_path"}."""
+    rows: list[dict[str, Any]] = []
+    for line_no, line in enumerate(path.read_text(encoding="utf-8").splitlines(), start=1):
+        if not line.strip():
+            continue
+        row = json.loads(line)
+        if "task" not in row:
+            raise ValueError(f"{path}:{line_no}: missing 'task'")
+        if "answer" not in row:
+            answer_path = row.get("answer_path")
+            if not answer_path:
+                raise ValueError(f"{path}:{line_no}: needs 'answer' or 'answer_path'")
+            row["answer"] = (path.parent / answer_path).read_text(encoding="utf-8")
+        rows.append(row)
+    return rows
+
+
+async def _grade_all(args: argparse.Namespace) -> int:
+    settings = GradingSettings(
+        judge_models=[m.strip() for m in args.judge_models.split(",") if m.strip()],
+        reasoning_effort=args.judge_reasoning_effort,
+        temperature=args.judge_temperature,
+        concurrency=args.concurrency,
+        adjudicator_model=None
+        if (args.adjudicator_model or "").lower() == "none"
+        else args.adjudicator_model,
+        **({"cache_dir": Path(args.cache_dir)} if args.cache_dir else {}),
+    )
+    rows = _read_answers(Path(args.answers))
+    out_root = Path(args.out)
+    summary_rows = []
+    failures = 0
+    for row in rows:
+        task_dir = Path(args.tasks) / row["task"]
+        rubric = load_rubric(task_dir / "tests" / "rubric.yaml")
+        result = await grade_answer(rubric, str(row["answer"]), settings)
+        out_dir = out_root / row["task"]
+        result.write(out_dir, extra={"answer_source": str(args.answers)})
+        (out_dir / "reward.json").write_text(
+            json.dumps(result.rewards(), indent=1), encoding="utf-8"
+        )
+        if result.ungradeable:
+            failures += 1
+        sc = result.score
+        summary_rows.append(
+            {
+                "task": row["task"],
+                "reward": sc.reward,
+                "pass": sc.passed,
+                "pass_clean": sc.pass_clean,
+                "q_final": sc.q_final,
+                "q": sc.q,
+                "p": sc.p,
+                "ungradeable": len(result.ungradeable),
+                "judge_cost_usd": round(result.usage.cost_usd, 4),
+            }
+        )
+        print(
+            f"{row['task']:32s} reward={sc.reward} score={_fmt(sc.q_final) or 'n/a'} "
+            f"(q={sc.q:.3f} p={_fmt(sc.p) or 'n/a'}) clean={sc.pass_clean}"
+        )
+    (out_root / "summary.json").write_text(json.dumps(summary_rows, indent=1), encoding="utf-8")
+    print(
+        f"wrote {out_root}/summary.json "
+        f"({len(summary_rows)} tasks, {failures} with ungradeable criteria)"
+    )
+    return 1 if failures else 0
+
+
+# ── pbench summary ──────────────────────────────────────────────────────
+
+
+def _trial_dirs(job_dir: Path) -> list[Path]:
+    return sorted(p.parent for p in job_dir.glob("*/result.json"))
+
+
+def _agent_cost(result: dict[str, Any]) -> float:
+    return float(((result.get("agent_result") or {}).get("cost_usd")) or 0.0)
+
+
+def summarise_job(job_dir: Path) -> dict[str, Any]:
+    tasks: list[dict[str, Any]] = []
+    categories: dict[str, dict[str, int]] = {}
+    case_law = {"at_max": 0, "total": 0, "verified_trials": 0, "trials": 0}
+    penalty = {"events": 0, "severity": 0.0, "misuse": 0, "not_covered": 0, "rubric_named": 0}
+    grading_versions: set[str] = set()
+    for trial in _trial_dirs(job_dir):
+        result = json.loads((trial / "result.json").read_text(encoding="utf-8"))
+        grading_path = trial / "verifier" / "grading.json"
+        grading = (
+            json.loads(grading_path.read_text(encoding="utf-8")) if grading_path.exists() else {}
+        )
+        rewards = (result.get("verifier_result") or {}).get("rewards") or {}
+        if grading:
+            grading_versions.add(grading_label(grading))
+        judge_cost = sum(float(u.get("cost_usd", 0)) for u in grading.get("usage", {}).values())
+        for cat, counts in grading.get("categories", {}).items():
+            entry = categories.setdefault(cat, {"+1": 0, "0": 0, "-1": 0, "total": 0})
+            for k in entry:
+                entry[k] += counts.get(k, 0)
+        cl = [c for c in grading.get("criteria", []) if c.get("grader") == "fr_case_law"]
+        if cl:
+            case_law["trials"] += 1
+            case_law["total"] += len(cl)
+            case_law["at_max"] += sum(1 for c in cl if c.get("at_max"))
+            if grading.get("groundedness_enabled") and not grading.get("groundedness_error"):
+                case_law["verified_trials"] += 1
+        pen = grading.get("penalty") or {}
+        penalty["events"] += sum(1 for e in pen.get("events", []) if e.get("scored"))
+        penalty["severity"] += float(pen.get("total_severity") or 0.0)
+        for k in ("misuse", "not_covered", "rubric_named"):
+            penalty[k] += int((pen.get("counts") or {}).get(k, 0))
+        agent_meta = ((result.get("agent_result") or {}).get("metadata") or {}).get(
+            "pbench_agent"
+        ) or {}
+        tasks.append(
+            {
+                "trial": trial.name,
+                "task": grading.get("task") or trial.name.split("__")[0],
+                "agent": (result.get("agent_info") or {}).get("name"),
+                "model": ((result.get("agent_info") or {}).get("model_info") or {}).get("name"),
+                "reward": rewards.get("reward"),
+                "groundedness": _groundedness_status(grading),
+                "pass_clean": grading.get("pass_clean"),
+                "q_final": grading.get("q_final"),
+                "q": grading.get("q"),
+                "p": grading.get("p"),
+                "integrity": grading.get("integrity"),
+                "ungradeable": len(grading.get("ungradeable", [])),
+                "exception": (result.get("exception_info") or {}).get("exception_type"),
+                "agent_cost_usd": round(_agent_cost(result), 4),
+                "judge_cost_usd": round(judge_cost, 4),
+                "tool_failures": sum((agent_meta.get("tools_failed") or {}).values()),
+            }
+        )
+    graded = [t for t in tasks if t["reward"] is not None and not t["exception"]]
+    # Trials graded without the groundedness check have no penalty: their reward is a
+    # more lenient number, reported under its own label and never averaged with verified
+    # ones; the penalty-dependent means are n/a as soon as one trial lacks them.
+    unverified = [t for t in graded if t["groundedness"] not in ("on", "none")]
+    verified = "yes" if not unverified else ("no" if len(unverified) == len(graded) else "partly")
+
+    def mean(key: str, *, strict: bool = False) -> float | None:
+        vals = [float(t[key]) for t in graded if t.get(key) is not None]
+        if strict and len(vals) != len(graded):
+            return None
+        return round(sum(vals) / len(vals), 4) if vals else None
+
+    return {
+        "job": job_dir.name,
+        "trials": len(tasks),
+        "graded": len(graded),
+        "failed": len(tasks) - len(graded),
+        "groundedness": verified,
+        "reward_label": "reward" if verified == "yes" else "reward (no-groundedness)",
+        "pass_rate": mean("reward"),
+        "pass_clean_rate": mean("pass_clean", strict=True),
+        "mean_score": mean("q_final", strict=True),
+        "mean_q": mean("q"),
+        "mean_p": mean("p", strict=True),
+        "mean_integrity": mean("integrity", strict=True),
+        "ungradeable_criteria": sum(t["ungradeable"] for t in tasks),
+        "agent_cost_usd": round(sum(t["agent_cost_usd"] for t in tasks), 4),
+        "judge_cost_usd": round(sum(t["judge_cost_usd"] for t in tasks), 4),
+        "tool_failures": sum(t["tool_failures"] for t in tasks),
+        "categories": {
+            k: {**v, "at_max_rate": round(v["+1"] / v["total"], 4) if v["total"] else None}
+            for k, v in sorted(categories.items())
+        },
+        "penalty": {**penalty, "severity": round(penalty["severity"], 4)},
+        "case_law": _case_law_summary(case_law),
+        "grading_versions": sorted(grading_versions),
+        "tasks": tasks,
+    }
+
+
+def _groundedness_status(grading: dict[str, Any]) -> str:
+    """The trial's groundedness status; derived for grading.json files that predate the field."""
+    status = grading.get("groundedness_status")
+    if isinstance(status, str):
+        return status
+    if grading.get("groundedness"):
+        return "on"
+    if grading.get("groundedness_error"):
+        return "error"
+    return "off"
+
+
+def grading_label(grading: dict[str, Any]) -> str:
+    """ "1", or "1 + judge_models" when settings differed from version 1."""
+    label = str(grading.get("grading_version") or "unrecorded")
+    changes = grading.get("grading_changes") or []
+    return f"{label} + {', '.join(changes)}" if changes else label
+
+
+def _case_law_summary(c: dict[str, int]) -> dict[str, Any] | None:
+    """Case-law criteria, reported apart: without the groundedness service they are judged by
+    the LLM judge alone, which cannot check that a cited decision exists."""
+    if not c["total"]:
+        return None
+    if c["verified_trials"] == c["trials"]:
+        verified = "yes"
+    elif c["verified_trials"] == 0:
+        verified = "no"
+    else:
+        verified = "partly"
+    return {
+        "at_max": c["at_max"],
+        "total": c["total"],
+        "at_max_rate": round(c["at_max"] / c["total"], 4),
+        "citations_verified": verified,
+    }
+
+
+def _fmt(v: Any) -> str:
+    if v is None:
+        return ""
+    return f"{v:.3f}" if isinstance(v, float) else str(v)
+
+
+def _na(v: Any) -> str:
+    return "n/a (no groundedness)" if v is None else str(v)
+
+
+def markdown_summary(summary: dict[str, Any]) -> str:
+    lines = [
+        f"## {summary['job']}",
+        "",
+        f"Trials: {summary['trials']} (graded {summary['graded']}, failed {summary['failed']}) · "
+        f"{summary['reward_label']} rate **{summary['pass_rate']}** · "
+        f"pass_clean {_na(summary['pass_clean_rate'])} · "
+        f"mean score {_na(summary['mean_score'])} "
+        f"(q {summary['mean_q']}, p {_na(summary['mean_p'])}) · "
+        f"citation integrity {_na(summary['mean_integrity'])} · "
+        f"ungradeable {summary['ungradeable_criteria']} · "
+        f"cost agent ${summary['agent_cost_usd']} + judge ${summary['judge_cost_usd']} · "
+        f"tool failures {summary['tool_failures']}",
+        "",
+        "Grading version: " + ("; ".join(summary.get("grading_versions") or []) or "none"),
+        "",
+        f"| task | {summary['reward_label']} | clean | score | q | p | integrity "
+        "| agent $ | judge $ | note |",
+        "|---|:---:|:---:|---:|---:|---:|---:|---:|---:|---|",
+    ]
+
+    def mark(v: Any) -> str:
+        return "" if v is None else ("✓" if v else "✗")
+
+    for t in summary["tasks"]:
+        note = t["exception"] or (f"{t['ungradeable']} ungradeable" if t["ungradeable"] else "")
+        if t.get("groundedness") not in ("on", "none", None) and not t["exception"]:
+            note = f"no groundedness ({t['groundedness']})" + (f"; {note}" if note else "")
+        lines.append(
+            f"| {t['task']} | {mark(t['reward'])} | {mark(t['pass_clean'])} | "
+            f"{_fmt(t['q_final'])} | {_fmt(t['q'])} | {_fmt(t['p'])} | {_fmt(t['integrity'])} | "
+            f"{t['agent_cost_usd']} | {t['judge_cost_usd']} | {note} |"
+        )
+    pen = summary.get("penalty") or {}
+    if pen:
+        lines += [
+            "",
+            f"Citation penalty: {pen['events']} scored events, total severity "
+            f"{pen['severity']}; recorded only: {pen['misuse']} misuse, "
+            f"{pen['not_covered']} outside register coverage, "
+            f"{pen['rubric_named']} named by the rubric.",
+        ]
+    if cl := summary.get("case_law"):
+        note = {
+            "yes": "citations verified by the groundedness service",
+            "no": "citations NOT verified: judged by the LLM judge alone",
+            "partly": "citations verified in some trials only",
+        }[cl["citations_verified"]]
+        lines += [
+            "",
+            f"Case-law criteria: {cl['at_max']}/{cl['total']} at their best value "
+            f"({cl['at_max_rate']}); {note}.",
+        ]
+    if summary["categories"]:
+        lines += [
+            "",
+            "| category | +1 | 0 | -1 | total | +1 rate |",
+            "|---|---:|---:|---:|---:|---:|",
+        ]
+        for k, v in summary["categories"].items():
+            lines.append(
+                f"| {k} | {v['+1']} | {v['0']} | {v['-1']} | {v['total']} | {v['at_max_rate']} |"
+            )
+    return "\n".join(lines)
+
+
+# ── pbench manifest ─────────────────────────────────────────────────────
+
+
+def dataset_pin(tasks_dir: Path) -> dict[str, Any]:
+    """Name, version and checksum of the dataset, from Harbor's ``dataset.toml`` next to tasks/.
+
+    The checksum is SHA-256 over the sorted ``name@digest`` lines of the tasks, where each digest
+    is Harbor's content hash of the task directory (``harbor add`` keeps them current).
+    """
+    import hashlib
+    import tomllib
+
+    toml_path = tasks_dir.parent / "dataset.toml"
+    if not toml_path.exists():
+        return {
+            "path": str(tasks_dir),
+            "task_count": sum(1 for p in tasks_dir.iterdir() if p.is_dir()),
+        }
+    data = tomllib.loads(toml_path.read_text(encoding="utf-8"))
+    tasks = sorted(f"{t['name']}@{t['digest']}" for t in data.get("tasks", []))
+    return {
+        "name": data.get("dataset", {}).get("name"),
+        "dataset_version": data.get("dataset", {}).get("version"),
+        "checksum": hashlib.sha256("\n".join(tasks).encode()).hexdigest(),
+        "task_count": len(tasks),
+    }
+
+
+def benchmark_manifest(
+    tasks_dir: Path | None, grading_version: str | None = None
+) -> dict[str, Any]:
+    """Everything that pins a benchmark result: code, dataset and grading version.
+
+    Harbor and the agent package are reported when installed (they are not needed to grade).
+    """
+    from importlib.metadata import PackageNotFoundError, version
+
+    from . import __version__
+
+    def installed(dist: str) -> str | None:
+        try:
+            return version(dist)
+        except PackageNotFoundError:
+            return None
+
+    dataset: dict[str, Any] | None = None
+    if tasks_dir is not None:
+        dataset = dataset_pin(tasks_dir)
+    grading = versions.get(grading_version) if grading_version else versions.current()
+    return {
+        "benchmark": "pergamon-bench",
+        "pbench_grader_version": __version__,
+        "pergamon_bench_version": installed("pergamon-bench"),
+        "harbor_version": installed("harbor"),
+        "dataset": dataset,
+        "grading": {**grading.as_dict(), "scoring": "weighted_legal"},
+    }
+
+
+# ── entry point ──────────────────────────────────────────────────────
+
+
+COMMANDS = ("grade", "manifest", "summary")
+
+
+def add_commands(sub: Any) -> None:
+    """Add grade / manifest / summary to an argparse subparsers object."""
+
+    g = sub.add_parser(
+        "grade", help="grade answers from a JSONL file against task rubrics (no Harbor)"
+    )
+    g.add_argument(
+        "--tasks", required=True, help="directory of Harbor task dirs (the dataset's tasks/)"
+    )
+    g.add_argument("--answers", required=True, help="JSONL with {task, answer|answer_path} rows")
+    g.add_argument("-o", "--out", default="graded", help="output directory")
+    g.add_argument("--judge-models", default=DEFAULT_JUDGE, help="comma-separated LiteLLM models")
+    g.add_argument("--judge-reasoning-effort", default=DEFAULT_JUDGE_REASONING_EFFORT)
+    g.add_argument("--judge-temperature", type=float, default=DEFAULT_JUDGE_TEMPERATURE)
+    g.add_argument(
+        "--adjudicator-model", default=DEFAULT_ADJUDICATOR_MODEL, help="`none` for no adjudicator"
+    )
+    g.add_argument("--concurrency", type=int, default=8)
+    g.add_argument(
+        "--cache-dir", default=None, help="judge cache directory (or PBENCH_JUDGE_CACHE_DIR)"
+    )
+
+    m = sub.add_parser(
+        "manifest", help="print the benchmark manifest (code, dataset, grading version)"
+    )
+    m.add_argument("--tasks", default=None, help="dataset tasks/ directory (reads ../dataset.toml)")
+    m.add_argument(
+        "--grading-version",
+        default=None,
+        choices=sorted(versions.load_versions()[1]),
+        help="a past grading version (default: the current one)",
+    )
+
+    s = sub.add_parser("summary", help="summarise a Harbor job directory graded by Pergamon Bench")
+    s.add_argument("job_dir")
+    s.add_argument("--markdown", action="store_true", help="print a markdown table instead of JSON")
+    s.add_argument("-o", "--out", default=None, help="also write the JSON summary to this file")
+
+
+def run(args: argparse.Namespace) -> int:
+    """Execute a parsed grade / manifest / summary command."""
+    if args.command == "grade":
+        return asyncio.run(_grade_all(args))
+    if args.command == "manifest":
+        tasks = Path(args.tasks) if args.tasks else None
+        print(
+            json.dumps(
+                benchmark_manifest(tasks, args.grading_version), indent=1, ensure_ascii=False
+            )
+        )
+        return 0
+    if args.command == "summary":
+        summary = summarise_job(Path(args.job_dir))
+        if args.out:
+            Path(args.out).write_text(json.dumps(summary, indent=1), encoding="utf-8")
+        print(markdown_summary(summary) if args.markdown else json.dumps(summary, indent=1))
+        return 0
+    return 2
+
+
+def main(argv: list[str] | None = None) -> int:
+    """Standalone use: ``python -m pbench_grader.cli <command> ...``."""
+    parser = argparse.ArgumentParser(prog="python -m pbench_grader.cli", description=__doc__)
+    add_commands(parser.add_subparsers(dest="command", required=True))
+    return run(parser.parse_args(argv))
+
+
+if __name__ == "__main__":
+    sys.exit(main())

pbench_grader/criteria/pass.py

@@ -0,0 +1,6 @@
+# rewardkit criteria file. The trial's single reward (reward.json `reward`): 1 when every
+# critical criterion is at its best value, else 0. The answer is judged once per trial;
+# see pbench_grader/rk.py. The graded score and the citation penalty are in grading.json.
+from pbench_grader import rk
+
+rk.register_pass(rk.tests_dir())

pbench_grader/criteria/reward.toml

@@ -0,0 +1,8 @@
+# How the criteria files in this directory combine into reward.json (rewardkit).
+# One output only: Harbor computes pass@k when every trial's reward.json has exactly one
+# key whose value is 0 or 1. pass.py is that key; rubric.py contributes nothing to it and
+# exists so reward-details.json lists every criterion.
+[[reward]]
+name = "reward"
+aggregation = "weighted-mean"
+weights = { pass = 1.0, rubric = 0.0 }

pbench_grader/criteria/rubric.py

@@ -0,0 +1,6 @@
+# rewardkit criteria file. One check per rubric criterion (at its best value), weighted by
+# importance. Weight 0 in reward.toml: it is here for reward-details.json, not the reward.
+# The answer is judged once per trial; see pbench_grader/rk.py.
+from pbench_grader import rk
+
+rk.register_rubric(rk.tests_dir())