verifiers 0.1.15.dev426__tar.gz → 0.1.15.dev428__tar.gz

{verifiers-0.1.15.dev426 → verifiers-0.1.15.dev428}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: verifiers
-Version: 0.1.15.dev426
+Version: 0.1.15.dev428
 Summary: Verifiers: Environments for LLM Reinforcement Learning
 Project-URL: Homepage, https://github.com/primeintellect-ai/verifiers
 Project-URL: Documentation, https://github.com/primeintellect-ai/verifiers

{verifiers-0.1.15.dev426 → verifiers-0.1.15.dev428}/tests/v1/test_judges.py

@@ -29,9 +29,9 @@ class QATask(vf.Task):
     answer: str = ""
 
 
-def make_trace(reply: str = "It is Paris.") -> vf.Trace:
+def make_trace(reply: str = "It is Paris.", answer: str = "Paris") -> vf.Trace:
     return vf.Trace(
-        task=QATask(idx=0, prompt="Capital of France?", answer="Paris"),
+        task=QATask(idx=0, prompt="Capital of France?", answer=answer),
         nodes=[
             MessageNode(
                 parent=None,
@@ -47,23 +47,31 @@ def make_trace(reply: str = "It is Paris.") -> vf.Trace:
 
 @pytest.fixture
 def fake_judge_model(monkeypatch):
-    """Fake the judge's model call, recording each prompt for assertions. Text calls reply
-    "yes" iff "Paris" appears in the response block of the rendered prompt; structured
-    (`schema`) calls verdict each `- name: text` criteria line "yes" iff it mentions Paris."""
+    """Fake the judge's model call, recording each prompt for assertions. Rubric calls (a JSON
+    `verdicts` instruction or a `schema`) reply one reasoned verdict per `- name: text` criterion,
+    "yes" iff it mentions Paris; other judges reply plain "yes"/"no" by the response block."""
     prompts: list[str] = []
 
     async def fake_complete(
         self, messages, *, trace=None, schema=None, parse=None, **sampling
     ):
         prompts.append(messages)
-        if schema is not None:
+        # Rubric calls carry criteria lines + a JSON `verdicts` instruction; reply one verdict per
+        # criterion (yes iff its text mentions Paris) with a reason. Other judges get plain yes/no.
+        if schema is not None or '"verdicts"' in messages:
             verdicts = [
-                {"name": name, "verdict": "yes" if "Paris" in text else "no"}
+                {
+                    "name": name,
+                    "reason": "cites Paris" if "Paris" in text else "no Paris",
+                    "verdict": "yes" if "Paris" in text else "no",
+                }
                 for name, text in re.findall(r"^- ([^:]+): (.+)$", messages, re.M)
             ]
             response = JudgeResponse(
-                text=json.dumps(verdicts),
-                parsed=schema.model_validate({"verdicts": verdicts}),
+                text=json.dumps({"verdicts": verdicts}),
+                parsed=schema.model_validate({"verdicts": verdicts})
+                if schema
+                else None,
             )
         else:
             response = JudgeResponse(
@@ -501,7 +509,7 @@ def test_rubric_rejects_bad_files(tmp_path):
 
 async def test_rubric_score(tmp_path, fake_judge_model):
     # verdicts: mentions_paris=1 (w=3), is_polite=0 (w=1) -> weighted mean 0.75, from ONE
-    # structured judge call; each verdict lands as a `<name>/<criterion>` metric.
+    # judge call; each verdict lands as a `<name>/<criterion>` metric.
     judge = rubric_judge(tmp_path)
     trace = make_trace()
     assert await judge.score(trace.task, trace) == 0.75
@@ -513,16 +521,73 @@ async def test_rubric_verdict_mismatch_raises(tmp_path, monkeypatch):
     # A reply that doesn't verdict exactly the rubric's criteria is a judge failure: raise
     # (-> rollout error), don't guess or silently score 0.
     async def wrong_names(self, messages, *, trace=None, schema=None, parse=None, **s):
-        verdicts = {"verdicts": [{"name": "typo", "verdict": "yes"}]}
-        return JudgeResponse(text="", parsed=schema.model_validate(verdicts))
+        verdicts = {"verdicts": [{"name": "typo", "reason": "x", "verdict": "yes"}]}
+        return JudgeResponse(text=json.dumps(verdicts), parsed=None)
 
     monkeypatch.setattr(Judge, "complete", wrong_names)
     judge = rubric_judge(tmp_path)
     trace = make_trace()
-    with pytest.raises(ValueError, match="expected the rubric"):
+    with pytest.raises(ValueError, match="expected the batch"):
         await judge.score(trace.task, trace)
 
 
+CHOICES_TOML = (
+    '[[criteria]]\nname = "depth"\ntext = "How thorough?"\n'
+    'choices = ["none", "partial", "good"]\n'
+)
+
+
+async def test_rubric_choices_normalize(tmp_path, monkeypatch):
+    # Ordered choices (worst→best) score by rank: "partial" of ["none","partial","good"] -> 0.5.
+    async def graded(self, messages, *, trace=None, schema=None, parse=None, **s):
+        v = {"verdicts": [{"name": "depth", "reason": "r", "verdict": "partial"}]}
+        return JudgeResponse(text=json.dumps(v), parsed=None)
+
+    monkeypatch.setattr(Judge, "complete", graded)
+    judge = rubric_judge(tmp_path, body=CHOICES_TOML, name="q")
+    trace = make_trace()
+    assert await judge.score(trace.task, trace) == 0.5
+    assert trace.metrics == {"q/depth": 0.5}
+
+
+async def test_rubric_off_menu_answer_raises(tmp_path, monkeypatch):
+    # A verdict that isn't one of the criterion's choices is a judge failure, not a 0.
+    async def off_menu(self, messages, *, trace=None, schema=None, parse=None, **s):
+        v = {"verdicts": [{"name": "depth", "reason": "r", "verdict": "maybe"}]}
+        return JudgeResponse(text=json.dumps(v), parsed=None)
+
+    monkeypatch.setattr(Judge, "complete", off_menu)
+    judge = rubric_judge(tmp_path, body=CHOICES_TOML)
+    trace = make_trace()
+    with pytest.raises(ValueError, match="expected one of"):
+        await judge.score(trace.task, trace)
+
+
+def test_rubric_choices_validation(tmp_path):
+    with pytest.raises(ValueError, match="at least two"):
+        rubric_judge(
+            tmp_path, body='[[criteria]]\nname = "x"\ntext = "t"\nchoices = ["only"]\n'
+        ).criteria
+    with pytest.raises(ValueError, match="duplicate options"):
+        rubric_judge(
+            tmp_path,
+            body='[[criteria]]\nname = "x"\ntext = "t"\nchoices = ["a", "a"]\n',
+        ).criteria
+
+
+async def test_rubric_reference_answer_optional(tmp_path, fake_judge_model):
+    # off by default: no reference block in the prompt.
+    t = make_trace()
+    await rubric_judge(tmp_path).score(t.task, t)
+    assert "Reference solution" not in fake_judge_model[-1]
+
+    # answer_field set: the task's gold answer is shown to the judge.
+    t = make_trace(answer="ZEBRA-GOLD")
+    await rubric_judge(tmp_path, answer_field="answer").score(t.task, t)
+    assert "Reference solution" in fake_judge_model[-1]
+    assert "ZEBRA-GOLD" in fake_judge_model[-1]
+
+
 # --- Taskset.score integration -------------------------------------------------------------
 
 

{verifiers-0.1.15.dev426 → verifiers-0.1.15.dev428}/verifiers/v1/cli/dashboard/eval.py

@@ -209,15 +209,24 @@ def Overview(config: EvalConfig) -> Table:
 
 
 def Progress(
-    rollouts: list[Rollout], start: float, page: tuple[int, int] | None = None
+    rollouts: list[Rollout],
+    start: float,
+    page: tuple[int, int] | None = None,
+    finished: list[Trace] | None = None,
 ) -> Group:
-    done = [r.trace for r in rollouts if r.phase == Phase.DONE]  # fully scored
+    # On resume, `finished` holds the kept on-disk rollouts (reloaded as finished traces); count
+    # them alongside this session's so progress, reward, err, and the breakdown cover the whole
+    # run. `rollouts` is only this session's (owed) work, so the total adds the kept ones back.
+    done = (finished or []) + [
+        r.trace for r in rollouts if r.phase == Phase.DONE
+    ]  # fully scored
+    total = len(finished or []) + len(rollouts)
     # Headline reward = mean over non-errored; when any errored, `format_mean` appends the
     # global avg (errored count as 0) in parens. `err` is the share that errored.
     reward = format_mean(done, lambda t: t.reward)
     err = f"{sum(t.has_error for t in done) / len(done):.2f}" if done else "—"
     stats = (
-        f"{len(done)}/{len(rollouts)} · {format_time(time.time() - start)} · "
+        f"{len(done)}/{total} · {format_time(time.time() - start)} · "
         f"reward {reward} · err {err}"
     )
     if page is not None:  # overflowing — show which page, and that the arrows page
@@ -226,7 +235,7 @@ def Progress(
     row.add_column(ratio=1)  # bar stretches to fill the width left of the stats
     row.add_column(justify="right", no_wrap=True)
     row.add_row(
-        ProgressBar(total=len(rollouts) or 1, completed=len(done)),
+        ProgressBar(total=total or 1, completed=len(done)),
         Text(stats),
     )
     breakdown = _breakdown(done)
@@ -561,7 +570,11 @@ def _paginate(
 
 
 def _render(
-    rollouts: list[Rollout], config: EvalConfig, start: float, pager: Pager
+    rollouts: list[Rollout],
+    config: EvalConfig,
+    start: float,
+    pager: Pager,
+    finished: list[Trace] | None = None,
 ) -> Group:
     now = time.time()
     warning = _warning(config)
@@ -570,10 +583,15 @@ def _render(
     # Measure the fixed top (header + progress + rule) so the rollout rows fill the rest of the
     # screen; page through them (timer, or the left/right arrows) when they'd overflow (rich would
     # otherwise truncate).
-    top = Group(header, Progress(rollouts, start), Rule(style="dim"))
+    top = Group(header, Progress(rollouts, start, finished=finished), Rule(style="dim"))
     rows_per_page = max(1, _CONSOLE.size.height - len(_CONSOLE.render_lines(top)) - 1)
     page_groups, index, count = _paginate(_groups(rollouts), rows_per_page, pager, now)
-    progress = Progress(rollouts, start, page=(index + 1, count) if count > 1 else None)
+    progress = Progress(
+        rollouts,
+        start,
+        page=(index + 1, count) if count > 1 else None,
+        finished=finished,
+    )
     return Group(
         header,
         progress,
@@ -583,11 +601,18 @@ def _render(
 
 
 @contextlib.asynccontextmanager
-async def dashboard(rollouts: list[Rollout], config: EvalConfig, start: float):
+async def dashboard(
+    rollouts: list[Rollout],
+    config: EvalConfig,
+    start: float,
+    finished: list[Trace] | None = None,
+):
     """Refresh the live eval view until the `with` block exits, then a final frame. Left/right
-    arrows page through rollout rows when they overflow the screen."""
+    arrows page through rollout rows when they overflow the screen. On resume, `finished` carries
+    the kept on-disk rollouts (reloaded as finished traces) so the counts and scores cover the
+    whole run, not just this session's re-run rollouts."""
     pager = Pager()
     async with live_view(
-        lambda: _render(rollouts, config, start, pager), on_key=pager.on_key
+        lambda: _render(rollouts, config, start, pager, finished), on_key=pager.on_key
     ):
         yield

{verifiers-0.1.15.dev426 → verifiers-0.1.15.dev428}/verifiers/v1/cli/eval/resume.py

@@ -16,7 +16,12 @@ from pathlib import Path
 
 from pydantic_core import from_json
 
+from verifiers.v1.cli.output import read_traces
 from verifiers.v1.configs.eval import EvalConfig
+from verifiers.v1.state import state_cls
+from verifiers.v1.task import WireTask
+from verifiers.v1.taskset import Taskset
+from verifiers.v1.trace import Trace
 
 
 def split_resume(argv: list[str]) -> tuple[Path | None, list[str]]:
@@ -116,6 +121,14 @@ def rewrite_results(resume_dir: Path, keep: list[int]) -> None:
     tmp.replace(path)
 
 
+def load_kept(resume_dir: Path, taskset: Taskset) -> list[Trace]:
+    """Reload the kept (good) traces as finished `Trace`s, so a resumed run's live dashboard counts
+    them toward the whole run (progress, reward, err, and the usage/time breakdown). Call *after*
+    `rewrite_results`, which leaves only the kept rows on disk. `WireTask` reads any taskset's saved
+    task without a runtime or its `Task` type (mirrors `replay`)."""
+    return read_traces(resume_dir, Trace[WireTask, state_cls(type(taskset))])
+
+
 def nothing_to_resume_msg(resume_dir: Path, num_tasks: int, num_rollouts: int) -> str:
     """The message shown (and then exit 0 - the run is already complete) when every selected
     rollout already completed without error."""

{verifiers-0.1.15.dev426 → verifiers-0.1.15.dev428}/verifiers/v1/cli/eval/runner.py

@@ -41,6 +41,10 @@ async def run_eval(env: Environment, config: EvalConfig) -> list[Trace]:
     # run only the owed rollouts. One lock serializes worker-thread appends from concurrent
     # rollouts while keeping large trace serialization off the event loop.
     owed: dict[str, int] | None = None
+    # On resume, the kept (good) on-disk rollouts, reloaded as finished traces so the live
+    # dashboard counts the whole run (progress, reward, err, usage/time) rather than only this
+    # session's re-run rollouts. Only the --rich dashboard reads them, so skip the load otherwise.
+    finished: list[Trace] = []
     if config.resume is not None:
         group = bool(discover_decorated(env.taskset, "group_reward"))
         keep, owed = resume.plan(
@@ -51,6 +55,8 @@ async def run_eval(env: Environment, config: EvalConfig) -> list[Trace]:
             raise SystemExit(0)
         tasks = [task for task in tasks if owed.get(task.idx)]
         resume.rewrite_results(out, keep)
+        if config.rich:
+            finished = resume.load_kept(out, env.taskset)
         logger.info(
             "resuming %s: %d task(s), %d rollout(s) owed",
             out,
@@ -84,7 +90,7 @@ async def run_eval(env: Environment, config: EvalConfig) -> list[Trace]:
         ]
         rollouts = [rollout for episode in episodes for rollout in episode.rollouts]
         display = (
-            dashboard(rollouts, config, start)
+            dashboard(rollouts, config, start, finished=finished)
             if config.rich
             else contextlib.nullcontext()
         )

verifiers-0.1.15.dev428/verifiers/v1/judges/rubric.py

@@ -0,0 +1,330 @@
+"""rubric — an off-the-shelf rubric judge, plugged from config alone.
+
+Reads a rubric — a list of criteria — from a JSON or TOML file and asks the judge model to grade
+each by picking one of its ordered `choices` (default `["no", "yes"]`, a binary check), scored to
+[0, 1] by rank — worst → best, so the first choice is 0.0 and the last 1.0 (all in one call by
+default, or batched `max_criteria`-at-a-time). Grading uses plain-text JSON by default, or
+structured output when `structured_output=true` (see `RubricJudgeConfig`). The reward is the
+weighted mean of the per-criterion scores (`sum(w*v) / sum(w)`, so it stays in [0, 1]); each score
+is also recorded as a `<name>/<criterion>` metric. Criterion weights come from the rubric file and
+are overridable from config (`weights`); the aggregate weighs into `trace.reward` via the judge's
+`weight`:
+
+    [[env.taskset.judges]]
+    id = "rubric"
+    path = "rubrics/quality.toml"
+    weight = 0.5
+    weights = { cites_sources = 2.0 }   # override the file's per-criterion weight
+
+with `rubrics/quality.toml` listing the criteria (JSON takes `{"criteria": [...]}` or a bare list):
+
+    [[criteria]]
+    name = "cites_sources"
+    text = "The response cites at least one specific source."
+    weight = 1.0
+    # choices default to ["no", "yes"]; give an ordered worst→best list for a graded criterion:
+    # choices = ["none", "partial", "thorough"]   # -> 0.0, 0.5, 1.0
+"""
+
+import asyncio
+import json
+import math
+import re
+import tomllib
+from functools import cached_property
+from pathlib import Path
+from typing import cast
+
+from pydantic import field_validator
+
+from verifiers.v1.judge import (
+    Judge,
+    JudgeConfig,
+    JudgeView,
+    judge_question,
+    judge_response,
+)
+from verifiers.v1.task import Task
+from verifiers.v1.trace import Trace
+from verifiers.v1.types import ID, StrictBaseModel
+
+# A sibling text file so it doubles as a starting point for a config `prompt_file`.
+RUBRIC_PROMPT = (Path(__file__).resolve().parent / "rubric.txt").read_text(
+    encoding="utf-8"
+)
+
+# Appended to the prompt when `structured_output` is off, so the reply is JSON we can parse. Each
+# criterion carries a one-sentence `reason` written *before* the verdict (chain-of-thought), so the
+# verdict follows from it and the reasoning is auditable in `trace.info["judge"]`.
+JSON_SUFFIX = (
+    "\n\nRespond with ONLY a JSON object and nothing else, in exactly this shape:\n"
+    '{"verdicts": [{"name": "<criterion name>", "reason": "<one sentence citing specific '
+    'evidence from the response>", "verdict": "<answer>"}, ...]}\n'
+    "with one entry per criterion, using each criterion's exact name. For each, first write the "
+    "one-sentence reason grounded in the response, then set verdict to exactly one of the options "
+    "listed in parentheses after that criterion."
+)
+
+
+def normalize_choice(choice: str, choices: list[str]) -> float:
+    """Score a chosen option to [0, 1] by rank. `choices` are ordered worst → best, so the first
+    scores 0.0, the last 1.0, and the rest evenly spaced (`choices` always has >= 2 entries)."""
+    return choices.index(choice) / (len(choices) - 1)
+
+
+def first_verdicts_object(text: str) -> dict | None:
+    """The first balanced JSON object in `text` that carries a `verdicts` key. Scanning for the
+    key (not just the first `{`) skips prose and, crucially, an echoed format example — the one
+    in `JSON_SUFFIX` fails to parse (its trailing `...` isn't JSON) and is passed over rather
+    than mistaken for the answer. Returns `None` if no such object is found."""
+    decoder = json.JSONDecoder()
+    idx = text.find("{")
+    while idx != -1:
+        try:
+            obj, _ = decoder.raw_decode(text[idx:])
+        except json.JSONDecodeError:
+            obj = None
+        if isinstance(obj, dict) and "verdicts" in obj:
+            return obj
+        idx = text.find("{", idx + 1)
+    return None
+
+
+class Criterion(StrictBaseModel):
+    """One rubric entry the judge grades by picking one of `choices`, scored to [0, 1] by rank.
+    Defaults to a binary yes/no check (1/0)."""
+
+    name: str
+    """Key for the criterion's metric (`<judge name>/<name>`) and its `weights` override."""
+    text: str
+    """The check itself, phrased so the first (best) choice means satisfied."""
+    weight: float = 1.0
+    """The criterion's share of the reward (overridable per name via `weights` in config)."""
+    choices: list[str] = ["no", "yes"]
+    """Allowed answers, ordered **worst → best**: the first scores 0.0, the last 1.0, the rest
+    evenly spaced by rank. Default `["no", "yes"]` is a binary check. Needs >= 2, no duplicates."""
+
+    @field_validator("choices")
+    @classmethod
+    def _check_choices(cls, v: list[str]) -> list[str]:
+        if len(v) < 2:
+            raise ValueError(f"`choices` needs at least two options, got {v}")
+        if len(set(v)) != len(v):
+            raise ValueError(f"`choices` has duplicate options: {v}")
+        return v
+
+
+class RubricJudgeConfig(JudgeConfig):
+    id: ID = "rubric"
+    """Pinned to the built-in, so a code-level default entry needs no explicit id."""
+    path: Path
+    """The rubric file (`.toml` or `.json`) listing the criteria — see the module docstring
+    for the shape. A relative path resolves against the eval's working directory."""
+    weights: dict[str, float] = {}
+    """Per-criterion weight overrides by criterion name (config wins over the file)."""
+    question_field: str = ""
+    """Task field to fill the prompt's `{question}`; empty = the task's prompt rendered as
+    text (`Task.prompt_text`)."""
+    answer_field: str = ""
+    """Optional task field holding a reference answer (e.g. a gold patch) to show the judge, like
+    the reference judge. Empty (default) shows none — `{reference}` renders blank. A list-valued
+    field is joined one item per line. Needs `{reference}` in the prompt (the built-in `rubric.txt`
+    has it)."""
+    view: JudgeView = "full_trace"
+    """How much of the rollout fills `{response}` (see `JudgeView`). Defaults to the whole
+    transcript — rubric criteria typically grade the process (tool use, citations,
+    intermediate steps), not just the final answer."""
+    max_criteria: int | None = None
+    """How many criteria to grade per judge call. `None` (default) grades all criteria in one
+    call. `1` sends one call per criterion (n independent judges); `k` batches them k-at-a-time.
+    Batches are graded concurrently and merged. Smaller batches trade more calls for focus/
+    robustness. Must be >= 1."""
+    structured_output: bool = False
+    """Grade via JSON-schema structured output (`response_format`) when `True`; grade via plain-text
+    output parsed as JSON when `False` (the default). Plain text is far more reliable on endpoints
+    whose structured decoding is flaky (e.g. GLM-5.2 and other non-OpenAI models on some providers
+    return empty completions for structured calls, especially over long transcripts); OpenAI models
+    handle either. Transient HTTP failures are already retried by the OpenAI client."""
+
+
+class CriterionVerdict(StrictBaseModel):
+    """One criterion's reply, matched back to the rubric by `name`: a short `reason`
+    (chain-of-thought, always recorded for auditability in `trace.info["judge"]`) written *before*
+    the `verdict`, so the verdict follows from it. `verdict` is one of the criterion's `choices`
+    (validated per criterion in `grade_batch`, since choices vary by criterion)."""
+
+    name: str
+    reason: str
+    verdict: str
+
+
+class RubricVerdicts(StrictBaseModel):
+    """The judge's reply: one reasoned verdict per rubric criterion."""
+
+    verdicts: list[CriterionVerdict]
+
+
+class RubricJudge(Judge[RubricVerdicts, RubricJudgeConfig]):
+    """Scores every rubric criterion 1/0 (in one judge call, or batched `max_criteria` at a
+    time) and rewards their weighted mean."""
+
+    prompt = RUBRIC_PROMPT
+    schema = RubricVerdicts
+
+    @cached_property
+    def criteria(self) -> list[Criterion]:
+        """The rubric, parsed once from `config.path` (TOML by suffix, else JSON) with the
+        config's `weights` overrides applied. Fails loudly on a missing/empty file, duplicate
+        criterion names, or an override naming no criterion."""
+        path = self.config.path
+        text = path.read_text(encoding="utf-8")
+        data = (
+            tomllib.loads(text) if path.suffix.lower() == ".toml" else json.loads(text)
+        )
+        items = data.get("criteria", []) if isinstance(data, dict) else data
+        criteria = [Criterion.model_validate(item) for item in items]
+        if not criteria:
+            raise ValueError(f"rubric file '{path}' lists no criteria")
+        names = [criterion.name for criterion in criteria]
+        if len(set(names)) != len(names):
+            raise ValueError(f"rubric file '{path}' has duplicate criterion names")
+        if unknown := set(self.config.weights) - set(names):
+            raise ValueError(
+                f"`weights` overrides name no criterion in '{path}': {sorted(unknown)}"
+            )
+        criteria = [
+            criterion.model_copy(
+                update={
+                    "weight": self.config.weights.get(criterion.name, criterion.weight)
+                }
+            )
+            for criterion in criteria
+        ]
+        if bad := [c.name for c in criteria if not 0 <= c.weight < math.inf]:
+            # A negative weight would invert a criterion (pushing the reward out of [0, 1]);
+            # NaN/inf (which json.loads accepts) would corrupt the weighted mean.
+            raise ValueError(
+                f"rubric '{path}' has negative or non-finite criterion weights: {bad}"
+            )
+        if sum(criterion.weight for criterion in criteria) <= 0:
+            raise ValueError(f"rubric '{path}' has no positive criterion weight")
+        return criteria
+
+    async def grade_batch(
+        self, task: Task, trace: Trace, batch: list[Criterion]
+    ) -> dict[str, float]:
+        """Grade one batch of criteria in a single judge call — structured output, or plain-text
+        JSON when `structured_output` is off (which avoids flaky structured decoding). Returns
+        `{name: score}`, each the chosen option normalized to [0, 1] by rank (best → worst),
+        matched back to the batch by name. `score` fans this out per batch."""
+
+        def render(
+            c: Criterion,
+        ) -> str:  # every criterion states its own allowed answers inline
+            return f"- {c.name}: {c.text} (answer one of, worst to best: {', '.join(c.choices)})"
+
+        reference = ""  # optional gold answer shown to the judge; blank unless `answer_field` set
+        if self.config.answer_field:
+            answer = getattr(task, self.config.answer_field, None)
+            if answer is None:
+                raise ValueError(
+                    f"rubric judge found no {self.config.answer_field!r} field on the task; "
+                    "point `answer_field` at the task's reference-answer field or leave it empty"
+                )
+            if isinstance(answer, (list, tuple)):
+                answer = "\n".join(str(item) for item in answer)
+            # A clearly-announced section (leading blank separates it from the Task block; the
+            # template's newline before Response closes it). Absent when `answer_field` is unset.
+            # Fence longer than any backtick run in the answer, so a patch/markdown containing ```
+            # can't close it early and spill into the prompt as instructions.
+            fence = "`" * (
+                max((len(m) for m in re.findall(r"`+", answer)), default=2) + 1
+            )
+            reference = (
+                "\nReference solution (a correct implementation of this task, for comparison):\n"
+                f"{fence}\n{answer}\n{fence}\n"
+            )
+
+        fields = dict(
+            question=judge_question(task, self.config.question_field),
+            response=judge_response(trace, self.config.view),
+            criteria="\n".join(render(c) for c in batch),
+            reference=reference,
+        )
+        if self.config.structured_output:
+            result = await self.evaluate(trace=trace, **fields)
+            verdicts = cast(RubricVerdicts, result.parsed).verdicts
+        else:
+            messages = cast(str, self.build_messages(**fields)) + JSON_SUFFIX
+            result = await self.complete(
+                messages, trace=trace
+            )  # no schema -> plain text
+            obj = first_verdicts_object(result.text)
+            if obj is None:
+                raise ValueError(
+                    f"judge returned no verdicts JSON object: {result.text!r}"
+                )
+            verdicts = RubricVerdicts.model_validate(obj).verdicts
+        # Exactly one verdict per criterion in the batch, matched by name — anything else is a
+        # judge failure and must error the rollout, not score the model (see `judge_verdict`).
+        by_criterion = {c.name: c for c in batch}
+        if sorted(v.name for v in verdicts) != sorted(by_criterion):
+            raise ValueError(
+                f"judge verdicts name {sorted(v.name for v in verdicts)}, expected the "
+                f"batch's {sorted(by_criterion)}"
+            )
+        scores: dict[str, float] = {}
+        for v in verdicts:
+            choices = by_criterion[v.name].choices
+            if (
+                v.verdict not in choices
+            ):  # an off-menu answer is a judge failure, not a 0
+                raise ValueError(
+                    f"judge answered {v.verdict!r} for '{v.name}', expected one of {choices}"
+                )
+            scores[v.name] = normalize_choice(v.verdict, choices)
+        return scores
+
+    async def score(self, task: Task, trace: Trace) -> float:
+        criteria = self.criteria
+        # Split into batches of `max_criteria` (None = one batch of all), then grade the batches
+        # concurrently (one judge call each) and merge.
+        k = self.config.max_criteria
+        if k is not None and k < 1:
+            raise ValueError(f"`max_criteria` must be >= 1 or None, got {k}")
+        batches = (
+            [criteria]
+            if k is None
+            else [criteria[i : i + k] for i in range(0, len(criteria), k)]
+        )
+        # Fan out one call per batch; if any fails, cancel the siblings so a judge failure
+        # doesn't keep billing the remaining batches.
+        tasks = [
+            asyncio.ensure_future(self.grade_batch(task, trace, batch))
+            for batch in batches
+        ]
+        try:
+            results = await asyncio.gather(*tasks)
+        except BaseException:
+            for t in tasks:
+                t.cancel()
+            await asyncio.gather(*tasks, return_exceptions=True)
+            raise
+        by_name: dict[str, float] = {}
+        for batch_verdicts in results:
+            by_name.update(batch_verdicts)
+        for criterion in criteria:
+            trace.record_metric(
+                f"{self.reward_name}/{criterion.name}", by_name[criterion.name]
+            )
+        total = sum(criterion.weight for criterion in criteria)
+        return sum(c.weight * by_name[c.name] for c in criteria) / total
+
+
+__all__ = [
+    "Criterion",
+    "CriterionVerdict",
+    "RubricJudge",
+    "RubricJudgeConfig",
+    "RubricVerdicts",
+]

{verifiers-0.1.15.dev426 → verifiers-0.1.15.dev428}/verifiers/v1/judges/rubric.txt

@@ -4,7 +4,7 @@ Task:
 ```
 {question}
 ```
-
+{reference}
 Response:
 ```
 {response}
@@ -13,4 +13,4 @@ Response:
 Criteria:
 {criteria}
 
-For each criterion, give a "yes" or "no" verdict under its exact name.
+For each criterion, answer under its exact name with exactly one of the options given in parentheses after it.