verifiers 0.4.1.dev56__tar.gz → 0.4.1.dev58__tar.gz

{verifiers-0.4.1.dev56 → verifiers-0.4.1.dev58}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: verifiers
-Version: 0.4.1.dev56
+Version: 0.4.1.dev58
 Summary: Verifiers: Environments for LLM Reinforcement Learning
 Project-URL: Homepage, https://github.com/primeintellect-ai/verifiers
 Project-URL: Documentation, https://github.com/primeintellect-ai/verifiers

{verifiers-0.4.1.dev56 → verifiers-0.4.1.dev58}/pyproject.toml

@@ -131,6 +131,8 @@ bash-interception = { path = "environments/bash_interception", editable = true }
 web-search-interception = { path = "environments/web_search_interception", editable = true }
 
 [tool.uv.exclude-newer-package]
+# CA bundle: security fixes (e.g. 2026.10.19 for the eMudhra compromise) must not wait out the cooldown
+certifi = false
 # Bounded cutoffs for the explicitly requested tool upgrades.
 # Full UTC timestamps: bare dates resolve to local-tz midnight and churn the
 # lockfile across timezones.

{verifiers-0.4.1.dev56 → verifiers-0.4.1.dev58}/verifiers/v1/__init__.py

@@ -60,6 +60,7 @@ from verifiers.v1.errors import (
     TaskError,
     ToolsetError,
     TunnelError,
+    UnscoredError,
 )
 from verifiers.v1.grading import grade_submission
 from verifiers.v1.graph import MessageNode
@@ -283,6 +284,7 @@ __all__ = [  # noqa: RUF022 - grouped by public API area
     "ToolsetError",
     "SandboxError",
     "TaskError",
+    "UnscoredError",
     "InterceptionError",
     "TunnelError",
     # clients

{verifiers-0.4.1.dev56 → verifiers-0.4.1.dev58}/verifiers/v1/envs/agentic_judge/env.py

@@ -18,8 +18,10 @@ from verifiers.v1.configs.scoring import resolve_scoring
 from verifiers.v1.judges.rubric import (
     Criterion,
     RubricVerdicts,
+    criterion_metrics,
     load_criteria,
     score_verdicts,
+    weighted_score,
 )
 from verifiers.v1.utils.compile import validate_pairing
 
@@ -45,10 +47,7 @@ SOLVED = Criterion(
 
 
 def _verdict_section(criteria: list[Criterion]) -> str:
-    listing = "\n".join(
-        f"- {c.name}: {c.text} (answer one of, worst to best: {', '.join(c.choices)})"
-        for c in criteria
-    )
+    listing = "\n".join(c.render() for c in criteria)
     return f"""\
 ## Your verdict
 
@@ -300,15 +299,16 @@ class AgenticJudgeEnv(vf.Env[AgenticJudgeEnvConfig]):
         verdicts = RubricVerdicts.model_validate(verdict.info.get("verdict")).verdicts
         criteria = self.config.task.criteria()
         scores = score_verdicts(verdicts, criteria, "the rubric's")
-        for criterion in criteria:
-            solution.record_metric(f"judge/{criterion.name}", scores[criterion.name])
+        solution.record_metrics(criterion_metrics("judge", verdicts, criteria, scores))
         if self.config.score.task_weight != 1.0:
             for reward in solution.rewards.values():
                 if reward is not None:
                     reward.weight *= self.config.score.task_weight
-        total = sum(criterion.weight for criterion in criteria)
-        reward = sum(c.weight * scores[c.name] for c in criteria) / total
-        solution.record_reward("judge", reward, weight=self.config.score.judge_weight)
+        solution.record_reward(
+            "judge",
+            weighted_score(scores, criteria),
+            weight=self.config.score.judge_weight,
+        )
 
     async def run(self, task: vf.Task, agents: vf.Agents) -> None:
         policy = resolve_scoring(task.scoring, self.config.solver.scoring)

{verifiers-0.4.1.dev56 → verifiers-0.4.1.dev58}/verifiers/v1/errors.py

@@ -69,6 +69,11 @@ class TaskError(RolloutError):
     """Task-authored code raised — `setup`, `finalize`, or a `@reward`/`@metric`."""
 
 
+class UnscoredError(RolloutError):
+    """Scoring completed but a reward came out `nan`, e.g. a judge's answer that scores
+    nothing: the rollout carries no reward signal, so consumers drop it like a failure."""
+
+
 class InterceptionError(RolloutError):
     """Communication with the host interception server failed."""
 

{verifiers-0.4.1.dev56 → verifiers-0.4.1.dev58}/verifiers/v1/judges/rubric.py

@@ -2,6 +2,7 @@
 
 import asyncio
 import json
+import math
 import re
 import tomllib
 from functools import cached_property
@@ -35,10 +36,6 @@ JSON_SUFFIX = (
 )
 
 
-def normalize_choice(choice: str, choices: list[str]) -> float:
-    return choices.index(choice) / (len(choices) - 1)
-
-
 def first_verdicts_object(text: str) -> dict | None:
     """The first balanced JSON object in `text` that carries a `verdicts` key. Scanning for the
     key (not just the first `{`) skips prose and, crucially, an echoed format example — the one
@@ -63,17 +60,44 @@ class Criterion(BaseModel):
     text: str
     weight: CriterionWeight = 1.0
     """The criterion's share of the reward (overridable per name via `weights` in config)."""
-    choices: list[str] = Field(default_factory=lambda: ["no", "yes"], min_length=2)
-    """Allowed answers, ordered **worst → best**: the first scores 0.0, the last 1.0, the rest
-    evenly spaced by rank. Default `["no", "yes"]` is a binary check. Needs >= 2, no duplicates."""
+    choices: list[str] | dict[str, float] = Field(
+        default_factory=lambda: ["no", "yes"], min_length=2
+    )
+    """Allowed answers. A list is ordered **worst → best**: the first scores 0.0, the last 1.0,
+    the rest evenly spaced by rank. A mapping gives each answer its own finite score, in any
+    order; `nan` scores nothing, which leaves the rollout unscored when the criterion
+    carries weight. Default `["no", "yes"]` is a binary check. Needs >= 2, no duplicates."""
 
     @field_validator("choices")
     @classmethod
-    def _check_choices(cls, v: list[str]) -> list[str]:
-        if len(set(v)) != len(v):
+    def _check_choices(
+        cls, v: list[str] | dict[str, float]
+    ) -> list[str] | dict[str, float]:
+        if isinstance(v, list) and len(set(v)) != len(v):
             raise ValueError(f"`choices` has duplicate options: {v}")
+        if isinstance(v, dict) and any(math.isinf(score) for score in v.values()):
+            raise ValueError(f"`choices` scores must be finite or nan: {v}")
         return v
 
+    def score(self, choice: str) -> float:
+        """The score of `choice`, one of `choices`; `nan` when it scores nothing."""
+        if isinstance(self.choices, dict):
+            return self.choices[choice]
+        return self.choices.index(choice) / (len(self.choices) - 1)
+
+    def render(self) -> str:
+        """The criterion as a prompt line, with its allowed answers."""
+        order = "" if isinstance(self.choices, dict) else ", worst to best"
+        return f"- {self.name}: {self.text} (answer one of{order}: {', '.join(self.choices)})"
+
+
+def weighted_score(scores: dict[str, float], criteria: list[Criterion]) -> float:
+    """The criteria's weighted mean score; `nan` when a weighted criterion scored nothing.
+    Zero-weight criteria never contribute, not even their `nan`."""
+    weighted = [c for c in criteria if c.weight]
+    total = sum(c.weight for c in weighted)
+    return sum(c.weight * scores[c.name] for c in weighted) / total
+
 
 CRITERIA_ADAPTER = TypeAdapter(list[Criterion])
 
@@ -174,15 +198,38 @@ def score_verdicts(
         )
     scores: dict[str, float] = {}
     for name, answer in answers.items():
-        choices = by_criterion[name].choices
-        if answer not in choices:
+        criterion = by_criterion[name]
+        if answer not in criterion.choices:
             raise ValueError(
-                f"judge answered {answer!r} for '{name}', expected one of {choices}"
+                f"judge answered {answer!r} for '{name}', expected one of "
+                f"{list(criterion.choices)}"
             )
-        scores[name] = normalize_choice(answer, choices)
+        scores[name] = criterion.score(answer)
     return scores
 
 
+def criterion_metrics(
+    prefix: str,
+    verdicts: list[CriterionVerdict],
+    criteria: list[Criterion],
+    scores: dict[str, float],
+) -> dict[str, float]:
+    """Each criterion's score as `<prefix>/<name>`, unless it scored nothing, and for a
+    criterion with scored choices (a mapping) an indicator per choice, `<prefix>/<name>/<choice>`,
+    since several choices can share a score."""
+    answers = {verdict.name: verdict.verdict for verdict in verdicts}
+    metrics: dict[str, float] = {}
+    for criterion in criteria:
+        if not math.isnan(scores[criterion.name]):
+            metrics[f"{prefix}/{criterion.name}"] = scores[criterion.name]
+        if isinstance(criterion.choices, dict):
+            for choice in criterion.choices:
+                metrics[f"{prefix}/{criterion.name}/{choice}"] = float(
+                    answers[criterion.name] == choice
+                )
+    return metrics
+
+
 class RubricJudge(Judge[RubricVerdicts, RubricJudgeConfig]):
     prompt = RUBRIC_PROMPT
     schema = RubricVerdicts
@@ -194,9 +241,6 @@ class RubricJudge(Judge[RubricVerdicts, RubricJudgeConfig]):
     async def grade_batch(
         self, task: TaskData, trace: Trace, batch: list[Criterion]
     ) -> dict[str, float]:
-        def render(c: Criterion) -> str:
-            return f"- {c.name}: {c.text} (answer one of, worst to best: {', '.join(c.choices)})"
-
         reference = ""
         if self.config.answer_field:
             answer = getattr(task, self.config.answer_field, None)
@@ -222,7 +266,7 @@ class RubricJudge(Judge[RubricVerdicts, RubricJudgeConfig]):
         fields = {
             "question": judge_question(task, self.config.question_field),
             "response": judge_response(trace, self.config.view),
-            "criteria": "\n".join(render(c) for c in batch),
+            "criteria": "\n".join(c.render() for c in batch),
             "reference": reference,
         }
         if self.config.structured_output:
@@ -238,7 +282,11 @@ class RubricJudge(Judge[RubricVerdicts, RubricJudgeConfig]):
                 )
             verdicts = RubricVerdicts.model_validate(obj).verdicts
         # A malformed verdict is a judge failure and must error the rollout, not score the model.
-        return score_verdicts(verdicts, batch, "the batch's")
+        scores = score_verdicts(verdicts, batch, "the batch's")
+        trace.record_metrics(
+            criterion_metrics(self.reward_name, verdicts, batch, scores)
+        )
+        return scores
 
     async def score(self, task: TaskData, trace: Trace) -> float:
         criteria = self.criteria
@@ -264,12 +312,7 @@ class RubricJudge(Judge[RubricVerdicts, RubricJudgeConfig]):
         by_name: dict[str, float] = {}
         for batch_verdicts in results:
             by_name.update(batch_verdicts)
-        for criterion in criteria:
-            trace.record_metric(
-                f"{self.reward_name}/{criterion.name}", by_name[criterion.name]
-            )
-        total = sum(criterion.weight for criterion in criteria)
-        return sum(c.weight * by_name[c.name] for c in criteria) / total
+        return weighted_score(by_name, criteria)
 
 
 __all__ = [
@@ -278,4 +321,6 @@ __all__ = [
     "RubricJudge",
     "RubricJudgeConfig",
     "RubricVerdicts",
+    "criterion_metrics",
+    "weighted_score",
 ]

{verifiers-0.4.1.dev56 → verifiers-0.4.1.dev58}/verifiers/v1/rollout.py

@@ -3,6 +3,7 @@
 import asyncio
 import contextlib
 import logging
+import math
 import time
 from collections.abc import Awaitable, Callable
 from contextlib import AsyncExitStack
@@ -19,6 +20,7 @@ from verifiers.v1.errors import (
     SandboxError,
     TaskError,
     ToolsetError,
+    UnscoredError,
     boundary,
 )
 from verifiers.v1.grading import (
@@ -908,6 +910,21 @@ class Rollout:
                         # the metrics read what it handed over and left behind.
                         await self._close_harness()
                         await self._score_harness(runtime, self._metrics_deadline())
+                # A nan reward scores nothing: keep it unscored (None) and fail the
+                # rollout after scoring, so it is never retried as a scoring failure.
+                unscored = [
+                    name
+                    for name, reward in trace.rewards.items()
+                    if reward is not None and math.isnan(reward.score)
+                ]
+                if unscored:
+                    for name in unscored:
+                        trace.rewards[name] = None
+                    self.fail(
+                        UnscoredError(
+                            f"reward {', '.join(unscored)} scored nothing (nan)"
+                        )
+                    )
                 trace.timing.scoring.end = time.time()
                 trace.notify()
         except Exception as e:  # noqa: BLE001 - finalize boundary records every rollout failure