verifiers 0.4.1.dev54__tar.gz → 0.4.1.dev58__tar.gz

{verifiers-0.4.1.dev54 → verifiers-0.4.1.dev58}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: verifiers
-Version: 0.4.1.dev54
+Version: 0.4.1.dev58
 Summary: Verifiers: Environments for LLM Reinforcement Learning
 Project-URL: Homepage, https://github.com/primeintellect-ai/verifiers
 Project-URL: Documentation, https://github.com/primeintellect-ai/verifiers

{verifiers-0.4.1.dev54 → verifiers-0.4.1.dev58}/pyproject.toml

@@ -131,6 +131,8 @@ bash-interception = { path = "environments/bash_interception", editable = true }
 web-search-interception = { path = "environments/web_search_interception", editable = true }
 
 [tool.uv.exclude-newer-package]
+# CA bundle: security fixes (e.g. 2026.10.19 for the eMudhra compromise) must not wait out the cooldown
+certifi = false
 # Bounded cutoffs for the explicitly requested tool upgrades.
 # Full UTC timestamps: bare dates resolve to local-tz midnight and churn the
 # lockfile across timezones.

{verifiers-0.4.1.dev54 → verifiers-0.4.1.dev58}/tests/integration/test_compose_scoring.py

@@ -42,6 +42,9 @@ services:
 
 class ComposeHarness(vf.Harness):
     async def launch(self, ctx, trace, runtime, endpoint, secret, mcp_urls, data):
+        if "build-main" in data.tags:
+            result = await runtime.run(["test", "-f", "/built-from-dockerfile"], {})
+            assert result.exit_code == 0, "workspace did not use its Dockerfile"
         return await runtime.run(
             [
                 "python",
@@ -65,6 +68,19 @@ class ComposeTask(HarborTask):
             assert all(
                 backend._host_backend.stopped for backend in SOLVERS[trace.id]
             ), "solver services survived into grading"
+            if "build-main" in self.data.tags:
+                result = await runtime.run(
+                    [
+                        "sh",
+                        "-c",
+                        "test -f /built-from-dockerfile && test ! -e /app/answer",
+                    ],
+                    {},
+                )
+                assert result.exit_code == 0, (
+                    "verifier was not rebuilt from clean inputs"
+                )
+                trace.info["verifier_rebuilt_from_dockerfile"] = True
             trace.info["solver_released_before_grading"] = True
 
     async def finalize_services(self, trace, targets):
@@ -80,24 +96,30 @@ class ComposeTask(HarborTask):
 
 
 @pytest.mark.parametrize(
-    ("mode", "collect_fails", "scoring_timeout"),
+    ("mode", "collect_fails", "scoring_timeout", "build_main"),
     [
-        ("shared", True, False),
-        ("isolated", False, False),
-        ("isolated", True, False),
-        ("isolated", False, True),
+        ("shared", True, False, False),
+        ("isolated", False, False, False),
+        ("isolated", True, False, False),
+        ("isolated", False, True, False),
+        ("shared", True, False, True),
+        ("isolated", False, False, True),
     ],
     ids=[
         "shared-skips-sidecars",
         "isolated-restores-sidecars",
         "isolated-collect-failure",
         "isolated-scoring-timeout",
+        "shared-builds-main",
+        "isolated-builds-main",
     ],
 )
 @pytest.mark.prime
 @pytest.mark.skipif(not os.environ.get("PRIME_API_KEY"), reason="needs PRIME_API_KEY")
-async def test_compose_scoring(tmp_path, mode, collect_fails, scoring_timeout):
-    task = compose_task(tmp_path, collect_fails, scoring_timeout, mode)
+async def test_compose_scoring(
+    tmp_path, mode, collect_fails, scoring_timeout, build_main
+):
+    task = compose_task(tmp_path, collect_fails, scoring_timeout, mode, build_main)
     config = vf.AgentConfig(
         model="unused",
         runtime=vf.PrimeConfig(labels=["vf-ci"]),
@@ -128,12 +150,23 @@ async def test_compose_scoring(tmp_path, mode, collect_fails, scoring_timeout):
         assert bool(trace.info.get("solver_released_before_grading")) == (
             mode == "isolated"
         )
+        assert bool(trace.info.get("verifier_rebuilt_from_dockerfile")) == (
+            mode == "isolated" and build_main
+        )
 
 
-def compose_task(tmp_path, collect_fails: bool, scoring_timeout: bool, mode: str):
+def compose_task(
+    tmp_path, collect_fails: bool, scoring_timeout: bool, mode: str, build_main: bool
+):
     environment = tmp_path / "environment"
     environment.mkdir()
-    (environment / "docker-compose.yaml").write_text(COMPOSE)
+    compose = COMPOSE
+    if build_main:
+        compose = compose.replace(f"    image: {IMAGE}\n", "", 1)
+        (environment / "Dockerfile").write_text(
+            f"FROM {IMAGE}\nRUN touch /built-from-dockerfile\n"
+        )
+    (environment / "docker-compose.yaml").write_text(compose)
     tests = tmp_path / "tests"
     tests.mkdir()
     (tests / "test.sh").write_text(
@@ -146,6 +179,7 @@ def compose_task(tmp_path, collect_fails: bool, scoring_timeout: bool, mode: str
     return ComposeTask(
         HarborData(
             prompt="Copy the sidecar answer",
+            tags=["build-main"] if build_main else [],
             task_dir=str(tmp_path),
             env={"COLLECTION_SCOPE": "workspace"},
             # Exercise agent overrides in both directions, not just task defaults.

{verifiers-0.4.1.dev54 → verifiers-0.4.1.dev58}/tests/unit/test_runtimes.py

@@ -300,7 +300,7 @@ async def test_compose(runtime, tmp_path) -> None:
     spec = ComposeSpec(directory=str(tmp_path), file="docker-compose.yaml")
     async with (
         asyncio.timeout(TIMEOUT),
-        compose_services(runtime, spec, {}, trust_compose=True) as (services, _, _),
+        compose_services(runtime, spec, {}, trust_compose=True) as (services, _, _, _),
     ):
         assert set(services) == {"main", "sidecar"}
         resolved = await services["main"].bind().run(["getent", "hosts", "sidecar"], {})

{verifiers-0.4.1.dev54 → verifiers-0.4.1.dev58}/verifiers/v1/__init__.py

@@ -60,6 +60,7 @@ from verifiers.v1.errors import (
     TaskError,
     ToolsetError,
     TunnelError,
+    UnscoredError,
 )
 from verifiers.v1.grading import grade_submission
 from verifiers.v1.graph import MessageNode
@@ -283,6 +284,7 @@ __all__ = [  # noqa: RUF022 - grouped by public API area
     "ToolsetError",
     "SandboxError",
     "TaskError",
+    "UnscoredError",
     "InterceptionError",
     "TunnelError",
     # clients

{verifiers-0.4.1.dev54 → verifiers-0.4.1.dev58}/verifiers/v1/envs/agentic_judge/env.py

@@ -18,8 +18,10 @@ from verifiers.v1.configs.scoring import resolve_scoring
 from verifiers.v1.judges.rubric import (
     Criterion,
     RubricVerdicts,
+    criterion_metrics,
     load_criteria,
     score_verdicts,
+    weighted_score,
 )
 from verifiers.v1.utils.compile import validate_pairing
 
@@ -45,10 +47,7 @@ SOLVED = Criterion(
 
 
 def _verdict_section(criteria: list[Criterion]) -> str:
-    listing = "\n".join(
-        f"- {c.name}: {c.text} (answer one of, worst to best: {', '.join(c.choices)})"
-        for c in criteria
-    )
+    listing = "\n".join(c.render() for c in criteria)
     return f"""\
 ## Your verdict
 
@@ -300,15 +299,16 @@ class AgenticJudgeEnv(vf.Env[AgenticJudgeEnvConfig]):
         verdicts = RubricVerdicts.model_validate(verdict.info.get("verdict")).verdicts
         criteria = self.config.task.criteria()
         scores = score_verdicts(verdicts, criteria, "the rubric's")
-        for criterion in criteria:
-            solution.record_metric(f"judge/{criterion.name}", scores[criterion.name])
+        solution.record_metrics(criterion_metrics("judge", verdicts, criteria, scores))
         if self.config.score.task_weight != 1.0:
             for reward in solution.rewards.values():
                 if reward is not None:
                     reward.weight *= self.config.score.task_weight
-        total = sum(criterion.weight for criterion in criteria)
-        reward = sum(c.weight * scores[c.name] for c in criteria) / total
-        solution.record_reward("judge", reward, weight=self.config.score.judge_weight)
+        solution.record_reward(
+            "judge",
+            weighted_score(scores, criteria),
+            weight=self.config.score.judge_weight,
+        )
 
     async def run(self, task: vf.Task, agents: vf.Agents) -> None:
         policy = resolve_scoring(task.scoring, self.config.solver.scoring)

{verifiers-0.4.1.dev54 → verifiers-0.4.1.dev58}/verifiers/v1/errors.py

@@ -69,6 +69,11 @@ class TaskError(RolloutError):
     """Task-authored code raised — `setup`, `finalize`, or a `@reward`/`@metric`."""
 
 
+class UnscoredError(RolloutError):
+    """Scoring completed but a reward came out `nan`, e.g. a judge's answer that scores
+    nothing: the rollout carries no reward signal, so consumers drop it like a failure."""
+
+
 class InterceptionError(RolloutError):
     """Communication with the host interception server failed."""
 

{verifiers-0.4.1.dev54 → verifiers-0.4.1.dev58}/verifiers/v1/harnesses/rlm/harness.py

@@ -69,7 +69,7 @@ class CompactionConfig(BaseConfig):
 class RLMHarnessConfig(HarnessConfig):
     dialect: Literal["chat_completions"] = "chat_completions"
     version: str = Field(
-        default="77c3489d1a7f918b75c31e195480ed3556af60a5", min_length=1
+        default="4cd7f399f5ba79c350d693af63144e2609062235", min_length=1
     )
     """Git ref (branch, tag, or commit) of nano-rlm to install. Must know every
     field this harness puts on the wire, i.e. be at least the default ref."""

{verifiers-0.4.1.dev54 → verifiers-0.4.1.dev58}/verifiers/v1/judges/rubric.py

@@ -2,6 +2,7 @@
 
 import asyncio
 import json
+import math
 import re
 import tomllib
 from functools import cached_property
@@ -35,10 +36,6 @@ JSON_SUFFIX = (
 )
 
 
-def normalize_choice(choice: str, choices: list[str]) -> float:
-    return choices.index(choice) / (len(choices) - 1)
-
-
 def first_verdicts_object(text: str) -> dict | None:
     """The first balanced JSON object in `text` that carries a `verdicts` key. Scanning for the
     key (not just the first `{`) skips prose and, crucially, an echoed format example — the one
@@ -63,17 +60,44 @@ class Criterion(BaseModel):
     text: str
     weight: CriterionWeight = 1.0
     """The criterion's share of the reward (overridable per name via `weights` in config)."""
-    choices: list[str] = Field(default_factory=lambda: ["no", "yes"], min_length=2)
-    """Allowed answers, ordered **worst → best**: the first scores 0.0, the last 1.0, the rest
-    evenly spaced by rank. Default `["no", "yes"]` is a binary check. Needs >= 2, no duplicates."""
+    choices: list[str] | dict[str, float] = Field(
+        default_factory=lambda: ["no", "yes"], min_length=2
+    )
+    """Allowed answers. A list is ordered **worst → best**: the first scores 0.0, the last 1.0,
+    the rest evenly spaced by rank. A mapping gives each answer its own finite score, in any
+    order; `nan` scores nothing, which leaves the rollout unscored when the criterion
+    carries weight. Default `["no", "yes"]` is a binary check. Needs >= 2, no duplicates."""
 
     @field_validator("choices")
     @classmethod
-    def _check_choices(cls, v: list[str]) -> list[str]:
-        if len(set(v)) != len(v):
+    def _check_choices(
+        cls, v: list[str] | dict[str, float]
+    ) -> list[str] | dict[str, float]:
+        if isinstance(v, list) and len(set(v)) != len(v):
             raise ValueError(f"`choices` has duplicate options: {v}")
+        if isinstance(v, dict) and any(math.isinf(score) for score in v.values()):
+            raise ValueError(f"`choices` scores must be finite or nan: {v}")
         return v
 
+    def score(self, choice: str) -> float:
+        """The score of `choice`, one of `choices`; `nan` when it scores nothing."""
+        if isinstance(self.choices, dict):
+            return self.choices[choice]
+        return self.choices.index(choice) / (len(self.choices) - 1)
+
+    def render(self) -> str:
+        """The criterion as a prompt line, with its allowed answers."""
+        order = "" if isinstance(self.choices, dict) else ", worst to best"
+        return f"- {self.name}: {self.text} (answer one of{order}: {', '.join(self.choices)})"
+
+
+def weighted_score(scores: dict[str, float], criteria: list[Criterion]) -> float:
+    """The criteria's weighted mean score; `nan` when a weighted criterion scored nothing.
+    Zero-weight criteria never contribute, not even their `nan`."""
+    weighted = [c for c in criteria if c.weight]
+    total = sum(c.weight for c in weighted)
+    return sum(c.weight * scores[c.name] for c in weighted) / total
+
 
 CRITERIA_ADAPTER = TypeAdapter(list[Criterion])
 
@@ -174,15 +198,38 @@ def score_verdicts(
         )
     scores: dict[str, float] = {}
     for name, answer in answers.items():
-        choices = by_criterion[name].choices
-        if answer not in choices:
+        criterion = by_criterion[name]
+        if answer not in criterion.choices:
             raise ValueError(
-                f"judge answered {answer!r} for '{name}', expected one of {choices}"
+                f"judge answered {answer!r} for '{name}', expected one of "
+                f"{list(criterion.choices)}"
             )
-        scores[name] = normalize_choice(answer, choices)
+        scores[name] = criterion.score(answer)
     return scores
 
 
+def criterion_metrics(
+    prefix: str,
+    verdicts: list[CriterionVerdict],
+    criteria: list[Criterion],
+    scores: dict[str, float],
+) -> dict[str, float]:
+    """Each criterion's score as `<prefix>/<name>`, unless it scored nothing, and for a
+    criterion with scored choices (a mapping) an indicator per choice, `<prefix>/<name>/<choice>`,
+    since several choices can share a score."""
+    answers = {verdict.name: verdict.verdict for verdict in verdicts}
+    metrics: dict[str, float] = {}
+    for criterion in criteria:
+        if not math.isnan(scores[criterion.name]):
+            metrics[f"{prefix}/{criterion.name}"] = scores[criterion.name]
+        if isinstance(criterion.choices, dict):
+            for choice in criterion.choices:
+                metrics[f"{prefix}/{criterion.name}/{choice}"] = float(
+                    answers[criterion.name] == choice
+                )
+    return metrics
+
+
 class RubricJudge(Judge[RubricVerdicts, RubricJudgeConfig]):
     prompt = RUBRIC_PROMPT
     schema = RubricVerdicts
@@ -194,9 +241,6 @@ class RubricJudge(Judge[RubricVerdicts, RubricJudgeConfig]):
     async def grade_batch(
         self, task: TaskData, trace: Trace, batch: list[Criterion]
     ) -> dict[str, float]:
-        def render(c: Criterion) -> str:
-            return f"- {c.name}: {c.text} (answer one of, worst to best: {', '.join(c.choices)})"
-
         reference = ""
         if self.config.answer_field:
             answer = getattr(task, self.config.answer_field, None)
@@ -222,7 +266,7 @@ class RubricJudge(Judge[RubricVerdicts, RubricJudgeConfig]):
         fields = {
             "question": judge_question(task, self.config.question_field),
             "response": judge_response(trace, self.config.view),
-            "criteria": "\n".join(render(c) for c in batch),
+            "criteria": "\n".join(c.render() for c in batch),
             "reference": reference,
         }
         if self.config.structured_output:
@@ -238,7 +282,11 @@ class RubricJudge(Judge[RubricVerdicts, RubricJudgeConfig]):
                 )
             verdicts = RubricVerdicts.model_validate(obj).verdicts
         # A malformed verdict is a judge failure and must error the rollout, not score the model.
-        return score_verdicts(verdicts, batch, "the batch's")
+        scores = score_verdicts(verdicts, batch, "the batch's")
+        trace.record_metrics(
+            criterion_metrics(self.reward_name, verdicts, batch, scores)
+        )
+        return scores
 
     async def score(self, task: TaskData, trace: Trace) -> float:
         criteria = self.criteria
@@ -264,12 +312,7 @@ class RubricJudge(Judge[RubricVerdicts, RubricJudgeConfig]):
         by_name: dict[str, float] = {}
         for batch_verdicts in results:
             by_name.update(batch_verdicts)
-        for criterion in criteria:
-            trace.record_metric(
-                f"{self.reward_name}/{criterion.name}", by_name[criterion.name]
-            )
-        total = sum(criterion.weight for criterion in criteria)
-        return sum(c.weight * by_name[c.name] for c in criteria) / total
+        return weighted_score(by_name, criteria)
 
 
 __all__ = [
@@ -278,4 +321,6 @@ __all__ = [
     "RubricJudge",
     "RubricJudgeConfig",
     "RubricVerdicts",
+    "criterion_metrics",
+    "weighted_score",
 ]

{verifiers-0.4.1.dev54 → verifiers-0.4.1.dev58}/verifiers/v1/rollout.py

@@ -3,6 +3,7 @@
 import asyncio
 import contextlib
 import logging
+import math
 import time
 from collections.abc import Awaitable, Callable
 from contextlib import AsyncExitStack
@@ -19,6 +20,7 @@ from verifiers.v1.errors import (
     SandboxError,
     TaskError,
     ToolsetError,
+    UnscoredError,
     boundary,
 )
 from verifiers.v1.grading import (
@@ -908,6 +910,21 @@ class Rollout:
                         # the metrics read what it handed over and left behind.
                         await self._close_harness()
                         await self._score_harness(runtime, self._metrics_deadline())
+                # A nan reward scores nothing: keep it unscored (None) and fail the
+                # rollout after scoring, so it is never retried as a scoring failure.
+                unscored = [
+                    name
+                    for name, reward in trace.rewards.items()
+                    if reward is not None and math.isnan(reward.score)
+                ]
+                if unscored:
+                    for name in unscored:
+                        trace.rewards[name] = None
+                    self.fail(
+                        UnscoredError(
+                            f"reward {', '.join(unscored)} scored nothing (nan)"
+                        )
+                    )
                 trace.timing.scoring.end = time.time()
                 trace.notify()
         except Exception as e:  # noqa: BLE001 - finalize boundary records every rollout failure

{verifiers-0.4.1.dev54 → verifiers-0.4.1.dev58}/verifiers/v1/runtimes/compose.py

@@ -85,7 +85,12 @@ async def compose_services(
     trust_compose: bool = False,
     setup_timeout: float | None = None,
 ) -> AsyncIterator[
-    tuple[dict[str, DockerBackend], Callable[[], Awaitable[str]], RuntimeBackend | None]
+    tuple[
+        dict[str, DockerBackend],
+        Callable[[], Awaitable[str]],
+        RuntimeBackend | None,
+        str | None,
+    ]
 ]:
     """Own one Compose attempt and lend its services until the context exits."""
     local = isinstance(config, DockerConfig)
@@ -186,9 +191,20 @@ async def compose_services(
                     }
                 }
             }
-            if {"image", "build"} & authored["services"].get(spec.workspace, {}).keys():
+            main = authored["services"].get(spec.workspace, {})
+            if {"image", "build"} & main.keys():
                 # A template default must not replace an authored image or skip its build.
                 del base["services"][spec.workspace]["image"]
+            elif (
+                not spec.image_override
+                and (
+                    directory / "task" / Path(spec.file).parent / "Dockerfile"
+                ).is_file()
+            ):
+                main = base["services"][spec.workspace]
+                del main["image"]
+                main["build"] = {"context": "${CONTEXT_DIR}"}
+                main["pull_policy"] = "build"
             (directory / "base.json").write_text(json.dumps(base))
             (directory / "env.json").write_text(
                 json.dumps({"services": {spec.workspace: {"environment": environment}}})
@@ -334,6 +350,9 @@ async def compose_services(
             backends,
             partial(compose, "stop", spec.workspace, timeout=60),
             host_backend,
+            backends[spec.workspace].config.image
+            if services[spec.workspace].get("build")
+            else None,
         )
     finally:
         await run_shielded(stack.aclose())

{verifiers-0.4.1.dev54 → verifiers-0.4.1.dev58}/verifiers/v1/runtimes/strategies.py

@@ -2,7 +2,7 @@
 
 import asyncio
 import logging
-from contextlib import AsyncExitStack
+from contextlib import AsyncExitStack, asynccontextmanager
 from functools import partial
 
 from verifiers.v1.runtimes.compose import compose_services
@@ -37,13 +37,19 @@ class _ContainerDeployment(_DeploymentStrategy):
         if d.borrowed_runtime is not None:
             raise ValueError("a composed deployment cannot borrow a single target")
         self._services = await d.enter(AsyncExitStack())
-        services, self._stop_main, _ = await self._services.enter_async_context(
+        (
+            services,
+            self._stop_main,
+            host,
+            built_image,
+        ) = await self._services.enter_async_context(
             compose_services(
                 d.backend.config, d.spec, d.env, trust_compose=d.config.trust_compose
             )
         )
         d.target_backends = dict(services)
         d.execution_backend = d.harness_backend = services[d.spec.workspace]
+        self._remote_built_image = built_image if host is not None else None
 
     async def stop_execution(self, d):
         await self._stop_main()
@@ -57,6 +63,22 @@ class _ContainerDeployment(_DeploymentStrategy):
                 "solver services teardown failed before grading", exc_info=True
             )
 
+    @asynccontextmanager
+    async def verifier(self, deployment, config, env):
+        if self._remote_built_image is None or config.image != self._remote_built_image:
+            async with super().verifier(deployment, config, env) as backend:
+                yield backend
+            return
+        # Rebuild from trusted task inputs on a fresh host: the solver's Docker tag
+        # is local to its released VM, and its modified container must never travel.
+        async with compose_services(
+            config,
+            deployment.spec,
+            env,
+            trust_compose=deployment.config.trust_compose,
+        ) as (services, _, _, _):
+            yield services[deployment.spec.workspace]
+
 
 class _PrivateNetworkDeployment(_DeploymentStrategy):
     async def start(self, d):

{verifiers-0.4.1.dev54 → verifiers-0.4.1.dev58}/verifiers/v1/tasksets/harbor/taskset.py

@@ -708,7 +708,11 @@ def resolve_image(
         import yaml
 
         main = yaml.safe_load(compose.read_text())["services"].get("main", {})
-        if main.get("image") or "build" in main:
+        if (
+            main.get("image")
+            or "build" in main
+            or (task_dir / "environment/Dockerfile").is_file()
+        ):
             return None
     section = "verifier.environment" if verifier else "environment"
     dockerfile = "tests/Dockerfile" if verifier else "environment/Dockerfile"