verifiers 0.4.1.dev54__tar.gz → 0.4.1.dev56__tar.gz

{verifiers-0.4.1.dev54 → verifiers-0.4.1.dev56}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: verifiers
-Version: 0.4.1.dev54
+Version: 0.4.1.dev56
 Summary: Verifiers: Environments for LLM Reinforcement Learning
 Project-URL: Homepage, https://github.com/primeintellect-ai/verifiers
 Project-URL: Documentation, https://github.com/primeintellect-ai/verifiers

{verifiers-0.4.1.dev54 → verifiers-0.4.1.dev56}/tests/integration/test_compose_scoring.py

@@ -42,6 +42,9 @@ services:
 
 class ComposeHarness(vf.Harness):
     async def launch(self, ctx, trace, runtime, endpoint, secret, mcp_urls, data):
+        if "build-main" in data.tags:
+            result = await runtime.run(["test", "-f", "/built-from-dockerfile"], {})
+            assert result.exit_code == 0, "workspace did not use its Dockerfile"
         return await runtime.run(
             [
                 "python",
@@ -65,6 +68,19 @@ class ComposeTask(HarborTask):
             assert all(
                 backend._host_backend.stopped for backend in SOLVERS[trace.id]
             ), "solver services survived into grading"
+            if "build-main" in self.data.tags:
+                result = await runtime.run(
+                    [
+                        "sh",
+                        "-c",
+                        "test -f /built-from-dockerfile && test ! -e /app/answer",
+                    ],
+                    {},
+                )
+                assert result.exit_code == 0, (
+                    "verifier was not rebuilt from clean inputs"
+                )
+                trace.info["verifier_rebuilt_from_dockerfile"] = True
             trace.info["solver_released_before_grading"] = True
 
     async def finalize_services(self, trace, targets):
@@ -80,24 +96,30 @@ class ComposeTask(HarborTask):
 
 
 @pytest.mark.parametrize(
-    ("mode", "collect_fails", "scoring_timeout"),
+    ("mode", "collect_fails", "scoring_timeout", "build_main"),
     [
-        ("shared", True, False),
-        ("isolated", False, False),
-        ("isolated", True, False),
-        ("isolated", False, True),
+        ("shared", True, False, False),
+        ("isolated", False, False, False),
+        ("isolated", True, False, False),
+        ("isolated", False, True, False),
+        ("shared", True, False, True),
+        ("isolated", False, False, True),
     ],
     ids=[
         "shared-skips-sidecars",
         "isolated-restores-sidecars",
         "isolated-collect-failure",
         "isolated-scoring-timeout",
+        "shared-builds-main",
+        "isolated-builds-main",
     ],
 )
 @pytest.mark.prime
 @pytest.mark.skipif(not os.environ.get("PRIME_API_KEY"), reason="needs PRIME_API_KEY")
-async def test_compose_scoring(tmp_path, mode, collect_fails, scoring_timeout):
-    task = compose_task(tmp_path, collect_fails, scoring_timeout, mode)
+async def test_compose_scoring(
+    tmp_path, mode, collect_fails, scoring_timeout, build_main
+):
+    task = compose_task(tmp_path, collect_fails, scoring_timeout, mode, build_main)
     config = vf.AgentConfig(
         model="unused",
         runtime=vf.PrimeConfig(labels=["vf-ci"]),
@@ -128,12 +150,23 @@ async def test_compose_scoring(tmp_path, mode, collect_fails, scoring_timeout):
         assert bool(trace.info.get("solver_released_before_grading")) == (
             mode == "isolated"
         )
+        assert bool(trace.info.get("verifier_rebuilt_from_dockerfile")) == (
+            mode == "isolated" and build_main
+        )
 
 
-def compose_task(tmp_path, collect_fails: bool, scoring_timeout: bool, mode: str):
+def compose_task(
+    tmp_path, collect_fails: bool, scoring_timeout: bool, mode: str, build_main: bool
+):
     environment = tmp_path / "environment"
     environment.mkdir()
-    (environment / "docker-compose.yaml").write_text(COMPOSE)
+    compose = COMPOSE
+    if build_main:
+        compose = compose.replace(f"    image: {IMAGE}\n", "", 1)
+        (environment / "Dockerfile").write_text(
+            f"FROM {IMAGE}\nRUN touch /built-from-dockerfile\n"
+        )
+    (environment / "docker-compose.yaml").write_text(compose)
     tests = tmp_path / "tests"
     tests.mkdir()
     (tests / "test.sh").write_text(
@@ -146,6 +179,7 @@ def compose_task(tmp_path, collect_fails: bool, scoring_timeout: bool, mode: str
     return ComposeTask(
         HarborData(
             prompt="Copy the sidecar answer",
+            tags=["build-main"] if build_main else [],
             task_dir=str(tmp_path),
             env={"COLLECTION_SCOPE": "workspace"},
             # Exercise agent overrides in both directions, not just task defaults.

{verifiers-0.4.1.dev54 → verifiers-0.4.1.dev56}/tests/unit/test_runtimes.py

@@ -300,7 +300,7 @@ async def test_compose(runtime, tmp_path) -> None:
     spec = ComposeSpec(directory=str(tmp_path), file="docker-compose.yaml")
     async with (
         asyncio.timeout(TIMEOUT),
-        compose_services(runtime, spec, {}, trust_compose=True) as (services, _, _),
+        compose_services(runtime, spec, {}, trust_compose=True) as (services, _, _, _),
     ):
         assert set(services) == {"main", "sidecar"}
         resolved = await services["main"].bind().run(["getent", "hosts", "sidecar"], {})

{verifiers-0.4.1.dev54 → verifiers-0.4.1.dev56}/verifiers/v1/harnesses/rlm/harness.py

@@ -69,7 +69,7 @@ class CompactionConfig(BaseConfig):
 class RLMHarnessConfig(HarnessConfig):
     dialect: Literal["chat_completions"] = "chat_completions"
     version: str = Field(
-        default="77c3489d1a7f918b75c31e195480ed3556af60a5", min_length=1
+        default="4cd7f399f5ba79c350d693af63144e2609062235", min_length=1
     )
     """Git ref (branch, tag, or commit) of nano-rlm to install. Must know every
     field this harness puts on the wire, i.e. be at least the default ref."""

{verifiers-0.4.1.dev54 → verifiers-0.4.1.dev56}/verifiers/v1/runtimes/compose.py

@@ -85,7 +85,12 @@ async def compose_services(
     trust_compose: bool = False,
     setup_timeout: float | None = None,
 ) -> AsyncIterator[
-    tuple[dict[str, DockerBackend], Callable[[], Awaitable[str]], RuntimeBackend | None]
+    tuple[
+        dict[str, DockerBackend],
+        Callable[[], Awaitable[str]],
+        RuntimeBackend | None,
+        str | None,
+    ]
 ]:
     """Own one Compose attempt and lend its services until the context exits."""
     local = isinstance(config, DockerConfig)
@@ -186,9 +191,20 @@ async def compose_services(
                     }
                 }
             }
-            if {"image", "build"} & authored["services"].get(spec.workspace, {}).keys():
+            main = authored["services"].get(spec.workspace, {})
+            if {"image", "build"} & main.keys():
                 # A template default must not replace an authored image or skip its build.
                 del base["services"][spec.workspace]["image"]
+            elif (
+                not spec.image_override
+                and (
+                    directory / "task" / Path(spec.file).parent / "Dockerfile"
+                ).is_file()
+            ):
+                main = base["services"][spec.workspace]
+                del main["image"]
+                main["build"] = {"context": "${CONTEXT_DIR}"}
+                main["pull_policy"] = "build"
             (directory / "base.json").write_text(json.dumps(base))
             (directory / "env.json").write_text(
                 json.dumps({"services": {spec.workspace: {"environment": environment}}})
@@ -334,6 +350,9 @@ async def compose_services(
             backends,
             partial(compose, "stop", spec.workspace, timeout=60),
             host_backend,
+            backends[spec.workspace].config.image
+            if services[spec.workspace].get("build")
+            else None,
         )
     finally:
         await run_shielded(stack.aclose())

{verifiers-0.4.1.dev54 → verifiers-0.4.1.dev56}/verifiers/v1/runtimes/strategies.py

@@ -2,7 +2,7 @@
 
 import asyncio
 import logging
-from contextlib import AsyncExitStack
+from contextlib import AsyncExitStack, asynccontextmanager
 from functools import partial
 
 from verifiers.v1.runtimes.compose import compose_services
@@ -37,13 +37,19 @@ class _ContainerDeployment(_DeploymentStrategy):
         if d.borrowed_runtime is not None:
             raise ValueError("a composed deployment cannot borrow a single target")
         self._services = await d.enter(AsyncExitStack())
-        services, self._stop_main, _ = await self._services.enter_async_context(
+        (
+            services,
+            self._stop_main,
+            host,
+            built_image,
+        ) = await self._services.enter_async_context(
             compose_services(
                 d.backend.config, d.spec, d.env, trust_compose=d.config.trust_compose
             )
         )
         d.target_backends = dict(services)
         d.execution_backend = d.harness_backend = services[d.spec.workspace]
+        self._remote_built_image = built_image if host is not None else None
 
     async def stop_execution(self, d):
         await self._stop_main()
@@ -57,6 +63,22 @@ class _ContainerDeployment(_DeploymentStrategy):
                 "solver services teardown failed before grading", exc_info=True
             )
 
+    @asynccontextmanager
+    async def verifier(self, deployment, config, env):
+        if self._remote_built_image is None or config.image != self._remote_built_image:
+            async with super().verifier(deployment, config, env) as backend:
+                yield backend
+            return
+        # Rebuild from trusted task inputs on a fresh host: the solver's Docker tag
+        # is local to its released VM, and its modified container must never travel.
+        async with compose_services(
+            config,
+            deployment.spec,
+            env,
+            trust_compose=deployment.config.trust_compose,
+        ) as (services, _, _, _):
+            yield services[deployment.spec.workspace]
+
 
 class _PrivateNetworkDeployment(_DeploymentStrategy):
     async def start(self, d):

{verifiers-0.4.1.dev54 → verifiers-0.4.1.dev56}/verifiers/v1/tasksets/harbor/taskset.py

@@ -708,7 +708,11 @@ def resolve_image(
         import yaml
 
         main = yaml.safe_load(compose.read_text())["services"].get("main", {})
-        if main.get("image") or "build" in main:
+        if (
+            main.get("image")
+            or "build" in main
+            or (task_dir / "environment/Dockerfile").is_file()
+        ):
             return None
     section = "verifier.environment" if verifier else "environment"
     dockerfile = "tests/Dockerfile" if verifier else "environment/Dockerfile"