verifiers 0.1.15.dev405__tar.gz → 0.1.15.dev407__tar.gz

{verifiers-0.1.15.dev405 → verifiers-0.1.15.dev407}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: verifiers
-Version: 0.1.15.dev405
+Version: 0.1.15.dev407
 Summary: Verifiers: Environments for LLM Reinforcement Learning
 Project-URL: Homepage, https://github.com/primeintellect-ai/verifiers
 Project-URL: Documentation, https://github.com/primeintellect-ai/verifiers

{verifiers-0.1.15.dev405 → verifiers-0.1.15.dev407}/verifiers/v1/GUIDE.md

@@ -66,7 +66,7 @@ class ReverseTaskset(vf.Taskset[ReverseTask, ReverseConfig]):
 
     @vf.reward()
     async def exact_match(self, task: ReverseTask, trace: vf.Trace) -> float:
-        return float(trace.assistant_messages[-1].content.strip() == task.answer)
+        return float(trace.last_reply == task.answer)
 
 
 __all__ = ["ReverseTaskset"]   # vf resolves the taskset by finding this Taskset subclass
@@ -250,7 +250,7 @@ VERIFY = (Path(__file__).parent / "verify.py").read_text()   # PEP 723 header de
 
 @vf.reward()
 async def verify(self, task, trace, runtime) -> float:
-    r = await runtime.run_uv_script(VERIFY, args=[task.answer, trace.assistant_messages[-1].content])
+    r = await runtime.run_uv_script(VERIFY, args=[task.answer, trace.last_reply])
     return float(r.stdout.strip() == "1.0")
 ```
 
@@ -317,7 +317,7 @@ a taskset `@vf.stop` fires. A stop is an `async (self, trace) -> bool` checked b
 ```python
 @vf.stop
 async def saw_answer(self, trace) -> bool:
-    last = trace.assistant_messages[-1].content or ""
+    last = trace.last_reply
     return "FINAL:" in last
 ```
 

{verifiers-0.1.15.dev405 → verifiers-0.1.15.dev407}/verifiers/v1/harness.py

@@ -8,6 +8,7 @@ runtime and the interception server are owned by the Rollout.
 
 import asyncio
 import logging
+import os
 from abc import ABC, abstractmethod
 from collections.abc import Mapping
 from typing import ClassVar, Generic, TypeVar
@@ -50,6 +51,11 @@ class HarnessConfig(BaseConfig):
     env: dict[str, str] = Field(default_factory=dict)
     """Additional environment variables for the harness program. Harness-owned endpoint,
     authentication, and model variables take precedence."""
+    forward_env: list[str] = Field(default_factory=list)
+    """Names of environment variables to forward into the harness program's runtime, for
+    secrets that must not be written into a checked-in config. Resolved from `os.environ` at
+    launch — names absent from the environment are skipped, and an explicit `env` value takes
+    precedence over a forwarded one."""
     disabled_tools: list[str] | None = None
     """Harness-specific tool names to disable."""
 
@@ -58,6 +64,14 @@ class HarnessConfig(BaseConfig):
         """The harness's package name (the id with any org / version stripped)."""
         return env_name(self.id)
 
+    @property
+    def resolved_env(self) -> dict[str, str]:
+        """`env` merged with the `forward_env` variables present in the current process
+        environment (explicit `env` wins). This is the environment the harness program runs
+        with; harnesses pass it to the runtime instead of `env` directly."""
+        forwarded = {k: os.environ[k] for k in self.forward_env if k in os.environ}
+        return {**forwarded, **self.env}
+
 
 ConfigT = TypeVar("ConfigT", bound=HarnessConfig)
 

{verifiers-0.1.15.dev405 → verifiers-0.1.15.dev407}/verifiers/v1/harnesses/bash/harness.py

@@ -33,7 +33,7 @@ class BashHarness(Harness[BashHarnessConfig]):
     SUPPORTS_MESSAGE_PROMPT = True
 
     async def setup(self, runtime: Runtime) -> None:
-        await runtime.prepare_uv_script(PROGRAM_SOURCE, self.config.env)
+        await runtime.prepare_uv_script(PROGRAM_SOURCE, self.config.resolved_env)
 
     async def launch(
         self,
@@ -46,7 +46,7 @@ class BashHarness(Harness[BashHarnessConfig]):
     ) -> ProgramResult:
         system_prompt, prompt = self.resolve_prompt(trace.task)
         system_prompt = "\n\n".join(p for p in (BASH_SYSTEM_PROMPT, system_prompt) if p)
-        env = {**self.config.env}
+        env = {**self.config.resolved_env}
         args = [
             f"--base-url={endpoint}",
             f"--api-key={secret}",
@@ -73,5 +73,7 @@ class BashHarness(Harness[BashHarnessConfig]):
             args.append(f"--prompt={prompt}")
         elif prompt is not None:
             env["INITIAL_MESSAGES"] = json.dumps([message_to_wire(m) for m in prompt])
-        program = await runtime.prepare_uv_script(PROGRAM_SOURCE, self.config.env)
+        program = await runtime.prepare_uv_script(
+            PROGRAM_SOURCE, self.config.resolved_env
+        )
         return await runtime.run_program([*program, *args], env)

{verifiers-0.1.15.dev405 → verifiers-0.1.15.dev407}/verifiers/v1/harnesses/bash_edit/harness.py

@@ -39,7 +39,7 @@ class BashEditHarness(Harness[BashEditHarnessConfig]):
     SUPPORTS_MESSAGE_PROMPT = True
 
     async def setup(self, runtime: Runtime) -> None:
-        await runtime.prepare_uv_script(PROGRAM_SOURCE, self.config.env)
+        await runtime.prepare_uv_script(PROGRAM_SOURCE, self.config.resolved_env)
 
     async def launch(
         self,
@@ -54,7 +54,7 @@ class BashEditHarness(Harness[BashEditHarnessConfig]):
         system_prompt = "\n\n".join(
             p for p in (BASH_EDIT_SYSTEM_PROMPT, system_prompt) if p
         )
-        env = {**self.config.env}
+        env = {**self.config.resolved_env}
         args = [
             f"--base-url={endpoint}",
             f"--api-key={secret}",
@@ -81,5 +81,7 @@ class BashEditHarness(Harness[BashEditHarnessConfig]):
             args.append(f"--prompt={prompt}")
         elif prompt is not None:
             env["INITIAL_MESSAGES"] = json.dumps([message_to_wire(m) for m in prompt])
-        program = await runtime.prepare_uv_script(PROGRAM_SOURCE, self.config.env)
+        program = await runtime.prepare_uv_script(
+            PROGRAM_SOURCE, self.config.resolved_env
+        )
         return await runtime.run_program([*program, *args], env)

{verifiers-0.1.15.dev405 → verifiers-0.1.15.dev407}/verifiers/v1/harnesses/codex/harness.py

@@ -75,7 +75,7 @@ class CodexHarness(Harness[CodexHarnessConfig]):
         _, prompt = self.resolve_prompt(trace.task)
         # codex authenticates to the interception server with the session secret (its provider
         # api key) and posts Responses calls to `{endpoint}/responses`.
-        env = {**self.config.env, KEY_VAR: secret}
+        env = {**self.config.resolved_env, KEY_VAR: secret}
         # Values are Codex feature names such as `shell_tool`; Codex owns validation.
         # https://developers.openai.com/codex/config-reference#features
         tool_config = [

{verifiers-0.1.15.dev405 → verifiers-0.1.15.dev407}/verifiers/v1/harnesses/default/harness.py

@@ -30,7 +30,7 @@ class DefaultHarness(Harness[DefaultHarnessConfig]):
     SUPPORTS_MESSAGE_PROMPT = True
 
     async def setup(self, runtime: Runtime) -> None:
-        await runtime.prepare_uv_script(PROGRAM_SOURCE, self.config.env)
+        await runtime.prepare_uv_script(PROGRAM_SOURCE, self.config.resolved_env)
 
     async def launch(
         self,
@@ -42,7 +42,7 @@ class DefaultHarness(Harness[DefaultHarnessConfig]):
         mcp_urls: dict[str, str],
     ) -> ProgramResult:
         system_prompt, prompt = self.resolve_prompt(trace.task)
-        env = {**self.config.env}
+        env = {**self.config.resolved_env}
         args = [
             f"--base-url={endpoint}",
             f"--api-key={secret}",
@@ -70,5 +70,7 @@ class DefaultHarness(Harness[DefaultHarnessConfig]):
             args.append(f"--prompt={prompt}")
         elif prompt is not None:
             env["INITIAL_MESSAGES"] = json.dumps([message_to_wire(m) for m in prompt])
-        program = await runtime.prepare_uv_script(PROGRAM_SOURCE, self.config.env)
+        program = await runtime.prepare_uv_script(
+            PROGRAM_SOURCE, self.config.resolved_env
+        )
         return await runtime.run_program([*program, *args], env)

{verifiers-0.1.15.dev405 → verifiers-0.1.15.dev407}/verifiers/v1/harnesses/kimi_code/harness.py

@@ -73,7 +73,7 @@ class KimiCodeHarness(Harness[KimiCodeHarnessConfig]):
     ) -> ProgramResult:
         _, prompt = self.resolve_prompt(trace.task)
         env = {
-            **self.config.env,
+            **self.config.resolved_env,
             "KIMI_CODE_HOME": KIMI_HOME,
             "KIMI_MODEL_NAME": ctx.model,
             "KIMI_MODEL_API_KEY": secret,

{verifiers-0.1.15.dev405 → verifiers-0.1.15.dev407}/verifiers/v1/harnesses/mini_swe_agent/harness.py

@@ -23,7 +23,7 @@ class MiniSWEAgentHarness(Harness[MiniSWEAgentHarnessConfig]):
 
     async def setup(self, runtime: Runtime) -> None:
         source = PROGRAM_SOURCE.replace("{version}", self.config.version)
-        await runtime.prepare_uv_script(source, self.config.env)
+        await runtime.prepare_uv_script(source, self.config.resolved_env)
 
     async def launch(
         self,
@@ -66,9 +66,9 @@ class MiniSWEAgentHarness(Harness[MiniSWEAgentHarnessConfig]):
             f"model.model_kwargs.api_key={secret}",
         ]
         env = {
-            **self.config.env,
+            **self.config.resolved_env,
             "MSWEA_CONFIGURED": "true",
             "MSWEA_SILENT_STARTUP": "true",
         }
-        program = await runtime.prepare_uv_script(source, self.config.env)
+        program = await runtime.prepare_uv_script(source, self.config.resolved_env)
         return await runtime.run_program([*program, *args], env)

{verifiers-0.1.15.dev405 → verifiers-0.1.15.dev407}/verifiers/v1/harnesses/rlm/harness.py

@@ -72,7 +72,7 @@ class RLMHarness(Harness[RLMHarnessConfig]):
         logger.info("rlm: ensuring rlm is installed (version=%s)", self.config.version)
         ensure = shlex.quote(f"[ -x {RLM_BIN} ] || ({install})")
         guarded = f"mkdir -p {RLM_DIR} && flock {RLM_DIR}/install.lock sh -c {ensure}"
-        env = {**self.config.env, "RLM_HOME": RLM_HOME}
+        env = {**self.config.resolved_env, "RLM_HOME": RLM_HOME}
         result = await runtime.run(["sh", "-c", guarded], env)
         if result.exit_code != 0:
             raise RuntimeError(f"rlm install failed: {result.stderr.strip()[-500:]}")
@@ -88,7 +88,7 @@ class RLMHarness(Harness[RLMHarnessConfig]):
     ) -> ProgramResult:
         system_prompt, prompt = self.resolve_prompt(trace.task)
         env = {
-            **self.config.env,
+            **self.config.resolved_env,
             "RLM_BASE_URL": endpoint,
             "RLM_API_KEY": secret,
             "RLM_MODEL": ctx.model,

{verifiers-0.1.15.dev405 → verifiers-0.1.15.dev407}/verifiers/v1/harnesses/terminus_2/harness.py

@@ -35,7 +35,7 @@ class Terminus2Harness(Harness[Terminus2HarnessConfig]):
                 "(--harness.runtime.type docker|prime|modal)."
             )
         source = PROGRAM_SOURCE.replace("{version}", self.config.version)
-        await runtime.prepare_uv_script(source, self.config.env)
+        await runtime.prepare_uv_script(source, self.config.resolved_env)
 
     async def launch(
         self,
@@ -55,7 +55,7 @@ class Terminus2Harness(Harness[Terminus2HarnessConfig]):
             )
         tmux_dir = f"/tmp/vf-terminus-2-{trace.id}"
         env = {
-            **self.config.env,
+            **self.config.resolved_env,
             "TMUX_TMPDIR": tmux_dir,
         }
         args = [
@@ -67,7 +67,7 @@ class Terminus2Harness(Harness[Terminus2HarnessConfig]):
         ]
         try:
             source = PROGRAM_SOURCE.replace("{version}", self.config.version)
-            program = await runtime.prepare_uv_script(source, self.config.env)
+            program = await runtime.prepare_uv_script(source, self.config.resolved_env)
             return await runtime.run_program([*program, *args], env)
         finally:
             # Harbor normally destroys its whole sandbox; this adapter borrows the

{verifiers-0.1.15.dev405 → verifiers-0.1.15.dev407}/verifiers/v1/trace.py

@@ -364,6 +364,15 @@ class Trace(StrictBaseModel, Generic[TaskT, StateT]):
             if n.sampled and isinstance(n.message, AssistantMessage)
         ]
 
+    @property
+    def last_reply(self) -> str:
+        """The model's final reply as plain text — the last assistant message's
+        `content` (whitespace-stripped), or ``""`` when there are no assistant
+        messages or the last one has no text. Convenience over
+        ``(assistant_messages[-1].content or "").strip()`` for last-turn scoring."""
+        msgs = self.assistant_messages
+        return (msgs[-1].content or "").strip() if msgs else ""
+
     @property
     def tool_messages(self) -> list[ToolMessage]:
         """The tool results in the latest full context — the main (last) branch's