verifiers 0.1.15.dev406__tar.gz → 0.1.15.dev408__tar.gz

{verifiers-0.1.15.dev406 → verifiers-0.1.15.dev408}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: verifiers
-Version: 0.1.15.dev406
+Version: 0.1.15.dev408
 Summary: Verifiers: Environments for LLM Reinforcement Learning
 Project-URL: Homepage, https://github.com/primeintellect-ai/verifiers
 Project-URL: Documentation, https://github.com/primeintellect-ai/verifiers

{verifiers-0.1.15.dev406 → verifiers-0.1.15.dev408}/verifiers/v1/cli/dashboard/eval.py

@@ -13,6 +13,7 @@ above a rule.
 import contextlib
 import time
 
+from pydantic import BaseModel
 from rich.console import Console, Group
 from rich.markup import escape
 from rich.progress_bar import ProgressBar
@@ -26,7 +27,12 @@ from verifiers.v1.configs.eval import EvalConfig
 from verifiers.v1.rollout import Phase, Rollout
 from verifiers.v1.trace import Trace
 from verifiers.v1.types import Usage
-from verifiers.v1.utils.format import format_count, format_mean, format_time
+from verifiers.v1.utils.format import (
+    format_count,
+    format_mean,
+    format_override,
+    format_time,
+)
 from verifiers.utils.pricing_utils import format_cost_usd
 
 # For sizing pages to the terminal: detects the real terminal height/width each access (the live
@@ -121,6 +127,54 @@ def _warning(config: EvalConfig) -> Text | None:
     return None
 
 
+def overrides(
+    config: BaseModel,
+    default: BaseModel | None = None,
+    skip: frozenset[str] = frozenset(),
+) -> list[str]:
+    """`field=value` segments for the fields the *user* customized, sorted, diffed against each
+    field's declared default. Not `model_fields_set`: a `--resume` run reloads its config via
+    `model_validate(config.toml)`, and that toml is dumped with `exclude_none` (every field), so
+    `model_fields_set` would flag them all. `default` is the reference instance, threaded through
+    recursion so a pinned nested default (a taskset's `user=UserConfig(colocated=True)`) reads as
+    unchanged. `skip` holds dotted paths (`harness.runtime.type`)."""
+    segments: list[str] = []
+    fields = type(config).model_fields
+    for field in sorted(fields):
+        if field in skip:
+            continue
+        value = getattr(config, field)
+        # `get_default` returns `PydanticUndefined` for a required field, so it always reads as set.
+        field_def = (
+            fields[field].get_default(call_default_factory=True)
+            if default is None
+            else getattr(default, field, None)
+        )
+        if isinstance(value, BaseModel):  # nested config: flatten as `field.<sub>`
+            child_skip = frozenset(
+                s[len(field) + 1 :] for s in skip if s.startswith(f"{field}.")
+            )
+            # A switched discriminator (e.g. subprocess→docker) is an override the per-field diff
+            # misses — within the new class `type` equals its own default — so surface it.
+            class_changed = type(value) is not type(field_def)
+            if (
+                class_changed
+                and "type" not in child_skip
+                and (discriminator := getattr(value, "type", None)) is not None
+            ):
+                segments.append(f"{field}.type={format_override(discriminator)}")
+            # A switched class is a new shape, so diff against its own defaults, not the instance.
+            segments.extend(
+                f"{field}.{seg}"
+                for seg in overrides(
+                    value, default=None if class_changed else field_def, skip=child_skip
+                )
+            )
+        elif value != field_def:
+            segments.append(f"{field}={format_override(value)}")
+    return segments
+
+
 def Overview(config: EvalConfig) -> Table:
     sampling = ", ".join(
         f"{k}={v}" for k, v in config.sampling.model_dump(exclude_none=True).items()
@@ -134,6 +188,16 @@ def Overview(config: EvalConfig) -> Table:
     )
     model = f"{config.model}  ({sampling})" if sampling else config.model
     grid.add_row("model", f"{model}  via {config.client.base_url}")
+    # Non-default knobs the user set, one row each when non-empty. `escape` the cell: an override
+    # value (or our `[...]`/`{...}` delimiters) can carry Rich markup that would otherwise be
+    # parsed as styling and dropped. `id` is in the `env` row; harness `runtime.type` too (hidden
+    # here), but only for the harness — a taskset's `user.runtime.type` has no other display.
+    if taskset_over := overrides(config.taskset, skip=frozenset({"id"})):
+        grid.add_row("taskset", escape("  ·  ".join(taskset_over)))
+    if harness_over := overrides(
+        config.harness, skip=frozenset({"id", "runtime.type"})
+    ):
+        grid.add_row("harness", escape("  ·  ".join(harness_over)))
     limits, timeouts = _aligned([_limits(config), _timeouts(config)])
     grid.add_row("limits", limits)
     grid.add_row("timeouts", timeouts)

{verifiers-0.1.15.dev406 → verifiers-0.1.15.dev408}/verifiers/v1/harness.py

@@ -8,6 +8,7 @@ runtime and the interception server are owned by the Rollout.
 
 import asyncio
 import logging
+import os
 from abc import ABC, abstractmethod
 from collections.abc import Mapping
 from typing import ClassVar, Generic, TypeVar
@@ -50,6 +51,11 @@ class HarnessConfig(BaseConfig):
     env: dict[str, str] = Field(default_factory=dict)
     """Additional environment variables for the harness program. Harness-owned endpoint,
     authentication, and model variables take precedence."""
+    forward_env: list[str] = Field(default_factory=list)
+    """Names of environment variables to forward into the harness program's runtime, for
+    secrets that must not be written into a checked-in config. Resolved from `os.environ` at
+    launch — names absent from the environment are skipped, and an explicit `env` value takes
+    precedence over a forwarded one."""
     disabled_tools: list[str] | None = None
     """Harness-specific tool names to disable."""
 
@@ -58,6 +64,14 @@ class HarnessConfig(BaseConfig):
         """The harness's package name (the id with any org / version stripped)."""
         return env_name(self.id)
 
+    @property
+    def resolved_env(self) -> dict[str, str]:
+        """`env` merged with the `forward_env` variables present in the current process
+        environment (explicit `env` wins). This is the environment the harness program runs
+        with; harnesses pass it to the runtime instead of `env` directly."""
+        forwarded = {k: os.environ[k] for k in self.forward_env if k in os.environ}
+        return {**forwarded, **self.env}
+
 
 ConfigT = TypeVar("ConfigT", bound=HarnessConfig)
 

{verifiers-0.1.15.dev406 → verifiers-0.1.15.dev408}/verifiers/v1/harnesses/bash/harness.py

@@ -33,7 +33,7 @@ class BashHarness(Harness[BashHarnessConfig]):
     SUPPORTS_MESSAGE_PROMPT = True
 
     async def setup(self, runtime: Runtime) -> None:
-        await runtime.prepare_uv_script(PROGRAM_SOURCE, self.config.env)
+        await runtime.prepare_uv_script(PROGRAM_SOURCE, self.config.resolved_env)
 
     async def launch(
         self,
@@ -46,7 +46,7 @@ class BashHarness(Harness[BashHarnessConfig]):
     ) -> ProgramResult:
         system_prompt, prompt = self.resolve_prompt(trace.task)
         system_prompt = "\n\n".join(p for p in (BASH_SYSTEM_PROMPT, system_prompt) if p)
-        env = {**self.config.env}
+        env = {**self.config.resolved_env}
         args = [
             f"--base-url={endpoint}",
             f"--api-key={secret}",
@@ -73,5 +73,7 @@ class BashHarness(Harness[BashHarnessConfig]):
             args.append(f"--prompt={prompt}")
         elif prompt is not None:
             env["INITIAL_MESSAGES"] = json.dumps([message_to_wire(m) for m in prompt])
-        program = await runtime.prepare_uv_script(PROGRAM_SOURCE, self.config.env)
+        program = await runtime.prepare_uv_script(
+            PROGRAM_SOURCE, self.config.resolved_env
+        )
         return await runtime.run_program([*program, *args], env)

{verifiers-0.1.15.dev406 → verifiers-0.1.15.dev408}/verifiers/v1/harnesses/bash_edit/harness.py

@@ -39,7 +39,7 @@ class BashEditHarness(Harness[BashEditHarnessConfig]):
     SUPPORTS_MESSAGE_PROMPT = True
 
     async def setup(self, runtime: Runtime) -> None:
-        await runtime.prepare_uv_script(PROGRAM_SOURCE, self.config.env)
+        await runtime.prepare_uv_script(PROGRAM_SOURCE, self.config.resolved_env)
 
     async def launch(
         self,
@@ -54,7 +54,7 @@ class BashEditHarness(Harness[BashEditHarnessConfig]):
         system_prompt = "\n\n".join(
             p for p in (BASH_EDIT_SYSTEM_PROMPT, system_prompt) if p
         )
-        env = {**self.config.env}
+        env = {**self.config.resolved_env}
         args = [
             f"--base-url={endpoint}",
             f"--api-key={secret}",
@@ -81,5 +81,7 @@ class BashEditHarness(Harness[BashEditHarnessConfig]):
             args.append(f"--prompt={prompt}")
         elif prompt is not None:
             env["INITIAL_MESSAGES"] = json.dumps([message_to_wire(m) for m in prompt])
-        program = await runtime.prepare_uv_script(PROGRAM_SOURCE, self.config.env)
+        program = await runtime.prepare_uv_script(
+            PROGRAM_SOURCE, self.config.resolved_env
+        )
         return await runtime.run_program([*program, *args], env)

{verifiers-0.1.15.dev406 → verifiers-0.1.15.dev408}/verifiers/v1/harnesses/codex/harness.py

@@ -75,7 +75,7 @@ class CodexHarness(Harness[CodexHarnessConfig]):
         _, prompt = self.resolve_prompt(trace.task)
         # codex authenticates to the interception server with the session secret (its provider
         # api key) and posts Responses calls to `{endpoint}/responses`.
-        env = {**self.config.env, KEY_VAR: secret}
+        env = {**self.config.resolved_env, KEY_VAR: secret}
         # Values are Codex feature names such as `shell_tool`; Codex owns validation.
         # https://developers.openai.com/codex/config-reference#features
         tool_config = [

{verifiers-0.1.15.dev406 → verifiers-0.1.15.dev408}/verifiers/v1/harnesses/default/harness.py

@@ -30,7 +30,7 @@ class DefaultHarness(Harness[DefaultHarnessConfig]):
     SUPPORTS_MESSAGE_PROMPT = True
 
     async def setup(self, runtime: Runtime) -> None:
-        await runtime.prepare_uv_script(PROGRAM_SOURCE, self.config.env)
+        await runtime.prepare_uv_script(PROGRAM_SOURCE, self.config.resolved_env)
 
     async def launch(
         self,
@@ -42,7 +42,7 @@ class DefaultHarness(Harness[DefaultHarnessConfig]):
         mcp_urls: dict[str, str],
     ) -> ProgramResult:
         system_prompt, prompt = self.resolve_prompt(trace.task)
-        env = {**self.config.env}
+        env = {**self.config.resolved_env}
         args = [
             f"--base-url={endpoint}",
             f"--api-key={secret}",
@@ -70,5 +70,7 @@ class DefaultHarness(Harness[DefaultHarnessConfig]):
             args.append(f"--prompt={prompt}")
         elif prompt is not None:
             env["INITIAL_MESSAGES"] = json.dumps([message_to_wire(m) for m in prompt])
-        program = await runtime.prepare_uv_script(PROGRAM_SOURCE, self.config.env)
+        program = await runtime.prepare_uv_script(
+            PROGRAM_SOURCE, self.config.resolved_env
+        )
         return await runtime.run_program([*program, *args], env)

{verifiers-0.1.15.dev406 → verifiers-0.1.15.dev408}/verifiers/v1/harnesses/kimi_code/harness.py

@@ -73,7 +73,7 @@ class KimiCodeHarness(Harness[KimiCodeHarnessConfig]):
     ) -> ProgramResult:
         _, prompt = self.resolve_prompt(trace.task)
         env = {
-            **self.config.env,
+            **self.config.resolved_env,
             "KIMI_CODE_HOME": KIMI_HOME,
             "KIMI_MODEL_NAME": ctx.model,
             "KIMI_MODEL_API_KEY": secret,

{verifiers-0.1.15.dev406 → verifiers-0.1.15.dev408}/verifiers/v1/harnesses/mini_swe_agent/harness.py

@@ -23,7 +23,7 @@ class MiniSWEAgentHarness(Harness[MiniSWEAgentHarnessConfig]):
 
     async def setup(self, runtime: Runtime) -> None:
         source = PROGRAM_SOURCE.replace("{version}", self.config.version)
-        await runtime.prepare_uv_script(source, self.config.env)
+        await runtime.prepare_uv_script(source, self.config.resolved_env)
 
     async def launch(
         self,
@@ -66,9 +66,9 @@ class MiniSWEAgentHarness(Harness[MiniSWEAgentHarnessConfig]):
             f"model.model_kwargs.api_key={secret}",
         ]
         env = {
-            **self.config.env,
+            **self.config.resolved_env,
             "MSWEA_CONFIGURED": "true",
             "MSWEA_SILENT_STARTUP": "true",
         }
-        program = await runtime.prepare_uv_script(source, self.config.env)
+        program = await runtime.prepare_uv_script(source, self.config.resolved_env)
         return await runtime.run_program([*program, *args], env)

{verifiers-0.1.15.dev406 → verifiers-0.1.15.dev408}/verifiers/v1/harnesses/rlm/harness.py

@@ -72,7 +72,7 @@ class RLMHarness(Harness[RLMHarnessConfig]):
         logger.info("rlm: ensuring rlm is installed (version=%s)", self.config.version)
         ensure = shlex.quote(f"[ -x {RLM_BIN} ] || ({install})")
         guarded = f"mkdir -p {RLM_DIR} && flock {RLM_DIR}/install.lock sh -c {ensure}"
-        env = {**self.config.env, "RLM_HOME": RLM_HOME}
+        env = {**self.config.resolved_env, "RLM_HOME": RLM_HOME}
         result = await runtime.run(["sh", "-c", guarded], env)
         if result.exit_code != 0:
             raise RuntimeError(f"rlm install failed: {result.stderr.strip()[-500:]}")
@@ -88,7 +88,7 @@ class RLMHarness(Harness[RLMHarnessConfig]):
     ) -> ProgramResult:
         system_prompt, prompt = self.resolve_prompt(trace.task)
         env = {
-            **self.config.env,
+            **self.config.resolved_env,
             "RLM_BASE_URL": endpoint,
             "RLM_API_KEY": secret,
             "RLM_MODEL": ctx.model,

{verifiers-0.1.15.dev406 → verifiers-0.1.15.dev408}/verifiers/v1/harnesses/terminus_2/harness.py

@@ -35,7 +35,7 @@ class Terminus2Harness(Harness[Terminus2HarnessConfig]):
                 "(--harness.runtime.type docker|prime|modal)."
             )
         source = PROGRAM_SOURCE.replace("{version}", self.config.version)
-        await runtime.prepare_uv_script(source, self.config.env)
+        await runtime.prepare_uv_script(source, self.config.resolved_env)
 
     async def launch(
         self,
@@ -55,7 +55,7 @@ class Terminus2Harness(Harness[Terminus2HarnessConfig]):
             )
         tmux_dir = f"/tmp/vf-terminus-2-{trace.id}"
         env = {
-            **self.config.env,
+            **self.config.resolved_env,
             "TMUX_TMPDIR": tmux_dir,
         }
         args = [
@@ -67,7 +67,7 @@ class Terminus2Harness(Harness[Terminus2HarnessConfig]):
         ]
         try:
             source = PROGRAM_SOURCE.replace("{version}", self.config.version)
-            program = await runtime.prepare_uv_script(source, self.config.env)
+            program = await runtime.prepare_uv_script(source, self.config.resolved_env)
             return await runtime.run_program([*program, *args], env)
         finally:
             # Harbor normally destroys its whole sandbox; this adapter borrows the

{verifiers-0.1.15.dev406 → verifiers-0.1.15.dev408}/verifiers/v1/utils/format.py

@@ -34,6 +34,15 @@ def format_count(n: int) -> str:
     return f"{n / 1e6:.1f}M"
 
 
+def format_override(value: object) -> str:
+    """A value as one compact segment: dict `{k=v,k=v}`, list/tuple `[a,b]`, else `str`."""
+    if isinstance(value, dict):
+        return "{" + ",".join(f"{k}={v}" for k, v in value.items()) + "}"
+    if isinstance(value, (list, tuple)):
+        return "[" + ",".join(str(v) for v in value) + "]"
+    return str(value)
+
+
 def format_mean(
     traces: list[Trace], value: Callable[[Trace], float], digits: int = 2
 ) -> str: