verifiers 0.2.1.dev59__tar.gz → 0.2.1.dev61__tar.gz

{verifiers-0.2.1.dev59 → verifiers-0.2.1.dev61}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: verifiers
-Version: 0.2.1.dev59
+Version: 0.2.1.dev61
 Summary: Verifiers: Environments for LLM Reinforcement Learning
 Project-URL: Homepage, https://github.com/primeintellect-ai/verifiers
 Project-URL: Documentation, https://github.com/primeintellect-ai/verifiers

{verifiers-0.2.1.dev59 → verifiers-0.2.1.dev61}/pyproject.toml

@@ -252,7 +252,7 @@ markers = [
     "modal: v1 e2e cases touching the modal runtime (local-only setup; excluded in CI)",
     "colocated: v1 e2e cases with a user/tool server colocated in the harness runtime",
     "null: v1 e2e cases on the null harness",
-    "default: v1 e2e cases on the default harness",
+    "bash: v1 e2e cases on the bash harness",
     "rlm: v1 e2e cases on the rlm harness",
     "kimi_code: v1 e2e cases on the kimi-code harness",
     "codex: v1 e2e cases on the codex harness",

{verifiers-0.2.1.dev59 → verifiers-0.2.1.dev61}/tests/v1/conftest.py

@@ -15,12 +15,12 @@ Every matrix value carries a pytest mark, so subsets select with `-m`:
     uv run pytest tests/v1 -n auto                                # the whole matrix (needs modal setup)
     uv run pytest tests/v1 -n auto -m "not prime and not modal"  # the CI matrix (host + docker only)
     uv run pytest tests/v1 -n auto -m docker                      # any case touching the docker runtime
-    uv run pytest tests/v1 -n auto -m default                     # only the default harness
+    uv run pytest tests/v1 -n auto -m bash                        # only the bash harness
     uv run pytest tests/v1 -n auto -m prime                       # only prime (real sandboxes; local)
     uv run pytest tests/v1 -n auto -m modal                       # only modal (needs local setup)
 
 Marks: runtimes `subprocess` / `docker` / `prime` / `modal`, placement `colocated`,
-harnesses `null` / `default` / `rlm` / `kimi_code` / `codex` / `claude_code`. A mark is applied per
+harnesses `null` / `bash` / `rlm` / `kimi_code` / `codex` / `claude_code`. A mark is applied per
 axis, so it selects every case touching that value on ANY axis; for one exact combination use `-k`
 on the test id (e.g. `-k "harness-in-docker-with-tool-in-subprocess"`). prime/modal provision real
 remote sandboxes (slow, infra-flaky, need setup), so they're local-only — CI runs
@@ -104,7 +104,7 @@ def tool_runtime(request) -> dict:
     return {"runtime": {"type": request.param}}
 
 
-# Harnesses, composed with the runtime fixtures, each carrying its harness mark (`-m default`, ...).
+# Harnesses, composed with the runtime fixtures, each carrying its harness mark (`-m bash`, ...).
 # Built-ins are bundled in the `harnesses` package; the agent CLIs (`rlm` / `kimi-code` / `codex` /
 # `claude-code`) install their dependencies at rollout. `compact` (an example harness) and
 # `terminus-2` (drives the host tmux) are excluded. `test_agentic` skips `null` (a chat loop with no
@@ -112,7 +112,7 @@ def tool_runtime(request) -> dict:
 @pytest.fixture(
     params=[
         pytest.param("null", marks=pytest.mark.null, id="null"),
-        pytest.param("default", marks=pytest.mark.default, id="default"),
+        pytest.param("bash", marks=pytest.mark.bash, id="bash"),
         pytest.param("rlm", marks=pytest.mark.rlm, id="rlm"),
         pytest.param("kimi-code", marks=pytest.mark.kimi_code, id="kimi-code"),
         pytest.param("codex", marks=pytest.mark.codex, id="codex"),

{verifiers-0.2.1.dev59 → verifiers-0.2.1.dev61}/verifiers/v1/__init__.py

@@ -105,9 +105,11 @@ from verifiers.v1.trace import (
     Branch,
     Error,
     EvalRunInfo,
+    GenerationSpan,
     ModelCall,
     RunInfo,
     TimeSpan,
+    TimeSplit,
     Timing,
     Trace,
     TraceTask,
@@ -184,6 +186,8 @@ __all__ = [
     "KeptTokens",
     "Timing",
     "TimeSpan",
+    "TimeSplit",
+    "GenerationSpan",
     "Error",
     # decorators
     "stop",

{verifiers-0.2.1.dev59 → verifiers-0.2.1.dev61}/verifiers/v1/cli/dashboard/eval.py

@@ -300,6 +300,7 @@ def _breakdown(done: list[Trace]) -> Table | None:
     have_cost = have_cached = have_reasoning = have_judge = False
     phase_secs: dict[str, float] = {}
     phase_count: dict[str, int] = {}
+    model_secs = harness_secs = 0.0
     for trace in done:
         prompt, completion, cached, reasoning, _ = _tokens(trace)
         total_in += prompt
@@ -326,6 +327,8 @@ def _breakdown(done: list[Trace]) -> Table | None:
             if span.end:  # phase was timed for this rollout
                 phase_secs[phase] = phase_secs.get(phase, 0.0) + span.duration
                 phase_count[phase] = phase_count.get(phase, 0) + 1
+        model_secs += trace.timing.generation.model.duration
+        harness_secs += trace.timing.generation.harness.duration
     if (
         total_in
         or total_out
@@ -353,11 +356,18 @@ def _breakdown(done: list[Trace]) -> Table | None:
                 cost += f" + {format_cost_usd(total_judge_cost)} judge"
             usage.append(cost)
         grid.add_row("usage", "  ·  ".join(usage))
-    time_segments = [
-        f"{phase} {format_time(phase_secs[phase] / phase_count[phase])}"
-        for phase in ("boot", "setup", "generation", "finalize", "scoring")
-        if phase_count.get(phase)
-    ]
+    time_segments = []
+    for phase in ("boot", "setup", "generation", "finalize", "scoring"):
+        count = phase_count.get(phase)
+        if not count:
+            continue
+        segment = f"{phase} {format_time(phase_secs[phase] / count)}"
+        if phase == "generation":
+            segment += (
+                f" (model {format_time(model_secs / count)}"
+                f" + harness {format_time(harness_secs / count)})"
+            )
+        time_segments.append(segment)
     if time_segments:
         grid.add_row("time", "  ·  ".join(time_segments))
     return grid if grid.row_count else None

{verifiers-0.2.1.dev59 → verifiers-0.2.1.dev61}/verifiers/v1/cli/debug.py

@@ -243,6 +243,7 @@ async def debug_task(task: Task, config: DebugConfig) -> tuple[Trace, bool]:
     except Exception as e:
         record_debug_error(trace, debug, e, setup_timeout, config.timeout.total)
     finally:
+        trace.split_generation()
         trace.info["debug"] = debug
         try:
             await runtime.stop()

{verifiers-0.2.1.dev59 → verifiers-0.2.1.dev61}/verifiers/v1/cli/init.py

@@ -235,7 +235,7 @@ A v1 verifiers environment, scaffolded with `init`.
 
 ```bash
 uv pip install -e .        # install this package (or register it in your project)
-uv run eval {dash} -n 3    # evaluate a few tasks with the default harness
+uv run eval {dash} -n 3    # evaluate a few tasks with the bash harness
 ```
 
 ## Layout

{verifiers-0.2.1.dev59 → verifiers-0.2.1.dev61}/verifiers/v1/dialects/chat.py

@@ -125,7 +125,7 @@ def parse_tools(raw: list[dict] | None) -> list[Tool] | None:
 
 
 # --- vf -> chat wire ----------------------------------------------------------
-# `message_to_wire` (chat-only): used by `extend` (user-sim turn injection), the default harness
+# `message_to_wire` (chat-only): used by `extend` (user-sim turn injection), the bash harness
 # (a Messages prompt), and the train client (its generate request). The proxy preserves its parsed
 # native JSON independently and does not use this serializer.
 

{verifiers-0.2.1.dev59 → verifiers-0.2.1.dev61}/verifiers/v1/env.py

@@ -87,11 +87,11 @@ def pool_serve_kwargs(pool: StaticPoolConfig | ElasticPoolConfig) -> dict:
 class EnvConfig(BaseConfig):
     """The taskset that loads tasks and the harness that runs them."""
 
-    # SerializeAsAny: these hold resolved subclasses (e.g. MathConfig, DefaultHarnessConfig);
+    # SerializeAsAny: these hold resolved subclasses (e.g. MathConfig, BashHarnessConfig);
     # without it model_dump() narrows to the base type and drops the subclass fields, so the
     # env-server subconfig the orchestrator writes would lose taskset/harness-specific knobs.
     taskset: SerializeAsAny[TasksetConfig] = TasksetConfig()
-    harness: SerializeAsAny[HarnessConfig] = HarnessConfig(id="default")
+    harness: SerializeAsAny[HarnessConfig] = HarnessConfig(id="bash")
     timeout: TimeoutConfig = TimeoutConfig()
     retries: RetryConfig = RetryConfig()
     max_turns: int | None = None

{verifiers-0.2.1.dev59 → verifiers-0.2.1.dev61}/verifiers/v1/harness.py

@@ -31,7 +31,7 @@ logger = logging.getLogger(__name__)
 
 
 class HarnessConfig(BaseConfig):
-    id: ID = "default"
+    id: ID = "bash"
     """Local package or Hub `org/name[@version]`, set through `--harness.id`."""
     runtime: RuntimeConfig = SubprocessConfig()
     """Runtime for the harness program; tool servers choose their placement separately."""

{verifiers-0.2.1.dev59 → verifiers-0.2.1.dev61}/verifiers/v1/harnesses/__init__.py

@@ -1,9 +1,9 @@
+from verifiers.v1.harnesses.bash import BashHarness, BashHarnessConfig
 from verifiers.v1.harnesses.claude_code import (
     ClaudeCodeHarness,
     ClaudeCodeHarnessConfig,
 )
 from verifiers.v1.harnesses.codex import CodexHarness, CodexHarnessConfig
-from verifiers.v1.harnesses.default import DefaultHarness, DefaultHarnessConfig
 from verifiers.v1.harnesses.kimi_code import KimiCodeHarness, KimiCodeHarnessConfig
 from verifiers.v1.harnesses.mini_swe_agent import (
     MiniSWEAgentHarness,
@@ -15,12 +15,12 @@ from verifiers.v1.harnesses.rlm import RLMHarness, RLMHarnessConfig
 from verifiers.v1.harnesses.terminus_2 import Terminus2Harness, Terminus2HarnessConfig
 
 __all__ = [
+    "BashHarness",
+    "BashHarnessConfig",
     "ClaudeCodeHarness",
     "ClaudeCodeHarnessConfig",
     "CodexHarness",
     "CodexHarnessConfig",
-    "DefaultHarness",
-    "DefaultHarnessConfig",
     "KimiCodeHarness",
     "KimiCodeHarnessConfig",
     "NullHarness",

verifiers-0.2.1.dev61/verifiers/v1/harnesses/bash/__init__.py

@@ -0,0 +1,6 @@
+from verifiers.v1.harnesses.bash.harness import (
+    BashHarness,
+    BashHarnessConfig,
+)
+
+__all__ = ["BashHarness", "BashHarnessConfig"]

{verifiers-0.2.1.dev59/verifiers/v1/harnesses/default → verifiers-0.2.1.dev61/verifiers/v1/harnesses/bash}/harness.py

@@ -25,7 +25,7 @@ SEARCH_PROMPT = (
 )
 
 
-class DefaultHarnessConfig(HarnessConfig):
+class BashHarnessConfig(HarnessConfig):
     edit: bool = True
     """Offer the local `edit` tool (single-occurrence string replacement in a file) alongside
     `bash`. On by default; set `--harness.edit false` for a bash-only agent."""
@@ -36,7 +36,7 @@ class DefaultHarnessConfig(HarnessConfig):
     the agent's `bash` subprocesses don't inherit it."""
 
 
-class DefaultHarness(Harness[DefaultHarnessConfig]):
+class BashHarness(Harness[BashHarnessConfig]):
     APPENDS_SYSTEM_PROMPT = True
     SUPPORTS_MCP = True
     SUPPORTS_USER_SIM = True
@@ -86,7 +86,7 @@ class DefaultHarness(Harness[DefaultHarnessConfig]):
                 serper_key = os.environ.get("SERPER_API_KEY")
             if not serper_key:
                 raise ValueError(
-                    "default search=true requires SERPER_API_KEY in the eval environment "
+                    "bash search=true requires SERPER_API_KEY in the eval environment "
                     "(the host env or --harness.env)"
                 )
             args += ["--search", f"--serper-key={serper_key}"]

{verifiers-0.2.1.dev59 → verifiers-0.2.1.dev61}/verifiers/v1/legacy.py

@@ -32,7 +32,16 @@ from verifiers.v1.serve.types import (
 )
 from verifiers.v1.task import WireTaskData
 from verifiers.v1 import graph
-from verifiers.v1.trace import Error, ModelCall, TimeSpan, Timing, Trace, TraceTask
+from verifiers.v1.trace import (
+    Error,
+    GenerationSpan,
+    ModelCall,
+    TimeSpan,
+    TimeSplit,
+    Timing,
+    Trace,
+    TraceTask,
+)
 from verifiers.v1.types import (
     AssistantMessage,
     Response,
@@ -185,7 +194,8 @@ def _to_v1_tokens(raw: Any) -> TurnTokens | None:
 
 def _timing(raw: Any) -> Timing:
     """Map the v0 timing record's generation/scoring durations onto a v1 ``Timing``
-    (we only have durations, so each span is encoded as start=0, end=duration)."""
+    (we only have durations, so each span is encoded as start=0, end=duration).
+    v0's per-turn ``model``/``env`` span collections carry the generation split."""
 
     def _dur(node: Any) -> float:
         if isinstance(node, dict):
@@ -198,7 +208,12 @@ def _timing(raw: Any) -> Timing:
 
     raw = raw or {}
     return Timing(
-        generation=TimeSpan(start=0.0, end=_dur(raw.get("generation"))),
+        generation=GenerationSpan(
+            start=0.0,
+            end=_dur(raw.get("generation")),
+            model=TimeSplit(duration=_dur(raw.get("model"))),
+            harness=TimeSplit(duration=_dur(raw.get("env"))),
+        ),
         scoring=TimeSpan(start=0.0, end=_dur(raw.get("scoring"))),
     )
 

{verifiers-0.2.1.dev59 → verifiers-0.2.1.dev61}/verifiers/v1/loaders.py

@@ -98,12 +98,12 @@ def judge_class(judge_id: str) -> type[Judge]:
 
 def default_harness_id(taskset_id: str) -> str:
     if not taskset_id:
-        return "default"
+        return "bash"
     try:
         module = import_taskset(taskset_id)
         _plugin_class(module, Harness, "harness")
     except (ModuleNotFoundError, TypeError, AttributeError):
-        return "default"
+        return "bash"
     return taskset_id
 
 

{verifiers-0.2.1.dev59 → verifiers-0.2.1.dev61}/verifiers/v1/rollout.py

@@ -249,6 +249,7 @@ class Rollout:
             ):
                 if span.start and not span.end:
                     span.end = now
+            trace.split_generation()
             try:
                 await runtime.stop()
             except Exception:

{verifiers-0.2.1.dev59 → verifiers-0.2.1.dev61}/verifiers/v1/trace.py

@@ -49,11 +49,30 @@ class TimeSpan(StrictBaseModel):
         return max(0.0, self.end - self.start) if self.end else 0.0
 
 
+class TimeSplit(StrictBaseModel):
+    """A span's share attributed to one side of a split: disjoint sub-intervals summed
+    into a duration, so there is no single start/end. Serialized, unlike a span's
+    derived `duration`, since it cannot be recomputed from two timestamps."""
+
+    duration: float = 0.0
+
+
+class GenerationSpan(TimeSpan):
+    """The generation span plus its split into time inside model calls (`model`) vs.
+    outside them (`harness`: harness logic, tools, user simulation). Stamped from the
+    recorded model calls' spans by `Trace.split_generation` when the span closes, with
+    `model.duration + harness.duration == duration` — concurrent calls (subagent forks)
+    are clamped to the span, saturating the model share."""
+
+    model: TimeSplit = Field(default_factory=TimeSplit)
+    harness: TimeSplit = Field(default_factory=TimeSplit)
+
+
 class Timing(StrictBaseModel):
     start: float = Field(default_factory=time.time)
     boot: TimeSpan = Field(default_factory=TimeSpan)
     setup: TimeSpan = Field(default_factory=TimeSpan)
-    generation: TimeSpan = Field(default_factory=TimeSpan)
+    generation: GenerationSpan = Field(default_factory=GenerationSpan)
     finalize: TimeSpan = Field(default_factory=TimeSpan)
     scoring: TimeSpan = Field(default_factory=TimeSpan)
 
@@ -537,6 +556,18 @@ class Trace(StrictBaseModel, Generic[DataT, StateT]):
         if self.stop_condition is None:
             self.stop_condition = condition
 
+    def split_generation(self) -> None:
+        """Stamp the closed generation span's model/harness split: model time is the
+        sum of the recorded calls' spans (clamped to the span), harness the complement.
+        Every path that closes the span calls this — a span without model calls (e.g.
+        a debug action) is all harness time."""
+        gen = self.timing.generation
+        if not gen.end:
+            return
+        model = sum(call.time.duration for call in self.calls)
+        gen.model.duration = min(model, gen.duration)
+        gen.harness.duration = gen.duration - gen.model.duration
+
     def capture_error(self, error: Exception) -> None:
         self.errors.append(
             Error(

verifiers-0.2.1.dev59/verifiers/v1/harnesses/default/__init__.py

@@ -1,6 +0,0 @@
-from verifiers.v1.harnesses.default.harness import (
-    DefaultHarness,
-    DefaultHarnessConfig,
-)
-
-__all__ = ["DefaultHarness", "DefaultHarnessConfig"]