verifiers 0.2.2.dev48__tar.gz → 0.2.2.dev50__tar.gz

{verifiers-0.2.2.dev48 → verifiers-0.2.2.dev50}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: verifiers
-Version: 0.2.2.dev48
+Version: 0.2.2.dev50
 Summary: Verifiers: Environments for LLM Reinforcement Learning
 Project-URL: Homepage, https://github.com/primeintellect-ai/verifiers
 Project-URL: Documentation, https://github.com/primeintellect-ai/verifiers

{verifiers-0.2.2.dev48 → verifiers-0.2.2.dev50}/pyproject.toml

@@ -136,8 +136,10 @@ kuhn-poker-v1 = { path = "environments/kuhn_poker_v1", editable = true }
 
 [tool.uv.exclude-newer-package]
 # Bounded cutoffs for the explicitly requested tool upgrades.
-ruff = "2026-07-27"
-ty = "2026-07-27"
+# Full UTC timestamps: bare dates resolve to local-tz midnight and churn the
+# lockfile across timezones.
+ruff = "2026-07-28T00:00:00Z"
+ty = "2026-07-28T00:00:00Z"
 # PrimeIntellect-published on PyPI (trusted publisher)
 prime-tunnel = false
 prime-sandboxes = false

{verifiers-0.2.2.dev48 → verifiers-0.2.2.dev50}/tests/v1/test_e2e.py

@@ -229,7 +229,7 @@ async def test_tool(run_v1, harness_runtime, tool_runtime, tmp_path):
     assert trace.reward == 1.0
     # The interception server captured the advertised tools onto the trace (for tool-use SFT):
     # the null harness offered the task's MCP tool as `echo_back`, schema included.
-    assert trace.tools is not None
+    assert trace.tools
     (echo_tool,) = [t for t in trace.tools if t.name == "echo_back"]
     assert "message" in echo_tool.parameters.get("properties", {})
 
@@ -355,9 +355,9 @@ async def test_multi_agent_env(run_v1, tmp_path):
         max_turns=2,
     )
     assert len(traces) == 2  # one episode, one trace per role
-    assert sorted(t.agent_name for t in traces) == ["a", "b"]
-    (b,) = [t for t in traces if t.agent_name == "b"]
-    assert b.trainable is False
+    assert sorted(t.agent.name for t in traces) == ["a", "b"]
+    (b,) = [t for t in traces if t.agent.name == "b"]
+    assert b.agent.trainable is False
     for trace in traces:
         assert trace.ok
         assert trace.reward == 1.0  # each seat's own task reward
@@ -385,7 +385,7 @@ async def test_env_id_best_of_n(run_v1, tmp_path):
         max_turns=2,
     )
     assert len(traces) == 2  # one episode, two attempts
-    assert all(t.agent_name == "agent" and t.ok for t in traces)
+    assert all(t.agent.name == "agent" and t.ok for t in traces)
     assert any(t.metrics["best"] == 1.0 for t in traces)
     assert all(t.metrics["pass_at_n"] == 1.0 for t in traces)  # echo always passes
 
@@ -401,6 +401,8 @@ async def test_env_id_agentic_judge(run_v1, tmp_path):
     model's call. Exercises the config surface too: a policy-only prompt
     override (the verdict contract is appended regardless) and
     reward-composition weights."""
+    policy = tmp_path / "judge_policy.txt"
+    policy.write_text("Check EMPIRICALLY that the agent echoed the word back.")
     traces = await run_v1(
         "echo-v1",
         harness=None,  # seats pin their own harness; there is no run-level one
@@ -414,20 +416,18 @@ async def test_env_id_agentic_judge(run_v1, tmp_path):
                 "harness": {"id": "bash"},
                 "max_output_tokens": 8192,
             },
-            "task": {
-                "prompt": "Check EMPIRICALLY that the agent echoed the word back.",
-            },
+            "task": {"prompt": str(policy)},
             "score": {"task_weight": 0.5},
         },
         output_dir=tmp_path,
         max_turns=10,
         rollout_timeout=600,
     )
-    assert sorted(t.agent_name for t in traces) == ["judge", "solver"]
-    (solver,) = [t for t in traces if t.agent_name == "solver"]
-    (judge,) = [t for t in traces if t.agent_name == "judge"]
+    assert sorted(t.agent.name for t in traces) == ["judge", "solver"]
+    (solver,) = [t for t in traces if t.agent.name == "solver"]
+    (judge,) = [t for t in traces if t.agent.name == "judge"]
     assert solver.ok and judge.ok
-    assert judge.trainable is False
+    assert judge.agent.trainable is False
     # The task's own reward keeps its raw score; the rescale lands on the weight.
     assert solver.rewards["echoed"].score == 1.0
     assert solver.rewards["echoed"].weight == 0.5
@@ -448,11 +448,11 @@ async def test_env_id_user_sim(run_v1, tmp_path):
         max_turns=6,
         rollout_timeout=300,
     )
-    assert sorted(t.agent_name for t in traces) == ["assistant", "user"]
-    (assistant,) = [t for t in traces if t.agent_name == "assistant"]
-    (user,) = [t for t in traces if t.agent_name == "user"]
+    assert sorted(t.agent.name for t in traces) == ["assistant", "user"]
+    (assistant,) = [t for t in traces if t.agent.name == "assistant"]
+    (user,) = [t for t in traces if t.agent.name == "user"]
     assert assistant.ok and user.ok
-    assert user.trainable is False
+    assert user.agent.trainable is False
     assert user.num_turns >= 1  # the modeled user actually spoke
     assert assistant.metrics["user_turns"] >= 1
     # `mask_prompt`: the scenario is hidden from the assistant's harness (the run's
@@ -465,7 +465,7 @@ async def test_env_id_user_sim(run_v1, tmp_path):
     from verifiers.v1.trace import WireTrace
 
     (record,) = read_episodes(tmp_path, WireTrace)
-    assert {t.agent_name for t in record.traces} == {"assistant", "user"}
+    assert {t.agent.name for t in record.traces} == {"assistant", "user"}
     assert record.id  # both traces are persisted under one durable episode identity
 
 
@@ -488,14 +488,14 @@ async def test_env_id_user_sim_with_tools(run_v1, tmp_path):
         max_tokens=8192,
         rollout_timeout=300,
     )
-    (assistant,) = [t for t in traces if t.agent_name == "assistant"]
-    (user,) = [t for t in traces if t.agent_name == "user"]
+    (assistant,) = [t for t in traces if t.agent.name == "assistant"]
+    (user,) = [t for t in traces if t.agent.name == "user"]
     assert assistant.ok and user.ok
     assert assistant.task.data.prompt is None  # the scenario stayed off the wire
     assert user.num_turns >= 1  # the modeled user actually drove the exchange
     assert assistant.rewards["echoed"].score == 1.0  # the tool ran, mid-conversation
     # The tool was advertised to the masked chat exactly as to any run.
-    assert assistant.tools is not None
+    assert assistant.tools
     assert any(tool.name == "echo_back" for tool in assistant.tools)
 
 
@@ -514,8 +514,8 @@ async def test_kuhn_poker_self_play(run_v1, tmp_path):
         max_tokens=8192,
         rollout_timeout=300,
     )
-    assert sorted(t.agent_name for t in traces) == ["player0", "player1"]
-    payoffs = {t.agent_name: t.rewards["payoff"].score for t in traces}
+    assert sorted(t.agent.name for t in traces) == ["player0", "player1"]
+    payoffs = {t.agent.name: t.rewards["payoff"].score for t in traces}
     assert payoffs["player0"] + payoffs["player1"] == 0  # zero-sum
     assert abs(payoffs["player0"]) in (1.0, 2.0)
     for trace in traces:
@@ -542,7 +542,7 @@ async def test_multi_agent_env_server(run_v1_server, tmp_path):
         max_turns=2,
     )
     assert len(traces) == 2
-    assert sorted(t.agent_name for t in traces) == ["a", "b"]
+    assert sorted(t.agent.name for t in traces) == ["a", "b"]
     for trace in traces:
         assert trace.ok
         assert trace.metrics["duet"] == 1.0

{verifiers-0.2.2.dev48 → verifiers-0.2.2.dev50}/tests/v1/test_graph.py

@@ -40,7 +40,8 @@ def test_routed_experts_attributed_and_aligned_across_turns():
     concatenates back to a `[tokens, layers, top_k]` array aligned 1:1 with `branch.token_ids` —
     and survives the base64 wire round-trip."""
     trace = vf.Trace(
-        task=vf.TraceTask(type="Task", data=vf.TaskData(idx=0, prompt="x"))
+        agent=vf.AgentInfo(config=vf.AgentConfig()),
+        task=vf.TraceTask(type="Task", data=vf.TaskData(idx=0, prompt="x")),
     )
     user = vf.UserMessage(content="u1")
     graph.prepare_turn(trace, [user]).commit(
@@ -94,7 +95,8 @@ def test_routed_experts_none_when_absent():
     """No routing captured (engine ran without `enable_return_routed_experts`) -> the branch
     reports None and the trainer simply skips replay."""
     trace = vf.Trace(
-        task=vf.TraceTask(type="Task", data=vf.TaskData(idx=0, prompt="x"))
+        agent=vf.AgentInfo(config=vf.AgentConfig()),
+        task=vf.TraceTask(type="Task", data=vf.TaskData(idx=0, prompt="x")),
     )
     graph.prepare_turn(trace, [vf.UserMessage(content="u1")]).commit(
         vf.Response(
@@ -128,7 +130,10 @@ def test_tool_call_hash_matches_v0_content_and_arguments_normalization():
 
 def test_reasoning_content_participates_in_graph_prefix_matching():
     task = vf.TaskData(idx=0, prompt="use a tool")
-    trace = vf.Trace(task=vf.TraceTask(type="Task", data=task))
+    trace = vf.Trace(
+        agent=vf.AgentInfo(config=vf.AgentConfig()),
+        task=vf.TraceTask(type="Task", data=task),
+    )
     user = vf.UserMessage(content="use a tool")
     call = vf.ToolCall(id="call_0", name="lookup", arguments="{}")
 
@@ -205,7 +210,8 @@ def test_renderer_level_break_forks_by_token_id():
 
     # Control: the prior turn re-renders to the same tokens -> stays one linear branch.
     linear = vf.Trace(
-        task=vf.TraceTask(type="Task", data=vf.TaskData(idx=0, prompt="x"))
+        agent=vf.AgentInfo(config=vf.AgentConfig()),
+        task=vf.TraceTask(type="Task", data=vf.TaskData(idx=0, prompt="x")),
     )
     first_turn(linear)
     second_turn(linear, [1, 2, 3, 4, 5, 6, 7])
@@ -214,7 +220,8 @@ def test_renderer_level_break_forks_by_token_id():
 
     # Break: the assistant turn retokenizes (4 -> 99), so prompt_ids diverge at that node.
     broken = vf.Trace(
-        task=vf.TraceTask(type="Task", data=vf.TaskData(idx=0, prompt="x"))
+        agent=vf.AgentInfo(config=vf.AgentConfig()),
+        task=vf.TraceTask(type="Task", data=vf.TaskData(idx=0, prompt="x")),
     )
     first_turn(broken)
     second_turn(broken, [1, 2, 3, 99, 5, 6, 7])
@@ -227,7 +234,10 @@ def test_renderer_level_break_forks_by_token_id():
 
 def test_prompt_supplied_assistant_messages_are_not_sampled_turns():
     task = vf.TaskData(idx=0, prompt="few-shot")
-    trace = vf.Trace(task=vf.TraceTask(type="Task", data=task))
+    trace = vf.Trace(
+        agent=vf.AgentInfo(config=vf.AgentConfig()),
+        task=vf.TraceTask(type="Task", data=task),
+    )
     fabricated = vf.AssistantMessage(
         content=None,
         tool_calls=[vf.ToolCall(id="call_0", name="lookup", arguments="{}")],

{verifiers-0.2.2.dev48 → verifiers-0.2.2.dev50}/tests/v1/test_judges.py

@@ -36,6 +36,7 @@ def make_trace(
     task_cls: type[QAData] = QAData,
 ) -> vf.Trace:
     return vf.Trace(
+        agent=vf.AgentInfo(config=vf.AgentConfig()),
         task=vf.TraceTask(
             type="Task",
             data=task_cls(idx=0, prompt="Capital of France?", answer=answer),
@@ -244,6 +245,7 @@ async def test_reference_score_messages_prompt(fake_judge_model):
         answer="Paris",
     )
     trace = vf.Trace(
+        agent=vf.AgentInfo(config=vf.AgentConfig()),
         task=vf.TraceTask(type="Task", data=task),
         nodes=[
             MessageNode(parent=None, message=UserMessage(content="q"), sampled=False),
@@ -269,6 +271,7 @@ async def test_reference_question_field(fake_judge_model):
         answer="Paris",
     )
     trace = vf.Trace(
+        agent=vf.AgentInfo(config=vf.AgentConfig()),
         task=vf.TraceTask(type="Task", data=task),
         nodes=[
             MessageNode(parent=None, message=UserMessage(content="q"), sampled=False),
@@ -293,6 +296,7 @@ def full_trace_fixture() -> vf.Trace:
     from verifiers.v1.types import ToolCall, ToolMessage
 
     return vf.Trace(
+        agent=vf.AgentInfo(config=vf.AgentConfig()),
         task=vf.TraceTask(
             type="Task", data=QAData(idx=0, prompt="Capital of France?", answer="Paris")
         ),
@@ -367,32 +371,20 @@ async def test_rubric_view_full_trace(tmp_path, fake_judge_model):
     assert all("SECRET REASONING" not in prompt for prompt in fake_judge_model)
 
 
-async def test_config_prompt_overrides_class_template(fake_judge_model):
-    judge = vf.ReferenceJudge(
-        vf.ReferenceJudgeConfig(prompt="Q:{question} A:{answer} R:{response}")
-    )
+async def test_config_prompt_overrides_class_template(tmp_path, fake_judge_model):
+    # Config `prompt` is a file path; the same {field} placeholders work.
+    file = tmp_path / "judge.txt"
+    file.write_text("Q:{question} A:{answer} R:{response}")
+    judge = vf.ReferenceJudge(vf.ReferenceJudgeConfig(prompt=file))
     assert judge.build_messages(question="q", answer="a", response="r") == "Q:q A:a R:r"
     # A template needn't use every evaluate field: score also passes {positive}/{negative},
     # which str.format ignores when the (custom) prompt doesn't reference them.
     trace = make_trace()
     assert await judge.score(trace.task.data, trace) == 1.0
     assert fake_judge_model[0] == "Q:Capital of France? A:Paris R:It is Paris."
-
-
-async def test_prompt_file(tmp_path, fake_judge_model):
-    # The prompt template can come from a file; the same {field} placeholders work.
-    file = tmp_path / "judge.txt"
-    file.write_text("Q:{question} A:{answer} R:{response}")
-    trace = make_trace()
-    judge = vf.ReferenceJudge(vf.ReferenceJudgeConfig(prompt_file=file))
-    assert await judge.score(trace.task.data, trace) == 1.0
-    assert fake_judge_model[0] == "Q:Capital of France? A:Paris R:It is Paris."
     # a bad path fails at judge construction, not mid-eval at score time
     with pytest.raises(FileNotFoundError):
-        vf.ReferenceJudge(vf.ReferenceJudgeConfig(prompt_file=tmp_path / "missing.txt"))
-    # inline and file prompts are mutually exclusive
-    with pytest.raises(ValueError, match="not both"):
-        vf.ReferenceJudgeConfig(prompt="inline", prompt_file=file)
+        vf.ReferenceJudge(vf.ReferenceJudgeConfig(prompt=tmp_path / "missing.txt"))
 
 
 # --- reference input/verdict knobs ------------------------------------------------------------

{verifiers-0.2.2.dev48 → verifiers-0.2.2.dev50}/tests/v1/test_trace.py

@@ -21,7 +21,8 @@ class MyState(vf.State):
 def test_bare_trace_round_trip():
     # The minimal trace: a base task, no nodes, no extras — dump and back into a plain Trace.
     tr = vf.Trace(
-        task=vf.TraceTask(type="Task", data=vf.TaskData(idx=3, prompt="hello"))
+        agent=vf.AgentInfo(config=vf.AgentConfig()),
+        task=vf.TraceTask(type="Task", data=vf.TaskData(idx=3, prompt="hello")),
     )
     rt = vf.Trace.model_validate(tr.model_dump())
     assert rt.id == tr.id
@@ -35,6 +36,7 @@ def test_custom_task_state_round_trip():
     # Custom data and state round-trip into the same parameterization. Data fields are
     # typed (not just `model_extra`); `state` is runtime-only and never crosses the wire.
     tr = vf.Trace[MyTask, MyState](
+        agent=vf.AgentInfo(config=vf.AgentConfig()),
         task=vf.TraceTask(type="MyTask", data=MyTask(idx=0, prompt="q", answer="gold")),
         state=MyState(score=7),
         nodes=[
@@ -59,6 +61,7 @@ def test_wire_trace_round_trip():
     # Two leaves off one root → 2 branches (a compaction-shaped trace), so the round-trip has to
     # carry node `parent` links for `num_branches` to survive.
     tr = vf.Trace[MyTask, vf.State](
+        agent=vf.AgentInfo(config=vf.AgentConfig()),
         task=vf.TraceTask(type="MyTask", data=MyTask(idx=0, prompt="q", answer="a")),
         tools=[vf.Tool(name="echo", description="", parameters={"type": "object"})],
         nodes=[

{verifiers-0.2.2.dev48 → verifiers-0.2.2.dev50}/verifiers/v1/agent.py

@@ -45,7 +45,7 @@ from verifiers.v1.types import (
     UserMessage,
 )
 from verifiers.v1.utils.compile import (
-    cap_remote_harness_timeout,
+    cap_remote_agent_timeout,
     resolve_runtime_config,
     validate_pairing,
 )
@@ -390,7 +390,7 @@ class Agent:
                 attempt + 1,
                 retry.max_retries,
                 delay,
-                trace.error.type if trace.error else "?",
+                trace.last_error.type if trace.last_error else "?",
             )
             await asyncio.sleep(delay)
         if history:
@@ -419,8 +419,8 @@ class Agent:
             # close() never runs — free the run's servers and owned runtime first.
             await run.abort()
             raise
-        if trace.runtime is not None:
-            trace.runtime.borrowed = runtime is not None
+        if trace.agent.runtime is not None:
+            trace.agent.runtime.borrowed = runtime is not None
         return trace
 
     @asynccontextmanager
@@ -482,8 +482,8 @@ class Agent:
             opened = await run.open()
             if not opened:
                 trace = await run.close()
-                if trace.runtime is not None:
-                    trace.runtime.borrowed = runtime is not None
+                if trace.agent.runtime is not None:
+                    trace.agent.runtime.borrowed = runtime is not None
         if not opened:
             failure = run.failure
             if failure is None:  # `open()` returning False always captures one.
@@ -499,8 +499,8 @@ class Agent:
             raise
         finally:
             trace = run.trace if run.closed else await interaction.close()
-            if trace.runtime is not None:
-                trace.runtime.borrowed = runtime is not None
+            if trace.agent.runtime is not None:
+                trace.agent.runtime.borrowed = runtime is not None
 
     def _rollout_params(
         self, task: Task, runtime: Runtime | None, shared_tools: dict
@@ -520,10 +520,10 @@ class Agent:
             self.harness, type(task), runtime_config, shared_tools=shared_tools
         )
         # Timeout precedence: agent-level wins, else the task's, else no limit.
-        harness_timeout = (
+        agent_timeout = (
             self.timeout.rollout
             if self.timeout.rollout is not None
-            else task.data.timeout.harness
+            else task.data.timeout.agent
         )
         return {
             "agent_config": self.config,
@@ -535,8 +535,8 @@ class Agent:
                 if self.timeout.setup is not None
                 else task.data.timeout.setup
             ),
-            "harness_timeout": cap_remote_harness_timeout(
-                harness_timeout, runtime_config, task
+            "agent_timeout": cap_remote_agent_timeout(
+                agent_timeout, runtime_config, task
             ),
             "finalize_timeout": (
                 self.timeout.finalize

{verifiers-0.2.2.dev48 → verifiers-0.2.2.dev50}/verifiers/v1/cli/dashboard/eval.py

@@ -302,7 +302,7 @@ def Progress(
     # run, a modeled user) are `trainable=False` and carry no rewards, so counting
     # them dilutes every mean with structural zeros. An all-untrainable run (every
     # role frozen) falls back to all traces rather than showing nothing.
-    scored = [t for t in done_traces if t.trainable] or done_traces
+    scored = [t for t in done_traces if t.agent.trainable] or done_traces
     total = len(slots)
     # Headline reward = mean over non-errored traces; when any errored, `format_mean` appends
     # the global avg (errored count as 0) in parens. `err` is the share of episodes that
@@ -367,13 +367,13 @@ def _breakdown(scored: list[Trace], done: list[Trace]) -> Table | None:
     # show); usage/time below still cover errored rollouts (their resources were spent regardless).
     has_clean = any(not t.has_error for t in done)
     score_rows = (("rewards", "rewards"), ("metrics", "metrics")) if has_clean else ()
-    by_agent: dict[str | None, list[Trace]] = {}
+    by_agent: dict[str, list[Trace]] = {}
     for trace in done:
-        by_agent.setdefault(trace.agent_name, []).append(trace)
+        by_agent.setdefault(trace.agent.name, []).append(trace)
     for label, source in score_rows:
         if len(by_agent) > 1:
             segments = [
-                f"[dim]{name or '—'}:[/dim] {means}"
+                f"[dim]{name}:[/dim] {means}"
                 for name, traces in by_agent.items()
                 if (means := _score_segments(traces, source)) is not None
             ]
@@ -505,7 +505,7 @@ def _stage(trace: Trace) -> str:
         stage = "boot"  # trace minted, first span not yet opened (an instant)
     # A boot stuck on a first-use platform image build reads differently from a
     # normal boot — it can sit there for ~10 minutes (prime runtime only).
-    if stage == "boot" and getattr(trace.runtime, "image_cached", None) is False:
+    if stage == "boot" and getattr(trace.agent.runtime, "image_cached", None) is False:
         return "build"
     return stage
 
@@ -564,14 +564,14 @@ def Rows(groups: list[list[RunSlot]], now: float, runtime_type: str) -> Table:
                     group_rows.append(("pending", [f"task {base}", *[""] * 7], "", ""))
                 continue
             for t in slot.traces:
-                label = f"{base} agent={t.agent_name}" if t.agent_name else base
+                label = f"{base} agent={t.agent.name}"
                 if slot.done:  # fully scored — reward is final
                     state = "error" if t.has_error else "success"
                     # A trace that recorded nothing shows no reward: a judge or
                     # modeled-user seat's `reward=0.00` would read as a score.
                     result = (
-                        t.error.type
-                        if t.has_error
+                        t.last_error.type
+                        if t.has_error and t.last_error
                         else (f"reward={t.reward:.2f}" if t.rewards else "")
                     )
                     if t.has_error:
@@ -582,7 +582,7 @@ def Rows(groups: list[list[RunSlot]], now: float, runtime_type: str) -> Table:
                             t.is_truncated
                         ):  # flag a clipped rollout next to its stop condition
                             stop = f"{stop} (truncated)".strip()
-                elif t.is_completed and (err := t.error) is not None:
+                elif t.is_completed and (err := t.last_error) is not None:
                     # An errored trace whose episode is still running its other
                     # traces (or `score()`) is already a failure — show it, don't
                     # let it sit as "scoring" until the whole episode lands.
@@ -592,12 +592,9 @@ def Rows(groups: list[list[RunSlot]], now: float, runtime_type: str) -> Table:
                 # The trace's own stamp, not the run-level runtime: a role's harness
                 # may resolve elsewhere (the judge env's sandboxed judge on a
                 # subprocess run).
-                if t.runtime is not None:
-                    runtime = (
-                        f"{t.runtime.type}({t.runtime.id})"
-                        if t.runtime.id
-                        else t.runtime.type
-                    )
+                if t.agent.runtime is not None:
+                    rt = t.agent.runtime
+                    runtime = f"{rt.type}({rt.id})" if rt.id else rt.type
                 else:
                     runtime = runtime_type
                 turns = t.num_turns
@@ -635,7 +632,7 @@ def Rows(groups: list[list[RunSlot]], now: float, runtime_type: str) -> Table:
                     f"{nbranches} branch{'es' * (nbranches != 1)}",
                     tokens,
                     f"{format_cost_usd(cost)}" if cost is not None else "",
-                    stop,  # stop condition (agent_completed / max_turns / harness_timeout), once done
+                    stop,  # stop condition (agent_completed / max_turns / error), once done
                 ]
                 # No start time yet (queued, not generating) → blank, not `now - 0` (~56 years).
                 elapsed = format_time(end - start) if start else ""

{verifiers-0.2.2.dev48 → verifiers-0.2.2.dev50}/verifiers/v1/cli/debug.py

@@ -101,9 +101,9 @@ def error_info(
 
 
 def capture_trace_error(trace: Trace, error: BaseException) -> None:
-    # CancelledError is a BaseException; Trace.capture_error accepts Exception.
+    # CancelledError is a BaseException; Trace.record_error accepts Exception.
     if isinstance(error, Exception):
-        trace.capture_error(error)
+        trace.record_error(error)
         return
     trace.errors.append(
         Error(

{verifiers-0.2.2.dev48 → verifiers-0.2.2.dev50}/verifiers/v1/cli/eval/runner.py

@@ -68,7 +68,7 @@ async def run_eval(env: Env, config: EvalConfig) -> list[Episode]:
 
     async def on_complete(episode: Episode) -> None:
         for trace in episode.traces:
-            trace.stamp(EvalRunInfo(id=config.uuid))
+            trace.record_run(EvalRunInfo(id=config.uuid))
         await append_episode(out, episode, write_lock)
 
     # Serving resources (shared tool servers, interception) come up once for the
@@ -240,7 +240,7 @@ async def run_eval_server(config: EvalConfig) -> list[Episode]:
                 )
             records = []
             for trace in traces:
-                trace.stamp(EvalRunInfo(id=config.uuid))
+                trace.record_run(EvalRunInfo(id=config.uuid))
                 await append_trace(out, trace, write_lock, env=config.env_id)
                 records.append(Episode.of(trace))
             return records
@@ -254,7 +254,7 @@ async def run_eval_server(config: EvalConfig) -> list[Episode]:
                     **payload,
                 )
             for trace in episode.traces:
-                trace.stamp(EvalRunInfo(id=config.uuid))
+                trace.record_run(EvalRunInfo(id=config.uuid))
             await append_episode(out, episode, write_lock)
             return [episode]
 

{verifiers-0.2.2.dev48 → verifiers-0.2.2.dev50}/verifiers/v1/cli/replay.py

@@ -165,7 +165,7 @@ async def run_replay(config: ReplayConfig, source: Path, out: Path) -> list[Trac
                     st.state, st.detail = "scored", f"reward {trace.reward:.3f}"
                 except Exception as exc:
                     st.state, st.detail = "error", type(exc).__name__
-                    trace.capture_error(exc)
+                    trace.record_error(exc)
                     if not config.rich:
                         logger.warning(
                             "replay: scoring failed for task %s",

{verifiers-0.2.2.dev48 → verifiers-0.2.2.dev50}/verifiers/v1/cli/validate.py

@@ -89,6 +89,12 @@ async def _run_gold(task: Task, config: ValidateConfig) -> ResultRow:
         trace = Trace(
             task=TraceTask(type=type(task).__name__, data=task.data),
             state=state_cls(type(task))(),
+            # No agent runs here — the info only records the runtime policy.
+            agent=vf.AgentInfo(
+                config=vf.AgentConfig(runtime=config.runtime),
+                name="validate",
+                trainable=False,
+            ),
         )
         await runtime.start()
         await asyncio.wait_for(
@@ -124,6 +130,12 @@ async def _run_setup(task: Task, config: ValidateConfig) -> ResultRow:
         trace = Trace(
             task=TraceTask(type=type(task).__name__, data=task.data),
             state=state_cls(type(task))(),
+            # No agent runs here — the info only records the runtime policy.
+            agent=vf.AgentInfo(
+                config=vf.AgentConfig(runtime=config.runtime),
+                name="validate",
+                trainable=False,
+            ),
         )
         await runtime.start()
         await asyncio.wait_for(

{verifiers-0.2.2.dev48 → verifiers-0.2.2.dev50}/verifiers/v1/configs/judge.py

@@ -5,7 +5,7 @@ from collections.abc import Sequence
 from pathlib import Path
 from typing import Any
 
-from pydantic import BaseModel, SerializeAsAny, model_validator
+from pydantic import BaseModel, SerializeAsAny
 
 from verifiers.v1.clients import BaseClientConfig
 from verifiers.v1.types import ID, SamplingConfig
@@ -20,15 +20,8 @@ class JudgeConfig(BaseClientConfig):
     weight: float = 1.0
     model: str = "openai/gpt-5.4-nano"
     sampling: SamplingConfig = SamplingConfig()
-    prompt: str | None = None
-    prompt_file: Path | None = None
-    """Prompt file override, mutually exclusive with `prompt`."""
-
-    @model_validator(mode="after")
-    def check_prompt_source(self) -> "JudgeConfig":
-        if self.prompt is not None and self.prompt_file is not None:
-            raise ValueError("set `prompt` or `prompt_file`, not both")
-        return self
+    prompt: Path | None = None
+    """File whose text overrides the judge's default prompt template."""
 
 
 Judges = list[SerializeAsAny[JudgeConfig]]

{verifiers-0.2.2.dev48 → verifiers-0.2.2.dev50}/verifiers/v1/configs/taskset.py

@@ -1,5 +1,7 @@
 """The taskset plugin's config: which rows load, under `--env.taskset.*`."""
 
+from pathlib import Path
+
 from pydantic import SerializeAsAny
 from pydantic_config import BaseConfig
 
@@ -14,6 +16,9 @@ class TasksetConfig(BaseConfig):
     positional `eval <taskset-id>`)."""
     task: SerializeAsAny[TaskConfig] = TaskConfig()
     """Config passed to each task, under `--env.taskset.task.*`."""
+    system_prompt: Path | None = None
+    """File whose text overrides each task's `TaskData.system_prompt` in
+    `Taskset.select` (e.g. a GEPA `best_system_prompt.txt`)."""
 
     @property
     def name(self) -> str:

{verifiers-0.2.2.dev48 → verifiers-0.2.2.dev50}/verifiers/v1/env.py

@@ -163,7 +163,7 @@ class Env(ABC, Generic[ConfigT]):
         """Cross-agent judgement — THE programmable judgement surface: plain
         imperative Python over the finished episode (per-trace judgement already
         ran on each trace's own task). `episode.traces` is the flat episode in
-        completion order, each trace's `agent_name` stamp naming its agent; attach
+        completion order, each trace's `agent.name` stamp naming its agent; attach
         signals via `record_reward`/`record_metric`, in program order. A raise
         fails the episode (the retryable unit) — validate strictly, never
         record a guess."""