verifiers 0.2.2.dev48__tar.gz → 0.2.2.dev57__tar.gz

{verifiers-0.2.2.dev48 → verifiers-0.2.2.dev57}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: verifiers
-Version: 0.2.2.dev48
+Version: 0.2.2.dev57
 Summary: Verifiers: Environments for LLM Reinforcement Learning
 Project-URL: Homepage, https://github.com/primeintellect-ai/verifiers
 Project-URL: Documentation, https://github.com/primeintellect-ai/verifiers

{verifiers-0.2.2.dev48 → verifiers-0.2.2.dev57}/pyproject.toml

@@ -136,8 +136,10 @@ kuhn-poker-v1 = { path = "environments/kuhn_poker_v1", editable = true }
 
 [tool.uv.exclude-newer-package]
 # Bounded cutoffs for the explicitly requested tool upgrades.
-ruff = "2026-07-27"
-ty = "2026-07-27"
+# Full UTC timestamps: bare dates resolve to local-tz midnight and churn the
+# lockfile across timezones.
+ruff = "2026-07-28T00:00:00Z"
+ty = "2026-07-28T00:00:00Z"
 # PrimeIntellect-published on PyPI (trusted publisher)
 prime-tunnel = false
 prime-sandboxes = false
@@ -232,6 +234,7 @@ markers = [
     "colocated: v1 e2e cases with a user/tool server colocated in the harness runtime",
     "null: v1 e2e cases on the null harness",
     "bash: v1 e2e cases on the bash harness",
+    "browser_use: v1 e2e cases on the browser_use harness",
     "rlm: v1 e2e cases on the rlm harness",
     "kimi_code: v1 e2e cases on the kimi-code harness",
     "pi: v1 e2e cases on the pi harness",

{verifiers-0.2.2.dev48 → verifiers-0.2.2.dev57}/tests/v1/conftest.py

@@ -135,10 +135,6 @@ def _eval_config(
     `{id: ...}` config; `runtime` places the `agent` seat's harness (an agent field, not a
     harness one); `model` overrides the default text model (e.g. a VLM for an image task).
 
-    `temperature=0` (greedy) makes the run reproducible; `max_tokens` is generous headroom,
-    not a target — these trivial tasks finish in a few hundred tokens, so capping tighter only
-    risks truncating the reasoning before the answer (which tanks the reward).
-
     `harness=None` leaves every seat on its own story — the multi-agent case: there
     is no run-level harness, so a single-agent test's `harness` lands on the `agent`
     seat and a multi-agent test pins its seats through `env` role fields instead."""
@@ -175,7 +171,6 @@ def _eval_config(
         num_rollouts=n,
         sampling={
             "max_tokens": max_tokens,
-            "temperature": 0,
             "reasoning_effort": reasoning_effort,
         },
         rich=False,
@@ -219,8 +214,8 @@ def run_v1_server():
 
 @pytest.fixture
 async def live_ctx():
-    """A live `ModelContext` (the e2e default model + endpoint, greedy) for driving
-    `Agent` directly — the agent-surface counterpart of `run_v1`."""
+    """A live `ModelContext` (the e2e default model + endpoint, provider-default
+    sampling) for driving `Agent` directly — the agent-surface counterpart of `run_v1`."""
     from verifiers.v1.clients import EvalClientConfig, ModelContext, resolve_client
     from verifiers.v1.types import SamplingConfig
 
@@ -229,7 +224,7 @@ async def live_ctx():
         yield ModelContext(
             model="deepseek/deepseek-v4-flash",
             client=client,
-            sampling=SamplingConfig(max_tokens=2048, temperature=0),
+            sampling=SamplingConfig(max_tokens=2048),
         )
     finally:
         await client.close()
@@ -252,7 +247,7 @@ def run_v0():
             legacy={"id": env_id, "args": args or {}},
             num_tasks=1,
             num_rollouts=n,
-            sampling={"max_tokens": max_tokens, "temperature": 0},
+            sampling={"max_tokens": max_tokens},
             rich=False,
             output_dir=output_dir,
         )

{verifiers-0.2.2.dev48 → verifiers-0.2.2.dev57}/tests/v1/fixtures/counter_tool_v1.py

@@ -24,7 +24,9 @@ class CounterTaskConfig(vf.TaskConfig):
 
 
 class CounterTask(vf.Task[vf.TaskData, CounterState, CounterTaskConfig]):
-    tools = (CounterToolset,)
+    @classmethod
+    def toolsets(cls, config: CounterTaskConfig) -> list[vf.Toolset]:
+        return [CounterToolset(config.tools)]
 
     @vf.reward(weight=1.0)
     async def counted(self, trace: vf.Trace) -> float:

{verifiers-0.2.2.dev48 → verifiers-0.2.2.dev57}/tests/v1/fixtures/echo_acp_resume_v1.py

@@ -28,7 +28,9 @@ class ACPResumeConfig(vf.TasksetConfig):
 
 
 class ACPResumeTask(vf.Task[vf.TaskData, vf.State, ACPResumeTaskConfig]):
-    tools = (ResumeToolset,)
+    @classmethod
+    def toolsets(cls, config: ACPResumeTaskConfig) -> list[vf.Toolset]:
+        return [ResumeToolset(config.tools)]
 
     @vf.reward(weight=1.0)
     async def resumed(self, trace: vf.Trace) -> float:

{verifiers-0.2.2.dev48 → verifiers-0.2.2.dev57}/tests/v1/fixtures/echo_tool_v1.py

@@ -29,7 +29,9 @@ class EchoToolTaskConfig(vf.TaskConfig):
 
 
 class EchoToolTask(vf.Task[vf.TaskData, vf.State, EchoToolTaskConfig]):
-    tools = (EchoToolset,)
+    @classmethod
+    def toolsets(cls, config: EchoToolTaskConfig) -> list[vf.Toolset]:
+        return [EchoToolset(config.tools)]
 
     @vf.reward(weight=1.0)
     async def echoed(self, trace: vf.Trace) -> float:

{verifiers-0.2.2.dev48 → verifiers-0.2.2.dev57}/tests/v1/fixtures/tool_response_image_v1.py

@@ -26,8 +26,16 @@ class VisionToolset(vf.Toolset[vf.ToolsetConfig]):
         ]
 
 
-class ToolResponseImageTask(vf.Task):
-    tools = (VisionToolset,)
+class ToolResponseImageTaskConfig(vf.TaskConfig):
+    tools: vf.ToolsetConfig = vf.ToolsetConfig()
+
+
+class ToolResponseImageTask(
+    vf.Task[vf.TaskData, vf.State, ToolResponseImageTaskConfig]
+):
+    @classmethod
+    def toolsets(cls, config: ToolResponseImageTaskConfig) -> list[vf.Toolset]:
+        return [VisionToolset(config.tools)]
 
     @vf.reward(weight=1.0)
     async def preserved_image_tool_result(self, trace: vf.Trace) -> float:
@@ -39,7 +47,13 @@ class ToolResponseImageTask(vf.Task):
         return 0.0
 
 
-class ToolResponseImageTaskset(vf.Taskset[ToolResponseImageTask, vf.TasksetConfig]):
+class ToolResponseImageConfig(vf.TasksetConfig):
+    task: ToolResponseImageTaskConfig = ToolResponseImageTaskConfig()
+
+
+class ToolResponseImageTaskset(
+    vf.Taskset[ToolResponseImageTask, ToolResponseImageConfig]
+):
     def load(self) -> list[ToolResponseImageTask]:
         return [
             ToolResponseImageTask(

{verifiers-0.2.2.dev48 → verifiers-0.2.2.dev57}/tests/v1/test_color_codeword_tasks.py

@@ -10,9 +10,7 @@ def test_color_images_are_rendered_once(monkeypatch) -> None:
         return render(image)
 
     monkeypatch.setattr(taskset, "image_data_url", counted)
-    tasks = taskset.ColorCodewordTaskset(taskset.ColorCodewordConfig()).select(
-        num_tasks=100
-    )
+    tasks = list(taskset.ColorCodewordTaskset(taskset.ColorCodewordConfig()).head(100))
 
     assert len(tasks) == 100
     assert calls == list(taskset.COLOR_RGB.values())

{verifiers-0.2.2.dev48 → verifiers-0.2.2.dev57}/tests/v1/test_e2e.py

@@ -109,7 +109,7 @@ async def test_single_turn(run_v1, harness, harness_runtime, tmp_path):
     assert trace.stop_condition == "agent_completed"
     assert trace.reward == 1.0
     # The seat's resolved identity rides the trace (policy metadata for trainers).
-    assert trace.agent is not None and trace.agent.config.sampling.temperature == 0
+    assert trace.agent is not None and trace.agent.config.sampling.max_tokens == 2048
     # Every sampled turn has one per-call record, linked to its assistant node.
     sampled = [i for i, n in enumerate(trace.nodes) if n.sampled]
     assert [c.node for c in trace.calls if c.error is None] == sampled
@@ -118,6 +118,26 @@ async def test_single_turn(run_v1, harness, harness_runtime, tmp_path):
         assert call.time.duration > 0
 
 
+@pytest.mark.e2e
+@pytest.mark.browser_use
+@pytest.mark.docker
+async def test_browser_use(run_v1, tmp_path):
+    """The browser_use harness runs in an explicitly browser-capable runtime."""
+    image = (
+        "mcr.microsoft.com/playwright/python:v1.61.0-noble@"
+        "sha256:a9731514f24121d1dcd25d58d0a38146646d290a5998fd80d3e533e7b5e21c69"
+    )
+    (trace,) = await run_v1(
+        "echo-v1",
+        harness="browser_use",
+        runtime={"type": "docker", "image": image},
+        output_dir=tmp_path,
+        max_turns=2,
+    )
+    assert trace.ok
+    assert trace.reward == 1.0
+
+
 @pytest.mark.e2e
 @pytest.mark.parametrize("harness_runtime", USER_RUNTIMES, indirect=True)
 async def test_user(run_v1, harness_runtime, tmp_path):
@@ -229,7 +249,7 @@ async def test_tool(run_v1, harness_runtime, tool_runtime, tmp_path):
     assert trace.reward == 1.0
     # The interception server captured the advertised tools onto the trace (for tool-use SFT):
     # the null harness offered the task's MCP tool as `echo_back`, schema included.
-    assert trace.tools is not None
+    assert trace.tools
     (echo_tool,) = [t for t in trace.tools if t.name == "echo_back"]
     assert "message" in echo_tool.parameters.get("properties", {})
 
@@ -335,6 +355,7 @@ async def test_agentic(run_v1, harness, harness_runtime, tmp_path):
         runtime={"type": harness_runtime},
         output_dir=tmp_path,
         max_turns=10,
+        max_tokens=8192,
     )
     assert trace.ok
     assert trace.num_turns >= 1  # ran a command, then finished
@@ -355,9 +376,9 @@ async def test_multi_agent_env(run_v1, tmp_path):
         max_turns=2,
     )
     assert len(traces) == 2  # one episode, one trace per role
-    assert sorted(t.agent_name for t in traces) == ["a", "b"]
-    (b,) = [t for t in traces if t.agent_name == "b"]
-    assert b.trainable is False
+    assert sorted(t.agent.name for t in traces) == ["a", "b"]
+    (b,) = [t for t in traces if t.agent.name == "b"]
+    assert b.agent.trainable is False
     for trace in traces:
         assert trace.ok
         assert trace.reward == 1.0  # each seat's own task reward
@@ -366,7 +387,7 @@ async def test_multi_agent_env(run_v1, tmp_path):
     # agent info (completion order — the gathered seats land in either order).
     (line,) = (tmp_path / "traces.jsonl").read_text().splitlines()
     row = json.loads(line)
-    assert row["env"] == "duet-v1"
+    assert row["env"] == {"id": "duet-v1"}
     by_name = {t["agent"]["name"]: t for t in row["traces"]}
     assert set(by_name) == {"a", "b"}
     assert by_name["a"]["agent"]["trainable"] is True
@@ -385,7 +406,7 @@ async def test_env_id_best_of_n(run_v1, tmp_path):
         max_turns=2,
     )
     assert len(traces) == 2  # one episode, two attempts
-    assert all(t.agent_name == "agent" and t.ok for t in traces)
+    assert all(t.agent.name == "agent" and t.ok for t in traces)
     assert any(t.metrics["best"] == 1.0 for t in traces)
     assert all(t.metrics["pass_at_n"] == 1.0 for t in traces)  # echo always passes
 
@@ -401,21 +422,21 @@ async def test_env_id_agentic_judge(run_v1, tmp_path):
     model's call. Exercises the config surface too: a policy-only prompt
     override (the verdict contract is appended regardless) and
     reward-composition weights."""
+    policy = tmp_path / "judge_policy.txt"
+    policy.write_text("Check EMPIRICALLY that the agent echoed the word back.")
     traces = await run_v1(
         "echo-v1",
-        harness=None,  # seats pin their own harness; there is no run-level one
+        harness=None,
         env={
             "id": "agentic-judge",
-            # The solver owns the shared box, so the container is pinned here.
             "solver": {"harness": {"id": "bash"}, "runtime": {"type": "docker"}},
-            # The judge reads the trace and reasons before it writes the
-            # verdict file; the shared 2048-token run cap truncates it mid-audit.
             "judge": {
                 "harness": {"id": "bash"},
                 "max_output_tokens": 8192,
             },
             "task": {
-                "prompt": "Check EMPIRICALLY that the agent echoed the word back.",
+                "prompt": {"path": str(policy)},
+                "hint": "Do not rely on README.md",
             },
             "score": {"task_weight": 0.5},
         },
@@ -423,15 +444,15 @@ async def test_env_id_agentic_judge(run_v1, tmp_path):
         max_turns=10,
         rollout_timeout=600,
     )
-    assert sorted(t.agent_name for t in traces) == ["judge", "solver"]
-    (solver,) = [t for t in traces if t.agent_name == "solver"]
-    (judge,) = [t for t in traces if t.agent_name == "judge"]
+    assert sorted(t.agent.name for t in traces) == ["judge", "solver"]
+    (solver,) = [t for t in traces if t.agent.name == "solver"]
+    (judge,) = [t for t in traces if t.agent.name == "judge"]
     assert solver.ok and judge.ok
-    assert judge.trainable is False
+    assert judge.agent.trainable is False
     # The task's own reward keeps its raw score; the rescale lands on the weight.
     assert solver.rewards["echoed"].score == 1.0
     assert solver.rewards["echoed"].weight == 0.5
-    assert isinstance(judge.info.get("verdict"), dict)  # scraped off the box
+    assert isinstance(judge.info.get("verdict"), dict)
     assert 0.0 <= solver.rewards["judge"].score <= 1.0
 
 
@@ -448,11 +469,11 @@ async def test_env_id_user_sim(run_v1, tmp_path):
         max_turns=6,
         rollout_timeout=300,
     )
-    assert sorted(t.agent_name for t in traces) == ["assistant", "user"]
-    (assistant,) = [t for t in traces if t.agent_name == "assistant"]
-    (user,) = [t for t in traces if t.agent_name == "user"]
+    assert sorted(t.agent.name for t in traces) == ["assistant", "user"]
+    (assistant,) = [t for t in traces if t.agent.name == "assistant"]
+    (user,) = [t for t in traces if t.agent.name == "user"]
     assert assistant.ok and user.ok
-    assert user.trainable is False
+    assert user.agent.trainable is False
     assert user.num_turns >= 1  # the modeled user actually spoke
     assert assistant.metrics["user_turns"] >= 1
     # `mask_prompt`: the scenario is hidden from the assistant's harness (the run's
@@ -465,7 +486,7 @@ async def test_env_id_user_sim(run_v1, tmp_path):
     from verifiers.v1.trace import WireTrace
 
     (record,) = read_episodes(tmp_path, WireTrace)
-    assert {t.agent_name for t in record.traces} == {"assistant", "user"}
+    assert {t.agent.name for t in record.traces} == {"assistant", "user"}
     assert record.id  # both traces are persisted under one durable episode identity
 
 
@@ -488,14 +509,14 @@ async def test_env_id_user_sim_with_tools(run_v1, tmp_path):
         max_tokens=8192,
         rollout_timeout=300,
     )
-    (assistant,) = [t for t in traces if t.agent_name == "assistant"]
-    (user,) = [t for t in traces if t.agent_name == "user"]
+    (assistant,) = [t for t in traces if t.agent.name == "assistant"]
+    (user,) = [t for t in traces if t.agent.name == "user"]
     assert assistant.ok and user.ok
     assert assistant.task.data.prompt is None  # the scenario stayed off the wire
     assert user.num_turns >= 1  # the modeled user actually drove the exchange
     assert assistant.rewards["echoed"].score == 1.0  # the tool ran, mid-conversation
     # The tool was advertised to the masked chat exactly as to any run.
-    assert assistant.tools is not None
+    assert assistant.tools
     assert any(tool.name == "echo_back" for tool in assistant.tools)
 
 
@@ -514,8 +535,8 @@ async def test_kuhn_poker_self_play(run_v1, tmp_path):
         max_tokens=8192,
         rollout_timeout=300,
     )
-    assert sorted(t.agent_name for t in traces) == ["player0", "player1"]
-    payoffs = {t.agent_name: t.rewards["payoff"].score for t in traces}
+    assert sorted(t.agent.name for t in traces) == ["player0", "player1"]
+    payoffs = {t.agent.name: t.rewards["payoff"].score for t in traces}
     assert payoffs["player0"] + payoffs["player1"] == 0  # zero-sum
     assert abs(payoffs["player0"]) in (1.0, 2.0)
     for trace in traces:
@@ -542,7 +563,7 @@ async def test_multi_agent_env_server(run_v1_server, tmp_path):
         max_turns=2,
     )
     assert len(traces) == 2
-    assert sorted(t.agent_name for t in traces) == ["a", "b"]
+    assert sorted(t.agent.name for t in traces) == ["a", "b"]
     for trace in traces:
         assert trace.ok
         assert trace.metrics["duet"] == 1.0

{verifiers-0.2.2.dev48 → verifiers-0.2.2.dev57}/tests/v1/test_graph.py

@@ -40,7 +40,8 @@ def test_routed_experts_attributed_and_aligned_across_turns():
     concatenates back to a `[tokens, layers, top_k]` array aligned 1:1 with `branch.token_ids` —
     and survives the base64 wire round-trip."""
     trace = vf.Trace(
-        task=vf.TraceTask(type="Task", data=vf.TaskData(idx=0, prompt="x"))
+        agent=vf.AgentInfo(config=vf.AgentConfig()),
+        task=vf.TraceTask(type="Task", data=vf.TaskData(idx=0, prompt="x")),
     )
     user = vf.UserMessage(content="u1")
     graph.prepare_turn(trace, [user]).commit(
@@ -94,7 +95,8 @@ def test_routed_experts_none_when_absent():
     """No routing captured (engine ran without `enable_return_routed_experts`) -> the branch
     reports None and the trainer simply skips replay."""
     trace = vf.Trace(
-        task=vf.TraceTask(type="Task", data=vf.TaskData(idx=0, prompt="x"))
+        agent=vf.AgentInfo(config=vf.AgentConfig()),
+        task=vf.TraceTask(type="Task", data=vf.TaskData(idx=0, prompt="x")),
     )
     graph.prepare_turn(trace, [vf.UserMessage(content="u1")]).commit(
         vf.Response(
@@ -128,7 +130,10 @@ def test_tool_call_hash_matches_v0_content_and_arguments_normalization():
 
 def test_reasoning_content_participates_in_graph_prefix_matching():
     task = vf.TaskData(idx=0, prompt="use a tool")
-    trace = vf.Trace(task=vf.TraceTask(type="Task", data=task))
+    trace = vf.Trace(
+        agent=vf.AgentInfo(config=vf.AgentConfig()),
+        task=vf.TraceTask(type="Task", data=task),
+    )
     user = vf.UserMessage(content="use a tool")
     call = vf.ToolCall(id="call_0", name="lookup", arguments="{}")
 
@@ -205,7 +210,8 @@ def test_renderer_level_break_forks_by_token_id():
 
     # Control: the prior turn re-renders to the same tokens -> stays one linear branch.
     linear = vf.Trace(
-        task=vf.TraceTask(type="Task", data=vf.TaskData(idx=0, prompt="x"))
+        agent=vf.AgentInfo(config=vf.AgentConfig()),
+        task=vf.TraceTask(type="Task", data=vf.TaskData(idx=0, prompt="x")),
     )
     first_turn(linear)
     second_turn(linear, [1, 2, 3, 4, 5, 6, 7])
@@ -214,7 +220,8 @@ def test_renderer_level_break_forks_by_token_id():
 
     # Break: the assistant turn retokenizes (4 -> 99), so prompt_ids diverge at that node.
     broken = vf.Trace(
-        task=vf.TraceTask(type="Task", data=vf.TaskData(idx=0, prompt="x"))
+        agent=vf.AgentInfo(config=vf.AgentConfig()),
+        task=vf.TraceTask(type="Task", data=vf.TaskData(idx=0, prompt="x")),
     )
     first_turn(broken)
     second_turn(broken, [1, 2, 3, 99, 5, 6, 7])
@@ -227,7 +234,10 @@ def test_renderer_level_break_forks_by_token_id():
 
 def test_prompt_supplied_assistant_messages_are_not_sampled_turns():
     task = vf.TaskData(idx=0, prompt="few-shot")
-    trace = vf.Trace(task=vf.TraceTask(type="Task", data=task))
+    trace = vf.Trace(
+        agent=vf.AgentInfo(config=vf.AgentConfig()),
+        task=vf.TraceTask(type="Task", data=task),
+    )
     fabricated = vf.AssistantMessage(
         content=None,
         tool_calls=[vf.ToolCall(id="call_0", name="lookup", arguments="{}")],

{verifiers-0.2.2.dev48 → verifiers-0.2.2.dev57}/tests/v1/test_judges.py

@@ -36,6 +36,7 @@ def make_trace(
     task_cls: type[QAData] = QAData,
 ) -> vf.Trace:
     return vf.Trace(
+        agent=vf.AgentInfo(config=vf.AgentConfig()),
         task=vf.TraceTask(
             type="Task",
             data=task_cls(idx=0, prompt="Capital of France?", answer=answer),
@@ -244,6 +245,7 @@ async def test_reference_score_messages_prompt(fake_judge_model):
         answer="Paris",
     )
     trace = vf.Trace(
+        agent=vf.AgentInfo(config=vf.AgentConfig()),
         task=vf.TraceTask(type="Task", data=task),
         nodes=[
             MessageNode(parent=None, message=UserMessage(content="q"), sampled=False),
@@ -269,6 +271,7 @@ async def test_reference_question_field(fake_judge_model):
         answer="Paris",
     )
     trace = vf.Trace(
+        agent=vf.AgentInfo(config=vf.AgentConfig()),
         task=vf.TraceTask(type="Task", data=task),
         nodes=[
             MessageNode(parent=None, message=UserMessage(content="q"), sampled=False),
@@ -293,6 +296,7 @@ def full_trace_fixture() -> vf.Trace:
     from verifiers.v1.types import ToolCall, ToolMessage
 
     return vf.Trace(
+        agent=vf.AgentInfo(config=vf.AgentConfig()),
         task=vf.TraceTask(
             type="Task", data=QAData(idx=0, prompt="Capital of France?", answer="Paris")
         ),
@@ -367,32 +371,20 @@ async def test_rubric_view_full_trace(tmp_path, fake_judge_model):
     assert all("SECRET REASONING" not in prompt for prompt in fake_judge_model)
 
 
-async def test_config_prompt_overrides_class_template(fake_judge_model):
-    judge = vf.ReferenceJudge(
-        vf.ReferenceJudgeConfig(prompt="Q:{question} A:{answer} R:{response}")
-    )
+async def test_config_prompt_overrides_class_template(tmp_path, fake_judge_model):
+    # Config `prompt` is a file path; the same {field} placeholders work.
+    file = tmp_path / "judge.txt"
+    file.write_text("Q:{question} A:{answer} R:{response}")
+    judge = vf.ReferenceJudge(vf.ReferenceJudgeConfig(prompt=file))
     assert judge.build_messages(question="q", answer="a", response="r") == "Q:q A:a R:r"
     # A template needn't use every evaluate field: score also passes {positive}/{negative},
     # which str.format ignores when the (custom) prompt doesn't reference them.
     trace = make_trace()
     assert await judge.score(trace.task.data, trace) == 1.0
     assert fake_judge_model[0] == "Q:Capital of France? A:Paris R:It is Paris."
-
-
-async def test_prompt_file(tmp_path, fake_judge_model):
-    # The prompt template can come from a file; the same {field} placeholders work.
-    file = tmp_path / "judge.txt"
-    file.write_text("Q:{question} A:{answer} R:{response}")
-    trace = make_trace()
-    judge = vf.ReferenceJudge(vf.ReferenceJudgeConfig(prompt_file=file))
-    assert await judge.score(trace.task.data, trace) == 1.0
-    assert fake_judge_model[0] == "Q:Capital of France? A:Paris R:It is Paris."
     # a bad path fails at judge construction, not mid-eval at score time
     with pytest.raises(FileNotFoundError):
-        vf.ReferenceJudge(vf.ReferenceJudgeConfig(prompt_file=tmp_path / "missing.txt"))
-    # inline and file prompts are mutually exclusive
-    with pytest.raises(ValueError, match="not both"):
-        vf.ReferenceJudgeConfig(prompt="inline", prompt_file=file)
+        vf.ReferenceJudge(vf.ReferenceJudgeConfig(prompt=tmp_path / "missing.txt"))
 
 
 # --- reference input/verdict knobs ------------------------------------------------------------

verifiers-0.2.2.dev57/tests/v1/test_taskset.py

@@ -0,0 +1,93 @@
+"""`Taskset` iteration and the `head`/`shuffle` views over list, generator,
+and `INFINITE` `load` implementations."""
+
+import itertools
+
+import pytest
+
+import verifiers.v1 as vf
+
+
+class CountTask(vf.Task[vf.TaskData]):
+    pass
+
+
+class InfiniteTaskset(vf.Taskset[CountTask, vf.TasksetConfig]):
+    INFINITE = True
+
+    def load(self):
+        for i in itertools.count():
+            yield CountTask(vf.TaskData(idx=i, prompt=f"task {i}"))
+
+
+class FiniteTaskset(vf.Taskset[CountTask, vf.TasksetConfig]):
+    def load(self):
+        for i in range(10):
+            yield CountTask(vf.TaskData(idx=i, prompt=f"task {i}"))
+
+
+def idxs(tasks) -> list[int]:
+    return [task.data.idx for task in tasks]
+
+
+def test_head_bounds_an_infinite_taskset() -> None:
+    tasks = list(InfiniteTaskset(vf.TasksetConfig()).head(5))
+    assert idxs(tasks) == [0, 1, 2, 3, 4]
+
+
+def test_iteration_materializes_a_finite_taskset() -> None:
+    taskset = FiniteTaskset(vf.TasksetConfig())
+    assert len(list(taskset)) == 10
+    assert idxs(taskset.head(4)) == [0, 1, 2, 3]
+
+
+def test_head_returns_the_same_taskset_type() -> None:
+    view = FiniteTaskset(vf.TasksetConfig()).head(4)
+    assert isinstance(view, FiniteTaskset)
+    assert view.task_type() is CountTask
+
+
+def test_shuffle_samples_the_whole_taskset_reproducibly() -> None:
+    taskset = FiniteTaskset(vf.TasksetConfig())
+    first = idxs(taskset.shuffle().head(5))
+    assert first == idxs(taskset.shuffle().head(5))
+    assert len(first) == 5 and set(first) <= set(range(10))
+    assert first != [0, 1, 2, 3, 4]  # sampled from the whole set, not the head
+
+
+def test_shuffle_seed_changes_the_sample() -> None:
+    taskset = FiniteTaskset(vf.TasksetConfig())
+    seeded = idxs(taskset.shuffle(seed=7).head(5))
+    assert seeded == idxs(taskset.shuffle(seed=7).head(5))  # reproducible per seed
+    assert seeded != idxs(taskset.shuffle().head(5))  # differs from the default seed
+
+
+def test_shuffle_raises_on_infinite() -> None:
+    with pytest.raises(ValueError, match="infinite"):
+        InfiniteTaskset(vf.TasksetConfig()).shuffle()
+
+
+def test_head_then_shuffle_bounds_an_infinite_taskset() -> None:
+    head = idxs(InfiniteTaskset(vf.TasksetConfig()).head(5).shuffle())
+    assert sorted(head) == [0, 1, 2, 3, 4]
+    assert head != [0, 1, 2, 3, 4]  # shuffled within the bounded head
+
+
+def test_head_only_builds_what_the_run_takes() -> None:
+    built: list[int] = []
+
+    class RecordingTaskset(InfiniteTaskset):
+        def load(self):
+            for i in itertools.count():
+                built.append(i)
+                yield CountTask(vf.TaskData(idx=i, prompt=f"task {i}"))
+
+    list(RecordingTaskset(vf.TasksetConfig()).head(3))
+    assert built == [0, 1, 2]
+
+
+def test_views_do_not_mutate_the_base_taskset() -> None:
+    taskset = InfiniteTaskset(vf.TasksetConfig())
+    view = taskset.head(3)
+    assert view.INFINITE is False and taskset.INFINITE is True
+    assert idxs(taskset.head(2)) == [0, 1]  # base iterates untransformed

{verifiers-0.2.2.dev48 → verifiers-0.2.2.dev57}/tests/v1/test_trace.py

@@ -21,7 +21,8 @@ class MyState(vf.State):
 def test_bare_trace_round_trip():
     # The minimal trace: a base task, no nodes, no extras — dump and back into a plain Trace.
     tr = vf.Trace(
-        task=vf.TraceTask(type="Task", data=vf.TaskData(idx=3, prompt="hello"))
+        agent=vf.AgentInfo(config=vf.AgentConfig()),
+        task=vf.TraceTask(type="Task", data=vf.TaskData(idx=3, prompt="hello")),
     )
     rt = vf.Trace.model_validate(tr.model_dump())
     assert rt.id == tr.id
@@ -35,6 +36,7 @@ def test_custom_task_state_round_trip():
     # Custom data and state round-trip into the same parameterization. Data fields are
     # typed (not just `model_extra`); `state` is runtime-only and never crosses the wire.
     tr = vf.Trace[MyTask, MyState](
+        agent=vf.AgentInfo(config=vf.AgentConfig()),
         task=vf.TraceTask(type="MyTask", data=MyTask(idx=0, prompt="q", answer="gold")),
         state=MyState(score=7),
         nodes=[
@@ -59,6 +61,7 @@ def test_wire_trace_round_trip():
     # Two leaves off one root → 2 branches (a compaction-shaped trace), so the round-trip has to
     # carry node `parent` links for `num_branches` to survive.
     tr = vf.Trace[MyTask, vf.State](
+        agent=vf.AgentInfo(config=vf.AgentConfig()),
         task=vf.TraceTask(type="MyTask", data=MyTask(idx=0, prompt="q", answer="a")),
         tools=[vf.Tool(name="echo", description="", parameters={"type": "object"})],
         nodes=[

{verifiers-0.2.2.dev48 → verifiers-0.2.2.dev57}/verifiers/v1/__init__.py

@@ -4,6 +4,12 @@ from pydantic_config import BaseConfig
 
 from verifiers.v1.acp import ACP
 from verifiers.v1.agent import Agent, Agents, Interaction, Segment, make_agent
+from verifiers.v1.artifacts import (
+    ARTIFACTS_DIR,
+    Artifact,
+    collect,
+    restore,
+)
 from verifiers.v1.clients import (
     BaseClientConfig,
     Client,
@@ -109,10 +115,10 @@ from verifiers.v1.taskset import Taskset
 from verifiers.v1.trace import (
     TRACE_VERSION,
     AgentInfo,
+    AgentSpan,
     Branch,
     Error,
     EvalRunInfo,
-    GenerationSpan,
     ModelCall,
     Reward,
     RunInfo,
@@ -138,7 +144,6 @@ from verifiers.v1.types import (
     Response,
     Sampling,
     SamplingConfig,
-    StrictBaseModel,
     SystemMessage,
     TextContentPart,
     Tool,
@@ -171,7 +176,6 @@ __all__ = [  # noqa: RUF022 - grouped by public API area
     "Response",
     "Sampling",
     "SamplingConfig",
-    "StrictBaseModel",
     "SystemMessage",
     "TextContentPart",
     "Tool",
@@ -207,7 +211,7 @@ __all__ = [  # noqa: RUF022 - grouped by public API area
     "Timing",
     "TimeSpan",
     "TimeSplit",
-    "GenerationSpan",
+    "AgentSpan",
     "Error",
     # decorators
     "stop",
@@ -297,6 +301,11 @@ __all__ = [  # noqa: RUF022 - grouped by public API area
     "PATCH_CAP_BYTES",
     "capture_patch",
     "resolve_head",
+    # grading artifacts
+    "ARTIFACTS_DIR",
+    "Artifact",
+    "collect",
+    "restore",
     # scoring
     "compare_stdout_results",
     "extract_boxed_answer",