verifiers 0.3.1.dev32__tar.gz → 0.3.1.dev33__tar.gz

{verifiers-0.3.1.dev32 → verifiers-0.3.1.dev33}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: verifiers
-Version: 0.3.1.dev32
+Version: 0.3.1.dev33
 Summary: Verifiers: Environments for LLM Reinforcement Learning
 Project-URL: Homepage, https://github.com/primeintellect-ai/verifiers
 Project-URL: Documentation, https://github.com/primeintellect-ai/verifiers

{verifiers-0.3.1.dev32 → verifiers-0.3.1.dev33}/tests/v1/fixtures/echo_v1.py

@@ -30,6 +30,10 @@ class EchoData(vf.TaskData):
 
 
 class EchoTask(vf.Task[EchoData]):
+    @property
+    def key(self) -> str:
+        return f"echo:{self.data.answer}"
+
     @vf.stop
     async def single_turn(self, trace: vf.Trace) -> bool:
         return trace.num_turns >= 1

{verifiers-0.3.1.dev32 → verifiers-0.3.1.dev33}/tests/v1/test_e2e.py

@@ -136,6 +136,8 @@ async def test_single_turn(run_v1, harness, harness_runtime, tmp_path):
     assert trace.num_turns == 1
     assert trace.stop_condition == "agent_completed"
     assert trace.reward == 1.0
+    assert trace.task.key == f"echo:{trace.task.data.answer}"
+    assert trace.task.hash is not None
     # The seat's resolved identity rides the trace (policy metadata for trainers).
     assert trace.agent is not None
     assert trace.agent.config.sampling.max_tokens == 2048

{verifiers-0.3.1.dev32 → verifiers-0.3.1.dev33}/tests/v1/test_taskset.py

@@ -37,7 +37,9 @@ def test_head_bounds_an_infinite_taskset() -> None:
 
 def test_iteration_materializes_a_finite_taskset() -> None:
     taskset = FiniteTaskset(vf.TasksetConfig())
-    assert len(list(taskset)) == 10
+    tasks = list(taskset)
+    assert len(tasks) == 10
+    assert all(task.key == task.hash for task in tasks)
     assert idxs(taskset.head(4)) == [0, 1, 2, 3]
 
 

{verifiers-0.3.1.dev32 → verifiers-0.3.1.dev33}/tests/v1/test_trace.py

@@ -22,12 +22,18 @@ def test_bare_trace_round_trip():
     # The minimal trace: a base task, no nodes, no extras — dump and back into a plain Trace.
     tr = vf.Trace(
         agent=vf.AgentInfo(config=vf.AgentConfig()),
-        task=vf.TraceTask(type="Task", data=vf.TaskData(idx=3, prompt="hello")),
+        task=vf.TraceTask(
+            type="Task",
+            data=vf.TaskData(idx=3, prompt="hello"),
+            key="dataset/example-3",
+            hash="content-digest",
+        ),
     )
     rt = vf.Trace.model_validate(tr.model_dump())
     assert rt.id == tr.id
     assert rt.task.type == "Task"
     assert rt.task.data.idx == 3 and rt.task.data.prompt == "hello"
+    assert rt.task.key == "dataset/example-3" and rt.task.hash == "content-digest"
     assert rt.num_turns == 0 and rt.num_branches == 0
     assert rt.reward == 0.0 and rt.errors == []
 

{verifiers-0.3.1.dev32 → verifiers-0.3.1.dev33}/verifiers/v1/cli/debug.py

@@ -193,7 +193,9 @@ async def run_action(runtime: Runtime, config: DebugConfig) -> dict[str, Any]:
 
 async def debug_task(task: Task, config: DebugConfig) -> tuple[Trace, bool]:
     trace = Trace(
-        task=TraceTask(type=type(task).__name__, data=task.data, hash=task.hash),
+        task=TraceTask(
+            type=type(task).__name__, data=task.data, key=task.key, hash=task.hash
+        ),
         state=state_cls(type(task))(),
         # No agent plays here (no model, no harness): the seat records the
         # runtime policy, and the provisioned box lands on it below (id, resolved

{verifiers-0.3.1.dev32 → verifiers-0.3.1.dev33}/verifiers/v1/cli/validate.py

@@ -247,7 +247,9 @@ async def _run_gold(task: Task, config: ValidateConfig) -> ResultRow:
     valid, exc = False, None
     try:
         trace = Trace(
-            task=TraceTask(type=type(task).__name__, data=task.data, hash=task.hash),
+            task=TraceTask(
+                type=type(task).__name__, data=task.data, key=task.key, hash=task.hash
+            ),
             state=state_cls(type(task))(),
             # No agent runs here — the info only records the runtime policy.
             agent=vf.AgentInfo(
@@ -288,7 +290,9 @@ async def _run_setup(task: Task, config: ValidateConfig) -> ResultRow:
     valid, exc = False, None
     try:
         trace = Trace(
-            task=TraceTask(type=type(task).__name__, data=task.data, hash=task.hash),
+            task=TraceTask(
+                type=type(task).__name__, data=task.data, key=task.key, hash=task.hash
+            ),
             state=state_cls(type(task))(),
             # No agent runs here — the info only records the runtime policy.
             agent=vf.AgentInfo(

{verifiers-0.3.1.dev32 → verifiers-0.3.1.dev33}/verifiers/v1/rollout.py

@@ -85,6 +85,7 @@ class Rollout:
             task=TraceTask(
                 type=type(task).__name__,
                 data=task.data,
+                key=task.key,
                 hash=task.hash,
             ),
             state=state_cls(type(task))(),

{verifiers-0.3.1.dev32 → verifiers-0.3.1.dev33}/verifiers/v1/task.py

@@ -144,10 +144,20 @@ class Task(Generic[DataT, StateT, ConfigT]):
 
     @property
     def hash(self) -> str:
-        """Content hash of the task's wire data — the identity that matches saved
-        rollouts to tasks; recorded on the trace as `task.hash`."""
+        """Content hash of the task's wire data, recorded on the trace as
+        ``task.hash``."""
         return task_key(self.data.model_dump(mode="json", exclude_none=True))
 
+    @property
+    def key(self) -> str:
+        """Stable task identity, recorded on the trace as ``task.key``.
+
+        The content hash is the default. Override this when the task data contains
+        run-local fields such as an autogenerated ``idx`` and the underlying dataset
+        provides a durable identifier. Keys must be unique within a taskset.
+        """
+        return self.hash
+
     def with_system_prompt(self, system_prompt: str) -> Self:
         clone = copy.copy(self)
         clone.data = self.data.model_copy(update={"system_prompt": system_prompt})

{verifiers-0.3.1.dev32 → verifiers-0.3.1.dev33}/verifiers/v1/trace.py

@@ -122,8 +122,10 @@ class TraceTask(BaseModel, Generic[DataT]):
     """The Task class name (`type(task).__name__`)."""
     data: DataT
     """The (immutable) row being solved."""
+    key: str | None = None
+    """Stable task identity (``Task.key``); defaults to the content hash unless overridden."""
     hash: str | None = None
-    """Content hash of `data` (`Task.hash`) — the task's identity across runs."""
+    """Content hash of ``data`` (``Task.hash``)."""
 
 
 class InterceptRecord(BaseModel):