verifiers 0.3.1.dev18__tar.gz → 0.3.1.dev19__tar.gz

{verifiers-0.3.1.dev18 → verifiers-0.3.1.dev19}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: verifiers
-Version: 0.3.1.dev18
+Version: 0.3.1.dev19
 Summary: Verifiers: Environments for LLM Reinforcement Learning
 Project-URL: Homepage, https://github.com/primeintellect-ai/verifiers
 Project-URL: Documentation, https://github.com/primeintellect-ai/verifiers

{verifiers-0.3.1.dev18 → verifiers-0.3.1.dev19}/verifiers/v1/cli/debug.py

@@ -193,7 +193,7 @@ async def run_action(runtime: Runtime, config: DebugConfig) -> dict[str, Any]:
 
 async def debug_task(task: Task, config: DebugConfig) -> tuple[Trace, bool]:
     trace = Trace(
-        task=TraceTask(type=type(task).__name__, data=task.data),
+        task=TraceTask(type=type(task).__name__, data=task.data, hash=task.hash),
         state=state_cls(type(task))(),
         # No agent plays here (no model, no harness): the seat records the
         # runtime policy, and the provisioned box lands on it below (id, resolved

{verifiers-0.3.1.dev18 → verifiers-0.3.1.dev19}/verifiers/v1/cli/eval/resume.py

@@ -19,9 +19,9 @@ from pathlib import Path
 from pydantic_core import from_json
 
 from verifiers.v1.cli.output import CONFIG_FILE, TRACES_FILE, sniff_episode
-from verifiers.v1.cli.resume import task_key
 from verifiers.v1.configs.cli.eval import EvalConfig
 from verifiers.v1.episode import Episode, WireEpisode
+from verifiers.v1.task import task_key
 from verifiers.v1.trace import WireTrace
 
 

{verifiers-0.3.1.dev18 → verifiers-0.3.1.dev19}/verifiers/v1/cli/eval/runner.py

@@ -12,7 +12,7 @@ from verifiers.v1.cli.output import (
     output_path,
     save_config,
 )
-from verifiers.v1.cli.resume import distribute, task_key
+from verifiers.v1.cli.resume import distribute
 from verifiers.v1.clients import ModelContext
 from verifiers.v1.configs.cli.eval import EvalConfig
 from verifiers.v1.env import Env, RunSlot
@@ -45,9 +45,7 @@ async def run_eval(env: Env, config: EvalConfig) -> list[Episode]:
     # Kept on-disk rollouts rejoin the run as finished episodes; only owed ones re-run.
     finished: list[Episode] = []
     if config.resume is not None:
-        keys = [
-            task_key(t.data.model_dump(mode="json", exclude_none=True)) for t in tasks
-        ]
+        keys = [task.hash for task in tasks]
         finished, owed = resume.load(out, keys, config.num_rollouts, env.complete)
         if not owed:  # already complete - report it and exit successfully
             print(resume.nothing_to_resume_msg(out, len(tasks), config.num_rollouts))
@@ -163,7 +161,7 @@ async def run_eval_server(config: EvalConfig) -> list[Episode]:
         client = EnvClient(address=address)
         await client.wait_for_server_startup(timeout=600)
         # A run dispatches — and resumes — tasks by content: the client owns them,
-        # and `task_key` is their identity.
+        # and `task.hash` is their identity.
         plan = [
             ({"task_data": task.data.model_dump(mode="json")}, config.num_rollouts)
             for task in tasks
@@ -171,10 +169,7 @@ async def run_eval_server(config: EvalConfig) -> list[Episode]:
         out = output_path(config)
         finished: list[Episode] = []
         if config.resume is not None:
-            keys = [
-                task_key(t.data.model_dump(mode="json", exclude_none=True))
-                for t in tasks
-            ]
+            keys = [task.hash for task in tasks]
             finished, owed = resume.load(out, keys, config.num_rollouts)
             counts = distribute(keys, owed, config.num_rollouts)
             if not owed:  # already complete - report it and exit successfully

{verifiers-0.3.1.dev18 → verifiers-0.3.1.dev19}/verifiers/v1/cli/resume.py

@@ -1,16 +1,8 @@
 """Resume primitives shared by eval-like CLIs."""
 
-import hashlib
-import json
-from collections.abc import Mapping
 from pathlib import Path
 
 
-def task_key(data: Mapping) -> str:
-    """Content identity for task wire data, independent of field order."""
-    return hashlib.sha256(json.dumps(data, sort_keys=True).encode()).hexdigest()
-
-
 def distribute(
     selected_keys: list[str], owed: dict[str, int], num_results: int
 ) -> list[int]:

{verifiers-0.3.1.dev18 → verifiers-0.3.1.dev19}/verifiers/v1/cli/validate.py

@@ -26,7 +26,7 @@ from verifiers.v1.cli.resolve import (
     references_config_file,
     with_positional_taskset,
 )
-from verifiers.v1.cli.resume import distribute, split_resume, task_key
+from verifiers.v1.cli.resume import distribute, split_resume
 from verifiers.v1.configs.cli.validate import ValidateConfig
 from verifiers.v1.runtimes import make_runtime
 from verifiers.v1.state import state_cls
@@ -247,7 +247,7 @@ async def _run_gold(task: Task, config: ValidateConfig) -> ResultRow:
     valid, exc = False, None
     try:
         trace = Trace(
-            task=TraceTask(type=type(task).__name__, data=task.data),
+            task=TraceTask(type=type(task).__name__, data=task.data, hash=task.hash),
             state=state_cls(type(task))(),
             # No agent runs here — the info only records the runtime policy.
             agent=vf.AgentInfo(
@@ -288,7 +288,7 @@ async def _run_setup(task: Task, config: ValidateConfig) -> ResultRow:
     valid, exc = False, None
     try:
         trace = Trace(
-            task=TraceTask(type=type(task).__name__, data=task.data),
+            task=TraceTask(type=type(task).__name__, data=task.data, hash=task.hash),
             state=state_cls(type(task))(),
             # No agent runs here — the info only records the runtime policy.
             agent=vf.AgentInfo(
@@ -380,9 +380,7 @@ async def run_validate(config: ValidateConfig) -> list[dict]:
     mode = validation_mode(config)
     checks = "gold+setup" if mode == "all" else mode
     out = output_path(config)
-    selected_keys = [
-        task_key(task.data.model_dump(mode="json", exclude_none=True)) for task in tasks
-    ]
+    selected_keys = [task.hash for task in tasks]
     if config.resume is None:
         save_run(config, out, len(tasks))
         rows: list[ResultRow] = []

{verifiers-0.3.1.dev18 → verifiers-0.3.1.dev19}/verifiers/v1/rollout.py

@@ -83,6 +83,7 @@ class Rollout:
             task=TraceTask(
                 type=type(task).__name__,
                 data=task.data,
+                hash=task.hash,
             ),
             state=state_cls(type(task))(),
             # The seat's resolved config, role overrides included — the agent

{verifiers-0.3.1.dev18 → verifiers-0.3.1.dev19}/verifiers/v1/task.py

@@ -10,7 +10,9 @@ initializes a task instance. Rides on `trace.task.data` in `traces.jsonl`.
 from __future__ import annotations
 
 import copy
+import hashlib
 import inspect
+import json
 import logging
 from collections.abc import Callable, Mapping
 from typing import TYPE_CHECKING, Any, ClassVar, Generic, Self
@@ -40,6 +42,11 @@ if TYPE_CHECKING:
 logger = logging.getLogger(__name__)
 
 
+def task_key(data: Mapping) -> str:
+    """Content identity for task wire data, independent of field order."""
+    return hashlib.sha256(json.dumps(data, sort_keys=True).encode()).hexdigest()
+
+
 class TaskResources(BaseModel):
     """Optional resource limits for the task."""
 
@@ -134,6 +141,12 @@ class Task(Generic[DataT, StateT, ConfigT]):
         self.data = data
         self.config = config if config is not None else self.config_type()()
 
+    @property
+    def hash(self) -> str:
+        """Content hash of the task's wire data — the identity that matches saved
+        rollouts to tasks; recorded on the trace as `task.hash`."""
+        return task_key(self.data.model_dump(mode="json", exclude_none=True))
+
     def with_system_prompt(self, system_prompt: str) -> Self:
         clone = copy.copy(self)
         clone.data = self.data.model_copy(update={"system_prompt": system_prompt})

{verifiers-0.3.1.dev18 → verifiers-0.3.1.dev19}/verifiers/v1/trace.py

@@ -121,6 +121,8 @@ class TraceTask(BaseModel, Generic[DataT]):
     """The Task class name (`type(task).__name__`)."""
     data: DataT
     """The (immutable) row being solved."""
+    hash: str | None = None
+    """Content hash of `data` (`Task.hash`) — the task's identity across runs."""
 
 
 class Reward(BaseModel):