verifiers 0.3.1.dev17__tar.gz → 0.3.1.dev19__tar.gz

{verifiers-0.3.1.dev17 → verifiers-0.3.1.dev19}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: verifiers
-Version: 0.3.1.dev17
+Version: 0.3.1.dev19
 Summary: Verifiers: Environments for LLM Reinforcement Learning
 Project-URL: Homepage, https://github.com/primeintellect-ai/verifiers
 Project-URL: Documentation, https://github.com/primeintellect-ai/verifiers

{verifiers-0.3.1.dev17 → verifiers-0.3.1.dev19}/verifiers/v1/cli/debug.py

@@ -193,7 +193,7 @@ async def run_action(runtime: Runtime, config: DebugConfig) -> dict[str, Any]:
 
 async def debug_task(task: Task, config: DebugConfig) -> tuple[Trace, bool]:
     trace = Trace(
-        task=TraceTask(type=type(task).__name__, data=task.data),
+        task=TraceTask(type=type(task).__name__, data=task.data, hash=task.hash),
         state=state_cls(type(task))(),
         # No agent plays here (no model, no harness): the seat records the
         # runtime policy, and the provisioned box lands on it below (id, resolved

{verifiers-0.3.1.dev17 → verifiers-0.3.1.dev19}/verifiers/v1/cli/eval/resume.py

@@ -19,9 +19,9 @@ from pathlib import Path
 from pydantic_core import from_json
 
 from verifiers.v1.cli.output import CONFIG_FILE, TRACES_FILE, sniff_episode
-from verifiers.v1.cli.resume import task_key
 from verifiers.v1.configs.cli.eval import EvalConfig
 from verifiers.v1.episode import Episode, WireEpisode
+from verifiers.v1.task import task_key
 from verifiers.v1.trace import WireTrace
 
 

{verifiers-0.3.1.dev17 → verifiers-0.3.1.dev19}/verifiers/v1/cli/eval/runner.py

@@ -12,7 +12,7 @@ from verifiers.v1.cli.output import (
     output_path,
     save_config,
 )
-from verifiers.v1.cli.resume import distribute, task_key
+from verifiers.v1.cli.resume import distribute
 from verifiers.v1.clients import ModelContext
 from verifiers.v1.configs.cli.eval import EvalConfig
 from verifiers.v1.env import Env, RunSlot
@@ -45,9 +45,7 @@ async def run_eval(env: Env, config: EvalConfig) -> list[Episode]:
     # Kept on-disk rollouts rejoin the run as finished episodes; only owed ones re-run.
     finished: list[Episode] = []
     if config.resume is not None:
-        keys = [
-            task_key(t.data.model_dump(mode="json", exclude_none=True)) for t in tasks
-        ]
+        keys = [task.hash for task in tasks]
         finished, owed = resume.load(out, keys, config.num_rollouts, env.complete)
         if not owed:  # already complete - report it and exit successfully
             print(resume.nothing_to_resume_msg(out, len(tasks), config.num_rollouts))
@@ -163,7 +161,7 @@ async def run_eval_server(config: EvalConfig) -> list[Episode]:
         client = EnvClient(address=address)
         await client.wait_for_server_startup(timeout=600)
         # A run dispatches — and resumes — tasks by content: the client owns them,
-        # and `task_key` is their identity.
+        # and `task.hash` is their identity.
         plan = [
             ({"task_data": task.data.model_dump(mode="json")}, config.num_rollouts)
             for task in tasks
@@ -171,10 +169,7 @@ async def run_eval_server(config: EvalConfig) -> list[Episode]:
         out = output_path(config)
         finished: list[Episode] = []
         if config.resume is not None:
-            keys = [
-                task_key(t.data.model_dump(mode="json", exclude_none=True))
-                for t in tasks
-            ]
+            keys = [task.hash for task in tasks]
             finished, owed = resume.load(out, keys, config.num_rollouts)
             counts = distribute(keys, owed, config.num_rollouts)
             if not owed:  # already complete - report it and exit successfully

{verifiers-0.3.1.dev17 → verifiers-0.3.1.dev19}/verifiers/v1/cli/resume.py

@@ -1,16 +1,8 @@
 """Resume primitives shared by eval-like CLIs."""
 
-import hashlib
-import json
-from collections.abc import Mapping
 from pathlib import Path
 
 
-def task_key(data: Mapping) -> str:
-    """Content identity for task wire data, independent of field order."""
-    return hashlib.sha256(json.dumps(data, sort_keys=True).encode()).hexdigest()
-
-
 def distribute(
     selected_keys: list[str], owed: dict[str, int], num_results: int
 ) -> list[int]:

{verifiers-0.3.1.dev17 → verifiers-0.3.1.dev19}/verifiers/v1/cli/validate.py

@@ -26,7 +26,7 @@ from verifiers.v1.cli.resolve import (
     references_config_file,
     with_positional_taskset,
 )
-from verifiers.v1.cli.resume import distribute, split_resume, task_key
+from verifiers.v1.cli.resume import distribute, split_resume
 from verifiers.v1.configs.cli.validate import ValidateConfig
 from verifiers.v1.runtimes import make_runtime
 from verifiers.v1.state import state_cls
@@ -247,7 +247,7 @@ async def _run_gold(task: Task, config: ValidateConfig) -> ResultRow:
     valid, exc = False, None
     try:
         trace = Trace(
-            task=TraceTask(type=type(task).__name__, data=task.data),
+            task=TraceTask(type=type(task).__name__, data=task.data, hash=task.hash),
             state=state_cls(type(task))(),
             # No agent runs here — the info only records the runtime policy.
             agent=vf.AgentInfo(
@@ -288,7 +288,7 @@ async def _run_setup(task: Task, config: ValidateConfig) -> ResultRow:
     valid, exc = False, None
     try:
         trace = Trace(
-            task=TraceTask(type=type(task).__name__, data=task.data),
+            task=TraceTask(type=type(task).__name__, data=task.data, hash=task.hash),
             state=state_cls(type(task))(),
             # No agent runs here — the info only records the runtime policy.
             agent=vf.AgentInfo(
@@ -380,9 +380,7 @@ async def run_validate(config: ValidateConfig) -> list[dict]:
     mode = validation_mode(config)
     checks = "gold+setup" if mode == "all" else mode
     out = output_path(config)
-    selected_keys = [
-        task_key(task.data.model_dump(mode="json", exclude_none=True)) for task in tasks
-    ]
+    selected_keys = [task.hash for task in tasks]
     if config.resume is None:
         save_run(config, out, len(tasks))
         rows: list[ResultRow] = []

{verifiers-0.3.1.dev17 → verifiers-0.3.1.dev19}/verifiers/v1/harnesses/openclaw/harness.py

@@ -8,7 +8,6 @@ import secrets
 import shlex
 from collections.abc import AsyncIterator
 from contextlib import asynccontextmanager
-from pathlib import Path
 
 from pydantic import Field
 
@@ -19,6 +18,7 @@ from verifiers.v1.harness import Harness
 from verifiers.v1.runtimes import DockerRuntime, ProgramResult, Runtime
 from verifiers.v1.task import TaskData
 from verifiers.v1.trace import Trace
+from verifiers.v1.utils.paths import CACHE_DIR
 
 logger = logging.getLogger(__name__)
 
@@ -40,7 +40,9 @@ async def _gateway_start_lock(runtime: Runtime) -> AsyncIterator[None]:
         yield
         return
     # Unrestricted Docker containers share the host network across server workers.
-    with Path("/tmp/vf-openclaw-gateway.lock").open("a") as lock:  # noqa: ASYNC230
+    lock_path = CACHE_DIR / "harnesses" / "openclaw" / "gateway.lock"
+    lock_path.parent.mkdir(parents=True, exist_ok=True)
+    with lock_path.open("a") as lock:
         while True:
             try:
                 fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)

{verifiers-0.3.1.dev17 → verifiers-0.3.1.dev19}/verifiers/v1/rollout.py

@@ -83,6 +83,7 @@ class Rollout:
             task=TraceTask(
                 type=type(task).__name__,
                 data=task.data,
+                hash=task.hash,
             ),
             state=state_cls(type(task))(),
             # The seat's resolved config, role overrides included — the agent

{verifiers-0.3.1.dev17 → verifiers-0.3.1.dev19}/verifiers/v1/runtimes/base.py

@@ -20,9 +20,14 @@ from pydantic_config import BaseConfig
 
 from verifiers.v1.errors import SandboxError
 from verifiers.v1.utils.aio import run_shielded
+from verifiers.v1.utils.paths import CACHE_DIR
 
 logger = logging.getLogger(__name__)
 
+# Digest-keyed cache of prepared PEP 723 scripts. Every runtime's `write` creates parent
+# directories, so the path also works inside containers and sandboxes.
+SCRIPTS_DIR = str(CACHE_DIR / "runtimes" / "scripts")
+
 # Ensure the latest `uv` is available for our PEP 723 scripts: prefer pip on Python images,
 # then fall back to the standalone installer (curl/wget), installing curl + CA certs when a
 # bare image has no downloader. Both paths install to ~/.local/bin, which we prepend to PATH.
@@ -277,7 +282,7 @@ class Runtime(ABC):
         """
         data = script.encode() if isinstance(script, str) else script
         digest = hashlib.sha256(data).hexdigest()
-        path = f"/tmp/vf-scripts/{digest}.py"
+        path = f"{SCRIPTS_DIR}/{digest}.py"
         if digest not in self._uv_interpreters:
             async with self._uv_script_locks.setdefault(digest, asyncio.Lock()):
                 if digest not in self._uv_interpreters:

{verifiers-0.3.1.dev17 → verifiers-0.3.1.dev19}/verifiers/v1/runtimes/subprocess.py

@@ -17,6 +17,7 @@ from verifiers.v1.runtimes.base import (
     Runtime,
     RuntimeProcess,
 )
+from verifiers.v1.utils.paths import CACHE_DIR
 
 _BACKGROUND_STOP_TIMEOUT = 5
 
@@ -89,8 +90,8 @@ class SubprocessRuntime(Runtime):
         self._background: list[asyncio.subprocess.Process] = []
 
     async def start(self) -> None:
-        self.workdir = Path("/tmp") / self.name
-        self.workdir.mkdir()
+        self.workdir = CACHE_DIR / "runtimes" / "subprocess" / self.name
+        self.workdir.mkdir(parents=True)
         self.info.id = str(self.workdir)
 
     async def run(self, argv: list[str], env: dict[str, str]) -> ProgramResult:

{verifiers-0.3.1.dev17 → verifiers-0.3.1.dev19}/verifiers/v1/serve/pool.py

@@ -26,8 +26,8 @@ import logging
 import multiprocessing as mp
 import os
 import signal
+import tempfile
 import threading
-import uuid
 from collections.abc import Callable
 
 import msgpack
@@ -90,7 +90,7 @@ class EnvServerPool:
         self.multiplex = multiplex
         self.elastic = elastic
         self.log_setup = log_setup
-        self.session = uuid.uuid4().hex[:12]
+        self._ipc_dir = tempfile.mkdtemp(prefix="vf-pool-")
         self.workers: list[dict] = []
         self._mpctx = mp.get_context("spawn")
         self._poller: zmq.asyncio.Poller | None = None
@@ -105,7 +105,7 @@ class EnvServerPool:
         self.address = self.frontend.getsockopt_string(zmq.LAST_ENDPOINT)
 
     def _worker_path(self, i: int) -> str:
-        return f"/tmp/vf-pool-{self.session}-{i}"
+        return f"{self._ipc_dir}/{i}"
 
     def _spawn_worker(self) -> None:
         i = len(self.workers)  # upscale-only, so the next index is the current count
@@ -241,6 +241,8 @@ class EnvServerPool:
                 w["dealer"].close()
             with contextlib.suppress(OSError):
                 os.unlink(self._worker_path(w["index"]))
+        with contextlib.suppress(OSError):
+            os.rmdir(self._ipc_dir)
         self.frontend.close()
         self.ctx.term()
         logger.info("EnvServerPool down")

{verifiers-0.3.1.dev17 → verifiers-0.3.1.dev19}/verifiers/v1/task.py

@@ -10,7 +10,9 @@ initializes a task instance. Rides on `trace.task.data` in `traces.jsonl`.
 from __future__ import annotations
 
 import copy
+import hashlib
 import inspect
+import json
 import logging
 from collections.abc import Callable, Mapping
 from typing import TYPE_CHECKING, Any, ClassVar, Generic, Self
@@ -40,6 +42,11 @@ if TYPE_CHECKING:
 logger = logging.getLogger(__name__)
 
 
+def task_key(data: Mapping) -> str:
+    """Content identity for task wire data, independent of field order."""
+    return hashlib.sha256(json.dumps(data, sort_keys=True).encode()).hexdigest()
+
+
 class TaskResources(BaseModel):
     """Optional resource limits for the task."""
 
@@ -134,6 +141,12 @@ class Task(Generic[DataT, StateT, ConfigT]):
         self.data = data
         self.config = config if config is not None else self.config_type()()
 
+    @property
+    def hash(self) -> str:
+        """Content hash of the task's wire data — the identity that matches saved
+        rollouts to tasks; recorded on the trace as `task.hash`."""
+        return task_key(self.data.model_dump(mode="json", exclude_none=True))
+
     def with_system_prompt(self, system_prompt: str) -> Self:
         clone = copy.copy(self)
         clone.data = self.data.model_copy(update={"system_prompt": system_prompt})

{verifiers-0.3.1.dev17 → verifiers-0.3.1.dev19}/verifiers/v1/trace.py

@@ -121,6 +121,8 @@ class TraceTask(BaseModel, Generic[DataT]):
     """The Task class name (`type(task).__name__`)."""
     data: DataT
     """The (immutable) row being solved."""
+    hash: str | None = None
+    """Content hash of `data` (`Task.hash`) — the task's identity across runs."""
 
 
 class Reward(BaseModel):