verifiers 0.3.1.dev31__tar.gz → 0.3.1.dev33__tar.gz

{verifiers-0.3.1.dev31 → verifiers-0.3.1.dev33}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: verifiers
-Version: 0.3.1.dev31
+Version: 0.3.1.dev33
 Summary: Verifiers: Environments for LLM Reinforcement Learning
 Project-URL: Homepage, https://github.com/primeintellect-ai/verifiers
 Project-URL: Documentation, https://github.com/primeintellect-ai/verifiers

{verifiers-0.3.1.dev31 → verifiers-0.3.1.dev33}/tests/v1/fixtures/echo_v1.py

@@ -30,6 +30,10 @@ class EchoData(vf.TaskData):
 
 
 class EchoTask(vf.Task[EchoData]):
+    @property
+    def key(self) -> str:
+        return f"echo:{self.data.answer}"
+
     @vf.stop
     async def single_turn(self, trace: vf.Trace) -> bool:
         return trace.num_turns >= 1

{verifiers-0.3.1.dev31 → verifiers-0.3.1.dev33}/tests/v1/test_e2e.py

@@ -136,6 +136,8 @@ async def test_single_turn(run_v1, harness, harness_runtime, tmp_path):
     assert trace.num_turns == 1
     assert trace.stop_condition == "agent_completed"
     assert trace.reward == 1.0
+    assert trace.task.key == f"echo:{trace.task.data.answer}"
+    assert trace.task.hash is not None
     # The seat's resolved identity rides the trace (policy metadata for trainers).
     assert trace.agent is not None
     assert trace.agent.config.sampling.max_tokens == 2048

{verifiers-0.3.1.dev31 → verifiers-0.3.1.dev33}/tests/v1/test_taskset.py

@@ -37,7 +37,9 @@ def test_head_bounds_an_infinite_taskset() -> None:
 
 def test_iteration_materializes_a_finite_taskset() -> None:
     taskset = FiniteTaskset(vf.TasksetConfig())
-    assert len(list(taskset)) == 10
+    tasks = list(taskset)
+    assert len(tasks) == 10
+    assert all(task.key == task.hash for task in tasks)
     assert idxs(taskset.head(4)) == [0, 1, 2, 3]
 
 

{verifiers-0.3.1.dev31 → verifiers-0.3.1.dev33}/tests/v1/test_trace.py

@@ -22,12 +22,18 @@ def test_bare_trace_round_trip():
     # The minimal trace: a base task, no nodes, no extras — dump and back into a plain Trace.
     tr = vf.Trace(
         agent=vf.AgentInfo(config=vf.AgentConfig()),
-        task=vf.TraceTask(type="Task", data=vf.TaskData(idx=3, prompt="hello")),
+        task=vf.TraceTask(
+            type="Task",
+            data=vf.TaskData(idx=3, prompt="hello"),
+            key="dataset/example-3",
+            hash="content-digest",
+        ),
     )
     rt = vf.Trace.model_validate(tr.model_dump())
     assert rt.id == tr.id
     assert rt.task.type == "Task"
     assert rt.task.data.idx == 3 and rt.task.data.prompt == "hello"
+    assert rt.task.key == "dataset/example-3" and rt.task.hash == "content-digest"
     assert rt.num_turns == 0 and rt.num_branches == 0
     assert rt.reward == 0.0 and rt.errors == []
 

{verifiers-0.3.1.dev31 → verifiers-0.3.1.dev33}/verifiers/v1/cli/debug.py

@@ -193,7 +193,9 @@ async def run_action(runtime: Runtime, config: DebugConfig) -> dict[str, Any]:
 
 async def debug_task(task: Task, config: DebugConfig) -> tuple[Trace, bool]:
     trace = Trace(
-        task=TraceTask(type=type(task).__name__, data=task.data, hash=task.hash),
+        task=TraceTask(
+            type=type(task).__name__, data=task.data, key=task.key, hash=task.hash
+        ),
         state=state_cls(type(task))(),
         # No agent plays here (no model, no harness): the seat records the
         # runtime policy, and the provisioned box lands on it below (id, resolved

{verifiers-0.3.1.dev31 → verifiers-0.3.1.dev33}/verifiers/v1/cli/validate.py

@@ -247,7 +247,9 @@ async def _run_gold(task: Task, config: ValidateConfig) -> ResultRow:
     valid, exc = False, None
     try:
         trace = Trace(
-            task=TraceTask(type=type(task).__name__, data=task.data, hash=task.hash),
+            task=TraceTask(
+                type=type(task).__name__, data=task.data, key=task.key, hash=task.hash
+            ),
             state=state_cls(type(task))(),
             # No agent runs here — the info only records the runtime policy.
             agent=vf.AgentInfo(
@@ -288,7 +290,9 @@ async def _run_setup(task: Task, config: ValidateConfig) -> ResultRow:
     valid, exc = False, None
     try:
         trace = Trace(
-            task=TraceTask(type=type(task).__name__, data=task.data, hash=task.hash),
+            task=TraceTask(
+                type=type(task).__name__, data=task.data, key=task.key, hash=task.hash
+            ),
             state=state_cls(type(task))(),
             # No agent runs here — the info only records the runtime policy.
             agent=vf.AgentInfo(

{verifiers-0.3.1.dev31 → verifiers-0.3.1.dev33}/verifiers/v1/configs/agent.py

@@ -6,7 +6,7 @@ from pydantic_config import BaseConfig
 from verifiers.v1.clients import ClientConfig
 from verifiers.v1.configs.harness import HarnessConfig, WireHarnessConfig
 from verifiers.v1.configs.retries import RetryConfig
-from verifiers.v1.runtimes import RuntimeConfig, SubprocessConfig
+from verifiers.v1.runtimes import PrimeConfig, RuntimeConfig
 from verifiers.v1.types import SamplingConfig
 
 
@@ -26,7 +26,7 @@ class TimeoutConfig(BaseConfig):
 class AgentConfig(BaseConfig):
     harness: SerializeAsAny[HarnessConfig] | None = None
     """The agent's program (None = the taskset's default harness)."""
-    runtime: RuntimeConfig = SubprocessConfig()
+    runtime: RuntimeConfig = PrimeConfig()
     """Runtime for the harness program — the policy each run provisions its box
     from; tool servers choose their placement separately."""
 

{verifiers-0.3.1.dev31 → verifiers-0.3.1.dev33}/verifiers/v1/configs/cli/debug.py

@@ -8,14 +8,14 @@ from pydantic_config import BaseConfig
 
 from verifiers.v1.configs.cli.validate import CheckTimeoutConfig
 from verifiers.v1.configs.taskset import TasksetConfig
-from verifiers.v1.runtimes import DockerConfig, RuntimeConfig
+from verifiers.v1.runtimes import PrimeConfig, RuntimeConfig
 
 
 class DebugConfig(BaseConfig):
     uuid: str = Field(default_factory=lambda: str(uuid4()), exclude=True)
     """Auto-generated run id, used as the default output directory leaf."""
     taskset: SerializeAsAny[TasksetConfig] = TasksetConfig()
-    runtime: RuntimeConfig = DockerConfig()
+    runtime: RuntimeConfig = PrimeConfig()
     """Where each task's setup hook and debug action run."""
     command: str | None = None
     """Inline shell command executed as `sh -lc <command>` after setup."""

{verifiers-0.3.1.dev31 → verifiers-0.3.1.dev33}/verifiers/v1/configs/cli/validate.py

@@ -7,7 +7,7 @@ from pydantic import AliasChoices, Field, SerializeAsAny, model_validator
 from pydantic_config import BaseConfig
 
 from verifiers.v1.configs.taskset import TasksetConfig
-from verifiers.v1.runtimes import DockerConfig, RuntimeConfig
+from verifiers.v1.runtimes import PrimeConfig, RuntimeConfig
 
 
 class CheckTimeoutConfig(BaseConfig):
@@ -23,7 +23,7 @@ class ValidateConfig(BaseConfig):
     """Auto-generated run id — the default output directory leaf. Excluded from the
     saved config so re-running it starts a fresh run."""
     taskset: SerializeAsAny[TasksetConfig] = TasksetConfig()
-    runtime: RuntimeConfig = DockerConfig()
+    runtime: RuntimeConfig = PrimeConfig()
     """Where each task's validation hooks run."""
     timeout: CheckTimeoutConfig = CheckTimeoutConfig()
     only_setup: bool = False

{verifiers-0.3.1.dev31 → verifiers-0.3.1.dev33}/verifiers/v1/configs/client.py

@@ -18,7 +18,7 @@ from pydantic import Field, model_validator
 from pydantic_config import BaseConfig
 from renderers import RendererConfig
 
-from verifiers.v1.utils.prime_config import load_prime_config
+from verifiers.v1.utils.prime import load_prime_config
 
 DEFAULT_PRIME_INFERENCE_URL = "https://api.pinference.ai/api/v1"
 

{verifiers-0.3.1.dev31 → verifiers-0.3.1.dev33}/verifiers/v1/interception/tunnel/prime.py

@@ -11,6 +11,7 @@ from typing import Literal
 from verifiers.v1.interception.tunnel.base import BaseTunnelConfig, Tunnel
 from verifiers.v1.runtimes.limiters import creation_limiter
 from verifiers.v1.utils.aio import run_shielded
+from verifiers.v1.utils.prime import ensure_prime_auth
 
 # The prime_tunnel service caps tunnel starts at 512/min per API token — a property of the
 # tunnel service, shared by every process for the user that opens one. One user-global
@@ -27,6 +28,10 @@ class PrimeTunnelConfig(BaseTunnelConfig):
 
 
 class PrimeTunnel(Tunnel[PrimeTunnelConfig]):
+    def __init__(self, config: PrimeTunnelConfig | None = None) -> None:
+        ensure_prime_auth()
+        super().__init__(config)
+
     @contextlib.asynccontextmanager
     async def expose(self, port: int) -> AsyncIterator[str]:
         """Bridge the host `port` to a public URL via prime_tunnel (frpc). Tunnel creation

{verifiers-0.3.1.dev31 → verifiers-0.3.1.dev33}/verifiers/v1/rollout.py

@@ -85,6 +85,7 @@ class Rollout:
             task=TraceTask(
                 type=type(task).__name__,
                 data=task.data,
+                key=task.key,
                 hash=task.hash,
             ),
             state=state_cls(type(task))(),

{verifiers-0.3.1.dev31 → verifiers-0.3.1.dev33}/verifiers/v1/runtimes/prime.py

@@ -31,6 +31,7 @@ from verifiers.v1.runtimes.base import (
     parse_gpu,
 )
 from verifiers.v1.runtimes.limiters import creation_limiter
+from verifiers.v1.utils.prime import ensure_prime_auth
 
 logger = logging.getLogger(__name__)
 
@@ -46,7 +47,7 @@ class PrimeConfig(NetworkPolicyConfig):
     ~10 minutes) and caches the result, so later sandboxes on the same ref start in
     seconds."""
     workdir: str = "/app"
-    vm: bool = False
+    vm: bool = True
     """Run as a micro-VM rather than a container (kernel features / stronger isolation)."""
     guaranteed: bool = False
     """Request guaranteed (vs best-effort) capacity."""
@@ -125,6 +126,7 @@ class PrimeRuntime(Runtime):
     is_local: ClassVar[bool] = False
 
     def __init__(self, config: PrimeConfig, name: str | None = None) -> None:
+        ensure_prime_auth()
         super().__init__(name)
         self.config = config
         self.info = PrimeRuntimeInfo(**config.model_dump())

{verifiers-0.3.1.dev31 → verifiers-0.3.1.dev33}/verifiers/v1/task.py

@@ -144,10 +144,20 @@ class Task(Generic[DataT, StateT, ConfigT]):
 
     @property
     def hash(self) -> str:
-        """Content hash of the task's wire data — the identity that matches saved
-        rollouts to tasks; recorded on the trace as `task.hash`."""
+        """Content hash of the task's wire data, recorded on the trace as
+        ``task.hash``."""
         return task_key(self.data.model_dump(mode="json", exclude_none=True))
 
+    @property
+    def key(self) -> str:
+        """Stable task identity, recorded on the trace as ``task.key``.
+
+        The content hash is the default. Override this when the task data contains
+        run-local fields such as an autogenerated ``idx`` and the underlying dataset
+        provides a durable identifier. Keys must be unique within a taskset.
+        """
+        return self.hash
+
     def with_system_prompt(self, system_prompt: str) -> Self:
         clone = copy.copy(self)
         clone.data = self.data.model_copy(update={"system_prompt": system_prompt})

{verifiers-0.3.1.dev31 → verifiers-0.3.1.dev33}/verifiers/v1/trace.py

@@ -122,8 +122,10 @@ class TraceTask(BaseModel, Generic[DataT]):
     """The Task class name (`type(task).__name__`)."""
     data: DataT
     """The (immutable) row being solved."""
+    key: str | None = None
+    """Stable task identity (``Task.key``); defaults to the content hash unless overridden."""
     hash: str | None = None
-    """Content hash of `data` (`Task.hash`) — the task's identity across runs."""
+    """Content hash of ``data`` (``Task.hash``)."""
 
 
 class InterceptRecord(BaseModel):

{verifiers-0.3.1.dev31 → verifiers-0.3.1.dev33}/verifiers/v1/utils/platform.py

@@ -17,7 +17,7 @@ import httpx
 from verifiers.v1.configs.cli.eval import EvalConfig
 from verifiers.v1.episode import Episode
 from verifiers.v1.trace import Trace
-from verifiers.v1.utils.prime_config import load_prime_config
+from verifiers.v1.utils.prime import load_prime_config
 
 logger = logging.getLogger(__name__)
 

verifiers-0.3.1.dev31/verifiers/v1/utils/prime_config.py → verifiers-0.3.1.dev33/verifiers/v1/utils/prime.py

@@ -1,5 +1,6 @@
 import json
 import logging
+import os
 from pathlib import Path
 
 logger = logging.getLogger(__name__)
@@ -17,3 +18,12 @@ def load_prime_config() -> dict:
     except (RuntimeError, json.JSONDecodeError, OSError) as e:
         logger.warning(f"Failed to load prime config: {e}")
     return {}
+
+
+def ensure_prime_auth() -> None:
+    """Exit when no Prime API key is configured (`prime login` or `$PRIME_API_KEY`)."""
+    if os.getenv("PRIME_API_KEY") or load_prime_config().get("api_key"):
+        return
+    raise SystemExit(
+        "not authenticated with prime - run `prime login` or set $PRIME_API_KEY"
+    )