verifiers 0.2.2.dev54__tar.gz → 0.2.2.dev55__tar.gz

{verifiers-0.2.2.dev54 → verifiers-0.2.2.dev55}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: verifiers
-Version: 0.2.2.dev54
+Version: 0.2.2.dev55
 Summary: Verifiers: Environments for LLM Reinforcement Learning
 Project-URL: Homepage, https://github.com/primeintellect-ai/verifiers
 Project-URL: Documentation, https://github.com/primeintellect-ai/verifiers

{verifiers-0.2.2.dev54 → verifiers-0.2.2.dev55}/verifiers/v1/cli/replay.py

@@ -31,7 +31,7 @@ from verifiers.v1.cli.resolve import narrow_taskset_config
 from verifiers.v1.configs.agent import WireAgentConfig
 from verifiers.v1.configs.cli.replay import ReplayConfig
 from verifiers.v1.state import state_cls
-from verifiers.v1.task import Task, WireTaskData, task_data_cls
+from verifiers.v1.task import Task, WireTaskData
 from verifiers.v1.trace import Trace
 from verifiers.v1.utils.interrupt import install_interrupt
 from verifiers.v1.utils.logging import setup_logging
@@ -76,7 +76,7 @@ async def run_replay(config: ReplayConfig, source: Path, out: Path) -> list[Trac
             "score() — multi-agent runs don't support replay"
         )
     task_cls = vf.task_type(config.taskset.id)
-    data_cls = task_data_cls(task_cls)
+    data_cls = task_cls.data_type()
     # `WireTaskData` reads any taskset's saved task without importing its Task type.
     # An episode may hold no traces (its env hooks failed before any agent ran);
     # there's nothing to re-score, so it drops out in the flatten. Each kept trace

{verifiers-0.2.2.dev54 → verifiers-0.2.2.dev55}/verifiers/v1/serve/server.py

@@ -22,7 +22,7 @@ from verifiers.v1.serve.types import (
     RunRequest,
     RunResponse,
 )
-from verifiers.v1.task import Task, task_data_cls
+from verifiers.v1.task import Task
 from verifiers.v1.types import SamplingConfig
 
 logger = logging.getLogger(__name__)
@@ -39,7 +39,7 @@ class EnvServer:
         self.taskset_id = config.taskset.id
         self.env = load_environment(config)
         self.task_cls = type(self.env.taskset).task_type()
-        self.data_cls = task_data_cls(self.task_cls)
+        self.data_cls = self.task_cls.data_type()
         # A dispatched task is its client-side model_dump(): a field excluded from
         # serialization would vanish on the wire and rebuild silently defaulted, so
         # refuse to serve such a taskset.

{verifiers-0.2.2.dev54 → verifiers-0.2.2.dev55}/verifiers/v1/task.py

@@ -1,25 +1,10 @@
-"""Task data, configuration, behavior, and scoring.
-
-`TaskData` is the wire half: a frozen pydantic model carrying everything a rollout's
-row IS — the base fields plus your typed, task-specific fields. It rides on
-`trace.task.data`, is what `traces.jsonl` stores, and what tool servers receive
-over the `/task` channel. Subclass it per dataset.
-
-`Task` is the behavior half: runtime prep (`setup`/`finalize`), server declarations
-(`tools`), well-formedness (`validate`), and per-trace judgement
-(`@reward`/`@metric` methods plus the plugged judges from `config.judges`, run by
-`score`). Subclass per dataset and parameterize `Task[MyData, MyState, MyConfig]`
-(all three default); judgement that compares sibling traces lives on
-`Env.finalize` instead.
-
-A Task instance is shared across its rollouts (`-r n` runs hold the same instance),
-so hooks must not stash per-rollout state on `self` — that lives on the trace
-(`trace.state`).
-
-On the wire only the data travels (plus the producing class's name,
-`trace.task.type`): a saved row reads back as `WireTaskData` without importing the
-taskset; a re-scoring consumer (`replay`) rebuilds the declared `TaskData` type and
-wraps it in the declared `Task` — one task type per taskset.
+"""Task data + behavior.
+
+`TaskData` is the wire half: a frozen pydantic model carrying the data which
+initializes a task instance. Rides on `trace.task.data` in `traces.jsonl`.
+
+`Task` is the behavior half: runtime prep (`setup`/`finalize`), tool declarations
+(`tools`), and scoring (`@reward`/`@metric`) methods.
 """
 
 from __future__ import annotations
@@ -52,6 +37,8 @@ logger = logging.getLogger(__name__)
 
 
 class TaskResources(BaseModel):
+    """Optional resource limits for the task."""
+
     model_config = ConfigDict(frozen=True)
 
     cpu: float | None = None
@@ -70,31 +57,35 @@ class TaskTimeout(BaseModel):
     model_config = ConfigDict(frozen=True)
 
     setup: float | None = None
+    """Timeout (in seconds) for the task's setup hook."""
     agent: float | None = None
+    """Timeout (in seconds) for the agent's solve attempt."""
     finalize: float | None = None
+    """Timeout (in seconds) for the task's finalize hook."""
     scoring: float | None = None
+    """Timeout (in seconds) for the task's scoring."""
 
 
 class TaskData(BaseModel):
-    """The task's wire half: one row's pure data, a frozen pydantic model. Subclass
-    per dataset to add typed task-specific fields next to the base fields; behavior
-    lives on `Task`, which wraps this (`self.data`)."""
-
     model_config = ConfigDict(frozen=True)
 
     idx: int | None = None
-    """Dataset row index — set by tasksets (selection, resume, and grouping key
-    on the eval path); `None` for ad-hoc tasks minted in scripts."""
+    """Taskset-autogenerated index."""
     name: str | None = None
+    """Optional human-readable task name."""
     description: str | None = None
+    """Optional human-readable task description."""
+
     prompt: str | Messages | None = None
-    """Initial user prompt; `None` means the user opens the conversation — run the
-    task through `agent.interaction()`, whose first `turn(message)` speaks first. (A
-    default, not just optional: the wire drops `None`s — `traces.jsonl` rows for
-    prompt-less tasks must read back.)"""
+    """Initial user prompt; unset if the user opens the conversation."""
     system_prompt: str | None = None
+    """Optional system prompt to prepend to the user prompt."""
+
     image: str | None = None
+    """Optional Docker image to use for the task. Only relevant for tasks that run in a container."""
     workdir: str | None = None
+    """Optional working directory to use for the task. Only relevant for tasks that run in a container."""
+
     network_allow: list[str] = Field(default_factory=lambda: ["*"])
     """Execution-time destinations requested by this task. `*` leaves the runtime
     allowlist unchanged; a concrete list replaces a wildcard or combines with existing
@@ -103,11 +94,13 @@ class TaskData(BaseModel):
     """Execution-time destinations denied by this task and combined with runtime
     blocks. Non-empty concrete allowlists cannot be combined with blocklists. Docker
     framework routes take precedence; ordinary Prime deny rules pass through unchanged."""
+
     artifacts: list[Artifact] = Field(default_factory=list)
     """Paths collected from one runtime and restored at the same locations in another,
     on top of the implicitly collected `/logs/artifacts/` convention dir. Declare
     runtime outputs that must cross that boundary. A declared path that is missing at
     collection time fails the rollout."""
+
     timeout: TaskTimeout = TaskTimeout()
     resources: TaskResources = TaskResources()
 
@@ -125,23 +118,10 @@ class WireTaskData(TaskData):
     model_config = ConfigDict(extra="allow")
 
 
-# No `default=`: an unparameterized `Trace`'s `task` field must serialize duck-typed
-# (a defaulted TypeVar narrows pydantic's serialization to the base `TaskData`, silently
-# dropping subclass fields from the wire).
 DataT = TypeVar("DataT", bound=TaskData)
 ConfigT = TypeVar("ConfigT", bound=TaskConfig, default=TaskConfig)
 
 
-def task_data_cls(cls: type) -> type[TaskData]:
-    """Resolve a task's `TaskData` specialization through its MRO, else `TaskData`."""
-    return concrete_type(cls, TaskData) or TaskData
-
-
-def task_config_cls(cls: type) -> type[TaskConfig]:
-    """Resolve a task's `TaskConfig` specialization through its MRO, else `TaskConfig`."""
-    return concrete_type(cls, TaskConfig) or TaskConfig
-
-
 def resolve_server_config(
     owner: str, config: BaseConfig, server_cls: type, *, sole: bool = True
 ) -> BaseConfig:
@@ -168,48 +148,20 @@ def resolve_server_config(
 
 
 class Task(Generic[DataT, StateT, ConfigT]):
-    """Behavior, lifecycle, servers, and scoring for one `TaskData` row.
-
-    Parameterize as `Task[MyData, MyState, MyConfig]`. Construction accepts the row
-    and an optional config; omitting config builds the declared config type. One task
-    instance is shared across a rollout group, so per-rollout state belongs on the trace.
-    """
-
     NEEDS_CONTAINER: ClassVar[bool] = False
+    """Whether the task needs a containerized environment (isolated filesystem, ...)."""
 
     tools: ClassVar[tuple[type[Toolset], ...]] = ()
 
     def __init__(self, data: DataT, config: ConfigT | None = None) -> None:
         self.data = data
-        self.config = config if config is not None else task_config_cls(type(self))()
+        self.config = config if config is not None else self.config_type()()
 
     def with_system_prompt(self, system_prompt: str) -> Self:
-        """A shallow copy of this task with `data.system_prompt` overridden. Copies the
-        instance instead of reconstructing via `type(self)(...)`, so a subclass with a
-        non-`(data, config)` constructor or extra load-time state keeps it. Used to apply the
-        config-layer / GEPA system prompt (see `TasksetConfig` and `verifiers.v1.gepa`)."""
         clone = copy.copy(self)
         clone.data = self.data.model_copy(update={"system_prompt": system_prompt})
         return clone
 
-    def plugged_judges(self) -> list[Judge]:
-        from verifiers.v1.loaders import load_judge
-
-        return [load_judge(config) for config in self.config.judges]
-
-    def server_config(self, server_cls: type) -> BaseConfig:
-        """The config a declared server class (`tools`) is built with (see
-        `resolve_server_config`). Override to pair explicitly."""
-        return resolve_server_config(
-            type(self).__name__,
-            self.config,
-            server_cls,
-            sole=len(set(type(self).tools)) == 1,
-        )
-
-    def tool_servers(self) -> list[Toolset]:
-        return [cls(self.server_config(cls)) for cls in type(self).tools]
-
     async def setup(self, trace: Trace, runtime: Runtime) -> None:
         return None
 
@@ -284,5 +236,29 @@ class Task(Generic[DataT, StateT, ConfigT]):
                 for key, value in items:
                     trace.record_reward(key, value, judge.config.weight)
 
+    @classmethod
+    def data_type(cls) -> type[TaskData]:
+        return concrete_type(cls, TaskData) or TaskData
+
+    @classmethod
+    def config_type(cls) -> type[TaskConfig]:
+        return concrete_type(cls, TaskConfig) or TaskConfig
+
+    def plugged_judges(self) -> list[Judge]:
+        from verifiers.v1.loaders import load_judge
+
+        return [load_judge(config) for config in self.config.judges]
+
+    def server_config(self, server_cls: type) -> BaseConfig:
+        return resolve_server_config(
+            type(self).__name__,
+            self.config,
+            server_cls,
+            sole=len(set(type(self).tools)) == 1,
+        )
+
+    def tool_servers(self) -> list[Toolset]:
+        return [cls(self.server_config(cls)) for cls in type(self).tools]
+
 
 TaskT = TypeVar("TaskT", bound=Task)

{verifiers-0.2.2.dev54 → verifiers-0.2.2.dev55}/verifiers/v1/taskset.py

@@ -103,8 +103,6 @@ class Taskset(ABC, Generic[TaskT, TasksetConfigT]):
         return concrete_type(cls, Task, origin=Taskset) or Task
 
     def server_config(self, server_cls: type) -> BaseConfig:
-        """The config a `tools` entry is built with, resolved off `self.config` (the
-        taskset config; see `resolve_server_config`). Override to pair explicitly."""
         return resolve_server_config(
             type(self).__name__,
             self.config,