verifiers 0.2.2.dev63__tar.gz → 0.2.2.dev65__tar.gz

{verifiers-0.2.2.dev63 → verifiers-0.2.2.dev65}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: verifiers
-Version: 0.2.2.dev63
+Version: 0.2.2.dev65
 Summary: Verifiers: Environments for LLM Reinforcement Learning
 Project-URL: Homepage, https://github.com/primeintellect-ai/verifiers
 Project-URL: Documentation, https://github.com/primeintellect-ai/verifiers

{verifiers-0.2.2.dev63 → verifiers-0.2.2.dev65}/tests/v1/conftest.py

@@ -41,8 +41,8 @@ import pytest
 import verifiers.v1 as vf
 from verifiers.v1.cli.eval.runner import run_eval
 from verifiers.v1.configs.cli.eval import EvalConfig
-from verifiers.v1.loaders import load_environment
 from verifiers.v1.trace import Trace
+from verifiers.v1.utils.loaders import load_environment
 
 # Fixture tasksets/envs (echo-v1, echo-agentic-v1, echo-v0, echo-multi-v0) live in
 # tests/v1/fixtures, added to the path via `pythonpath` in pyproject so the v1 loader and the

{verifiers-0.2.2.dev63 → verifiers-0.2.2.dev65}/tests/v1/test_judges.py

@@ -11,8 +11,8 @@ from pydantic import Field
 import verifiers.v1 as vf
 from verifiers.v1.graph import MessageNode
 from verifiers.v1.judge import Judge, JudgeResponse
-from verifiers.v1.loaders import judge_class, judge_config_type, load_judge
 from verifiers.v1.types import AssistantMessage, UserMessage
+from verifiers.v1.utils.loaders import judge_class, judge_config_type, load_judge
 
 RUBRIC_TOML = """
 [[criteria]]

{verifiers-0.2.2.dev63 → verifiers-0.2.2.dev65}/verifiers/v1/__init__.py

@@ -4,12 +4,6 @@ from pydantic_config import BaseConfig
 
 from verifiers.v1.acp import ACP
 from verifiers.v1.agent import Agent, Agents, Interaction, Segment, make_agent
-from verifiers.v1.artifacts import (
-    ARTIFACTS_DIR,
-    Artifact,
-    collect,
-    restore,
-)
 from verifiers.v1.clients import (
     BaseClientConfig,
     Client,
@@ -34,7 +28,6 @@ from verifiers.v1.configs.serve import (
 )
 from verifiers.v1.configs.task import TaskConfig
 from verifiers.v1.configs.taskset import TasksetConfig
-from verifiers.v1.decorators import metric, reward, stop, tool
 from verifiers.v1.env import Env
 from verifiers.v1.envs.single_agent import SingleAgentEnv, SingleAgentEnvConfig
 from verifiers.v1.episode import Episode, WireEpisode
@@ -59,24 +52,6 @@ from verifiers.v1.judges import (
     RubricJudge,
     RubricJudgeConfig,
 )
-from verifiers.v1.loaders import (
-    default_harness_id,
-    env_config_type,
-    environment_class,
-    harness_config_type,
-    import_environment,
-    import_harness,
-    import_judge,
-    import_taskset,
-    judge_config_type,
-    load_environment,
-    load_harness,
-    load_judge,
-    load_taskset,
-    resolve_env_config,
-    task_type,
-    taskset_config_type,
-)
 from verifiers.v1.mcp import (
     SharedToolsetConfig,
     Toolset,
@@ -91,24 +66,6 @@ from verifiers.v1.runtimes import (
     RuntimeInfo,
     SubprocessConfig,
 )
-from verifiers.v1.scoring import (
-    compare_stdout_results as compare_stdout_results,
-)
-from verifiers.v1.scoring import (
-    extract_boxed_answer as extract_boxed_answer,
-)
-from verifiers.v1.scoring import (
-    parse_judge_choice as parse_judge_choice,
-)
-from verifiers.v1.scoring import (
-    parse_pytest_outcomes as parse_pytest_outcomes,
-)
-from verifiers.v1.scoring import (
-    read_answer_file_or_last_reply as read_answer_file_or_last_reply,
-)
-from verifiers.v1.scoring import (
-    verify_boxed_math_answer as verify_boxed_math_answer,
-)
 from verifiers.v1.state import State, StateT
 from verifiers.v1.task import Task, TaskData, TaskResources, TaskTimeout, WireTaskData
 from verifiers.v1.taskset import Taskset
@@ -153,6 +110,13 @@ from verifiers.v1.types import (
     Usage,
     UserMessage,
 )
+from verifiers.v1.utils.artifacts import (
+    ARTIFACTS_DIR,
+    Artifact,
+    collect,
+    restore,
+)
+from verifiers.v1.utils.decorators import metric, reward, stop, tool
 from verifiers.v1.utils.git import (
     PATCH_CAP_BYTES as PATCH_CAP_BYTES,
 )
@@ -162,6 +126,42 @@ from verifiers.v1.utils.git import (
 from verifiers.v1.utils.git import (
     resolve_head as resolve_head,
 )
+from verifiers.v1.utils.loaders import (
+    default_harness_id,
+    env_config_type,
+    environment_class,
+    harness_config_type,
+    import_environment,
+    import_harness,
+    import_judge,
+    import_taskset,
+    judge_config_type,
+    load_environment,
+    load_harness,
+    load_judge,
+    load_taskset,
+    resolve_env_config,
+    task_type,
+    taskset_config_type,
+)
+from verifiers.v1.utils.score import (
+    compare_stdout_results as compare_stdout_results,
+)
+from verifiers.v1.utils.score import (
+    extract_boxed_answer as extract_boxed_answer,
+)
+from verifiers.v1.utils.score import (
+    parse_judge_choice as parse_judge_choice,
+)
+from verifiers.v1.utils.score import (
+    parse_pytest_outcomes as parse_pytest_outcomes,
+)
+from verifiers.v1.utils.score import (
+    read_answer_file_or_last_reply as read_answer_file_or_last_reply,
+)
+from verifiers.v1.utils.score import (
+    verify_boxed_math_answer as verify_boxed_math_answer,
+)
 
 __all__ = [  # noqa: RUF022 - grouped by public API area
     # types

{verifiers-0.2.2.dev63 → verifiers-0.2.2.dev65}/verifiers/v1/agent.py

@@ -24,7 +24,6 @@ from verifiers.v1.configs.agent import AgentConfig, TimeoutConfig
 from verifiers.v1.harness import Harness
 from verifiers.v1.interception import Interception, InterceptionServer
 from verifiers.v1.mcp import SharedToolServer
-from verifiers.v1.retries import backoff, trace_should_retry
 from verifiers.v1.rollout import RolloutRun, RolloutTimeouts, _as_messages
 from verifiers.v1.runtimes import (
     NetworkPolicyConfig,
@@ -49,6 +48,7 @@ from verifiers.v1.utils.compile import (
     resolve_runtime_config,
     validate_pairing,
 )
+from verifiers.v1.utils.retries import backoff, trace_should_retry
 
 __all__ = ["Agent", "AgentConfig", "Agents", "TimeoutConfig", "make_agent"]
 
@@ -241,7 +241,7 @@ class Agent:
         client: Client | None = None,
         interception: Interception | None = None,
     ) -> None:
-        from verifiers.v1.loaders import harness_config_type, load_harness
+        from verifiers.v1.utils.loaders import harness_config_type, load_harness
 
         if config.model is None:
             raise ValueError(

{verifiers-0.2.2.dev63 → verifiers-0.2.2.dev65}/verifiers/v1/cli/dashboard/eval.py

@@ -29,7 +29,7 @@ from verifiers.v1.utils.install import env_name
 from verifiers.v1.utils.interrupt import cleaning_up
 
 if TYPE_CHECKING:
-    from verifiers.v1.push import PushState
+    from verifiers.v1.utils.platform import PushState
 
 # For sizing pages to the terminal: detects the real terminal height/width each access (the live
 # view writes to the same terminal). Reused so we don't rebuild it every refresh tick.
@@ -160,7 +160,7 @@ def _warning(config: EvalConfig) -> Text | None:
     """A local-runtime caution when any code-running seat resolves to the subprocess
     runtime (the tool-less chat loops are exempt), shown above the overview rather
     than as a row in it."""
-    from verifiers.v1.loaders import harness_class
+    from verifiers.v1.utils.loaders import harness_class
 
     if any(
         getattr(config.env, role).runtime.type == "subprocess"

{verifiers-0.2.2.dev63 → verifiers-0.2.2.dev65}/verifiers/v1/cli/debug.py

@@ -24,12 +24,12 @@ from verifiers.v1.cli.resolve import (
     with_positional_taskset,
 )
 from verifiers.v1.configs.cli.debug import DebugConfig
-from verifiers.v1.decorators import invoke
 from verifiers.v1.runtimes import ProgramResult, Runtime, make_runtime
 from verifiers.v1.state import state_cls
 from verifiers.v1.task import Task
 from verifiers.v1.trace import AgentInfo, Error, Trace, TraceTask
 from verifiers.v1.utils.compile import resolve_runtime_config
+from verifiers.v1.utils.decorators import invoke
 from verifiers.v1.utils.interrupt import install_interrupt
 from verifiers.v1.utils.logging import setup_logging
 

{verifiers-0.2.2.dev63 → verifiers-0.2.2.dev65}/verifiers/v1/cli/eval/main.py

@@ -120,7 +120,7 @@ def main(argv: list[str] | None = None) -> None:
         # disk. Exit on the conventional Ctrl-C code without a traceback.
         raise SystemExit(130)
     if config.push and not rich:
-        from verifiers.v1.push import push_traces
+        from verifiers.v1.utils.platform import push_traces
 
         push_traces(episodes, config)
     if not rich:  # --rich is the whole output; otherwise dump each trace as JSON

{verifiers-0.2.2.dev63 → verifiers-0.2.2.dev65}/verifiers/v1/cli/eval/runner.py

@@ -3,6 +3,7 @@
 import asyncio
 import contextlib
 import logging
+import random
 import time
 
 from verifiers.v1.cli.dashboard import dashboard
@@ -17,8 +18,8 @@ from verifiers.v1.clients import ModelContext, resolve_client
 from verifiers.v1.configs.cli.eval import EvalConfig
 from verifiers.v1.env import Env, RunSlot
 from verifiers.v1.episode import Episode
+from verifiers.v1.taskset import SEED
 from verifiers.v1.trace import EvalRunInfo
-from verifiers.v1.utils.sampling import sample
 
 logger = logging.getLogger(__name__)
 
@@ -86,7 +87,7 @@ async def run_eval(env: Env, config: EvalConfig) -> list[Episode]:
         slots = [RunSlot.finished(episode) for episode in finished] + planned
         push_state = None
         if config.push and config.rich:
-            from verifiers.v1.push import PushState
+            from verifiers.v1.utils.platform import PushState
 
             push_state = PushState()
         display = (
@@ -102,7 +103,7 @@ async def run_eval(env: Env, config: EvalConfig) -> list[Episode]:
             if (
                 push_state is not None
             ):  # upload off the event loop so the view keeps refreshing
-                from verifiers.v1.push import push_traces
+                from verifiers.v1.utils.platform import push_traces
 
                 push_state.started = True
                 await asyncio.to_thread(push_traces, episodes, config, push_state)
@@ -136,7 +137,7 @@ async def run_eval_server(config: EvalConfig) -> list[Episode]:
     )
     tasks = []
     if not legacy:
-        from verifiers.v1.loaders import load_taskset
+        from verifiers.v1.utils.loaders import load_taskset
 
         # The client owns the taskset: load it here, once — the server (and its pool
         # workers) never load data, they rebuild each dispatched task from its request.
@@ -184,7 +185,10 @@ async def run_eval_server(config: EvalConfig) -> list[Episode]:
         if legacy:
             info = await client.info()
             group_scored = info.requires_group_scoring
-            idxs = sample(list(range(info.num_tasks)), config.shuffle, config.num_tasks)
+            idxs = list(range(info.num_tasks))
+            if config.shuffle:
+                random.Random(SEED).shuffle(idxs)
+            idxs = idxs[: config.num_tasks]
             plan = [({"task_idx": idx}, config.num_rollouts) for idx in idxs]
         else:
             group_scored = False

{verifiers-0.2.2.dev63 → verifiers-0.2.2.dev65}/verifiers/v1/cli/validate.py

@@ -20,12 +20,12 @@ from verifiers.v1.cli.resolve import (
     with_positional_taskset,
 )
 from verifiers.v1.configs.cli.validate import ValidateConfig
-from verifiers.v1.decorators import invoke
 from verifiers.v1.runtimes import make_runtime
 from verifiers.v1.state import state_cls
 from verifiers.v1.task import Task
 from verifiers.v1.trace import Trace, TraceTask
 from verifiers.v1.utils.compile import resolve_runtime_config
+from verifiers.v1.utils.decorators import invoke
 from verifiers.v1.utils.interrupt import install_interrupt
 from verifiers.v1.utils.logging import setup_logging
 

{verifiers-0.2.2.dev63 → verifiers-0.2.2.dev65}/verifiers/v1/configs/agent.py

@@ -57,7 +57,10 @@ class AgentConfig(BaseConfig):
         stays None = the taskset's default). The lazy import keeps class-body
         `AgentConfig()` defaults constructible while this module initializes."""
         if isinstance(data, dict) and data.get("harness") is not None:
-            from verifiers.v1.loaders import harness_config_type, narrow_plugin_field
+            from verifiers.v1.utils.loaders import (
+                harness_config_type,
+                narrow_plugin_field,
+            )
 
             narrow_plugin_field(data, "harness", harness_config_type, "bash")
         return data

{verifiers-0.2.2.dev63 → verifiers-0.2.2.dev65}/verifiers/v1/configs/cli/debug.py

@@ -52,7 +52,7 @@ class DebugConfig(BaseConfig):
     @model_validator(mode="before")
     @classmethod
     def resolve_taskset(cls, data):
-        from verifiers.v1.loaders import narrow_plugin_field, taskset_config_type
+        from verifiers.v1.utils.loaders import narrow_plugin_field, taskset_config_type
 
         narrow_plugin_field(data, "taskset", taskset_config_type)
         return data

{verifiers-0.2.2.dev63 → verifiers-0.2.2.dev65}/verifiers/v1/configs/cli/env.py

@@ -35,7 +35,7 @@ def resolve_env_field(data: dict, narrowed: "type[EnvConfig] | None" = None) ->
                     raw.model_dump() if isinstance(raw, BaseConfig) else raw
                 )
             return data
-        from verifiers.v1.loaders import resolve_env_config
+        from verifiers.v1.utils.loaders import resolve_env_config
 
         data["env"] = resolve_env_config(raw)
     except ValidationError as e:

{verifiers-0.2.2.dev63 → verifiers-0.2.2.dev65}/verifiers/v1/configs/cli/replay.py

@@ -50,7 +50,7 @@ class ReplayConfig(BaseConfig):
     @model_validator(mode="before")
     @classmethod
     def _resolve_taskset(cls, data):
-        from verifiers.v1.loaders import narrow_plugin_field, taskset_config_type
+        from verifiers.v1.utils.loaders import narrow_plugin_field, taskset_config_type
 
         if isinstance(data, dict) and not data.get("taskset"):
             # The base layer is usually a saved eval config, which keeps its

{verifiers-0.2.2.dev63 → verifiers-0.2.2.dev65}/verifiers/v1/configs/cli/validate.py

@@ -54,7 +54,7 @@ class ValidateConfig(BaseConfig):
     @model_validator(mode="before")
     @classmethod
     def _resolve_taskset(cls, data):
-        from verifiers.v1.loaders import narrow_plugin_field, taskset_config_type
+        from verifiers.v1.utils.loaders import narrow_plugin_field, taskset_config_type
 
         narrow_plugin_field(data, "taskset", taskset_config_type)
         return data

{verifiers-0.2.2.dev63 → verifiers-0.2.2.dev65}/verifiers/v1/configs/env.py

@@ -100,7 +100,10 @@ class EnvConfig(BaseConfig):
         """Narrow `taskset` to its concrete config type by `id`; lazy import for
         the same reason as `AgentConfig._resolve_harness`."""
         if isinstance(data, dict) and data.get("taskset") is not None:
-            from verifiers.v1.loaders import narrow_plugin_field, taskset_config_type
+            from verifiers.v1.utils.loaders import (
+                narrow_plugin_field,
+                taskset_config_type,
+            )
 
             narrow_plugin_field(data, "taskset", taskset_config_type)
         return data
@@ -157,7 +160,7 @@ def _declared_agent_configs(config: EnvConfig) -> dict[str, AgentConfig]:
 def default_agent_harness(taskset_id: str) -> HarnessConfig:
     """What an unpinned role's `harness=None` resolves to: the taskset's bundled
     harness when it ships one, else the built-in `bash`."""
-    from verifiers.v1.loaders import default_harness_id, harness_config_type
+    from verifiers.v1.utils.loaders import default_harness_id, harness_config_type
 
     ident = default_harness_id(taskset_id)
     return harness_config_type(ident).model_validate({"id": ident})

{verifiers-0.2.2.dev63 → verifiers-0.2.2.dev65}/verifiers/v1/configs/judge.py

@@ -33,7 +33,7 @@ def judge_key(config: JudgeConfig) -> str:
 
 
 def resolve_judges(entries: Sequence[Any]) -> list[JudgeConfig]:
-    from verifiers.v1.loaders import judge_config_type
+    from verifiers.v1.utils.loaders import judge_config_type
 
     resolved = []
     for entry in entries:

{verifiers-0.2.2.dev63 → verifiers-0.2.2.dev65}/verifiers/v1/env.py

@@ -29,12 +29,12 @@ from verifiers.v1.interception import (
     requires_tunnel,
 )
 from verifiers.v1.mcp import SharedToolServer, serve_shared
-from verifiers.v1.retries import run_episode_with_retry
 from verifiers.v1.runtimes import SubprocessConfig, runtime_is_local
 from verifiers.v1.task import Task
 from verifiers.v1.trace import Error, Trace
 from verifiers.v1.utils.generic import concrete_type
 from verifiers.v1.utils.memory import trim_memory_periodically
+from verifiers.v1.utils.retries import run_episode_with_retry
 
 logger = logging.getLogger(__name__)
 
@@ -81,7 +81,7 @@ class Env(ABC, Generic[ConfigT]):
     task the agent actually receives — an env-minted task carries its own needs."""
 
     def __init__(self, config: ConfigT) -> None:
-        from verifiers.v1.loaders import load_harness, load_taskset
+        from verifiers.v1.utils.loaders import load_harness, load_taskset
 
         config_cls = concrete_type(type(self), EnvConfig, origin=Env) or EnvConfig
         if not isinstance(config, config_cls):

{verifiers-0.2.2.dev63 → verifiers-0.2.2.dev65}/verifiers/v1/harness.py

@@ -8,11 +8,11 @@ from typing import TYPE_CHECKING, ClassVar, Generic, TypeVar
 
 from verifiers.v1.clients import ModelContext
 from verifiers.v1.configs.harness import HarnessConfig
-from verifiers.v1.decorators import discover_decorated, invoke_all
 from verifiers.v1.errors import HarnessError, SandboxError, boundary
 from verifiers.v1.runtimes import ProgramResult, Runtime
 from verifiers.v1.task import TaskData
 from verifiers.v1.types import Messages
+from verifiers.v1.utils.decorators import discover_decorated, invoke_all
 
 if TYPE_CHECKING:
     # Annotation-only: `Trace` appears in signatures only, so this module stays

{verifiers-0.2.2.dev63 → verifiers-0.2.2.dev65}/verifiers/v1/harnesses/openclaw/harness.py

@@ -309,7 +309,9 @@ class OpenClawHarness(Harness[OpenClawHarnessConfig]):
                 'while kill -0 "$gateway_pid" 2>/dev/null; do sleep 0.1; done; '
                 f"rm -f {shlex.quote(pid_path)}' EXIT; "
                 f'{shlex.quote(binary)} acp --url "ws://127.0.0.1:{gateway_port}" '
-                f"--token {shlex.quote(trace.id)} --no-prefix-cwd"
+                f"--token {shlex.quote(trace.id)} "
+                f"--session {shlex.quote(f'agent:main:acp-bridge:{trace.id}')} "
+                "--no-prefix-cwd"
             ),
         ]
         # OpenClaw rejects ACP per-session MCP declarations; the isolated Gateway

{verifiers-0.2.2.dev63 → verifiers-0.2.2.dev65}/verifiers/v1/harnesses/rlm/harness.py

@@ -10,12 +10,12 @@ from pydantic import Field, model_validator
 
 from verifiers.v1.clients import ModelContext
 from verifiers.v1.configs.harness import HarnessConfig
-from verifiers.v1.decorators import metric
 from verifiers.v1.dialects.chat import message_to_wire
 from verifiers.v1.harness import Harness
 from verifiers.v1.runtimes import ProgramResult, Runtime
 from verifiers.v1.task import TaskData
 from verifiers.v1.trace import Trace
+from verifiers.v1.utils.decorators import metric
 
 logger = logging.getLogger(__name__)
 

{verifiers-0.2.2.dev63 → verifiers-0.2.2.dev65}/verifiers/v1/interception/tunnel/prime.py

@@ -36,7 +36,7 @@ class PrimeTunnel(Tunnel[PrimeTunnelConfig]):
         from prime_tunnel import Tunnel as TunnelClient
 
         from verifiers.v1.errors import TunnelError
-        from verifiers.v1.retries import retrying
+        from verifiers.v1.utils.retries import retrying
 
         label = f"host tunnel (port {port})"
         try:

{verifiers-0.2.2.dev63 → verifiers-0.2.2.dev65}/verifiers/v1/judge.py

@@ -40,7 +40,7 @@ for a pure call (e.g. in tests).
 
 A judge can also be *plugged* rather than called from task code: a judge with an `id` and a
 `score` implementation is a plugin (like a taskset or harness — see `verifiers.v1.judges` for the
-built-ins and `verifiers.v1.loaders` for resolution). Its config lives on `TaskConfig.judges`
+built-ins and `verifiers.v1.utils.loaders` for resolution). Its config lives on `TaskConfig.judges`
 only — judges are config, never row data (`--taskset.task.judges`; a taskset config may
 pre-plug them as class defaults) — and `Task.score` builds and runs it after the task's own
 `@reward`s.
@@ -61,9 +61,9 @@ from verifiers.v1.configs.judge import (
     judge_key,
 )
 from verifiers.v1.dialects.chat import message_to_wire
-from verifiers.v1.scoring import parse_judge_choice
 from verifiers.v1.types import Messages, Usage
 from verifiers.v1.utils.generic import concrete_type
+from verifiers.v1.utils.score import parse_judge_choice
 
 if TYPE_CHECKING:
     from verifiers.v1.task import TaskData

{verifiers-0.2.2.dev63 → verifiers-0.2.2.dev65}/verifiers/v1/legacy.py

@@ -507,11 +507,12 @@ def _legacy_output_dir(config) -> Path:
 async def run_legacy_eval(config) -> list[Episode]:
     """Run a legacy environment in process and return v1 episode records."""
     import asyncio
+    import random
 
     from verifiers import load_environment
     from verifiers.v1.cli.output import append_trace, save_config
+    from verifiers.v1.taskset import SEED
     from verifiers.v1.utils.install import ensure_installed, env_name
-    from verifiers.v1.utils.sampling import sample
 
     # Install from the env hub on demand for an `org/name[@version]` id (a local id is
     # already importable), then load by module name.
@@ -523,7 +524,10 @@ async def run_legacy_eval(config) -> list[Episode]:
     ):  # post-load knobs (max_total_completion_tokens, …)
         env.set_kwargs(**config.legacy.extra_env_kwargs)
     dataset = env.get_eval_dataset()  # the eval split (falls back to train when unset)
-    idxs = sample(list(range(len(dataset))), config.shuffle, config.num_tasks)
+    idxs = list(range(len(dataset)))
+    if config.shuffle:
+        random.Random(SEED).shuffle(idxs)
+    idxs = idxs[: config.num_tasks]
 
     client = _eval_client(config.client, config.model)
     sampling_args = config.sampling.model_dump(exclude_none=True)

{verifiers-0.2.2.dev63 → verifiers-0.2.2.dev65}/verifiers/v1/mcp/toolset.py

@@ -4,10 +4,10 @@ from typing import TYPE_CHECKING
 
 from pydantic_config import BaseConfig
 
-from verifiers.v1.decorators import discover_decorated
 from verifiers.v1.mcp.server import ConfigT, ServerBase
 from verifiers.v1.runtimes import RuntimeConfig, SubprocessConfig
 from verifiers.v1.state import StateT
+from verifiers.v1.utils.decorators import discover_decorated
 
 if TYPE_CHECKING:
     from mcp.server.fastmcp import FastMCP

{verifiers-0.2.2.dev63 → verifiers-0.2.2.dev65}/verifiers/v1/rollout.py

@@ -27,7 +27,6 @@ from dataclasses import dataclass
 
 from verifiers.v1.clients import ModelContext
 from verifiers.v1.configs.agent import AgentConfig
-from verifiers.v1.decorators import discover_decorated, invoke
 from verifiers.v1.dialects import parse_message
 from verifiers.v1.errors import (
     HarnessError,
@@ -54,6 +53,7 @@ from verifiers.v1.state import state_cls
 from verifiers.v1.task import Task, TaskData
 from verifiers.v1.trace import AgentInfo, Trace, TraceTask
 from verifiers.v1.types import Messages
+from verifiers.v1.utils.decorators import discover_decorated, invoke
 
 logger = logging.getLogger(__name__)
 

{verifiers-0.2.2.dev63 → verifiers-0.2.2.dev65}/verifiers/v1/serve/pool.py

@@ -334,7 +334,7 @@ def serve_env(
             if (
                 "config_data" in server_kwargs
             ):  # rebuild the env config for an in-process server
-                from verifiers.v1.loaders import resolve_env_config
+                from verifiers.v1.utils.loaders import resolve_env_config
 
                 server_kwargs = {**server_kwargs}
                 server_kwargs["config"] = resolve_env_config(

{verifiers-0.2.2.dev63 → verifiers-0.2.2.dev65}/verifiers/v1/serve/server.py

@@ -12,7 +12,6 @@ from verifiers.v1.clients import ModelContext, resolve_client
 from verifiers.v1.clients.client import Client
 from verifiers.v1.clients.config import ClientConfig
 from verifiers.v1.configs.env import EnvConfig
-from verifiers.v1.loaders import load_environment
 from verifiers.v1.serve.types import (
     BaseResponse,
     HealthResponse,
@@ -24,6 +23,7 @@ from verifiers.v1.serve.types import (
 )
 from verifiers.v1.task import Task
 from verifiers.v1.types import SamplingConfig
+from verifiers.v1.utils.loaders import load_environment
 
 logger = logging.getLogger(__name__)
 

{verifiers-0.2.2.dev63 → verifiers-0.2.2.dev65}/verifiers/v1/state.py

@@ -1,8 +1,4 @@
-"""Mutable state shared within one rollout.
-
-Tool servers synchronize it through the interception state channel. It is excluded
-from serialized traces.
-"""
+"""Mutable rollout-level state shared across tool server + host."""
 
 from pydantic import BaseModel, ConfigDict, Field
 from typing_extensions import TypeVar
@@ -12,6 +8,7 @@ from verifiers.v1.utils.generic import concrete_type
 
 class State(BaseModel):
     model_config = ConfigDict(ser_json_inf_nan="constants")
+
     artifacts: dict[str, bytes] = Field(default_factory=dict)
 
 

{verifiers-0.2.2.dev63 → verifiers-0.2.2.dev65}/verifiers/v1/task.py

@@ -18,12 +18,12 @@ from typing import TYPE_CHECKING, ClassVar, Generic, Self
 from pydantic import BaseModel, ConfigDict, Field
 from typing_extensions import TypeVar
 
-from verifiers.v1.artifacts import Artifact
 from verifiers.v1.configs.task import TaskConfig
-from verifiers.v1.decorators import discover_decorated, invoke_all
 from verifiers.v1.errors import TaskError, boundary
 from verifiers.v1.state import StateT
 from verifiers.v1.types import Messages, content_text
+from verifiers.v1.utils.artifacts import Artifact
+from verifiers.v1.utils.decorators import discover_decorated, invoke_all
 from verifiers.v1.utils.generic import concrete_type
 
 if TYPE_CHECKING:
@@ -217,7 +217,7 @@ class Task(Generic[DataT, StateT, ConfigT]):
         return concrete_type(cls, TaskConfig) or TaskConfig
 
     def plugged_judges(self) -> list[Judge]:
-        from verifiers.v1.loaders import load_judge
+        from verifiers.v1.utils.loaders import load_judge
 
         return [load_judge(config) for config in self.config.judges]
 

{verifiers-0.2.2.dev63 → verifiers-0.2.2.dev65}/verifiers/v1/taskset.py

@@ -26,11 +26,12 @@ from typing_extensions import TypeVar
 from verifiers.v1.configs.taskset import TasksetConfig
 from verifiers.v1.task import Task, TaskT
 from verifiers.v1.utils.generic import concrete_type
-from verifiers.v1.utils.sampling import SEED
 
 if TYPE_CHECKING:
     from verifiers.v1.mcp import Toolset
 
+SEED = 0  # fixed so `--shuffle` samples the same items every run (reproducible)
+
 TasksetConfigT = TypeVar("TasksetConfigT", bound=TasksetConfig, default=TasksetConfig)
 
 

{verifiers-0.2.2.dev63 → verifiers-0.2.2.dev65}/verifiers/v1/tasksets/harbor/taskset.py

@@ -24,14 +24,14 @@ from pathlib import Path
 
 from pydantic import BaseModel, Field
 
-from verifiers.v1.artifacts import Artifact, collect
 from verifiers.v1.configs.taskset import TasksetConfig
-from verifiers.v1.decorators import reward
 from verifiers.v1.errors import SandboxError
 from verifiers.v1.runtimes import Runtime
 from verifiers.v1.task import Task, TaskData, TaskResources, TaskTimeout
 from verifiers.v1.taskset import Taskset
 from verifiers.v1.trace import Trace
+from verifiers.v1.utils.artifacts import Artifact, collect
+from verifiers.v1.utils.decorators import reward
 
 CACHE = Path.home() / ".cache" / "harbor"
 HARBOR_INSTALL_HINT = "uv sync --python 3.12 --extra harbor"

{verifiers-0.2.2.dev63 → verifiers-0.2.2.dev65}/verifiers/v1/tasksets/lean/taskset.py

@@ -16,7 +16,6 @@ from pydantic_config import BaseConfig
 
 from verifiers.v1.configs.task import TaskConfig
 from verifiers.v1.configs.taskset import TasksetConfig
-from verifiers.v1.decorators import reward
 from verifiers.v1.runtimes import Runtime
 from verifiers.v1.state import State
 from verifiers.v1.task import Task, TaskData, TaskResources
@@ -28,6 +27,7 @@ from verifiers.v1.tasksets.lean.scoring import (
     protected_signature_substring_present,
 )
 from verifiers.v1.trace import Trace
+from verifiers.v1.utils.decorators import reward
 
 # Lean v4.27 with Mathlib v4.27.
 DEFAULT_DOCKER_IMAGE = "team-clyvldofb0000gg1kx39rgzjq/lean-tactic:mathlib-v4.27.0-v3"