verifiers 0.2.2.dev33__py3-none-any.whl → 0.2.2.dev35__py3-none-any.whl

verifiers/__init__.py

@@ -1,3 +1,5 @@
+# ruff: noqa
+
 from importlib.metadata import PackageNotFoundError
 from importlib.metadata import version as _version
 

verifiers/cli/commands/build.py

@@ -1,3 +1,5 @@
+# ruff: noqa
+
 """Environment build command module for external hosts."""
 
 from verifiers.scripts.build import main

verifiers/cli/commands/eval.py

@@ -1,3 +1,5 @@
+# ruff: noqa
+
 """Evaluation command module for external hosts."""
 
 from verifiers.scripts.eval import (

verifiers/cli/commands/gepa.py

@@ -1,3 +1,5 @@
+# ruff: noqa
+
 """GEPA command module for external hosts."""
 
 from verifiers.scripts.gepa import main

verifiers/cli/commands/init.py

@@ -1,3 +1,5 @@
+# ruff: noqa
+
 """Environment initialization command module for external hosts."""
 
 from verifiers.scripts.init import main

verifiers/cli/commands/install.py

@@ -1,3 +1,5 @@
+# ruff: noqa
+
 """Environment install command module for external hosts."""
 
 from verifiers.scripts.install import main

verifiers/cli/plugins/prime.py

@@ -1,3 +1,5 @@
+# ruff: noqa
+
 """Prime-hosted command plugin contract."""
 
 from dataclasses import dataclass

verifiers/clients/__init__.py

@@ -1,3 +1,5 @@
+# ruff: noqa
+
 from verifiers.clients.anthropic_messages_client import AnthropicMessagesClient
 from verifiers.clients.client import Client
 from verifiers.clients.nemorl_chat_completions_client import (

verifiers/clients/anthropic_messages_client.py

@@ -1,3 +1,5 @@
+# ruff: noqa
+
 import functools
 import json
 import time

verifiers/clients/client.py

@@ -1,3 +1,5 @@
+# ruff: noqa
+
 import logging
 from abc import ABC, abstractmethod
 from typing import TYPE_CHECKING, Generic, TypeVar

verifiers/clients/nemorl_chat_completions_client.py

@@ -108,8 +108,8 @@ def _attach_trajectory_tokens_to_prompt(
             continue
         msg = prompt[i]
         if (prompt_ids := tokens.get("prompt_ids")) is not None:
-            msg.prompt_token_ids = list(prompt_ids)
+            msg.prompt_token_ids = list(prompt_ids)  # ty:ignore[invalid-assignment]
         if (completion_ids := tokens.get("completion_ids")) is not None:
-            msg.generation_token_ids = list(completion_ids)
+            msg.generation_token_ids = list(completion_ids)  # ty:ignore[invalid-assignment]
         if (completion_logprobs := tokens.get("completion_logprobs")) is not None:
-            msg.generation_log_probs = list(completion_logprobs)
+            msg.generation_log_probs = list(completion_logprobs)  # ty:ignore[invalid-assignment]

verifiers/clients/openai_chat_completions_client.py

@@ -1,3 +1,5 @@
+# ruff: noqa
+
 import functools
 from collections.abc import Iterable, Mapping
 from typing import Any, TypeAlias, cast
@@ -231,7 +233,7 @@ class OpenAIChatCompletionsClient(
                     role="assistant",
                     content=cast(Any, normalize_content(message.content)),
                     tool_calls=cast(Any, oai_tool_calls),
-                    reasoning_content=message.reasoning_content,  # type: ignore[arg-type]
+                    reasoning_content=message.reasoning_content,  # type: ignore[arg-type]  # ty:ignore[invalid-key]
                 )
             elif isinstance(message, ToolMessage):
                 return ChatCompletionToolMessageParam(

verifiers/clients/openai_chat_completions_token_client.py

@@ -1,3 +1,5 @@
+# ruff: noqa
+
 from collections.abc import Mapping
 from typing import Any, Optional, cast
 
@@ -288,7 +290,7 @@ class OpenAIChatCompletionsTokenClient(OpenAIChatCompletionsClient):
         if tool_call_ids:
             dummy_assistant: OpenAIChatMessage = ChatCompletionAssistantMessageParam(
                 role="assistant",
-                reasoning_content="",  # type: ignore[typeddict-unknown-key]
+                reasoning_content="",  # type: ignore[typeddict-unknown-key]  # ty:ignore[invalid-key]
                 tool_calls=[
                     ChatCompletionMessageFunctionToolCallParam(
                         id=tc_id,
@@ -301,7 +303,7 @@ class OpenAIChatCompletionsTokenClient(OpenAIChatCompletionsClient):
         else:
             dummy_assistant: OpenAIChatMessage = ChatCompletionAssistantMessageParam(
                 role="assistant",
-                reasoning_content="",  # type: ignore[typeddict-unknown-key]
+                reasoning_content="",  # type: ignore[typeddict-unknown-key]  # ty:ignore[invalid-key]
                 content="x",
             )
 

verifiers/clients/openai_completions_client.py

@@ -1,3 +1,5 @@
+# ruff: noqa
+
 from openai import (
     AsyncOpenAI,
 )

verifiers/clients/openai_responses_client.py

@@ -1,3 +1,5 @@
+# ruff: noqa
+
 import time
 from collections.abc import Iterable, Mapping
 from typing import Any, TypeAlias

verifiers/clients/renderer_client.py

@@ -1,3 +1,5 @@
+# ruff: noqa
+
 """Renderer-based client.
 
 All tokenization happens client-side via a Renderer from the renderers package.

verifiers/decorators.py

@@ -1,3 +1,5 @@
+# ruff: noqa
+
 import inspect
 from typing import Any, Callable, Literal, TypeVar, overload
 

verifiers/envs/env_group.py

@@ -1,3 +1,5 @@
+# ruff: noqa
+
 import json
 from collections.abc import Mapping
 from typing import TYPE_CHECKING, Any, cast, final
@@ -292,7 +294,7 @@ class EnvGroup(vf.Environment):
         return dataset
 
     @final
-    async def run_rollout(  # type: ignore[override]
+    async def run_rollout(  # type: ignore[override]  # ty:ignore[override-of-final-method]
         self,
         input: RolloutInput,
         client: Client | ClientConfig,
@@ -328,10 +330,10 @@ class EnvGroup(vf.Environment):
             state_columns,
             env.env_client,
         )
-        return _set_info_route(output, route)  # type: ignore[return-value]
+        return _set_info_route(output, route)  # type: ignore[return-value]  # ty:ignore[invalid-return-type]
 
     @final
-    async def run_group(  # type: ignore[override]
+    async def run_group(  # type: ignore[override]  # ty:ignore[override-of-final-method]
         self,
         group_inputs: list[RolloutInput],
         client: Client | ClientConfig,
@@ -340,7 +342,7 @@ class EnvGroup(vf.Environment):
         max_retries: int = 0,
         state_columns: list[str] | None = None,
         env_client: EnvClient | None = None,
-    ) -> list[vf.RolloutOutput]:
+    ) -> list[vf.RolloutOutput]:  # ty:ignore[invalid-method-override]
         target_env_client = env_client or self.env_client
         if target_env_client is not None:
             if not isinstance(client, ClientConfig):
@@ -381,7 +383,7 @@ class EnvGroup(vf.Environment):
             state_columns,
             env.env_client,
         )
-        return [_set_info_route(output, route) for output in outputs]  # type: ignore[return-value]
+        return [_set_info_route(output, route) for output in outputs]  # type: ignore[return-value]  # ty:ignore[invalid-return-type]
 
     @final
     async def rollout(
@@ -415,7 +417,7 @@ class EnvGroup(vf.Environment):
             child_input["info"] = info
         else:
             child_input.pop("info", None)
-        return env_name, child_input, route  # type: ignore[return-value]
+        return env_name, child_input, route  # type: ignore[return-value]  # ty:ignore[invalid-return-type]
 
     def _input_env_route(self, input: RolloutInput) -> tuple[str, ...]:
         info = _info_dict(input.get("info"))

verifiers/envs/environment.py

@@ -1,3 +1,5 @@
+# ruff: noqa
+
 import asyncio
 import atexit
 import json
@@ -160,7 +162,7 @@ class Environment(ABC):
 
         if dataset is not None:
             if callable(dataset):
-                self.dataset_source: DatasetBuilder | None = cast(
+                self.dataset_source: DatasetBuilder | None = cast(  # ty:ignore[unsupported-operator]
                     DatasetBuilder, dataset
                 )
             else:
@@ -171,7 +173,7 @@ class Environment(ABC):
 
         if eval_dataset is not None:
             if callable(eval_dataset):
-                self.eval_dataset_source: DatasetBuilder | None = cast(
+                self.eval_dataset_source: DatasetBuilder | None = cast(  # ty:ignore[unsupported-operator]
                     DatasetBuilder, eval_dataset
                 )
             else:
@@ -292,7 +294,7 @@ class Environment(ABC):
                 if few_shot:
                     messages.extend(few_shot)
                 messages.append({"role": "user", "content": prompt_str})
-                return messages
+                return messages  # ty:ignore[invalid-return-type]
 
             if answer_key == "answer":
                 dataset = dataset.map(

verifiers/envs/experimental/__init__.py

@@ -1,3 +1,5 @@
+# ruff: noqa
+
 from verifiers.envs.experimental.sandbox_mixin import SandboxMixin
 
 __all__ = [

verifiers/envs/experimental/cli_agent_env.py

@@ -1,3 +1,5 @@
+# ruff: noqa
+
 import asyncio
 import logging
 import os

verifiers/envs/experimental/composable/README.md

@@ -30,13 +30,13 @@ from verifiers.envs.experimental.composable.harnesses.opencode import opencode_h
 from verifiers.envs.experimental.composable import ComposableEnv
 
 # Create a taskset
-taskset = R2EGymTaskSet()                 # 4578 SWE instances
+taskset = R2EGymTaskSet()  # 4578 SWE instances
 
 # Explore instances
-task = taskset[0]                          # one Task
-task.prompt                                # the problem statement
-task.sandbox_spec                          # SandboxSpec(image=..., cpu=4, ...)
-task.sandbox_spec.image                    # per-instance docker image
+task = taskset[0]  # one Task
+task.prompt  # the problem statement
+task.sandbox_spec  # SandboxSpec(image=..., cpu=4, ...)
+task.sandbox_spec.image  # per-instance docker image
 
 # Slice and filter
 small = taskset.take(100)
@@ -46,9 +46,9 @@ filtered = taskset.filter(lambda ex: ...)
 results = await taskset.validate(
     concurrency=50,
     out_path="outputs/validate.jsonl",
-    max_retries=2,       # retry on vf.InfraError
+    max_retries=2,  # retry on vf.InfraError
     sandbox_client_max_workers=100,  # optional sandbox client worker cap override
-    resume=True,         # skip indices already in out_path
+    resume=True,  # skip indices already in out_path
 )
 
 # Run with an agent
@@ -135,7 +135,9 @@ class MySWERubric(vf.Rubric):
         sandbox_client = state["sandbox_client"]
         sandbox_id = state["sandbox_id"]
         # Run tests in the sandbox
-        test_output = await self.taskset._run_tests(sandbox_client, sandbox_id, state, ...)
+        test_output = await self.taskset._run_tests(
+            sandbox_client, sandbox_id, state, ...
+        )
         return float(self.taskset._calculate_reward(test_output, info))
 
     @vf.cleanup

verifiers/envs/experimental/composable/__init__.py

@@ -1,3 +1,5 @@
+# ruff: noqa
+
 from verifiers.envs.experimental.composable.task import (
     SandboxSpec,
     Task,

verifiers/envs/experimental/composable/_filter.py

@@ -1,3 +1,5 @@
+# ruff: noqa
+
 """Helper for the base ``TaskSet.filter_fn`` kwarg.
 
 ``filter_fn`` accepts a Python expression string (typically a lambda) that

verifiers/envs/experimental/composable/composable_env.py

@@ -1,3 +1,5 @@
+# ruff: noqa
+
 """ComposableEnv — a CliAgentEnv that delegates to a TaskSet + Harness.
 
 Subclasses ``CliAgentEnv`` and overrides its hooks to delegate to the

verifiers/envs/experimental/composable/harness.py

@@ -1,3 +1,5 @@
+# ruff: noqa
+
 """Harness — agent-side configuration for ComposableEnv.
 
 A Harness declares how to install and run an agent binary, and where it

verifiers/envs/experimental/composable/harnesses/__init__.py

@@ -1,3 +1,5 @@
+# ruff: noqa
+
 from verifiers.envs.experimental.composable.harnesses.rlm import (
     DEFAULT_RLM_EXEC_TIMEOUT,
     DEFAULT_RLM_MAX_TURNS,

verifiers/envs/experimental/composable/harnesses/mini_swe_agent.py

@@ -1,3 +1,5 @@
+# ruff: noqa
+
 """mini-SWE-agent harness configuration."""
 
 from pathlib import PurePosixPath

verifiers/envs/experimental/composable/harnesses/rlm.py

@@ -1,3 +1,5 @@
+# ruff: noqa
+
 """RLM agent harness: install script, run command, and harness factory."""
 
 import hashlib

verifiers/envs/experimental/composable/sandbox_debug_env.py

@@ -1,3 +1,5 @@
+# ruff: noqa
+
 """No-agent debugger for sandbox-backed TaskSet instances."""
 
 import shlex

verifiers/envs/experimental/composable/swe_debug_env.py

@@ -1,3 +1,5 @@
+# ruff: noqa
+
 """Deprecated SWE-specific debug environment wrappers."""
 
 from typing import Any

verifiers/envs/experimental/composable/task.py

@@ -1,3 +1,5 @@
+# ruff: noqa
+
 """Task, TaskSet, SandboxTaskSet, and SandboxSpec — WHAT to solve.
 
 A **Task** is a single, fully-bound problem instance.
@@ -367,7 +369,7 @@ class TaskSet:
         try:
             from tqdm.auto import tqdm
         except ImportError:  # pragma: no cover
-            tqdm = None  # type: ignore[assignment]
+            tqdm = None  # type: ignore[assignment]  # ty:ignore[invalid-assignment]
 
         import verifiers as vf
 
@@ -519,7 +521,7 @@ class TaskSet:
                 state: State = {  # type: ignore[assignment]
                     "info": info,
                     "answer": row.get("answer", ""),
-                }
+                }  # ty:ignore[invalid-assignment]
                 try:
                     valid = await self.validate_instance(state)
                     return valid, None, state
@@ -541,7 +543,7 @@ class TaskSet:
                 state: State = {  # type: ignore[assignment]
                     "info": info,
                     "answer": row.get("answer", ""),
-                }
+                }  # ty:ignore[invalid-assignment]
                 spec = self.get_sandbox_spec(info)
                 # validate() runs without a SandboxMixin, so resolve
                 # spec.timeout_minutes=None (its "auto-derive at rollout

verifiers/envs/experimental/composable/tasksets/__init__.py

@@ -1,3 +1,5 @@
+# ruff: noqa
+
 from verifiers.envs.experimental.composable.tasksets.swe.swe_tasksets import (
     make_multiswe_taskset,
     make_openswe_taskset,

verifiers/envs/experimental/composable/tasksets/cp/__init__.py

@@ -1,3 +1,5 @@
+# ruff: noqa
+
 from .cp_task import CPRubric, CPTaskSet
 
 __all__ = ["CPTaskSet", "CPRubric"]

verifiers/envs/experimental/composable/tasksets/cp/cp_task.py

@@ -1,3 +1,5 @@
+# ruff: noqa
+
 """Competitive Programming TaskSet.
 
 Usage::

verifiers/envs/experimental/composable/tasksets/cp/test_utils.py

@@ -1,3 +1,5 @@
+# ruff: noqa
+
 # modifed from https://github.com/hendrycks/apps/blob/main/eval/testing_util.py
 # https://github.com/NovaSky-AI/SkyThought/blob/main/skythought/skythought_evals/tasks/taco/taco_util.py
 import asyncio
@@ -317,7 +319,7 @@ async def run_func_call(
                     )
                 try:
                     if isinstance(exec_outputs[0], tuple):
-                        exec_outputs = [list(x) for x in exec_outputs]
+                        exec_outputs = [list(x) for x in exec_outputs]  # ty:ignore[invalid-argument-type]
                         tmp_result = tmp_result or (
                             exec_outputs == test_case_outputs_parsed[0]
                         )

verifiers/envs/experimental/composable/tasksets/harbor/__init__.py

@@ -1,3 +1,5 @@
+# ruff: noqa
+
 from .harbor import (
     HarborDatasetRubric,
     HarborDatasetTaskSet,

verifiers/envs/experimental/composable/tasksets/harbor/harbor.py

@@ -1,3 +1,5 @@
+# ruff: noqa
+
 import json
 import logging
 import tarfile

verifiers/envs/experimental/composable/tasksets/harbor/terminal_lego.py

@@ -1,3 +1,5 @@
+# ruff: noqa
+
 """Terminal-Lego taskset built on the composable Harbor task layout."""
 
 import contextlib

verifiers/envs/experimental/composable/tasksets/math/math_task.py

@@ -1,3 +1,5 @@
+# ruff: noqa
+
 """Math TaskSet — QA-style math problems with sandbox-based scoring.
 
 Usage::

verifiers/envs/experimental/composable/tasksets/swe/__init__.py

@@ -1,3 +1,5 @@
+# ruff: noqa
+
 from .swe_tasksets import (
     make_multiswe_taskset,
     make_openswe_taskset,

verifiers/envs/experimental/composable/tasksets/swe/multi_swe/taskset.py

@@ -1,3 +1,5 @@
+# ruff: noqa
+
 import logging
 import shlex
 import tempfile

verifiers/envs/experimental/composable/tasksets/swe/openswe/taskset.py

@@ -1,3 +1,5 @@
+# ruff: noqa
+
 import logging
 import re
 import tempfile

verifiers/envs/experimental/composable/tasksets/swe/r2e_gym/taskset.py

@@ -1,3 +1,5 @@
+# ruff: noqa
+
 import json
 import logging
 import tempfile

verifiers/envs/experimental/composable/tasksets/swe/scale_swe/taskset.py

@@ -1,3 +1,5 @@
+# ruff: noqa
+
 """Scale-SWE TaskSet.
 
 Scale-SWE (https://huggingface.co/datasets/AweAI-Team/Scale-SWE) is a

verifiers/envs/experimental/composable/tasksets/swe/shared/test_patch.py

@@ -1,3 +1,5 @@
+# ruff: noqa
+
 """Revert-agent-test-edits + re-apply-test_patch helper.
 
 Mirrors the canonical SWE-bench pattern used by upstream's harness before

verifiers/envs/experimental/composable/tasksets/swe/swe_bench/taskset.py

@@ -1,3 +1,5 @@
+# ruff: noqa
+
 import json
 import logging
 import re

verifiers/envs/experimental/composable/tasksets/swe/swe_lego/taskset.py

@@ -1,3 +1,5 @@
+# ruff: noqa
+
 import json
 import logging
 import re

verifiers/envs/experimental/composable/tasksets/swe/swe_rebench_v2/taskset.py

@@ -1,3 +1,5 @@
+# ruff: noqa
+
 """SWE-rebench-V2 TaskSet.
 
 SWE-rebench-V2 (https://huggingface.co/datasets/nebius/SWE-rebench-V2) is a

verifiers/envs/experimental/composable/tasksets/swe/swe_smith/taskset.py

@@ -1,3 +1,5 @@
+# ruff: noqa
+
 """SWE-Smith multilingual TaskSet.
 
 SWE-Smith (https://github.com/SWE-bench/SWE-smith) is a synthetic bug-fix

verifiers/envs/experimental/gym_env.py

@@ -1,3 +1,5 @@
+# ruff: noqa
+
 from collections.abc import Callable
 from typing import Any, Protocol, TypeAlias, cast
 

verifiers/envs/experimental/harbor_env/__init__.py

@@ -1,3 +1,5 @@
+# ruff: noqa
+
 from .env import HarborEnv
 from .mcp import (
     NETWORK_TRANSPORTS,

verifiers/envs/experimental/harbor_env/env.py

@@ -1,3 +1,5 @@
+# ruff: noqa
+
 import json
 import logging
 import tarfile

verifiers/envs/experimental/harbor_env/mcp.py

@@ -180,7 +180,7 @@ class HarborMCPMixin:
         for name in list(jobs):
             try:
                 pid_file = shlex.quote(self._mcp_pid_file(name))
-                await self.sandbox_client.execute_command(  # type: ignore[attr-defined]
+                await self.sandbox_client.execute_command(  # type: ignore[attr-defined]  # ty:ignore[unresolved-attribute]
                     sandbox_id,
                     f'kill -9 -"$(cat {pid_file} 2>/dev/null)" 2>/dev/null; rm -f {pid_file}',
                     working_dir=None,
@@ -203,7 +203,7 @@ class HarborMCPMixin:
                 f"MCP server {server.name!r} has no port in its URL {server.url!r}"
             )
 
-        job = await self.sandbox_client.start_background_job(  # type: ignore[attr-defined]
+        job = await self.sandbox_client.start_background_job(  # type: ignore[attr-defined]  # ty:ignore[unresolved-attribute]
             sandbox_id=sandbox_id,
             command=(
                 f"setsid sh -c {shlex.quote(command)} & "
@@ -239,7 +239,7 @@ class HarborMCPMixin:
         consecutive_failures = 0
 
         while True:
-            status = await self.sandbox_client.get_background_job(  # type: ignore[attr-defined]
+            status = await self.sandbox_client.get_background_job(  # type: ignore[attr-defined]  # ty:ignore[unresolved-attribute]
                 sandbox_id, job
             )
             if getattr(status, "completed", False):
@@ -251,7 +251,7 @@ class HarborMCPMixin:
                     f"Stderr:\n{stderr}"
                 )
 
-            result = await self.sandbox_client.execute_command(  # type: ignore[attr-defined]
+            result = await self.sandbox_client.execute_command(  # type: ignore[attr-defined]  # ty:ignore[unresolved-attribute]
                 sandbox_id, health_cmd, working_dir=None, timeout=probe_timeout
             )
             if getattr(result, "exit_code", 1) == 0:
@@ -280,7 +280,7 @@ class HarborMCPMixin:
             if consecutive_failures >= hc.retries:
                 log_tail = ""
                 try:
-                    status = await self.sandbox_client.get_background_job(  # type: ignore[attr-defined]
+                    status = await self.sandbox_client.get_background_job(  # type: ignore[attr-defined]  # ty:ignore[unresolved-attribute]
                         sandbox_id, job
                     )
                     log_tail = (getattr(status, "stderr", "") or "").strip()[-2000:]
@@ -313,7 +313,7 @@ class HarborMCPMixin:
             f"echo {shlex.quote(f'127.0.0.1 {h}')} >> /etc/hosts)"
             for h in sorted(hosts)
         )
-        result = await self.sandbox_client.execute_command(  # type: ignore[attr-defined]
+        result = await self.sandbox_client.execute_command(  # type: ignore[attr-defined]  # ty:ignore[unresolved-attribute]
             sandbox_id, statements, working_dir=None
         )
         exit_code = getattr(result, "exit_code", 0)

verifiers/envs/experimental/mcp_env.py

@@ -1,3 +1,5 @@
+# ruff: noqa
+
 import asyncio
 import logging
 import threading

verifiers/envs/experimental/opencode_env.py

@@ -1,3 +1,5 @@
+# ruff: noqa
+
 import asyncio
 import json
 import logging

verifiers/envs/experimental/opencode_rlm_env.py

@@ -1,3 +1,5 @@
+# ruff: noqa
+
 """
 OpenCode RLM Environment.
 

verifiers/envs/experimental/sandbox_mixin.py

@@ -1,3 +1,5 @@
+# ruff: noqa
+
 import asyncio
 import io
 import logging

verifiers/envs/experimental/utils/file_locks.py

@@ -1,3 +1,5 @@
+# ruff: noqa
+
 from collections.abc import Iterator
 from contextlib import contextmanager
 import fcntl