verifiers 0.1.15.dev4__tar.gz → 0.1.15.dev5__tar.gz

{verifiers-0.1.15.dev4 → verifiers-0.1.15.dev5}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: verifiers
-Version: 0.1.15.dev4
+Version: 0.1.15.dev5
 Summary: Verifiers: Environments for LLM Reinforcement Learning
 Project-URL: Homepage, https://github.com/primeintellect-ai/verifiers
 Project-URL: Documentation, https://github.com/primeintellect-ai/verifiers

{verifiers-0.1.15.dev4 → verifiers-0.1.15.dev5}/tests/test_v1_harbor_cli.py

@@ -9,10 +9,18 @@ from uuid import uuid4
 
 import pytest
 
+import verifiers as root_vf
 import verifiers.v1 as vf
 from verifiers.v1.packages.harnesses.pi import pi_mcp_json, pi_models_json
+from verifiers.v1.packages.harnesses.terminus_2 import (
+    DEFAULT_API_BASE_URL,
+    DEFAULT_HARBOR_PACKAGE,
+    DEFAULT_MODEL_NAME,
+    Terminus2,
+    terminus_2_agent_script,
+)
 from verifiers.v1.packages.tasksets.harbor import harbor_reward
-from verifiers.v1.utils.program_utils import merge_task_program
+from verifiers.v1.utils.program_utils import merge_task_program, merge_task_sandbox
 
 
 def write_harbor_task(root: Path, name: str = "task-a") -> Path:
@@ -85,6 +93,13 @@ def test_harbor_taskset_loads_package_tasks_with_program_patch(
     assert task["sandbox"]["memory_gb"] == 2.0
     assert task["sandbox"]["disk_size_gb"] == 8.0
     assert task["sandbox"]["command_timeout"] == 600
+    assert "network_access" not in task["sandbox"]
+    assert (
+        merge_task_sandbox({"network_access": False, "scope": "rollout"}, task)[
+            "network_access"
+        ]
+        is False
+    )
     assert task["harbor"]["test_timeout"] == 300.0
     assert task["program"]["files"] == {
         "/task/instruction.md": {"task": "instruction"},
@@ -200,6 +215,8 @@ def test_packaged_harbor_and_opencode_imports_are_reexported() -> None:
     assert vf.OpenCode is OpenCode
     assert vf.OpenCodeConfig is OpenCodeConfig
     assert vf.Pi is Pi
+    assert vf.Terminus2 is Terminus2
+    assert root_vf.Terminus2 is Terminus2
     assert vf.HarborTaskset is HarborTaskset
 
 
@@ -254,6 +271,45 @@ def test_pi_harness_writes_intercepted_model_and_mcp_config() -> None:
     assert mcp["mcpServers"]["verifiers-tools"]["command"] == "python3"
 
 
+def test_terminus_2_harness_builds_sandbox_program() -> None:
+    harness = vf.Terminus2(
+        system_prompt="extra system prompt",
+        agent_workdir="/workspace",
+        max_turns=7,
+        python_version="3.12",
+    )
+    program = cast(dict[str, object], harness.program)
+    command = cast(list[object], program["command"])
+    setup = cast(str, program["setup"])
+    files = cast(dict[str, object], program["files"])
+    artifacts = cast(dict[str, object], program["artifacts"])
+    env = cast(dict[str, object], program.get("env", {}))
+
+    assert isinstance(harness, vf.Harness)
+    assert "/terminus_2/instruction.md" in files
+    assert "/terminus_2/system_prompt.txt" in files
+    assert "apt-get -o Acquire::Retries=3 update" in setup
+    assert "apt-get -o Acquire::Retries=3 install" in setup
+    assert "git" not in setup
+    assert "terminus_2_log" in artifacts
+    assert "OPENAI_MODEL" not in env
+
+    run_script = cast(str, command[2])
+    assert "TERMINUS_2_WORKDIR=/workspace" in run_script
+    assert f"--with {DEFAULT_HARBOR_PACKAGE}" in run_script
+    assert "git+https://github.com" not in run_script
+    assert "max_turns=7" in run_script
+
+    script = terminus_2_agent_script(max_turns=7)
+    compile(script, "terminus_2_agent.py", "exec")
+    assert DEFAULT_MODEL_NAME in script
+    assert DEFAULT_API_BASE_URL in script
+    assert "OPENAI_MODEL" not in script
+    assert "PRIME_API_KEY" in script
+    assert "async def prepare_logs_for_host(self) -> None" in script
+    assert "max_turns=7" in script
+
+
 def test_task_program_merges_into_command_program_without_collisions() -> None:
     harness = vf.Harness(
         program={

{verifiers-0.1.15.dev4 → verifiers-0.1.15.dev5}/verifiers/__init__.py

@@ -1,4 +1,4 @@
-__version__ = "0.1.15.dev4"
+__version__ = "0.1.15.dev5"
 
 import importlib
 import os
@@ -93,6 +93,7 @@ __all__ = [
     "Pi",
     "RLM",
     "RLMConfig",
+    "Terminus2",
     "Environment",
     "MultiTurnEnv",
     "SingleTurnEnv",
@@ -217,6 +218,7 @@ _LAZY_IMPORTS = {
     "Pi": "verifiers.v1:Pi",
     "RLM": "verifiers.v1:RLM",
     "RLMConfig": "verifiers.v1:RLMConfig",
+    "Terminus2": "verifiers.v1:Terminus2",
     "get_messages": "verifiers.v1:get_messages",
     "add_metric": "verifiers.v1:add_metric",
     "add_reward": "verifiers.v1:add_reward",
@@ -311,6 +313,7 @@ if TYPE_CHECKING:
         ProgramConfig,
         RLM,
         RLMConfig,
+        Terminus2,
         SandboxConfig,
         Task,
         TaskRow,

{verifiers-0.1.15.dev4 → verifiers-0.1.15.dev5}/verifiers/v1/README.md

@@ -537,8 +537,8 @@ signature.
 Reusable CLI programs should be packaged as `Harness` subclasses. Package
 implementations live under `verifiers.v1.packages` while the v1 API stabilizes,
 and are re-exported from `verifiers.v1` for normal use. `OpenCode`, `Pi`,
-`MiniSWEAgent`, and `RLM` are bundled `Harness` leaf wrappers for common
-coding-agent CLIs.
+`MiniSWEAgent`, `Terminus2`, and `RLM` are bundled `Harness` leaf wrappers for
+common coding-agent CLIs.
 
 ```python
 import verifiers as vf
@@ -560,6 +560,8 @@ endpoint and, when tools are enabled, installs the Pi MCP adapter and writes a
 project `.mcp.json`. Neither side needs to know the other's private fields.
 `MiniSWEAgent` owns mini-swe-agent installation, config layering, endpoint env,
 and log/trajectory artifacts.
+`Terminus2` owns Harbor Terminus agent installation, endpoint env, and log
+artifacts.
 `RLM` follows the same boundary for recursive LLM runs: `HarborTaskset` owns
 the task directory and tests, while `RLM` owns RLM installation, optional skill
 upload to `/rlm/skills`, endpoint wiring, and trajectory filtering.

{verifiers-0.1.15.dev4 → verifiers-0.1.15.dev5}/verifiers/v1/RE_MIGRATION.md

@@ -548,8 +548,8 @@ Use this for:
 Migration:
 
 1. Use `vf.HarborTaskset` for Harbor-format task directories.
-2. Use `vf.OpenCode()`, `vf.Pi()`, `vf.MiniSWEAgent()`, or `vf.RLM()` for the
-   command harness.
+2. Use `vf.OpenCode()`, `vf.Pi()`, `vf.MiniSWEAgent()`, `vf.Terminus2()`, or
+   `vf.RLM()` for the command harness.
 3. Put task-owned uploads and sandbox overrides on the taskset.
 4. Keep scoring as reward/metric functions on the taskset.
 
@@ -580,6 +580,8 @@ Gotchas:
   adapter setup, and log artifacts.
 - `MiniSWEAgent` owns mini-swe-agent installation, config layering, endpoint
   env, and log/trajectory artifacts.
+- `Terminus2` owns Harbor Terminus agent installation, endpoint env, and log
+  artifacts.
 - `RLM` owns RLM installation, optional `/task/rlm-skills` upload, endpoint
   wiring, and trajectory filtering.
 - `task.program` is the merge point for task-owned program files/env/setup.

{verifiers-0.1.15.dev4 → verifiers-0.1.15.dev5}/verifiers/v1/__init__.py

@@ -41,6 +41,7 @@ from .packages.harnesses import (
     Pi,
     RLM,
     RLMConfig,
+    Terminus2,
 )
 from .utils.scoring_utils import (
     add_metric,
@@ -97,6 +98,7 @@ __all__ = [
     "ProgramConfig",
     "RLM",
     "RLMConfig",
+    "Terminus2",
     "SandboxConfig",
     "State",
     "Task",

{verifiers-0.1.15.dev4 → verifiers-0.1.15.dev5}/verifiers/v1/config.py

@@ -367,15 +367,20 @@ class EnvConfig(Config):
         return base
 
 
-def sandbox_config_mapping(value: object | None) -> ConfigData | None:
+def sandbox_config_mapping(
+    value: object | None, *, fill_defaults: bool = True
+) -> ConfigData | None:
     if value is None:
         return None
     if isinstance(value, SandboxConfig):
-        return value.model_dump(exclude_none=True)
+        return value.model_dump(exclude_none=True, exclude_unset=not fill_defaults)
     if isinstance(value, Mapping):
         mapping = cast(ConfigMap, value)
         prefer = mapping.get("prefer")
         if prefer is not None and prefer != "program":
             raise ValueError("sandbox.prefer must be 'program'.")
-        return SandboxConfig.from_config(mapping).model_dump(exclude_none=True)
+        sandbox = SandboxConfig.from_config(mapping).model_dump(exclude_none=True)
+        if fill_defaults:
+            return sandbox
+        return {key: sandbox[key] for key in mapping if key in sandbox}
     raise TypeError("Sandbox config must be a mapping.")

{verifiers-0.1.15.dev4 → verifiers-0.1.15.dev5}/verifiers/v1/packages/harnesses/__init__.py

@@ -3,6 +3,7 @@ from .mini_swe_agent import MiniSWEAgent
 from .opencode import OpenCode
 from .pi import Pi
 from .rlm import RLM
+from .terminus_2 import Terminus2
 
 __all__ = [
     "MiniSWEAgent",
@@ -11,4 +12,5 @@ __all__ = [
     "Pi",
     "RLM",
     "RLMConfig",
+    "Terminus2",
 ]

verifiers-0.1.15.dev5/verifiers/v1/packages/harnesses/terminus_2.py

@@ -0,0 +1,286 @@
+import shlex
+from pathlib import PurePosixPath
+
+from typing_extensions import Unpack
+
+from .command import HarnessKwargs, command_program, command_sandbox
+from ...config import SandboxConfig
+from ...harness import Harness
+from ...utils.prompt_utils import (
+    state_system_prompt_text,
+    task_text as task_instruction_text,
+)
+from ...types import ConfigMap, ProgramMap, ProgramOptionMap, ProgramValue, PromptInput
+
+DEFAULT_AGENT_WORKDIR = "/app"
+DEFAULT_INSTRUCTION_PATH = "/terminus_2/instruction.md"
+DEFAULT_SYSTEM_PROMPT_PATH = "/terminus_2/system_prompt.txt"
+DEFAULT_LOG_PATH = "/logs/agent/terminus_2.log"
+DEFAULT_HARBOR_PACKAGE = "harbor==0.6.6"
+DEFAULT_PYTHON_VERSION = "3.12"
+DEFAULT_MODEL_NAME = "openai/gpt-4.1-mini"
+DEFAULT_API_BASE_URL = "https://api.pinference.ai/api/v1"
+
+
+class Terminus2(Harness):
+    def __init__(
+        self,
+        *,
+        agent_workdir: str = DEFAULT_AGENT_WORKDIR,
+        instruction_path: str = DEFAULT_INSTRUCTION_PATH,
+        system_prompt_path: str = DEFAULT_SYSTEM_PROMPT_PATH,
+        log_path: str = DEFAULT_LOG_PATH,
+        harbor_package: str = DEFAULT_HARBOR_PACKAGE,
+        python_version: str = DEFAULT_PYTHON_VERSION,
+        model_name: str = DEFAULT_MODEL_NAME,
+        api_base_url: str = DEFAULT_API_BASE_URL,
+        system_prompt: PromptInput | None = None,
+        sandbox: bool | ConfigMap | SandboxConfig = True,
+        program: ProgramMap | None = None,
+        max_turns: int | None = 4,
+        **kwargs: Unpack[HarnessKwargs],
+    ):
+        files: dict[str, ProgramValue] = {
+            instruction_path: task_instruction_text,
+        }
+        if system_prompt is not None:
+            files[system_prompt_path] = state_system_prompt_text
+        artifacts: ProgramOptionMap = {
+            "terminus_2_log": {
+                "path": log_path,
+                "format": "text",
+                "optional": True,
+            }
+        }
+        command = [
+            "bash",
+            "-lc",
+            build_terminus_2_run_script(
+                agent_workdir=agent_workdir,
+                instruction_path=instruction_path,
+                system_prompt_path=system_prompt_path
+                if system_prompt is not None
+                else None,
+                log_path=log_path,
+                harbor_package=harbor_package,
+                python_version=python_version,
+                model_name=model_name,
+                api_base_url=api_base_url,
+                max_turns=max_turns,
+            ),
+        ]
+        super().__init__(
+            program=command_program(
+                command=command,
+                sandbox=sandbox,
+                files=files,
+                setup=build_terminus_2_install_script(),
+                artifacts=artifacts,
+                program=program,
+            ),
+            sandbox=command_sandbox(sandbox),
+            system_prompt=system_prompt,
+            max_turns=max_turns,
+            **kwargs,
+        )
+
+
+def build_terminus_2_install_script() -> str:
+    return """\
+set -e
+apt-get -o Acquire::Retries=3 update -qq
+apt-get -o Acquire::Retries=3 install -y -qq curl ca-certificates > /dev/null 2>&1
+if ! command -v uv >/dev/null 2>&1; then
+  curl -LsSf https://astral.sh/uv/install.sh | sh
+fi
+"""
+
+
+def build_terminus_2_run_script(
+    *,
+    agent_workdir: str = DEFAULT_AGENT_WORKDIR,
+    instruction_path: str = DEFAULT_INSTRUCTION_PATH,
+    system_prompt_path: str | None = DEFAULT_SYSTEM_PROMPT_PATH,
+    log_path: str = DEFAULT_LOG_PATH,
+    harbor_package: str = DEFAULT_HARBOR_PACKAGE,
+    python_version: str = DEFAULT_PYTHON_VERSION,
+    model_name: str = DEFAULT_MODEL_NAME,
+    api_base_url: str = DEFAULT_API_BASE_URL,
+    max_turns: int | None = 4,
+) -> str:
+    log_dir = str(PurePosixPath(log_path).parent)
+    agent_script = terminus_2_agent_script(
+        instruction_path=instruction_path,
+        system_prompt_path=system_prompt_path,
+        log_dir=log_dir,
+        model_name=model_name,
+        api_base_url=api_base_url,
+        max_turns=max_turns,
+    )
+    return f"""\
+set -eo pipefail
+export PATH="$HOME/.local/bin:$PATH"
+
+TERMINUS_2_WORKDIR="${{AGENT_WORKDIR:-}}"
+if [[ -z "$TERMINUS_2_WORKDIR" ]]; then
+    TERMINUS_2_WORKDIR={shlex.quote(agent_workdir)}
+fi
+export AGENT_WORKDIR="$TERMINUS_2_WORKDIR"
+
+mkdir -p {shlex.quote(log_dir)} "$TERMINUS_2_WORKDIR"
+cd "$TERMINUS_2_WORKDIR"
+uv --no-config run --no-project --quiet \
+  --python {shlex.quote(python_version)} \
+  --with {shlex.quote(harbor_package)} \
+  python - <<'PY' 2>&1 | tee -a {shlex.quote(log_path)}
+{agent_script}
+PY
+"""
+
+
+def terminus_2_agent_script(
+    *,
+    instruction_path: str = DEFAULT_INSTRUCTION_PATH,
+    system_prompt_path: str | None = DEFAULT_SYSTEM_PROMPT_PATH,
+    log_dir: str = "/logs/agent",
+    model_name: str = DEFAULT_MODEL_NAME,
+    api_base_url: str = DEFAULT_API_BASE_URL,
+    max_turns: int | None = 4,
+) -> str:
+    system_prompt_block = ""
+    if system_prompt_path is not None:
+        system_prompt_block = f"""\
+    system_prompt_path = Path({system_prompt_path!r})
+    if system_prompt_path.exists() and system_prompt_path.stat().st_size > 0:
+        instruction = system_prompt_path.read_text() + "\\n\\n" + instruction
+"""
+    return f"""\
+from __future__ import annotations
+
+import asyncio
+import logging
+import os
+import shutil
+import subprocess
+from pathlib import Path
+
+from harbor.agents.terminus_2 import Terminus2
+from harbor.environments.base import BaseEnvironment, ExecResult
+from harbor.models.agent.context import AgentContext
+from harbor.models.environment_type import EnvironmentType
+from harbor.models.trial.paths import TrialPaths
+
+
+class LocalEnvironment(BaseEnvironment):
+    def __init__(self, workdir: Path, logs_dir: Path):
+        self.workdir = workdir
+        self.trial_paths = TrialPaths(trial_dir=logs_dir)
+        self.trial_paths.mkdir()
+        self.default_user = None
+        self.session_id = "local"
+        self.logger = logging.getLogger(__name__)
+
+    @staticmethod
+    def type() -> EnvironmentType:
+        return EnvironmentType.DOCKER
+
+    @property
+    def is_mounted(self) -> bool:
+        return True
+
+    @property
+    def supports_gpus(self) -> bool:
+        return False
+
+    @property
+    def can_disable_internet(self) -> bool:
+        return False
+
+    def _validate_definition(self):
+        pass
+
+    async def start(self, force_build: bool) -> None:
+        pass
+
+    async def stop(self, delete: bool):
+        pass
+
+    async def prepare_logs_for_host(self) -> None:
+        pass
+
+    async def upload_file(self, source_path, target_path):
+        shutil.copy(source_path, target_path)
+
+    async def upload_dir(self, source_dir, target_dir):
+        shutil.copytree(source_dir, target_dir, dirs_exist_ok=True)
+
+    async def download_file(self, source_path, target_path):
+        shutil.copy(source_path, target_path)
+
+    async def download_dir(self, source_dir, target_dir):
+        shutil.copytree(source_dir, target_dir, dirs_exist_ok=True)
+
+    async def exec(
+        self,
+        command: str,
+        cwd: str | None = None,
+        env: dict | None = None,
+        timeout_sec: int | None = None,
+        user: str | int | None = None,
+    ) -> ExecResult:
+        del user
+        try:
+            result = subprocess.run(
+                command,
+                shell=True,
+                cwd=cwd or str(self.workdir),
+                env={{**os.environ, **(env or {{}})}},
+                capture_output=True,
+                text=True,
+                timeout=timeout_sec,
+            )
+        except subprocess.TimeoutExpired:
+            return ExecResult(stdout="", stderr="Command timed out", return_code=124)
+        return ExecResult(
+            stdout=result.stdout,
+            stderr=result.stderr,
+            return_code=result.returncode,
+        )
+
+
+async def main() -> None:
+    workdir = Path(os.environ.get("AGENT_WORKDIR") or {DEFAULT_AGENT_WORKDIR!r})
+    logs_dir = Path({log_dir!r})
+    instruction = Path({instruction_path!r}).read_text()
+{system_prompt_block}    env = LocalEnvironment(workdir=workdir, logs_dir=logs_dir)
+    if "OPENAI_API_KEY" not in os.environ and "PRIME_API_KEY" in os.environ:
+        os.environ["OPENAI_API_KEY"] = os.environ["PRIME_API_KEY"]
+    api_base = os.environ.get("OPENAI_BASE_URL") or {api_base_url!r}
+    agent = Terminus2(
+        logs_dir=logs_dir,
+        model_name={model_name!r},
+        api_base=api_base,
+        max_turns={max_turns!r},
+    )
+    await agent.setup(env)
+    await agent.run(instruction, env, AgentContext())
+
+
+asyncio.run(main())
+"""
+
+
+__all__ = [
+    "DEFAULT_AGENT_WORKDIR",
+    "DEFAULT_API_BASE_URL",
+    "DEFAULT_HARBOR_PACKAGE",
+    "DEFAULT_INSTRUCTION_PATH",
+    "DEFAULT_LOG_PATH",
+    "DEFAULT_MODEL_NAME",
+    "DEFAULT_PYTHON_VERSION",
+    "DEFAULT_SYSTEM_PROMPT_PATH",
+    "Terminus2",
+    "build_terminus_2_install_script",
+    "build_terminus_2_run_script",
+    "terminus_2_agent_script",
+]

{verifiers-0.1.15.dev4 → verifiers-0.1.15.dev5}/verifiers/v1/packages/tasksets/harbor.py

@@ -199,6 +199,22 @@ class HarborTaskset(Taskset):
         verifier_config = config.get("verifier", {}) or {}
         instruction = instruction_path.read_text().strip()
         task_remote_dir = self.task_dir.rstrip("/") or "/task"
+        sandbox = {
+            "image": environment.get("docker_image") or self.docker_image,
+            "cpu_cores": parse_number(environment.get("cpus"), self.cpu_cores),
+            "memory_gb": parse_gb(environment.get("memory"), self.memory_gb),
+            "disk_size_gb": parse_gb(environment.get("storage"), self.disk_size_gb),
+            "timeout_minutes": self.timeout_minutes,
+            "command_timeout": int(
+                parse_number(
+                    agent_config.get("timeout_sec"), self.agent_timeout_seconds
+                )
+            ),
+            "workdir": self.workdir,
+            "scope": self.scope,
+        }
+        if "allow_internet" in environment:
+            sandbox["network_access"] = bool(environment["allow_internet"])
         return {
             "example_id": index,
             "task_name": task_dir.name,
@@ -206,21 +222,7 @@ class HarborTaskset(Taskset):
             "task_toml": task_toml_path.read_text(),
             "task_dir": str(task_dir),
             "prompt": [{"role": "user", "content": instruction}],
-            "sandbox": {
-                "image": environment.get("docker_image") or self.docker_image,
-                "cpu_cores": parse_number(environment.get("cpus"), self.cpu_cores),
-                "memory_gb": parse_gb(environment.get("memory"), self.memory_gb),
-                "disk_size_gb": parse_gb(environment.get("storage"), self.disk_size_gb),
-                "network_access": bool(environment.get("allow_internet", True)),
-                "timeout_minutes": self.timeout_minutes,
-                "command_timeout": int(
-                    parse_number(
-                        agent_config.get("timeout_sec"), self.agent_timeout_seconds
-                    )
-                ),
-                "workdir": self.workdir,
-                "scope": self.scope,
-            },
+            "sandbox": sandbox,
             "program": {
                 "files": {
                     f"{task_remote_dir}/instruction.md": {"task": "instruction"},

{verifiers-0.1.15.dev4 → verifiers-0.1.15.dev5}/verifiers/v1/task.py

@@ -37,7 +37,9 @@ class Task(dict):
         if "sandbox" in self and not isinstance(self["sandbox"], Mapping):
             raise TypeError("task.sandbox must be a mapping.")
         if "sandbox" in self:
-            super().__setitem__("sandbox", sandbox_config_mapping(self["sandbox"]))
+            super().__setitem__(
+                "sandbox", sandbox_config_mapping(self["sandbox"], fill_defaults=False)
+            )
         if "program" in self and not isinstance(self["program"], Mapping):
             raise TypeError("task.program must be a mapping.")
         if "max_turns" in self and (