vis-agent 0.2.43.dev7641__tar.gz → 0.2.43.dev7649__tar.gz

{vis_agent-0.2.43.dev7641 → vis_agent-0.2.43.dev7649}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: vis-agent
-Version: 0.2.43.dev7641
+Version: 0.2.43.dev7649
 Summary: The Vis Python SDK — author extensions and control local or remote engines.
 Project-URL: Homepage, https://vis.blockether.com
 Project-URL: Documentation, https://vis.blockether.com
@@ -15,15 +15,14 @@ Classifier: Intended Audience :: Developers
 Classifier: Programming Language :: Python :: 3
 Classifier: Topic :: Software Development :: Build Tools
 Classifier: Topic :: Software Development :: Libraries :: Python Modules
-Requires-Python: >=3.11
+Requires-Python: >=3.12
+Requires-Dist: harbor==0.24.0
 Requires-Dist: jsonschema<5,>=4.23
 Requires-Dist: packaging>=24
 Requires-Dist: pydantic<3,>=2.13.5
 Requires-Dist: referencing>=0.28.4
 Provides-Extra: docs
 Requires-Dist: pdoc==16.0.0; extra == 'docs'
-Provides-Extra: harbor
-Requires-Dist: harbor==0.24.0; (python_version >= '3.12') and extra == 'harbor'
 Description-Content-Type: text/markdown
 
 # vis-agent
@@ -34,7 +33,7 @@ Python SDK for Vis extensions and agent sessions.
 pip install vis-agent
 ```
 
-Requires Python 3.11+. Install the Vis executable and configure a provider
+Requires Python 3.12+. Install the Vis executable and configure a provider
 separately for local use. The execution-layer API below is unreleased: build
 the SDK and engine from the same source revision.
 

{vis_agent-0.2.43.dev7641 → vis_agent-0.2.43.dev7649}/README.md

@@ -6,7 +6,7 @@ Python SDK for Vis extensions and agent sessions.
 pip install vis-agent
 ```
 
-Requires Python 3.11+. Install the Vis executable and configure a provider
+Requires Python 3.12+. Install the Vis executable and configure a provider
 separately for local use. The execution-layer API below is unreleased: build
 the SDK and engine from the same source revision.
 

{vis_agent-0.2.43.dev7641 → vis_agent-0.2.43.dev7649}/contracts/schema/config.json

@@ -158,10 +158,16 @@
           "type": "integer",
           "minimum": 1
         },
+        "is_tool_calls_supported": {
+          "type": "boolean"
+        },
         "is_tool_call": {
           "type": "boolean"
         },
-        "is_vision": {
+        "is_vision_supported": {
+          "type": "boolean"
+        },
+        "is_image_input": {
           "type": "boolean"
         },
         "is_reasoning": {
@@ -255,6 +261,9 @@
         "is_stateless": {
           "type": "boolean"
         },
+        "is_vision_supported": {
+          "type": "boolean"
+        },
         "is_image_input": {
           "type": "boolean"
         },

{vis_agent-0.2.43.dev7641 → vis_agent-0.2.43.dev7649}/pyproject.toml

@@ -8,13 +8,20 @@ build-backend = "hatchling.build"
 
 [project]
 name = "vis-agent"
-version = "0.2.43.dev7641"
+version = "0.2.43.dev7649"
 description = "The Vis Python SDK — author extensions and control local or remote engines."
 readme = "README.md"
-requires-python = ">=3.11"
+requires-python = ">=3.12"
 license = "Apache-2.0"
 license-files = ["LICENSE"]
-dependencies = ["jsonschema>=4.23,<5", "packaging>=24", "pydantic>=2.13.5,<3", "referencing>=0.28.4"]
+# Harbor runs the tasks of `blockether.vis.evals`: the agent, host environment, check and judge.
+dependencies = [
+  "harbor==0.24.0",
+  "jsonschema>=4.23,<5",
+  "packaging>=24",
+  "pydantic>=2.13.5,<3",
+  "referencing>=0.28.4",
+]
 authors = [{ name = "Blockether" }]
 keywords = ["vis", "agent", "extension", "automation", "human-in-the-loop"]
 classifiers = [
@@ -27,8 +34,6 @@ classifiers = [
 
 [project.optional-dependencies]
 docs = ["pdoc==16.0.0"]
-# The Harbor agent, host environment, check and judge of `blockether.vis.evals`.
-harbor = ["harbor==0.24.0; python_version >= '3.12'"]
 
 [project.urls]
 Homepage = "https://vis.blockether.com"

{vis_agent-0.2.43.dev7641 → vis_agent-0.2.43.dev7649}/src/blockether/vis/evals/__init__.py

@@ -1,5 +1,6 @@
-"""Evaluation tools: read Vis runs and Harbor jobs, and compare their results.
+"""Evaluation tools: write and run Harbor tasks, read the results and compare them.
 
+Write a Vis task with `write_task`, and run tasks with the Vis agent with `run_tasks`.
 Read one Vis run with `load_trajectory` or `parse_trajectory`. Read the cases of a
 Harbor dataset with `read_tasks`, and the trials of a job or round with `read_trials`.
 Count the trials with `case_results` or `summarize`. Seal a finished round with
@@ -46,10 +47,12 @@ from .harbor import (
     read_tasks,
     read_trial,
     read_trials,
+    run_tasks,
     seal_digest,
     seal_round,
     seal_rows,
     summarize,
+    write_task,
 )
 from .labels import (
     KAPPA_REVIEW,
@@ -171,6 +174,7 @@ __all__ = [
     "read_trials",
     "repeat_stats",
     "round_summary",
+    "run_tasks",
     "sample_agreement",
     "save_label",
     "save_stage_label",
@@ -184,4 +188,5 @@ __all__ = [
     "task_attempts",
     "task_note",
     "wilson_interval",
+    "write_task",
 ]

{vis_agent-0.2.43.dev7641 → vis_agent-0.2.43.dev7649}/src/blockether/vis/evals/agent.py

@@ -1,9 +1,8 @@
 """Run the Vis CLI as a Harbor agent on the host.
 
 Select it with the Harbor agent import path ``blockether.vis.evals.agent:VisAgent``.
-It runs only in ``blockether.vis.evals.environment:HostEnvironment``. Install the
-``harbor`` extra of the SDK first. Each trial starts one CLI process with its own
-database. With ``--persist`` the
+It runs only in ``blockether.vis.evals.environment:HostEnvironment``. Each trial
+starts one CLI process with its own database. With ``--persist`` the
 CLI runs the gateway in the same process, so no shared gateway and no operator
 extension changes the measurement. The agent writes these files to its logs
 directory:

{vis_agent-0.2.43.dev7641 → vis_agent-0.2.43.dev7649}/src/blockether/vis/evals/bench.py

@@ -400,7 +400,7 @@ def cause_check(
         ),
         source="diagnose()",
         noun="causes",
-        labeller="label()",
+        labeller="save_label()",
         fix="Correct the diagnose() rules first.",
     )
 

{vis_agent-0.2.43.dev7641 → vis_agent-0.2.43.dev7649}/src/blockether/vis/evals/check.py

@@ -2,7 +2,7 @@
 
 Run it with ``python -m blockether.vis.evals.check TASK_DIR --provider P --model M``.
 ``TASK_DIR`` is one task or a directory of tasks, for example ``e2e`` of the Vis
-repository. Install the ``harbor`` extra first. Each check makes paid model calls.
+repository. Each check makes paid model calls.
 
 ``harbor check`` runs Claude Code in Docker. This module uses the same rubric,
 prompt and result validation of Harbor. The Vis agent runs each check in the host

{vis_agent-0.2.43.dev7641 → vis_agent-0.2.43.dev7649}/src/blockether/vis/evals/environment.py

@@ -29,7 +29,7 @@ from harbor.environments.base import BaseEnvironment, ExecResult
 from harbor.environments.capabilities import EnvironmentCapabilities
 from harbor.models.trial.paths import EnvironmentPaths
 
-from .grade import WORKSPACE_FILESYSTEM_FILE
+from .harbor import WORKSPACE_FILESYSTEM_FILE
 
 WORKSPACE_DIRECTORY = "workspace"
 GIT_SETUP = (
@@ -81,10 +81,10 @@ class HostEnvironment(BaseEnvironment):
         paths = EnvironmentPaths.for_os(self.os)
         table = {
             paths.logs_dir: self.root / "logs",
-            paths.agent_dir: self.trial_paths.agent_dir,
-            paths.user_agent_dir: self.trial_paths.user_agent_dir,
-            paths.verifier_dir: self.trial_paths.verifier_dir,
-            paths.artifacts_dir: self.trial_paths.artifacts_dir,
+            paths.agent_dir: self.trial_paths.agent_dir.resolve(),
+            paths.user_agent_dir: self.trial_paths.user_agent_dir.resolve(),
+            paths.verifier_dir: self.trial_paths.verifier_dir.resolve(),
+            paths.artifacts_dir: self.trial_paths.artifacts_dir.resolve(),
             paths.tests_dir: self.root / "tests",
             paths.solution_dir: self.root / "solution",
             paths.default_skills_dir: self.root / "skills",
@@ -121,8 +121,8 @@ class HostEnvironment(BaseEnvironment):
             "VIS_HARBOR_TASK": self.environment_dir.parent.name,
             "VIS_HARBOR_WORKSPACE": str(self.workspace),
             "VIS_HARBOR_TESTS": str(self.host_path(paths.tests_dir)),
-            "VIS_HARBOR_AGENT_LOGS": str(self.trial_paths.agent_dir),
-            "VIS_HARBOR_VERIFIER_LOGS": str(self.trial_paths.verifier_dir),
+            "VIS_HARBOR_AGENT_LOGS": str(self.trial_paths.agent_dir.resolve()),
+            "VIS_HARBOR_VERIFIER_LOGS": str(self.trial_paths.verifier_dir.resolve()),
             "VIS_HARBOR_PYTHON": sys.executable,
         }
 

{vis_agent-0.2.43.dev7641 → vis_agent-0.2.43.dev7649}/src/blockether/vis/evals/grade.py

@@ -37,7 +37,6 @@ STDERR_FILE = "stderr.txt"
 TRAJECTORY_FILE = "trajectory.json"
 RUN_FILE = "run.json"
 REWARD_FILE = "reward.json"
-WORKSPACE_FILESYSTEM_FILE = "workspace_filesystem.json"
 GRADE_FILE = "grade.json"
 
 

{vis_agent-0.2.43.dev7641 → vis_agent-0.2.43.dev7649}/src/blockether/vis/evals/harbor.py

@@ -1,21 +1,25 @@
-"""Read Harbor jobs: trials, cases, a summary with the cost, and round seals.
+"""Write and run Harbor tasks, and read Harbor jobs: trials, cases, a summary and seals.
 
-A Harbor job directory holds `config.json`, `result.json` and one directory for each
-trial. A trial directory holds `result.json`, `agent/trajectory.json` and
-`verifier/reward.json`. A round directory holds one or more job directories.
+A Harbor task is a folder with `task.toml`, `instruction.md`, `environment/` and
+`tests/`. `write_task` writes a Vis task, and `run_tasks` runs tasks with the Vis
+agent on this host. A Harbor job directory holds `config.json`, `result.json` and
+one directory for each trial. A trial directory holds `result.json`,
+`agent/trajectory.json` and `verifier/reward.json`. A round directory holds one or
+more job directories.
 
-This module reads the JSON files directly, so it does not need Harbor itself.
+The read functions read the JSON files directly. Only `run_tasks` imports Harbor.
 """
 
 from __future__ import annotations
 
 import hashlib
 import json
+import re
 import time
 import tomllib
 from collections.abc import Iterable, Mapping, Sequence
 from dataclasses import dataclass
-from pathlib import Path
+from pathlib import Path, PurePosixPath
 from types import MappingProxyType
 from typing import Any
 
@@ -29,15 +33,29 @@ __all__ = [
     "read_tasks",
     "read_trial",
     "read_trials",
+    "run_tasks",
     "seal_digest",
     "seal_round",
     "seal_rows",
     "summarize",
+    "write_task",
 ]
 
 RESULT_FILE = "result.json"
 TASK_FILE = "task.toml"
 INSTRUCTION_FILE = "instruction.md"
+CHECKS_FILE = "checks.json"
+TEST_SCRIPT_FILE = "test.sh"
+WORKSPACE_FILESYSTEM_FILE = "workspace_filesystem.json"
+WORKSPACE_DIRECTORY = "workspace"
+TASK_NAME = re.compile(r"[a-z0-9]+(?:-[a-z0-9]+)*")
+AGENT = "blockether.vis.evals.agent:VisAgent"
+ENVIRONMENT = "blockether.vis.evals.environment:HostEnvironment"
+GRADER_SCRIPT = """#!/usr/bin/env bash
+# Grade the trial. The Vis host environment sets the VIS_HARBOR_* paths.
+set -euo pipefail
+exec "$VIS_HARBOR_PYTHON" -m blockether.vis.evals.grade "$VIS_HARBOR_TESTS/checks.json"
+"""
 
 
 @dataclass(frozen=True, slots=True)
@@ -334,6 +352,25 @@ def _seconds(table: object) -> float | None:
     return float(value)
 
 
+def _read_task(config: Path) -> HarborTask:
+    info = tomllib.loads(config.read_text(encoding="utf-8"))
+    instruction = config.parent / INSTRUCTION_FILE
+    environment = info.get("environment")
+    gpus = environment.get("gpus") if isinstance(environment, dict) else None
+    metadata = info.get("metadata")
+    return HarborTask(
+        name=config.parent.name,
+        instruction=instruction.read_text(encoding="utf-8")
+        if instruction.is_file()
+        else "",
+        agent_timeout_sec=_seconds(info.get("agent")),
+        verifier_timeout_sec=_seconds(info.get("verifier")),
+        gpus=gpus if isinstance(gpus, int) and not isinstance(gpus, bool) else 0,
+        metadata=MappingProxyType(dict(metadata) if isinstance(metadata, dict) else {}),
+        path=config.parent,
+    )
+
+
 def read_tasks(dataset: Path | str) -> tuple[HarborTask, ...]:
     """Read the tasks of a Harbor dataset directory, sorted by name.
 
@@ -344,28 +381,164 @@ def read_tasks(dataset: Path | str) -> tuple[HarborTask, ...]:
     root = Path(dataset)
     if not root.is_dir():
         raise FileNotFoundError(f"no Harbor dataset directory at {root}")
-    tasks = []
-    for config in sorted(root.glob(f"*/{TASK_FILE}")):
-        info = tomllib.loads(config.read_text(encoding="utf-8"))
-        instruction = config.parent / INSTRUCTION_FILE
-        environment = info.get("environment")
-        gpus = environment.get("gpus") if isinstance(environment, dict) else None
-        metadata = info.get("metadata")
-        tasks.append(
-            HarborTask(
-                name=config.parent.name,
-                instruction=instruction.read_text(encoding="utf-8")
-                if instruction.is_file()
-                else "",
-                agent_timeout_sec=_seconds(info.get("agent")),
-                verifier_timeout_sec=_seconds(info.get("verifier")),
-                gpus=gpus
-                if isinstance(gpus, int) and not isinstance(gpus, bool)
-                else 0,
-                metadata=MappingProxyType(
-                    dict(metadata) if isinstance(metadata, dict) else {}
-                ),
-                path=config.parent,
-            )
+    return tuple(_read_task(config) for config in sorted(root.glob(f"*/{TASK_FILE}")))
+
+
+def _toml_value(key: str, value: object) -> str:
+    if isinstance(value, bool):
+        return "true" if value else "false"
+    if isinstance(value, (int, float)):
+        return repr(value)
+    if isinstance(value, str):
+        return json.dumps(value)
+    raise ValueError(f"metadata `{key}` must be a string, a number or a boolean")
+
+
+def _seed_path(name: str) -> PurePosixPath:
+    path = PurePosixPath(name)
+    if not name or path.is_absolute() or ".." in path.parts or name != str(path):
+        raise ValueError(f"seed file path must stay inside the workspace: {name!r}")
+    return path
+
+
+def write_task(
+    dataset: Path | str,
+    name: str,
+    instruction: str,
+    checks: Mapping[str, Any],
+    *,
+    files: Mapping[str, str] | None = None,
+    agent_timeout_sec: float = 300.0,
+    verifier_timeout_sec: float = 300.0,
+    metadata: Mapping[str, str | int | float | bool] | None = None,
+    artifacts: Sequence[str] = (),
+    filesystem_roots: Mapping[str, str] | None = None,
+) -> HarborTask:
+    """Write the Vis task `name` in the folder `dataset` and return it.
+
+    `name` is lowercase words joined by hyphens. `instruction` is the prompt of the
+    agent. `checks` is the `tests/checks.json` object that the Vis grader reads, for
+    example `{"want": {"m.py": ["def c("]}}`. `files` maps each seed file path to its
+    text. The paths are relative to `environment/workspace/`. `artifacts` lists the
+    workspace files that Harbor copies to `artifacts/` in the trial folder after the
+    check, for example the call record of a mock extension. `filesystem_roots` maps each
+    filesystem root id to a workspace folder; the trial registers them in `vis.yml`. The
+    task gets `tests/test.sh`, which runs the grader. An existing task folder, an invalid
+    name, empty checks or a path outside the workspace raise an error.
+    """
+    if not TASK_NAME.fullmatch(name):
+        raise ValueError(
+            f"task name must be lowercase words joined by hyphens: {name!r}"
         )
-    return tuple(tasks)
+    if not isinstance(checks, Mapping) or not checks:
+        raise ValueError("checks must be a nonempty object")
+    if not instruction.strip():
+        raise ValueError("instruction must not be empty")
+    seeds = {_seed_path(path): text for path, text in (files or {}).items()}
+    kept = [str(_seed_path(path)) for path in artifacts]
+    roots = {
+        root: str(_seed_path(path)) for root, path in (filesystem_roots or {}).items()
+    }
+    lines = [
+        f"{key} = {_toml_value(key, value)}" for key, value in (metadata or {}).items()
+    ]
+    # Harbor reads `artifacts` only as a root key, so it comes before every table.
+    root = f"artifacts = {json.dumps(kept)}\n\n" if kept else ""
+    tables = [
+        ("metadata", lines),
+        ("agent", [f"timeout_sec = {float(agent_timeout_sec)!r}"]),
+        ("verifier", [f"timeout_sec = {float(verifier_timeout_sec)!r}"]),
+    ]
+    task = Path(dataset) / name
+    task.mkdir(parents=True, exist_ok=False)
+    (task / TASK_FILE).write_text(
+        root
+        + "\n".join(
+            f"[{table}]\n" + "".join(f"{line}\n" for line in rows)
+            for table, rows in tables
+        ),
+        encoding="utf-8",
+    )
+    (task / INSTRUCTION_FILE).write_text(
+        instruction.rstrip("\n") + "\n", encoding="utf-8"
+    )
+    workspace = task / "environment" / WORKSPACE_DIRECTORY
+    workspace.mkdir(parents=True)
+    for path, text in seeds.items():
+        target = workspace.joinpath(*path.parts)
+        target.parent.mkdir(parents=True, exist_ok=True)
+        target.write_text(text, encoding="utf-8")
+    if roots:
+        (workspace.parent / WORKSPACE_FILESYSTEM_FILE).write_text(
+            json.dumps(roots, indent=2, ensure_ascii=False) + "\n", encoding="utf-8"
+        )
+    tests = task / "tests"
+    tests.mkdir()
+    (tests / CHECKS_FILE).write_text(
+        json.dumps(checks, indent=2, ensure_ascii=False) + "\n", encoding="utf-8"
+    )
+    script = tests / TEST_SCRIPT_FILE
+    script.write_text(GRADER_SCRIPT, encoding="utf-8")
+    script.chmod(0o755)
+    return _read_task(task / TASK_FILE)
+
+
+def run_tasks(
+    path: Path | str,
+    model: str,
+    *,
+    jobs_dir: Path | str = "jobs",
+    job_name: str | None = None,
+    attempts: int = 1,
+    concurrency: int = 1,
+    agent_options: Mapping[str, str] | None = None,
+) -> Path:
+    """Run the tasks in `path` with the Vis agent on this host and return the job folder.
+
+    `path` is one task folder or a dataset folder of tasks. `model` is the
+    `provider/model` route of the agent. Harbor writes the job to
+    `jobs_dir/job_name`. Without `job_name`, Harbor names the job with the time.
+    `attempts` is the number of trials for each task. `concurrency` is the number
+    of trials that run at the same time. `agent_options` are the agent options of
+    `harbor run --ak`, for example `{"source": "/path/to/vis"}`. The call sets
+    `HARBOR_TELEMETRY=off`, so Harbor sends no usage data. Each trial makes paid
+    model calls. Call it outside a running event loop.
+    """
+    import asyncio
+    import os
+
+    from harbor.job import Job
+    from harbor.models.job.config import JobConfig
+    from harbor.models.trial.config import AgentConfig, EnvironmentConfig, TaskConfig
+
+    root = Path(path)
+    if (root / TASK_FILE).is_file():
+        folders = [root]
+    else:
+        folders = [task.path for task in read_tasks(root)]
+    if not folders:
+        raise ValueError(f"no Harbor tasks in {root}")
+    os.environ["HARBOR_TELEMETRY"] = "off"
+    config = JobConfig(
+        jobs_dir=Path(jobs_dir),
+        n_attempts=attempts,
+        n_concurrent_trials=concurrency,
+        quiet=True,
+        agents=[
+            AgentConfig(
+                import_path=AGENT, model_name=model, kwargs=dict(agent_options or {})
+            )
+        ],
+        environment=EnvironmentConfig(import_path=ENVIRONMENT),
+        tasks=[TaskConfig(path=folder) for folder in folders],
+        datasets=[],
+    )
+    if job_name is not None:
+        config = config.model_copy(update={"job_name": job_name})
+
+    async def run() -> Path:
+        job = await Job.create(config)
+        await job.run()
+        return job.job_dir
+
+    return asyncio.run(run())

{vis_agent-0.2.43.dev7641 → vis_agent-0.2.43.dev7649}/src/blockether/vis/evals/judge.py

@@ -2,8 +2,7 @@
 
 Run it with ``python -m blockether.vis.evals.judge PATH --provider P --model M``.
 ``PATH`` is one trial directory or a Harbor job directory, for example a job of
-``harbor run``. Install the ``harbor`` extra first. Each judgement makes paid model
-calls.
+``harbor run``. Each judgement makes paid model calls.
 
 ``harbor analyze`` runs Claude Code in Docker. This module uses the same prompt,
 result validation and ``analysis.json`` file of Harbor. The Vis agent runs each

{vis_agent-0.2.43.dev7641 → vis_agent-0.2.43.dev7649}/src/blockether/vis/extension.py

@@ -2940,20 +2940,33 @@ class ProviderStatus(_ProviderValue):
 
 @dataclass(frozen=True, slots=True)
 class ProviderModel(_ProviderValue):
-    """Model metadata returned by enrichment. extra keeps additional router fields."""
+    """Model metadata returned by enrichment. extra keeps additional router fields.
+
+    is_tool_call and is_image_input are old names of is_tool_calls_supported and
+    is_vision_supported. They still work. A new name wins over its old name.
+    """
 
     name: str
     context: int | None = None
-    is_tool_call: bool | None = None
-    is_image_input: bool | None = None
+    is_tool_calls_supported: bool | None = None
+    is_vision_supported: bool | None = None
     extra: Mapping[str, ProviderJSON] | None = None
+    is_tool_call: bool | None = field(default=None, repr=False, compare=False)
+    is_image_input: bool | None = field(default=None, repr=False, compare=False)
 
     def __post_init__(self):
         _provider_text(self.name, "model name", required=True)
         _provider_number(self.context, "context", integer=True, minimum=1)
-        for name in ("is_tool_call", "is_image_input"):
-            if getattr(self, name) is not None:
-                _provider_bool(getattr(self, name), name)
+        for new, old in (
+            ("is_tool_calls_supported", "is_tool_call"),
+            ("is_vision_supported", "is_image_input"),
+        ):
+            for name in (new, old):
+                if getattr(self, name) is not None:
+                    _provider_bool(getattr(self, name), name)
+            if getattr(self, new) is None:
+                object.__setattr__(self, new, getattr(self, old))
+            object.__setattr__(self, old, None)
         _provider_extra(self)
 
 

{vis_agent-0.2.43.dev7641 → vis_agent-0.2.43.dev7649}/tests/conftest.py

@@ -1,6 +1,5 @@
 """Make the package importable from the checkout, and give every test a clean host."""
 
-import importlib.util
 import os
 import sys
 from pathlib import Path
@@ -15,15 +14,6 @@ sys.modules.pop("blockether.vis._outside", None)
 import blockether.vis.extension as vis  # noqa: E402
 import pytest  # noqa: E402  (the path above is what makes `vis` importable)
 
-# The Harbor agent, environment, check and judge need the `harbor` extra.
-if importlib.util.find_spec("harbor") is None:
-    collect_ignore = [
-        "test_evals_agent.py",
-        "test_evals_check.py",
-        "test_evals_environment.py",
-        "test_evals_judge.py",
-    ]
-
 
 @pytest.fixture(autouse=True)
 def outside_home(tmp_path, monkeypatch):

{vis_agent-0.2.43.dev7641 → vis_agent-0.2.43.dev7649}/tests/test_engine.py

@@ -72,7 +72,7 @@ def credential():
         llm_headers=__CREDENTIAL_HEADERS__)
 
 def enrich(provider, router_opts):
-    return [vis.ProviderModel(m["name"], context=200000, is_tool_call=True)
+    return [vis.ProviderModel(m["name"], context=200000, is_tool_calls_supported=True)
             for m in provider["models"]]
 
 vis.register_extension(vis.Extension(name="sdk-provider", description="Typed provider integration",

{vis_agent-0.2.43.dev7641 → vis_agent-0.2.43.dev7649}/tests/test_evals_environment.py

@@ -13,7 +13,7 @@ from pathlib import Path
 
 import pytest
 from blockether.vis.evals.environment import HostEnvironment
-from blockether.vis.evals.grade import WORKSPACE_FILESYSTEM_FILE
+from blockether.vis.evals.harbor import WORKSPACE_FILESYSTEM_FILE
 from harbor.models.task.config import EnvironmentConfig
 from harbor.models.trial.paths import TrialPaths
 
@@ -126,6 +126,26 @@ def test_exec_sets_the_paths_and_runs_in_the_workspace(started):
     ]
 
 
+def test_relative_trial_paths_reach_the_trial_directory(tmp_path, monkeypatch):
+    # `harbor run --path tasks --jobs-dir jobs` gives relative trial paths.
+    # Commands run in the workspace, so a relative verifier path lost the reward.
+    monkeypatch.chdir(tmp_path)
+    environment = _environment(tmp_path)
+    environment.trial_paths = TrialPaths(Path("jobs") / "trial")
+    environment.trial_paths.mkdir()
+    asyncio.run(environment.start(force_build=False))
+    try:
+        command = 'echo 1 > /logs/verifier/reward.txt; echo "$VIS_HARBOR_VERIFIER_LOGS"'
+        result = asyncio.run(environment.exec(command))
+        verifier = tmp_path.resolve() / "jobs" / "trial" / "verifier"
+        assert result.return_code == 0
+        assert result.stdout.strip() == str(verifier)
+        assert (verifier / "reward.txt").read_text() == "1\n"
+        assert environment.path_table()["/logs/agent"].is_absolute()
+    finally:
+        asyncio.run(environment.stop(delete=True))
+
+
 def test_exec_maps_the_cwd_and_returns_the_exit_code(started):
     result = asyncio.run(started.exec("pwd -P; echo oops >&2; exit 3", cwd="/tests"))
     assert result.stdout.strip() == str(started.root / "tests")

{vis_agent-0.2.43.dev7641 → vis_agent-0.2.43.dev7649}/tests/test_evals_guards.py

@@ -11,11 +11,8 @@ from pathlib import Path
 
 import evals_atif as atif
 from blockether.vis.evals import grade
-from blockether.vis.evals.grade import (
-    WORKSPACE_FILESYSTEM_FILE,
-    cache_metric_failures,
-    usage_percent,
-)
+from blockether.vis.evals.grade import cache_metric_failures, usage_percent
+from blockether.vis.evals.harbor import WORKSPACE_FILESYSTEM_FILE
 
 PROVIDER = "test-provider"
 TASKS = Path(__file__).resolve().parents[3] / "e2e"