vis-agent 0.2.42.dev7622__tar.gz → 0.2.42.dev7627__tar.gz

{vis_agent-0.2.42.dev7622 → vis_agent-0.2.42.dev7627}/.gitignore

@@ -117,6 +117,3 @@ own_your_harness.pptx
 # Call journal written by the e2e schema-probe fixture when it runs from this
 # checkout instead of a temporary workspace.
 /schema-calls.jsonl
-
-# The uv environment of the e2e Harbor project.
-/e2e/.venv/

{vis_agent-0.2.42.dev7622 → vis_agent-0.2.42.dev7627}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: vis-agent
-Version: 0.2.42.dev7622
+Version: 0.2.42.dev7627
 Summary: The Vis Python SDK — author extensions and control local or remote engines.
 Project-URL: Homepage, https://vis.blockether.com
 Project-URL: Documentation, https://vis.blockether.com
@@ -22,6 +22,8 @@ Requires-Dist: pydantic<3,>=2.13.5
 Requires-Dist: referencing>=0.28.4
 Provides-Extra: docs
 Requires-Dist: pdoc==16.0.0; extra == 'docs'
+Provides-Extra: harbor
+Requires-Dist: harbor==0.24.0; (python_version >= '3.12') and extra == 'harbor'
 Description-Content-Type: text/markdown
 
 # vis-agent

{vis_agent-0.2.42.dev7622 → vis_agent-0.2.42.dev7627}/contracts/schema/evals.json

@@ -211,10 +211,12 @@
       "type": "object",
       "properties": {
         "package": {
-          "type": "string"
+          "type": "string",
+          "description": "The Harbor Hub package of the board, for example terminal-bench/terminal-bench."
         },
         "name": {
-          "type": "string"
+          "type": "string",
+          "description": "The board name in the package, for example 4-0-0."
         },
         "summary": {
           "type": "object",
@@ -227,10 +229,14 @@
           ]
         }
       },
+      "required": [
+        "package",
+        "name"
+      ],
       "additionalProperties": false
     },
     "bench_report": {
-      "description": "The score of one Terminal-Bench round. Rates are in percent. Intervals are 95% intervals.",
+      "description": "The score of one benchmark round. Rates are in percent. Intervals are 95% intervals.",
       "type": "object",
       "properties": {
         "dataset": {

{vis_agent-0.2.42.dev7622 → vis_agent-0.2.42.dev7627}/pyproject.toml

@@ -8,7 +8,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "vis-agent"
-version = "0.2.42.dev7622"
+version = "0.2.42.dev7627"
 description = "The Vis Python SDK — author extensions and control local or remote engines."
 readme = "README.md"
 requires-python = ">=3.11"
@@ -27,6 +27,8 @@ classifiers = [
 
 [project.optional-dependencies]
 docs = ["pdoc==16.0.0"]
+# The Harbor agent, host environment, check and judge of `blockether.vis.evals`.
+harbor = ["harbor==0.24.0; python_version >= '3.12'"]
 
 [project.urls]
 Homepage = "https://vis.blockether.com"

{vis_agent-0.2.42.dev7622 → vis_agent-0.2.42.dev7627}/src/blockether/vis/evals/_gateway.py

@@ -61,6 +61,13 @@ def _word(body: Mapping, key: str, default: str) -> str:
     return default if value is None else value
 
 
+def _required_text(body: Mapping, key: str) -> str:
+    value = _text(body, key)
+    if not value:
+        raise ValueError(f"`{key}` is required.")
+    return value
+
+
 def _cases(body: Mapping, key: str) -> dict[tuple[str, ...], tuple[int, int]]:
     rows = body.get(key)
     if not isinstance(rows, list):
@@ -133,8 +140,8 @@ def _agreement(body: Mapping):
 def _leaderboard(body: Mapping):
     summary = body.get("summary")
     return read_leaderboard(
-        _word(body, "package", "terminal-bench/terminal-bench"),
-        _word(body, "name", "4-0-0"),
+        _required_text(body, "package"),
+        _required_text(body, "name"),
         None if summary is None else _mapping(body, "summary"),
         _text(body, "round_name"),
     )

vis_agent-0.2.42.dev7627/src/blockether/vis/evals/agent.py

@@ -0,0 +1,341 @@
+"""Run the Vis CLI as a Harbor agent on the host.
+
+Select it with the Harbor agent import path ``blockether.vis.evals.agent:VisAgent``.
+It runs only in ``blockether.vis.evals.environment:HostEnvironment``. Install the
+``harbor`` extra of the SDK first. Each trial starts one CLI process with its own
+database. With ``--persist`` the
+CLI runs the gateway in the same process, so no shared gateway and no operator
+extension changes the measurement. The agent writes these files to its logs
+directory:
+
+- ``trace.jsonl``: the ``--full-trace-json-stream`` output.
+- ``stderr.txt``: the error output of the CLI.
+- ``trajectory.json``: the ATIF trajectory of the session.
+- ``run.json``: the provider, the model, the exit code and the wall time.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import functools
+import json
+import os
+import shlex
+import shutil
+import subprocess
+import tempfile
+import time
+from pathlib import Path
+
+from harbor.agents.base import BaseAgent
+from harbor.agents.capabilities import AgentCapabilities
+from harbor.agents.options import AgentOptions
+from harbor.environments.base import BaseEnvironment
+from harbor.models.agent.context import AgentContext
+from pydantic import Field
+
+from .environment import HostEnvironment
+from .grade import RUN_FILE, STDERR_FILE, TRACE_FILE, TRAJECTORY_FILE
+from .trajectory import load_trajectory as load_vis_trajectory
+
+DEFAULT_COMMAND = "vis-agent"
+CLASSPATH_TIMEOUT_SEC = 120
+_NATIVE_MAGIC = (
+    b"\x7fELF",
+    b"\xcf\xfa\xed\xfe",
+    b"\xfe\xed\xfa\xcf",
+    b"\xca\xfe\xba\xbe",
+)
+_SOURCE_ENGINE = (
+    'cd "$VIS_HARBOR_SOURCE" && "$VIS_HARBOR_ENGINE" -Scp "$VIS_HARBOR_CLASSPATH" '
+    '-J-Duser.dir="$VIS_HARBOR_WORKSPACE" -M:vis'
+)
+_NATIVE_ENGINE = '"$VIS_HARBOR_ENGINE" -Duser.dir="$VIS_HARBOR_WORKSPACE"'
+_INSTALLED_ENGINE = '"$VIS_HARBOR_ENGINE"'
+
+
+class VisOptions(AgentOptions):
+    """The options that ``--ak key=value`` sets."""
+
+    provider: str | None = Field(
+        default=None, description="Vis provider. Wins over the provider in --model."
+    )
+    model: str | None = Field(
+        default=None, description="Vis model. Wins over the model in --model."
+    )
+    reasoning_effort: str | None = Field(
+        default=None, description="Value of the CLI option --reasoning-effort."
+    )
+    source: str | None = Field(
+        default=None, description="Vis checkout to run from source with Clojure."
+    )
+    clojure: str | None = Field(default=None, description="Clojure CLI executable.")
+    native_bin: str | None = Field(
+        default=None, description="Raw native Vis executable to run."
+    )
+    command: str | None = Field(
+        default=None, description="Shell prefix that replaces the engine, for tests."
+    )
+
+
+@functools.cache
+def source_classpath(clojure: str, source: Path) -> str:
+    """Resolve the :vis classpath once, with absolute entries for any user.dir."""
+    classpath = subprocess.check_output(
+        [clojure, "-Spath", "-A:vis"],
+        cwd=source,
+        text=True,
+        timeout=CLASSPATH_TIMEOUT_SEC,
+    ).strip()
+    return os.pathsep.join(
+        entry if os.path.isabs(entry) else str(source / entry)
+        for entry in classpath.split(os.pathsep)
+    )
+
+
+@functools.cache
+def source_commit(source: Path) -> str | None:
+    """Return the commit of the checkout ``source``, or None when git cannot tell."""
+    try:
+        completed = subprocess.run(
+            ["git", "rev-parse", "--short=12", "HEAD"],
+            cwd=source,
+            capture_output=True,
+            text=True,
+            timeout=30,
+            check=True,
+        )
+    except (OSError, subprocess.SubprocessError):
+        return None
+    return completed.stdout.strip() or None
+
+
+class ProviderUnavailableError(RuntimeError):
+    """The provider refused the turn, so the trial measures nothing about Vis."""
+
+
+def provider_failure(trace: Path) -> str | None:
+    """Return the provider error that ended the turn in ``trace``, or None.
+
+    A provider quota or outage ends the turn with an error block whose code
+    starts with ``provider_``. Such a trial is an infrastructure error, not a
+    wrong answer.
+    """
+    try:
+        lines = trace.read_text(encoding="utf-8").splitlines()
+    except OSError:
+        return None
+    for line in reversed(lines):
+        try:
+            frame = json.loads(line)
+        except ValueError:
+            continue
+        if not isinstance(frame, dict) or frame.get("event") != "result":
+            continue
+        payload = frame.get("payload")
+        content = payload.get("content") if isinstance(payload, dict) else None
+        for block in content if isinstance(content, list) else ():
+            if not isinstance(block, dict) or block.get("type") != "error":
+                continue
+            code = str(block.get("code"))
+            if code.startswith("provider_"):
+                return f"{code}: {block.get('title') or 'provider error'}"
+        return None
+    return None
+
+
+def check_native_binary(path: str) -> None:
+    """Refuse a launcher script: the measurement must use the native image."""
+    with open(path, "rb") as executable:
+        magic = executable.read(4)
+    if magic not in _NATIVE_MAGIC:
+        raise ValueError("native_bin must be a raw native executable, not a launcher")
+
+
+class VisAgent(BaseAgent):
+    """Harbor agent that runs one Vis CLI session in the host workspace.
+
+    The provider and the model come from the first source that sets them:
+
+    1. The ``provider`` and ``model`` options (``--ak``).
+    2. ``--model provider/model``.
+    3. ``VIS_PROVIDER`` and ``VIS_MODEL`` in ``--ae`` or in the host environment.
+
+    The engine comes from the first option that is set: ``command``, ``native_bin``
+    (``VIS_HARBOR_NATIVE_BIN``), ``source`` (``VIS_HARBOR_SOURCE``). Without them, the
+    agent runs the installed ``vis-agent`` CLI.
+    """
+
+    capabilities = AgentCapabilities(atif=True)
+    options_model = VisOptions
+
+    def __init__(self, logs_dir: Path, model_name: str | None = None, **kwargs) -> None:
+        super().__init__(logs_dir=logs_dir, model_name=model_name, **kwargs)
+        options = self.options
+        if not isinstance(options, VisOptions):
+            raise TypeError("VisAgent needs VisOptions")
+        provider = (
+            options.provider
+            or self._parsed_model_provider
+            or self._setting("VIS_PROVIDER")
+        )
+        model = options.model or self._parsed_model_name or self._setting("VIS_MODEL")
+        if not provider or not model:
+            raise ValueError(
+                "VisAgent needs a provider and a model: use --model provider/model"
+            )
+        self.provider, self.model = provider, model
+        self.reasoning_effort = options.reasoning_effort or self._setting(
+            "VIS_REASONING_EFFORT"
+        )
+        self.clojure = (
+            options.clojure or self._setting("VIS_HARBOR_CLOJURE") or "clojure"
+        )
+        native_bin = options.native_bin or self._setting("VIS_HARBOR_NATIVE_BIN")
+        self.native_bin = os.path.abspath(native_bin) if native_bin else None
+        source = options.source or self._setting("VIS_HARBOR_SOURCE")
+        self.source = Path(source).resolve() if source else None
+        self.engine_override = options.command
+        self.classpath: str | None = None
+        self.model_name = f"{self.provider}/{self.model}"
+        self._init_model_info()
+
+    @staticmethod
+    def name() -> str:
+        return "vis"
+
+    def version(self) -> str | None:
+        if self.engine_override or self.native_bin or self.source is None:
+            return None
+        return source_commit(self.source)
+
+    def _setting(self, name: str) -> str | None:
+        value = self._extra_env.get(name, os.environ.get(name, ""))
+        return value.strip() or None
+
+    def vis_arguments(self) -> list[str]:
+        """Return the CLI options before ``--trajectory`` and the prompt."""
+        arguments = [
+            "--full-trace-json-stream",
+            "--provider",
+            self.provider,
+            "--model",
+            self.model,
+        ]
+        if self.reasoning_effort:
+            arguments += ["--reasoning-effort", self.reasoning_effort]
+        return [*arguments, "--persist"]
+
+    def shell_command(self) -> str:
+        """Return the bash command of one trial.
+
+        Paths and the prompt stay in environment variables. Then the path
+        translation of the host environment cannot change them.
+        """
+        if self.engine_override:
+            engine = self.engine_override
+        elif self.native_bin:
+            engine = _NATIVE_ENGINE
+        elif self.source is not None:
+            engine = _SOURCE_ENGINE
+        else:
+            engine = _INSTALLED_ENGINE
+        arguments = " ".join(shlex.quote(item) for item in self.vis_arguments())
+        return (
+            "unset VIS_GATEWAY_URL; "
+            f"{engine} {arguments} "
+            f'--trajectory "$VIS_HARBOR_AGENT_LOGS/{TRAJECTORY_FILE}" '
+            '"$VIS_HARBOR_PROMPT" '
+            f'> "$VIS_HARBOR_AGENT_LOGS/{TRACE_FILE}" '
+            f'2> "$VIS_HARBOR_AGENT_LOGS/{STDERR_FILE}"'
+        )
+
+    def process_env(self, instruction: str, runtime: Path) -> dict[str, str]:
+        """Return the variables of the CLI process, with a database for this trial."""
+        env = dict(self._extra_env)
+        env["VIS_DB_PATH"] = str(runtime / "vis.mdb")
+        env["VIS_EXTENSIONS_DIR"] = self._setting("VIS_EXTENSIONS_DIR") or str(
+            runtime / "extensions"
+        )
+        env["VIS_HARBOR_PROMPT"] = instruction
+        if self.native_bin:
+            env["VIS_HARBOR_ENGINE"] = self.native_bin
+        elif self.source is not None:
+            env["VIS_HARBOR_ENGINE"] = self.clojure
+            env["VIS_HARBOR_SOURCE"] = str(self.source)
+        else:
+            env["VIS_HARBOR_ENGINE"] = DEFAULT_COMMAND
+        if self.classpath is not None:
+            env["VIS_HARBOR_CLASSPATH"] = self.classpath
+        return env
+
+    async def setup(self, environment: BaseEnvironment) -> None:
+        if not isinstance(environment, HostEnvironment):
+            raise TypeError(
+                "VisAgent runs only in blockether.vis.evals.environment:HostEnvironment"
+            )
+        if self.engine_override:
+            return
+        if self.native_bin:
+            check_native_binary(self.native_bin)
+            return
+        if self.source is not None:
+            self.classpath = await asyncio.to_thread(
+                source_classpath, self.clojure, self.source
+            )
+            return
+        if shutil.which(DEFAULT_COMMAND) is None:
+            raise FileNotFoundError(
+                f"{DEFAULT_COMMAND} is not on PATH: install Vis, or set source or native_bin"
+            )
+
+    async def run(
+        self, instruction: str, environment: BaseEnvironment, context: AgentContext
+    ) -> None:
+        runtime = Path(tempfile.mkdtemp(prefix="vis_harbor_runtime_"))
+        record = {
+            "provider": self.provider,
+            "model": self.model,
+            "reasoning_effort": self.reasoning_effort,
+            "exit_code": None,
+            "wall_s": None,
+            "is_cancelled": False,
+        }
+        started = time.monotonic()
+        try:
+            result = await environment.exec(
+                self.shell_command(), env=self.process_env(instruction, runtime)
+            )
+            record["exit_code"] = result.return_code
+        except asyncio.CancelledError:
+            record["is_cancelled"] = True
+            raise
+        finally:
+            record["wall_s"] = round(time.monotonic() - started, 3)
+            self.logs_dir.mkdir(parents=True, exist_ok=True)
+            (self.logs_dir / RUN_FILE).write_text(json.dumps(record, indent=2) + "\n")
+            shutil.rmtree(runtime, ignore_errors=True)
+            self.populate_context_post_run(context)
+        if failure := provider_failure(self.logs_dir / TRACE_FILE):
+            raise ProviderUnavailableError(failure)
+
+    def populate_context_post_run(self, context: AgentContext) -> None:
+        """Copy the token and cost totals of the trajectory into ``context``."""
+        path = self.logs_dir / TRAJECTORY_FILE
+        if not path.is_file():
+            return
+        # Harbor calls this hook outside the error handling of a trial. An exception
+        # here stops the whole job, so a trajectory that cannot be read only loses
+        # its totals.
+        try:
+            trajectory = load_vis_trajectory(path, strict=False)
+        except Exception as error:
+            self.logger.warning("Cannot read %s: %s", path, error)
+            return
+        metrics = trajectory.final_metrics
+        if metrics is None:
+            return
+        context.n_input_tokens = metrics.total_prompt_tokens
+        context.n_cache_tokens = metrics.total_cached_tokens
+        context.n_output_tokens = metrics.total_completion_tokens
+        context.cost_usd = metrics.total_cost_usd

{vis_agent-0.2.42.dev7622 → vis_agent-0.2.42.dev7627}/src/blockether/vis/evals/bench.py

@@ -1,4 +1,4 @@
-"""Score a Terminal-Bench round and find where it loses tasks.
+"""Score a Harbor benchmark round and find where it loses tasks.
 
 `round_summary` reads a round summary from Harbor job folders: the trials, the task
 outcomes and the usage of the round. Give that summary to `bench_report` for
@@ -78,7 +78,7 @@ class OutcomeCount:
 
 @dataclass(frozen=True)
 class BenchReport:
-    """The Terminal-Bench score of one round, with its usage.
+    """The benchmark score of one round, with its usage.
 
     `pass_rate_percent` divides solved tasks by scored tasks, by the latest scored
     attempt of each task. `strict_pass_rate_percent` divides them by all dataset tasks,

vis_agent-0.2.42.dev7627/src/blockether/vis/evals/check.py

@@ -0,0 +1,211 @@
+"""Check Harbor tasks with the ``harbor check`` rubric and the Vis agent.
+
+Run it with ``python -m blockether.vis.evals.check TASK_DIR --provider P --model M``.
+``TASK_DIR`` is one task or a directory of tasks, for example ``e2e`` of the Vis
+repository. Install the ``harbor`` extra first. Each check makes paid model calls.
+
+``harbor check`` runs Claude Code in Docker. This module uses the same rubric,
+prompt and result validation of Harbor. The Vis agent runs each check in the host
+environment, so the check needs no Docker and no Anthropic key. The reviewed task
+is in ``task/`` of the trial workspace, and the agent writes ``check-result.json``
+next to it. The job directory holds ``check_report.json``.
+"""
+
+from __future__ import annotations
+
+import argparse
+import asyncio
+import shutil
+import sys
+import tempfile
+from collections.abc import Sequence
+from datetime import UTC, datetime
+from pathlib import Path
+
+from harbor.analyze import checker
+from harbor.analyze.models import (
+    QualityCheckResult,
+    build_check_response_model,
+    load_rubric,
+)
+from harbor.cli.quality_checker.models import CheckReport
+from harbor.models.job.config import JobConfig
+from harbor.models.trial.config import AgentConfig, EnvironmentConfig, TaskConfig
+
+from .environment import WORKSPACE_DIRECTORY
+
+AGENT = "blockether.vis.evals.agent:VisAgent"
+ENVIRONMENT = "blockether.vis.evals.environment:HostEnvironment"
+DEFAULT_WORKERS = 5
+TASK_DIRECTORY = "task"
+DEFAULT_JOBS = "jobs/vis-check"
+REPORT_FILE = "check_report.json"
+TEST_SCRIPT = """#!/usr/bin/env bash
+# Validate the check result. The host environment sets the VIS_HARBOR_* paths.
+if "$VIS_HARBOR_PYTHON" "$VIS_HARBOR_TESTS/validate.py" check-result.json; then
+  echo 1 > "$VIS_HARBOR_VERIFIER_LOGS/reward.txt"
+else
+  echo 0 > "$VIS_HARBOR_VERIFIER_LOGS/reward.txt"
+fi
+"""
+
+
+def prompt_template() -> str:
+    """Return Harbor's check prompt with the reviewed task in ``task/`` of the workspace."""
+    return (
+        (checker.PROMPTS_DIR / "check.txt")
+        .read_text()
+        .replace("{task_path}", TASK_DIRECTORY)
+    )
+
+
+def assemble(task_dir: Path, rubric, response_model, dest: Path) -> Path:
+    """Build the check task of ``task_dir`` in ``dest`` for the host environment.
+
+    Harbor's verifier script writes to the container paths ``/logs`` and ``/tests``.
+    The host environment translates only commands, so the wrapper gets a script
+    that reads the ``VIS_HARBOR_*`` paths.
+    """
+    wrapper = checker.assemble_check_task(
+        task_dir=task_dir,
+        rubric=rubric,
+        template=prompt_template(),
+        output_schema=response_model.model_json_schema(),
+        dest=dest,
+    )
+    environment = wrapper / "environment"
+    workspace = environment / WORKSPACE_DIRECTORY
+    workspace.mkdir()
+    (environment / TASK_DIRECTORY).rename(workspace / TASK_DIRECTORY)
+    script = wrapper / "tests" / "test.sh"
+    script.write_text(TEST_SCRIPT)
+    script.chmod(0o755)
+    return wrapper
+
+
+def job_config(
+    wrappers: Sequence[Path],
+    provider: str,
+    model: str,
+    jobs_dir: Path,
+    job_name: str,
+    workers: int,
+) -> JobConfig:
+    """Return the Harbor job that checks each wrapper task once with the Vis agent."""
+    return JobConfig(
+        job_name=job_name,
+        jobs_dir=jobs_dir,
+        n_concurrent_trials=workers,
+        n_attempts=1,
+        quiet=True,
+        agents=[AgentConfig(import_path=AGENT, model_name=f"{provider}/{model}")],
+        environment=EnvironmentConfig(import_path=ENVIRONMENT),
+        tasks=[TaskConfig(path=wrapper) for wrapper in wrappers],
+        datasets=[],
+    )
+
+
+async def check(
+    path: Path,
+    provider: str,
+    model: str,
+    jobs_dir: Path = Path(DEFAULT_JOBS),
+    job_name: str | None = None,
+    workers: int = DEFAULT_WORKERS,
+) -> tuple[CheckReport, Path]:
+    """Check the task or the tasks in ``path``. Return the report and the job directory."""
+    from harbor.job import Job
+
+    task_dirs = checker._resolve_task_dirs(Path(path), None, None, None)
+    rubric = load_rubric(None)
+    response_model = build_check_response_model(rubric)
+    name = job_name or datetime.now(UTC).strftime("check-%Y%m%d-%H%M%S")
+    tmp = Path(tempfile.mkdtemp(prefix="vis-harbor-check-"))
+    try:
+        names = {}
+        wrappers = []
+        for task_dir in task_dirs:
+            wrapper = assemble(
+                task_dir,
+                rubric,
+                response_model,
+                tmp / f"check-{task_dir.resolve().name}",
+            )
+            wrappers.append(wrapper)
+            names[str(wrapper.resolve())] = task_dir.name
+        config = job_config(wrappers, provider, model, Path(jobs_dir), name, workers)
+        job = await Job.create(config)
+        job_result = await job.run()
+        results = []
+        for trial_result in job_result.trial_results:
+            wrapper_path = trial_result.config.task.path
+            task_name = names.get(
+                str(wrapper_path.resolve()) if wrapper_path else "",
+                trial_result.trial_name,
+            )
+            trial_dir = job.job_dir / trial_result.trial_name
+            try:
+                result = checker._extract_check_result(
+                    trial_result, trial_dir, response_model
+                )
+                result.task_name = task_name
+            except (ValueError, RuntimeError) as error:
+                result = QualityCheckResult(task_name=task_name, error=str(error))
+            results.append(result)
+        results.sort(key=lambda result: result.task_name or "")
+        report = CheckReport(results=results)
+        (job.job_dir / REPORT_FILE).write_text(report.model_dump_json(indent=2))
+        return report, job.job_dir
+    finally:
+        shutil.rmtree(tmp, ignore_errors=True)
+
+
+def print_report(report: CheckReport, job_dir: Path) -> None:
+    """Print the report with the tables of ``harbor check``."""
+    from harbor.cli.analyze import _render_check_summary, _render_checks_table, console
+
+    if len(report.results) == 1:
+        result = report.results[0]
+        if result.error:
+            console.print(f"{result.task_name}: {result.error}")
+        else:
+            _render_checks_table(
+                f"Task Quality Checks: {result.task_name}", result.checks
+            )
+    else:
+        _render_check_summary(report)
+    console.print(f"Report: {job_dir / REPORT_FILE}")
+
+
+def main(argv: Sequence[str] | None = None) -> int:
+    """Check the tasks of ``argv``. Return 1 if a check has no valid result."""
+    parser = argparse.ArgumentParser(prog="python -m blockether.vis.evals.check")
+    parser.add_argument(
+        "path", type=Path, help="a task directory or a directory of tasks"
+    )
+    parser.add_argument("--provider", required=True)
+    parser.add_argument("--model", required=True)
+    parser.add_argument("--jobs-dir", type=Path, default=Path(DEFAULT_JOBS))
+    parser.add_argument("--job-name")
+    parser.add_argument("--workers", type=int, default=DEFAULT_WORKERS)
+    args = parser.parse_args(argv)
+    try:
+        report, job_dir = asyncio.run(
+            check(
+                args.path,
+                provider=args.provider,
+                model=args.model,
+                jobs_dir=args.jobs_dir,
+                job_name=args.job_name,
+                workers=args.workers,
+            )
+        )
+    except (FileNotFoundError, ValueError) as error:
+        print(error, file=sys.stderr)
+        return 1
+    print_report(report, job_dir)
+    return 1 if any(result.error for result in report.results) else 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())