vis-agent 0.2.41.dev7619__tar.gz → 0.2.42.dev7622__tar.gz

{vis_agent-0.2.41.dev7619 → vis_agent-0.2.42.dev7622}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: vis-agent
-Version: 0.2.41.dev7619
+Version: 0.2.42.dev7622
 Summary: The Vis Python SDK — author extensions and control local or remote engines.
 Project-URL: Homepage, https://vis.blockether.com
 Project-URL: Documentation, https://vis.blockether.com

{vis_agent-0.2.41.dev7619 → vis_agent-0.2.42.dev7622}/pyproject.toml

@@ -8,7 +8,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "vis-agent"
-version = "0.2.41.dev7619"
+version = "0.2.42.dev7622"
 description = "The Vis Python SDK — author extensions and control local or remote engines."
 readme = "README.md"
 requires-python = ">=3.11"

{vis_agent-0.2.41.dev7619 → vis_agent-0.2.42.dev7622}/src/blockether/vis/evals/__init__.py

@@ -1,14 +1,15 @@
 """Evaluation tools: read Vis runs and Harbor jobs, and compare their results.
 
-Read one Vis run with `load_trajectory` or `parse_trajectory`. Read the trials of a
-Harbor job or round with `read_trials`, then count them with `case_results` or
-`summarize`. Seal a finished round with `seal_round`. Compare two results with
-`compare_cases`, or test paired changes with `paired_change` and `mcnemar_exact`.
-Score repeated attempts with `task_attempts` and `repeat_stats`. Summarize a
-Terminal-Bench round from its Harbor job folders with `round_summary`, then score and
-diagnose it with `bench_report` and `diagnose`, and keep human cause
-labels with `save_label`. Place a round on a Harbor Hub board with `read_leaderboard`. Check automatic
-causes against human labels with `agreement`, and judge stages with `stage_agreement`. For installation and sessions, use the
+Read one Vis run with `load_trajectory` or `parse_trajectory`. Read the cases of a
+Harbor dataset with `read_tasks`, and the trials of a job or round with `read_trials`.
+Count the trials with `case_results` or `summarize`. Seal a finished round with
+`seal_round`. Compare two results with `compare_cases`, or test paired changes with
+`paired_change` and `mcnemar_exact`. Score repeated attempts with `task_attempts` and
+`repeat_stats`. Summarize a round from its Harbor job folders with `round_summary`,
+then score and diagnose it with `bench_report` and `diagnose`. Keep human cause labels
+with `save_label`. Place a round on a Harbor Hub board with `read_leaderboard`. Check
+automatic causes against human labels with `agreement`, and judge stages with
+`stage_agreement`. For installation and sessions, use the
 [Python SDK guide](https://vis.blockether.com/python-sdk.html).
 """
 
@@ -38,9 +39,11 @@ from .bench import (
 )
 from .compare import CaseChange, CaseComparison, RouteTest, compare_cases
 from .harbor import (
+    HarborTask,
     HarborTrial,
     RunSummary,
     case_results,
+    read_tasks,
     read_trial,
     read_trials,
     seal_digest,
@@ -117,6 +120,7 @@ __all__ = [
     "Diagnosis",
     "Finding",
     "FunctionCall",
+    "HarborTask",
     "HarborTrial",
     "Leaderboard",
     "LeaderboardRow",
@@ -162,6 +166,7 @@ __all__ = [
     "paired_change",
     "parse_trajectory",
     "read_leaderboard",
+    "read_tasks",
     "read_trial",
     "read_trials",
     "repeat_stats",

{vis_agent-0.2.41.dev7619 → vis_agent-0.2.42.dev7622}/src/blockether/vis/evals/bench.py

@@ -359,7 +359,7 @@ def attempt_cause(trial: Mapping, summary: Mapping, limits: Mapping[str, float])
     """Give the `diagnose` cause of one attempt, or `solved`.
 
     The causes are `dead_tools`, `agent_errors`, `agent_timeouts`, `unscored`,
-    `early_stops` and `other`.
+    `early_stops` and `other`. Empty `limits` use the `agent_limits` of the summary.
     """
     outcome = attempt_outcome(trial)
     if outcome == "solved":
@@ -373,7 +373,7 @@ def attempt_cause(trial: Mapping, summary: Mapping, limits: Mapping[str, float])
     if outcome == "unscored":
         return "unscored"
     seconds = trial.get("agent_seconds")
-    limit = limits.get(_short(trial.get("task")))
+    limit = (limits or summary.get("agent_limits") or {}).get(_short(trial.get("task")))
     if seconds and limit and seconds / limit <= EARLY_STOP_SHARE:
         return "early_stops"
     return "other"
@@ -430,7 +430,8 @@ def diagnose(
     """Find where a round loses tasks, from its `round_summary` data.
 
     `limits` gives the agent time limit of each task, from `agent_limits`. Without
-    limits, no attempt counts as an early stop. `labels` maps a trial name to a human
+    limits, it uses the `agent_limits` of the summary, and without those no attempt
+    counts as an early stop. `labels` maps a trial name to a human
     label with a `cause` value.
     """
     outcomes = {
@@ -440,7 +441,7 @@ def diagnose(
     scored = int(summary.get("scored_tasks") or 0)
     solved = len(outcomes.get("solved", ()))
     latest = latest_trials(summary)
-    limits = limits or {}
+    limits = limits or summary.get("agent_limits") or {}
 
     def notes(outcome: str) -> list[TaskNote]:
         return [

{vis_agent-0.2.41.dev7619 → vis_agent-0.2.42.dev7622}/src/blockether/vis/evals/harbor.py

@@ -12,6 +12,7 @@ from __future__ import annotations
 import hashlib
 import json
 import time
+import tomllib
 from collections.abc import Iterable, Mapping, Sequence
 from dataclasses import dataclass
 from pathlib import Path
@@ -21,9 +22,11 @@ from typing import Any
 from .stats import cluster_interval
 
 __all__ = [
+    "HarborTask",
     "HarborTrial",
     "RunSummary",
     "case_results",
+    "read_tasks",
     "read_trial",
     "read_trials",
     "seal_digest",
@@ -33,6 +36,8 @@ __all__ = [
 ]
 
 RESULT_FILE = "result.json"
+TASK_FILE = "task.toml"
+INSTRUCTION_FILE = "instruction.md"
 
 
 @dataclass(frozen=True, slots=True)
@@ -301,3 +306,66 @@ def seal_round(
     }
     body["digest"] = seal_digest(body)
     return body
+
+
+@dataclass(frozen=True, slots=True)
+class HarborTask:
+    """One Harbor task, read from its `task.toml` and `instruction.md`.
+
+    `name` is the task folder name, as in `case_results`. `instruction` is the prompt
+    that the agent gets. The timeouts are in seconds, or None when `task.toml` gives
+    none. `gpus` is the GPU count that the environment needs. `metadata` holds the
+    `[metadata]` table, for example the `lang` of an e2e scenario.
+    """
+
+    name: str
+    instruction: str
+    agent_timeout_sec: float | None
+    verifier_timeout_sec: float | None
+    gpus: int
+    metadata: Mapping[str, Any]
+    path: Path
+
+
+def _seconds(table: object) -> float | None:
+    value = table.get("timeout_sec") if isinstance(table, Mapping) else None
+    if isinstance(value, bool) or not isinstance(value, (int, float)):
+        return None
+    return float(value)
+
+
+def read_tasks(dataset: Path | str) -> tuple[HarborTask, ...]:
+    """Read the tasks of a Harbor dataset directory, sorted by name.
+
+    Each task is a folder with a `task.toml`. Other folders do not count. A directory
+    that does not exist raises `FileNotFoundError`. Compile the e2e scenarios into a
+    dataset with `python -m vis_harbor.dataset --out DIR`, then read it here.
+    """
+    root = Path(dataset)
+    if not root.is_dir():
+        raise FileNotFoundError(f"no Harbor dataset directory at {root}")
+    tasks = []
+    for config in sorted(root.glob(f"*/{TASK_FILE}")):
+        info = tomllib.loads(config.read_text(encoding="utf-8"))
+        instruction = config.parent / INSTRUCTION_FILE
+        environment = info.get("environment")
+        gpus = environment.get("gpus") if isinstance(environment, dict) else None
+        metadata = info.get("metadata")
+        tasks.append(
+            HarborTask(
+                name=config.parent.name,
+                instruction=instruction.read_text(encoding="utf-8")
+                if instruction.is_file()
+                else "",
+                agent_timeout_sec=_seconds(info.get("agent")),
+                verifier_timeout_sec=_seconds(info.get("verifier")),
+                gpus=gpus
+                if isinstance(gpus, int) and not isinstance(gpus, bool)
+                else 0,
+                metadata=MappingProxyType(
+                    dict(metadata) if isinstance(metadata, dict) else {}
+                ),
+                path=config.parent,
+            )
+        )
+    return tuple(tasks)

{vis_agent-0.2.41.dev7619 → vis_agent-0.2.42.dev7622}/src/blockether/vis/evals/rounds.py

@@ -16,13 +16,12 @@ from collections import Counter
 from datetime import datetime
 from pathlib import Path
 
-from .bench import attempt_outcome
+from .bench import agent_limits, attempt_outcome
 from .trajectory import Step, Trajectory, load_trajectory
 
 __all__ = ["round_summary"]
 
 _TRAJECTORY = "agent/trajectory.json"
-_PREFIX = "terminal-bench/"
 _KIND = re.compile(r"[A-Za-z0-9_.:-]{1,100}")
 
 
@@ -148,8 +147,23 @@ def _tests(path: Path) -> dict | None:
     return {key: summary[key] for key in keys if _count(summary.get(key))} or None
 
 
-def _trial(result_path: Path, root: Path, model: str | None) -> dict:
-    result = json.loads(result_path.read_text(encoding="utf-8"))
+def _results(jobs: Path) -> list[tuple[Path, dict]]:
+    """Read each trial `result.json` one or two levels below `jobs`, in path order.
+
+    A job writes its own `result.json` without `trial_name`. It is not a trial.
+    """
+    found = []
+    for path in sorted({*jobs.glob("*/result.json"), *jobs.glob("*/*/result.json")}):
+        try:
+            result = json.loads(path.read_text(encoding="utf-8"))
+        except (OSError, ValueError):
+            continue
+        if isinstance(result, dict) and result.get("trial_name"):
+            found.append((path, result))
+    return found
+
+
+def _trial(result_path: Path, result: dict, root: Path, model: str | None) -> dict:
     folder = result_path.parent
     agent = result.get("agent_result") or {}
     verifier = result.get("verifier_result") or {}
@@ -207,6 +221,7 @@ def _trial(result_path: Path, root: Path, model: str | None) -> dict:
     check = result.get("verifier") or {}
     return {
         "job": folder.parent.name,
+        "dataset": result.get("source"),
         "task": result.get("task_name"),
         "trial": result.get("trial_name"),
         "status": status,
@@ -253,7 +268,16 @@ def _usage_totals(trials: list[dict]) -> dict:
 
 
 def _name(task) -> str | None:
-    return task.removeprefix(_PREFIX) if isinstance(task, str) else None
+    """Name a task without its dataset, as `bench_report` and `diagnose` do."""
+    return task.rsplit("/", 1)[-1] if isinstance(task, str) and task else None
+
+
+def _dataset(trials: list[dict], dataset_dir: Path | str | None) -> str:
+    """Name the dataset: the most common Harbor `source` of the trials, or the folder."""
+    sources = Counter(trial["dataset"] for trial in trials if trial["dataset"])
+    if sources:
+        return sources.most_common(1)[0][0]
+    return Path(dataset_dir).name if dataset_dir is not None else "unknown"
 
 
 def round_summary(
@@ -261,22 +285,24 @@ def round_summary(
     dataset_dir: Path | str | None = None,
     *,
     model: str | None = None,
-    dataset: str = "terminal-bench/terminal-bench@4.0.0",
+    dataset: str | None = None,
 ) -> dict:
     """Summarize every trial under `jobs_dir`, as `bench_report` and `diagnose` read it.
 
-    `dataset_dir` holds one folder with a `task.toml` for each task. It gives the
-    total, the GPU tasks and the tasks without an attempt. Without it, the total is
-    the attempted tasks. `model` is the model name or the `provider/model` route of
-    the trajectory agent. An attempt of another model does not count. With no
-    `model`, any model counts. Each task counts once, by its latest scored attempt.
+    `jobs_dir` is one Harbor job folder or a folder of jobs. `dataset_dir` holds one
+    folder with a `task.toml` for each task. It gives the total, the GPU tasks, the
+    tasks without an attempt and the `agent_limits` that `diagnose` uses. Without it,
+    the total is the attempted tasks. `dataset` names the dataset. Without it, the
+    name is the `source` that Harbor wrote in the trials. `model` is the model name or
+    the `provider/model` route of the agent. An attempt of another model does not
+    count. With no `model`, any model counts. Each task counts once, by its latest
+    scored attempt.
     """
     jobs = Path(jobs_dir)
     root = jobs.parent
-    trials = [
-        _trial(path, root, model) for path in sorted(jobs.glob("*/*/result.json"))
-    ]
+    trials = [_trial(path, result, root, model) for path, result in _results(jobs)]
     attempted = {name for trial in trials if (name := _name(trial["task"]))}
+    limits = {} if dataset_dir is None else agent_limits(dataset_dir)
     if dataset_dir is None:
         tasks, gpu_tasks = sorted(attempted), []
     else:
@@ -299,7 +325,7 @@ def round_summary(
         (trial for trial in trials if trial["scored"] and _name(trial["task"])),
         key=lambda trial: trial["finished_at"],
     ):
-        scored[trial["task"].removeprefix(_PREFIX)] = trial
+        scored[str(_name(trial["task"]))] = trial
     model_tasks = {
         name
         for trial in trials
@@ -315,10 +341,11 @@ def round_summary(
         if streak and streak["iterations"] > 1:
             repeated.append({"trial": f"{trial['job']}/{trial['trial']}", **streak})
     return {
-        "dataset": dataset,
+        "dataset": dataset or _dataset(trials, dataset_dir),
         "total_dataset_tasks": len(tasks),
         "gpu_required_tasks": gpu_tasks,
         "unattempted_tasks": [task for task in tasks if task not in attempted],
+        "agent_limits": limits,
         "attempts": len(trials),
         "verified_attempts": len(verified),
         "exception_attempts": sum(trial["status"] == "exception" for trial in trials),

{vis_agent-0.2.41.dev7619 → vis_agent-0.2.42.dev7622}/tests/test_evals_harbor.py

@@ -6,6 +6,7 @@ import json
 import pytest
 from blockether.vis.evals import (
     case_results,
+    read_tasks,
     read_trial,
     read_trials,
     seal_digest,
@@ -226,3 +227,29 @@ def test_seal_round_has_one_row_for_each_trial_and_a_digest(tmp_path):
     assert seal["digest"] == seal_digest(seal) == expected
     seal["rows"][0]["reward"] = 0.0
     assert seal_digest(seal) != seal["digest"]
+
+
+def test_read_tasks_reads_the_task_folders_of_a_dataset(tmp_path):
+    task = tmp_path / "dataset" / "fix-git"
+    task.mkdir(parents=True)
+    (task / "task.toml").write_text(
+        '[metadata]\nlang = "py"\n[agent]\ntimeout_sec = 600\n'
+        "[verifier]\ntimeout_sec = 300.0\n[environment]\ngpus = 1\n"
+    )
+    (task / "instruction.md").write_text("Fix the repository.")
+    bare = tmp_path / "dataset" / "bare"
+    bare.mkdir()
+    (bare / "task.toml").write_text("")
+    (tmp_path / "dataset" / "notes").mkdir()
+
+    found = {task.name: task for task in read_tasks(tmp_path / "dataset")}
+
+    assert sorted(found) == ["bare", "fix-git"]
+    assert found["fix-git"].instruction == "Fix the repository."
+    assert found["fix-git"].agent_timeout_sec == 600.0
+    assert found["fix-git"].verifier_timeout_sec == 300.0
+    assert found["fix-git"].gpus == 1
+    assert found["fix-git"].metadata == {"lang": "py"}
+    assert (found["bare"].instruction, found["bare"].agent_timeout_sec) == ("", None)
+    with pytest.raises(FileNotFoundError):
+        read_tasks(tmp_path / "missing")

{vis_agent-0.2.41.dev7619 → vis_agent-0.2.42.dev7622}/tests/test_evals_rounds.py

@@ -208,3 +208,45 @@ def test_round_summary_reads_the_harbor_model_without_a_trajectory(tmp_path):
     assert trials["terminal-bench/silent"]["scored"] is False
     assert trials["terminal-bench/other"]["model_attempt"] is False
     assert summary["solved_tasks"] == ["solved"]
+
+
+def test_round_summary_reads_one_e2e_job_folder(tmp_path):
+    # An e2e run writes one Harbor job. Its tasks have no dataset prefix, and the job
+    # writes its own result.json next to the trial folders.
+    job = tmp_path / "e2e-job"
+    (job / "quick__1" / "agent").mkdir(parents=True)
+    (job / "result.json").write_text(json.dumps({"id": "job", "stats": {}}))
+    result = {
+        "task_name": "quick",
+        "trial_name": "quick__1",
+        "source": "vis-e2e",
+        "started_at": "2026-01-02T00:00:00Z",
+        "finished_at": "2026-01-02T00:02:00Z",
+        "agent_execution": {
+            "started_at": "2026-01-02T00:00:00Z",
+            "finished_at": "2026-01-02T00:01:00Z",
+        },
+        "agent_info": {
+            "name": "vis",
+            "model_info": {"name": MODEL, "provider": PROVIDER},
+        },
+        "agent_result": {"n_input_tokens": 10, "n_output_tokens": 5},
+        "verifier_result": {"rewards": {"reward": 0.0}},
+    }
+    (job / "quick__1" / "result.json").write_text(json.dumps(result))
+    tasks = tmp_path / "dataset"
+    for name in ("quick", "unrun"):
+        (tasks / name).mkdir(parents=True)
+        (tasks / name / "task.toml").write_text("[agent]\ntimeout_sec = 600\n")
+
+    summary = round_summary(job, tasks)
+
+    assert summary["dataset"] == "vis-e2e"
+    assert summary["attempts"] == 1
+    assert summary["scored_tasks"] == 1
+    assert summary["unattempted_tasks"] == ["unrun"]
+    assert summary["agent_limits"] == {"quick": 600.0, "unrun": 600.0}
+    assert summary["trials"][0]["dataset"] == "vis-e2e"
+    finding = next(f for f in diagnose(summary).findings if f.kind == "early_stops")
+    assert finding.tasks[0].budget_used == 0.1
+    assert round_summary(job, dataset="scenarios")["dataset"] == "scenarios"