vis-agent 0.2.42.dev7626__tar.gz → 0.2.42.dev7627__tar.gz

{vis_agent-0.2.42.dev7626 → vis_agent-0.2.42.dev7627}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: vis-agent
-Version: 0.2.42.dev7626
+Version: 0.2.42.dev7627
 Summary: The Vis Python SDK — author extensions and control local or remote engines.
 Project-URL: Homepage, https://vis.blockether.com
 Project-URL: Documentation, https://vis.blockether.com

{vis_agent-0.2.42.dev7626 → vis_agent-0.2.42.dev7627}/pyproject.toml

@@ -8,7 +8,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "vis-agent"
-version = "0.2.42.dev7626"
+version = "0.2.42.dev7627"
 description = "The Vis Python SDK — author extensions and control local or remote engines."
 readme = "README.md"
 requires-python = ">=3.11"

{vis_agent-0.2.42.dev7626 → vis_agent-0.2.42.dev7627}/src/blockether/vis/evals/rounds.py

@@ -11,12 +11,12 @@ from __future__ import annotations
 
 import json
 import re
-import tomllib
 from collections import Counter
 from datetime import datetime
 from pathlib import Path
 
 from .bench import agent_limits, attempt_outcome
+from .harbor import read_tasks
 from .trajectory import Step, Trajectory, load_trajectory
 
 __all__ = ["round_summary"]
@@ -306,13 +306,9 @@ def round_summary(
     if dataset_dir is None:
         tasks, gpu_tasks = sorted(attempted), []
     else:
-        folders = sorted(path for path in Path(dataset_dir).iterdir() if path.is_dir())
-        tasks = [folder.name for folder in folders]
-        gpu_tasks = []
-        for folder in folders:
-            info = tomllib.loads((folder / "task.toml").read_text(encoding="utf-8"))
-            if (info.get("environment", {}).get("gpus") or 0) > 0:
-                gpu_tasks.append(folder.name)
+        dataset_tasks = read_tasks(dataset_dir)
+        tasks = [task.name for task in dataset_tasks]
+        gpu_tasks = [task.name for task in dataset_tasks if task.gpus > 0]
     verified = [trial for trial in trials if trial["status"] == "verified"]
     rewards = [
         trial["reward"]

{vis_agent-0.2.42.dev7626 → vis_agent-0.2.42.dev7627}/tests/test_evals_rounds.py

@@ -250,3 +250,22 @@ def test_round_summary_reads_one_e2e_job_folder(tmp_path):
     finding = next(f for f in diagnose(summary).findings if f.kind == "early_stops")
     assert finding.tasks[0].budget_used == 0.1
     assert round_summary(job, dataset="scenarios")["dataset"] == "scenarios"
+
+
+def test_round_summary_counts_only_task_folders_of_the_dataset(tmp_path):
+    # The repository e2e directory is the dataset, and it also holds tool caches.
+    tasks = tmp_path / "e2e"
+    for name in ("one", "two"):
+        (tasks / name).mkdir(parents=True)
+        (tasks / name / "task.toml").write_text("[environment]\ngpus = 0\n")
+    (tasks / "two" / "task.toml").write_text("[environment]\ngpus = 1\n")
+    for cache in (".pytest_cache", "__pycache__", ".venv"):
+        (tasks / cache).mkdir()
+    jobs = tmp_path / "jobs"
+    jobs.mkdir()
+
+    summary = round_summary(jobs, tasks)
+
+    assert summary["total_dataset_tasks"] == 2
+    assert summary["unattempted_tasks"] == ["one", "two"]
+    assert summary["gpu_required_tasks"] == ["two"]