vis-agent 0.3.3.dev7953__tar.gz → 0.3.4.dev7992__tar.gz

{vis_agent-0.3.3.dev7953 → vis_agent-0.3.4.dev7992}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: vis-agent
-Version: 0.3.3.dev7953
+Version: 0.3.4.dev7992
 Summary: The Vis Python SDK — author extensions and control local or remote engines.
 Project-URL: Homepage, https://vis.blockether.com
 Project-URL: Documentation, https://vis.blockether.com

{vis_agent-0.3.3.dev7953 → vis_agent-0.3.4.dev7992}/contracts/schema/agents.json

@@ -192,7 +192,7 @@
       "additionalProperties": false
     },
     "grant": {
-      "$comment": "Add file rules to the grant of a subagent. A child without write rules writes only the granted rules after a write grant.",
+      "$comment": "Add file rules to the grant of a subagent. A child without write rules already writes where its parent writes, so a write grant changes nothing for it. A write rule under a read_only or deny rule of the child fails.",
       "type": "object",
       "properties": {
         "session_id": {
@@ -238,6 +238,10 @@
         "task": {
           "$ref": "#/$defs/spawn/properties/task"
         },
+        "files": {
+          "$comment": "Replace the grant of the child for the follow-up task.",
+          "$ref": "#/$defs/files"
+        },
         "timeout_secs": {
           "$ref": "#/$defs/wait/properties/timeout_secs"
         }
@@ -377,6 +381,10 @@
           "type": "integer",
           "minimum": 0
         },
+        "question": {
+          "type": "string",
+          "minLength": 1
+        },
         "usage": {
           "type": [
             "object",

{vis_agent-0.3.3.dev7953 → vis_agent-0.3.4.dev7992}/contracts/schema/evals.json

@@ -580,7 +580,7 @@
       "additionalProperties": false
     },
     "agreement": {
-      "description": "The agreement of automatic results with human labels, with Cohen kappa and advice.",
+      "description": "The agreement of automatic results with human labels, with Cohen kappa, Krippendorff alpha and advice.",
       "type": "object",
       "properties": {
         "labelled": {
@@ -598,6 +598,12 @@
             "null"
           ]
         },
+        "alpha": {
+          "type": [
+            "number",
+            "null"
+          ]
+        },
         "disagreements": {
           "type": "array",
           "items": {
@@ -612,6 +618,7 @@
         "labelled",
         "agreement_percent",
         "kappa",
+        "alpha",
         "disagreements",
         "advice"
       ],

{vis_agent-0.3.3.dev7953 → vis_agent-0.3.4.dev7992}/pyproject.toml

@@ -8,7 +8,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "vis-agent"
-version = "0.3.3.dev7953"
+version = "0.3.4.dev7992"
 description = "The Vis Python SDK — author extensions and control local or remote engines."
 readme = "README.md"
 requires-python = ">=3.12"

vis_agent-0.3.4.dev7992/release-tag

@@ -0,0 +1 @@
+beta-a96abba81be1cb50ca4ae87bc76d6453711765db

{vis_agent-0.3.3.dev7953 → vis_agent-0.3.4.dev7992}/src/blockether/vis/engine/__init__.py

@@ -4,7 +4,7 @@ For installation, workflows and lifecycle choices, use the
 [Python SDK guide](https://vis.blockether.com/python-sdk.html).
 """
 
-from ._agent import Agent, StructuredOutputError
+from ._agent import Agent, StructuredOutputError, final_answer
 from ._agents import Subagent, SubagentRecord
 from ._client import (
     Event,
@@ -70,5 +70,6 @@ __all__ = [
     "Turn",
     "VisTimeout",
     "ensure_installed",
+    "final_answer",
     "install",
 ]

{vis_agent-0.3.3.dev7953 → vis_agent-0.3.4.dev7992}/src/blockether/vis/engine/_agent.py

@@ -239,7 +239,13 @@ def _syntax_message(answer: str, error: json.JSONDecodeError) -> str:
     return f"not valid JSON: {error.msg} at {where}"
 
 
-def _final_answer(turn: dict) -> str | None:
+def final_answer(turn: dict) -> str | None:
+    """Return the final answer text of the turn record `turn`, or None.
+
+    A turn record has no `answer` field. The answer is the Markdown of the last
+    `prose` block in `turn["content"]`. A turn without that block, or with only
+    blank text in it, gives None.
+    """
     content = turn.get("content")
     if not isinstance(content, list):
         return None
@@ -397,7 +403,7 @@ def _run(
                 {"turn": turn, "answer": None, "errors": [_turn_problem(turn)]}
             )
             raise StructuredOutputError(f"{name} turn did not complete", attempts)
-        answer = _final_answer(turn)
+        answer = final_answer(turn)
         result, problems = contract.validate(answer)
         if not problems:
             return result

{vis_agent-0.3.3.dev7953 → vis_agent-0.3.4.dev7992}/src/blockether/vis/engine/_council.py

@@ -316,15 +316,27 @@ class Council:
         return self._records("/cancel", {"session_id": session_id}, "cancel")[0]
 
     def resume(
-        self, session_id: str, task: str, *, timeout_secs: int = 240
+        self,
+        session_id: str,
+        task: str,
+        *,
+        files: dict[str, list[str]] | None = None,
+        timeout_secs: int = 240,
     ) -> SubagentRecord:
         """Give a finished subagent a follow-up task in its own session, then wait for it.
 
         The parent must be active, as for ``spawn``: an idle parent gets
         ``inactive-session``. The child keeps its whole context. A child without
         iterations left needs ``extend`` first. A running child needs ``wait``.
+        ``files`` replaces the file grant of the child for the follow-up task.
         """
-        body = {"session_id": session_id, "task": task, "timeout_secs": timeout_secs}
+        body: dict[str, Any] = {
+            "session_id": session_id,
+            "task": task,
+            "timeout_secs": timeout_secs,
+        }
+        if files is not None:
+            body["files"] = files
         return self._records("/resume", body, "resume", timeout_secs)[0]
 
     def extend(self, session_id: str, iterations: int) -> SubagentRecord:
@@ -343,8 +355,11 @@ class Council:
     def grant(self, session_id: str, files: dict[str, list[str]]) -> SubagentRecord:
         """Add ``write``, ``read_only`` or ``deny`` file rules to the grant of a subagent.
 
-        A grant gives at most your own access. The record has a ``warning`` when an
-        active sibling can write the same files.
+        A grant gives at most your own access. A write rule that the child can already
+        write changes nothing. A write rule under a ``read_only`` or ``deny`` rule of the
+        child fails: use ``resume`` with ``files`` for a new scope. The record has a
+        ``warning`` for unchanged write rules and for an active sibling that can write
+        the same files.
         """
         body = {"session_id": session_id, "files": files}
         return self._records("/grant", body, "grant")[0]

{vis_agent-0.3.3.dev7953 → vis_agent-0.3.4.dev7992}/src/blockether/vis/evals/__init__.py

@@ -1,7 +1,9 @@
 """Evaluation tools: write and run Harbor tasks, read the results and compare them.
 
 Write a Vis task with `write_task`, and run tasks with the Vis agent with `run_tasks`.
-Read one Vis run with `load_trajectory` or `parse_trajectory`. Read the cases of a
+Give `write_task` a list of user messages for a task with several turns.
+Read one Vis run with `load_trajectory` or `parse_trajectory`, and join the turns of
+one session with `merge_trajectories`. Read the cases of a
 Harbor dataset with `read_tasks`, and the trials of a job or round with `read_trials`.
 Count the trials with `case_results` or `summarize`. Seal a finished round with
 `seal_round`. Compare two results with `compare_cases`, or test paired changes with
@@ -10,7 +12,9 @@ Count the trials with `case_results` or `summarize`. Seal a finished round with
 then score and diagnose it with `bench_report` and `diagnose`. Keep human cause labels
 with `save_label`. Place a round on a Harbor Hub board with `read_leaderboard`. Check
 automatic causes against human labels with `agreement`, and judge stages with
-`stage_agreement`. For installation and sessions, use the
+`stage_agreement`. Estimate pass@k and pass^k with `pass_at_k` and `pass_all_k`, and
+the agreement of several raters with `krippendorff_alpha`. Build all of it for one
+round with `round_report`, or run `python -m blockether.vis.evals.report`. For installation and sessions, use the
 [Python SDK guide](https://vis.blockether.com/python-sdk.html).
 """
 
@@ -51,19 +55,24 @@ from .harbor import (
     seal_digest,
     seal_round,
     seal_rows,
+    split_turns,
     summarize,
     write_task,
 )
 from .labels import (
+    ALPHA_RELIABLE,
+    ALPHA_TENTATIVE,
     KAPPA_REVIEW,
     KAPPA_TRUST,
     MIN_LABELS,
     STAGES,
     Agreement,
     agreement,
+    alpha_advice,
     kappa_advice,
     majority,
     sample_agreement,
+    sample_alpha,
 )
 from .leaderboard import (
     LEADERBOARD_URL,
@@ -75,6 +84,14 @@ from .leaderboard import (
     standing,
 )
 from .repeats import RepeatStats, repeat_stats, task_attempts
+from .reports import (
+    REPORT_FILE,
+    RepeatRow,
+    RoundReport,
+    render_report,
+    round_report,
+    write_report,
+)
 from .rounds import round_summary
 from .stages import (
     StageAgreement,
@@ -91,8 +108,11 @@ from .stats import (
     cohen_kappa,
     detectable_change,
     effective_size,
+    krippendorff_alpha,
     mcnemar_exact,
     paired_change,
+    pass_all_k,
+    pass_at_k,
     wilson_interval,
 )
 from .trajectory import (
@@ -103,10 +123,13 @@ from .trajectory import (
     Trajectory,
     TrajectoryMetrics,
     load_trajectory,
+    merge_trajectories,
     parse_trajectory,
 )
 
 __all__ = [
+    "ALPHA_RELIABLE",
+    "ALPHA_TENTATIVE",
     "CAUSES",
     "EARLY_STOP_SHARE",
     "KAPPA_REVIEW",
@@ -115,6 +138,7 @@ __all__ = [
     "MIN_LABELS",
     "NEAR_MISS_RATIO",
     "NOTE_LIMIT",
+    "REPORT_FILE",
     "STAGES",
     "Agreement",
     "BenchReport",
@@ -128,7 +152,9 @@ __all__ = [
     "Leaderboard",
     "LeaderboardRow",
     "OutcomeCount",
+    "RepeatRow",
     "RepeatStats",
+    "RoundReport",
     "RouteTest",
     "RunSummary",
     "StageAgreement",
@@ -143,6 +169,7 @@ __all__ = [
     "VisStanding",
     "agent_limits",
     "agreement",
+    "alpha_advice",
     "attempt_cause",
     "attempt_outcome",
     "bench_report",
@@ -159,6 +186,7 @@ __all__ = [
     "judge_errors",
     "judged_samples",
     "kappa_advice",
+    "krippendorff_alpha",
     "latest_trials",
     "leaderboard_row",
     "load_labels",
@@ -166,21 +194,28 @@ __all__ = [
     "load_trajectory",
     "majority",
     "mcnemar_exact",
+    "merge_trajectories",
     "paired_change",
     "parse_trajectory",
+    "pass_all_k",
+    "pass_at_k",
     "read_leaderboard",
     "read_tasks",
     "read_trial",
     "read_trials",
+    "render_report",
     "repeat_stats",
+    "round_report",
     "round_summary",
     "run_tasks",
     "sample_agreement",
+    "sample_alpha",
     "save_label",
     "save_stage_label",
     "seal_digest",
     "seal_round",
     "seal_rows",
+    "split_turns",
     "stage_agreement",
     "standing",
     "summarize",
@@ -188,5 +223,6 @@ __all__ = [
     "task_attempts",
     "task_note",
     "wilson_interval",
+    "write_report",
     "write_task",
 ]

{vis_agent-0.3.3.dev7953 → vis_agent-0.3.4.dev7992}/src/blockether/vis/evals/agent.py

@@ -11,6 +11,12 @@ directory:
 - ``stderr.txt``: the error output of the CLI.
 - ``trajectory.json``: the ATIF trajectory of the session.
 - ``run.json``: the provider, the model, the exit code and the wall time.
+
+A line with only ``<!-- vis:turn -->`` in the instruction starts the next user
+turn. The agent then sends each message as one CLI turn of the same session, with
+``--session-id``. ``turns/<n>.json`` keeps the trajectory of turn n, and
+``trajectory.json`` joins all turns. Other Harbor agents get the whole text as one
+prompt.
 """
 
 from __future__ import annotations
@@ -36,9 +42,12 @@ from pydantic import Field
 from ..engine._install import ensure_installed
 from .environment import HostEnvironment
 from .grade import RUN_FILE, STDERR_FILE, TRACE_FILE, TRAJECTORY_FILE
+from .harbor import split_turns
 from .trajectory import load_trajectory as load_vis_trajectory
+from .trajectory import merge_trajectories
 
 DEFAULT_COMMAND = "vis-agent"
+TURNS_DIR = "turns"
 CLASSPATH_TIMEOUT_SEC = 120
 _NATIVE_MAGIC = (
     b"\x7fELF",
@@ -76,6 +85,10 @@ class VisOptions(AgentOptions):
     command: str | None = Field(
         default=None, description="Shell prefix that replaces the engine, for tests."
     )
+    append_system: str | None = Field(
+        default=None,
+        description="File whose text each trial adds to the system prompt.",
+    )
 
 
 @functools.cache
@@ -114,12 +127,20 @@ class ProviderUnavailableError(RuntimeError):
     """The provider refused the turn, so the trial measures nothing about Vis."""
 
 
+SCORED_PROVIDER_CODES = frozenset({"provider_context_overflow"})
+"""Provider error codes that the agent causes, so the trial keeps its score.
+
+A conversation that grows past the context window shows how the agent manages
+its context, for example that it did not fold. Such a trial fails its checks.
+"""
+
+
 def provider_failure(trace: Path) -> str | None:
     """Return the provider error that ended the turn in ``trace``, or None.
 
     A provider quota or outage ends the turn with an error block whose code
     starts with ``provider_``. Such a trial is an infrastructure error, not a
-    wrong answer.
+    wrong answer. The codes in ``SCORED_PROVIDER_CODES`` are not returned.
     """
     try:
         lines = trace.read_text(encoding="utf-8").splitlines()
@@ -138,12 +159,21 @@ def provider_failure(trace: Path) -> str | None:
             if not isinstance(block, dict) or block.get("type") != "error":
                 continue
             code = str(block.get("code"))
-            if code.startswith("provider_"):
+            if code.startswith("provider_") and code not in SCORED_PROVIDER_CODES:
                 return f"{code}: {block.get('title') or 'provider error'}"
         return None
     return None
 
 
+def _session_id(trajectory: Path) -> str | None:
+    """Return the session id of the trajectory file, or None."""
+    try:
+        value = json.loads(trajectory.read_text(encoding="utf-8")).get("session_id")
+    except (OSError, ValueError, AttributeError):
+        return None
+    return value if isinstance(value, str) and value else None
+
+
 def check_native_binary(path: str) -> None:
     """Refuse a launcher script: the measurement must use the native image."""
     with open(path, "rb") as executable:
@@ -165,6 +195,10 @@ class VisAgent(BaseAgent):
     (``VIS_HARBOR_NATIVE_BIN``), ``source`` (``VIS_HARBOR_SOURCE``). Without them, the
     agent runs the ``vis-agent`` CLI of the SDK's release. When the installed CLI is
     missing or another release, setup downloads that release once into the SDK cache.
+
+    ``append_system`` (``VIS_HARBOR_APPEND_SYSTEM``) names a file. Each trial writes
+    its text to ``.vis/APPEND_SYSTEM.md`` in the workspace, so two runs can compare a
+    prompt rule on the same tasks. A task that has that file already is an error.
     """
 
     capabilities = AgentCapabilities(atif=True)
@@ -197,6 +231,12 @@ class VisAgent(BaseAgent):
         source = options.source or self._setting("VIS_HARBOR_SOURCE")
         self.source = Path(source).resolve() if source else None
         self.engine_override = options.command
+        append_system = options.append_system or self._setting(
+            "VIS_HARBOR_APPEND_SYSTEM"
+        )
+        self.append_system = (
+            Path(append_system).read_text(encoding="utf-8") if append_system else None
+        )
         self.classpath: str | None = None
         self.installed_command: str | None = None
         self.model_name = f"{self.provider}/{self.model}"
@@ -229,10 +269,12 @@ class VisAgent(BaseAgent):
         return [*arguments, "--persist"]
 
     def shell_command(self) -> str:
-        """Return the bash command of one trial.
+        """Return the bash command of one CLI turn.
 
         Paths and the prompt stay in environment variables. Then the path
-        translation of the host environment cannot change them.
+        translation of the host environment cannot change them. A later turn
+        continues the session in ``VIS_HARBOR_SESSION_ID``. Each turn appends to the
+        trace and the error output.
         """
         if self.engine_override:
             engine = self.engine_override
@@ -246,15 +288,30 @@ class VisAgent(BaseAgent):
         return (
             "unset VIS_GATEWAY_URL; "
             f"{engine} {arguments} "
-            f'--trajectory "$VIS_HARBOR_AGENT_LOGS/{TRAJECTORY_FILE}" '
+            '${VIS_HARBOR_SESSION_ID:+--session-id "$VIS_HARBOR_SESSION_ID"} '
+            '--trajectory "$VIS_HARBOR_AGENT_LOGS/$VIS_HARBOR_TRAJECTORY" '
             '"$VIS_HARBOR_PROMPT" '
-            f'> "$VIS_HARBOR_AGENT_LOGS/{TRACE_FILE}" '
-            f'2> "$VIS_HARBOR_AGENT_LOGS/{STDERR_FILE}"'
+            f'>> "$VIS_HARBOR_AGENT_LOGS/{TRACE_FILE}" '
+            f'2>> "$VIS_HARBOR_AGENT_LOGS/{STDERR_FILE}"'
         )
 
-    def process_env(self, instruction: str, runtime: Path) -> dict[str, str]:
-        """Return the variables of the CLI process, with a database for this trial."""
+    def process_env(
+        self,
+        instruction: str,
+        runtime: Path,
+        *,
+        trajectory: str = TRAJECTORY_FILE,
+        session_id: str | None = None,
+    ) -> dict[str, str]:
+        """Return the variables of one CLI turn, with a database for this trial.
+
+        ``trajectory`` is the trajectory path relative to the agent logs. With
+        ``session_id``, the turn continues that session.
+        """
         env = dict(self._extra_env)
+        env["VIS_HARBOR_TRAJECTORY"] = trajectory
+        if session_id:
+            env["VIS_HARBOR_SESSION_ID"] = session_id
         env["VIS_DB_PATH"] = str(runtime / "vis.mdb")
         env["VIS_EXTENSIONS_DIR"] = self._setting("VIS_EXTENSIONS_DIR") or str(
             runtime / "extensions"
@@ -291,6 +348,11 @@ class VisAgent(BaseAgent):
     async def run(
         self, instruction: str, environment: BaseEnvironment, context: AgentContext
     ) -> None:
+        turns = split_turns(instruction)
+        if self.append_system is not None:
+            if not isinstance(environment, HostEnvironment):
+                raise TypeError("append_system needs the HostEnvironment workspace")
+            self._write_append_system(Path(environment.workspace))
         runtime = Path(tempfile.mkdtemp(prefix="vis_harbor_runtime_"))
         record = {
             "provider": self.provider,
@@ -301,23 +363,81 @@ class VisAgent(BaseAgent):
             "is_cancelled": False,
         }
         started = time.monotonic()
+        if len(turns) > 1:
+            record["turns"] = []
         try:
-            result = await environment.exec(
-                self.shell_command(), env=self.process_env(instruction, runtime)
-            )
-            record["exit_code"] = result.return_code
+            if len(turns) == 1:
+                result = await environment.exec(
+                    self.shell_command(), env=self.process_env(instruction, runtime)
+                )
+                record["exit_code"] = result.return_code
+            else:
+                await self._run_turns(turns, environment, runtime, record)
         except asyncio.CancelledError:
             record["is_cancelled"] = True
             raise
         finally:
             record["wall_s"] = round(time.monotonic() - started, 3)
             self.logs_dir.mkdir(parents=True, exist_ok=True)
+            if len(turns) > 1:
+                self._join_turns()
             (self.logs_dir / RUN_FILE).write_text(json.dumps(record, indent=2) + "\n")
             shutil.rmtree(runtime, ignore_errors=True)
             self.populate_context_post_run(context)
         if failure := provider_failure(self.logs_dir / TRACE_FILE):
             raise ProviderUnavailableError(failure)
 
+    def _write_append_system(self, workspace: Path) -> None:
+        target = workspace / ".vis" / "APPEND_SYSTEM.md"
+        if target.exists():
+            raise ValueError(
+                "append_system cannot replace the .vis/APPEND_SYSTEM.md of the task"
+            )
+        target.parent.mkdir(parents=True, exist_ok=True)
+        target.write_text(self.append_system or "", encoding="utf-8")
+
+    async def _run_turns(
+        self,
+        turns: list[str],
+        environment: BaseEnvironment,
+        runtime: Path,
+        record: dict,
+    ) -> None:
+        """Send each message as one turn of one session; stop at a failed turn."""
+        (self.logs_dir / TURNS_DIR).mkdir(parents=True, exist_ok=True)
+        session_id = None
+        for number, message in enumerate(turns, start=1):
+            name = f"{TURNS_DIR}/{number}.json"
+            env = self.process_env(
+                message, runtime, trajectory=name, session_id=session_id
+            )
+            result = await environment.exec(self.shell_command(), env=env)
+            record["exit_code"] = result.return_code
+            record["turns"].append({"exit_code": result.return_code})
+            if result.return_code != 0 or provider_failure(self.logs_dir / TRACE_FILE):
+                return
+            if session_id is None:
+                session_id = _session_id(self.logs_dir / name)
+            if session_id is None:
+                raise RuntimeError("the first turn wrote no persisted session id")
+
+    def _join_turns(self) -> None:
+        """Write ``trajectory.json`` from the trajectories of the finished turns."""
+        documents = []
+        number = 1
+        while (path := self.logs_dir / TURNS_DIR / f"{number}.json").is_file():
+            try:
+                documents.append(json.loads(path.read_text(encoding="utf-8")))
+            except (OSError, ValueError) as error:
+                self.logger.warning("Cannot read %s: %s", path, error)
+                break
+            number += 1
+        if documents:
+            merged = merge_trajectories(documents)
+            (self.logs_dir / TRAJECTORY_FILE).write_text(
+                json.dumps(merged, indent=2) + "\n", encoding="utf-8"
+            )
+
     def populate_context_post_run(self, context: AgentContext) -> None:
         """Copy the token and cost totals of the trajectory into ``context``."""
         path = self.logs_dir / TRAJECTORY_FILE

{vis_agent-0.3.3.dev7953 → vis_agent-0.3.4.dev7992}/src/blockether/vis/evals/bench.py

@@ -87,8 +87,9 @@ class BenchReport:
     `_ci95` value is a 95% Wilson interval in percent. A second round of the same size
     cannot detect a change below `detectable_change_points`. `attempts_per_task` is the
     most common scored attempt count of a task. When it is more than one,
-    `pass_at_k_percent` and `pass_all_k_percent` give the share of tasks that solved at
-    least one or all of their first attempts, up to that count. Otherwise they are None.
+    `pass_at_k_percent` and `pass_all_k_percent` give the unbiased pass@k and pass^k with
+    that count as k: the chance that at least one or all of k attempts solve a task.
+    Each task uses all its scored attempts. Otherwise they are None.
     `unstable_tasks` lists tasks with mixed results. `estimated_metered_cost_usd` and
     `cost_per_solved_usd` use metered API prices. They are not subscription charges.
     """

{vis_agent-0.3.3.dev7953 → vis_agent-0.3.4.dev7992}/src/blockether/vis/evals/environment.py

@@ -37,7 +37,7 @@ GIT_SETUP = (
     ("git", "config", "user.email", "t@t.co"),
     ("git", "config", "user.name", "t"),
     ("git", "add", "-A"),
-    ("git", "commit", "-qm", "init"),
+    ("git", "commit", "-q", "--allow-empty", "-m", "init"),
 )
 _SAFE_PATH = re.compile(r"[\w./-]+")
 _BEFORE_PATH = r"(?<![\w./-])"

{vis_agent-0.3.3.dev7953 → vis_agent-0.3.4.dev7992}/src/blockether/vis/evals/grade.py

@@ -25,6 +25,8 @@ import hashlib
 import json
 import os
 import re
+import subprocess
+import sys
 from collections.abc import Iterable, Mapping
 from dataclasses import dataclass
 from pathlib import Path
@@ -38,6 +40,7 @@ TRAJECTORY_FILE = "trajectory.json"
 RUN_FILE = "run.json"
 REWARD_FILE = "reward.json"
 GRADE_FILE = "grade.json"
+ANSWER_PROGRAM_TIMEOUT_SEC = 120
 
 
 TERMINAL_ACTIVITY_STATES = {"succeeded", "failed", "cancelled"}
@@ -1040,6 +1043,32 @@ def _workspace_text(work: str, name: str) -> str:
         return ""
 
 
+def answer_program_problem(program: str, answer: str, work: str) -> str | None:
+    """Run the Python file ``program`` on the final answer; return its failure.
+
+    The program gets the answer on its standard input and runs in the workspace
+    ``work``. Exit code 0 accepts the answer, and the result is None. Otherwise the
+    result names the last line of the program's error output.
+    """
+    try:
+        completed = subprocess.run(
+            [sys.executable, program],
+            input=answer,
+            capture_output=True,
+            text=True,
+            cwd=work,
+            timeout=ANSWER_PROGRAM_TIMEOUT_SEC,
+            check=False,
+        )
+    except subprocess.TimeoutExpired:
+        return f"answer program ran longer than {ANSWER_PROGRAM_TIMEOUT_SEC} s"
+    if completed.returncode == 0:
+        return None
+    lines = (completed.stderr or completed.stdout).strip().splitlines()
+    reason = lines[-1] if lines else f"exit code {completed.returncode}"
+    return f"answer program rejected the answer: {reason}"
+
+
 def grade_trial(
     sc: Mapping[str, Any], work: str | os.PathLike[str], trial: TrialFiles
 ) -> Grade:
@@ -1186,6 +1215,10 @@ def grade_trial(
     for s in sc.get("want_answer") or []:
         if s not in answer:
             checks.fail("answer", f"answer missing {s!r}")
+    if program := sc.get("want_answer_program"):
+        checks.use("answer")
+        if problem := answer_program_problem(program, answer, work):
+            checks.fail("answer", problem)
     if sc.get("want_forms"):
         checks.use("forms")
     for needle in sc.get("want_forms") or []:
@@ -1545,6 +1578,8 @@ def main(argv: list[str] | None = None) -> int:
         if getattr(args, name) is None:
             parser.error(f"--{name.replace('_', '-')} is required outside Vis Harbor")
     checks = json.loads(args.checks.read_text())
+    if program := checks.get("want_answer_program"):
+        checks["want_answer_program"] = str(args.checks.parent / program)
     if args.task:
         checks["id"] = args.task
     grade = grade_trial(checks, args.workspace, load_trial(args.agent_logs))