vis-agent 0.2.38.dev7554__tar.gz → 0.2.38.dev7565__tar.gz

{vis_agent-0.2.38.dev7554 → vis_agent-0.2.38.dev7565}/.gitignore

@@ -117,3 +117,6 @@ own_your_harness.pptx
 # Call journal written by the e2e schema-probe fixture when it runs from this
 # checkout instead of a temporary workspace.
 /schema-calls.jsonl
+
+# The uv environment of the e2e Harbor project.
+/e2e/.venv/

{vis_agent-0.2.38.dev7554 → vis_agent-0.2.38.dev7565}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: vis-agent
-Version: 0.2.38.dev7554
+Version: 0.2.38.dev7565
 Summary: The Vis Python SDK — author extensions and control local or remote engines.
 Project-URL: Homepage, https://vis.blockether.com
 Project-URL: Documentation, https://vis.blockether.com

{vis_agent-0.2.38.dev7554 → vis_agent-0.2.38.dev7565}/pyproject.toml

@@ -8,7 +8,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "vis-agent"
-version = "0.2.38.dev7554"
+version = "0.2.38.dev7565"
 description = "The Vis Python SDK — author extensions and control local or remote engines."
 readme = "README.md"
 requires-python = ">=3.11"

{vis_agent-0.2.38.dev7554 → vis_agent-0.2.38.dev7565}/src/blockether/vis/activity.py

@@ -84,6 +84,7 @@ class ActivityRow:
     resources: tuple[ActivityResource, ...]
     evidence: tuple[ActivityEvidence, ...]
     argument_key: str | None = None
+    read_key: str | None = None
     handle_id: str | None = None
     group_token: str | None = None
     duration_ms: int | None = None

vis_agent-0.2.38.dev7565/src/blockether/vis/evals/__init__.py

@@ -0,0 +1,66 @@
+"""Evaluation tools: read Vis runs and Harbor jobs, and compare their results.
+
+For the reader of one Vis run, see `blockether.vis.evals.trajectory`. For Harbor jobs
+and round seals, see `blockether.vis.evals.harbor`. For the statistics, see
+`blockether.vis.evals.stats`.
+"""
+
+from .harbor import (
+    HarborTrial,
+    RunSummary,
+    case_results,
+    read_trial,
+    read_trials,
+    seal_digest,
+    seal_round,
+    seal_rows,
+    summarize,
+)
+from .stats import (
+    benjamini_hochberg,
+    cluster_interval,
+    cohen_kappa,
+    detectable_change,
+    effective_size,
+    mcnemar_exact,
+    paired_change,
+    wilson_interval,
+)
+from .trajectory import (
+    FunctionCall,
+    Step,
+    StepMetrics,
+    ToolCall,
+    Trajectory,
+    TrajectoryMetrics,
+    load_trajectory,
+    parse_trajectory,
+)
+
+__all__ = [
+    "FunctionCall",
+    "HarborTrial",
+    "RunSummary",
+    "Step",
+    "StepMetrics",
+    "ToolCall",
+    "Trajectory",
+    "TrajectoryMetrics",
+    "benjamini_hochberg",
+    "case_results",
+    "cluster_interval",
+    "cohen_kappa",
+    "detectable_change",
+    "effective_size",
+    "load_trajectory",
+    "mcnemar_exact",
+    "paired_change",
+    "parse_trajectory",
+    "read_trial",
+    "read_trials",
+    "seal_digest",
+    "seal_round",
+    "seal_rows",
+    "summarize",
+    "wilson_interval",
+]

vis_agent-0.2.38.dev7565/src/blockether/vis/evals/harbor.py

@@ -0,0 +1,303 @@
+"""Read Harbor jobs: trials, cases, a summary with the cost, and round seals.
+
+A Harbor job directory holds `config.json`, `result.json` and one directory for each
+trial. A trial directory holds `result.json`, `agent/trajectory.json` and
+`verifier/reward.json`. A round directory holds one or more job directories.
+
+This module reads the JSON files directly, so it does not need Harbor itself.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import json
+import time
+from collections.abc import Iterable, Mapping, Sequence
+from dataclasses import dataclass
+from pathlib import Path
+from types import MappingProxyType
+from typing import Any
+
+from .stats import cluster_interval
+
+__all__ = [
+    "HarborTrial",
+    "RunSummary",
+    "case_results",
+    "read_trial",
+    "read_trials",
+    "seal_digest",
+    "seal_round",
+    "seal_rows",
+    "summarize",
+]
+
+RESULT_FILE = "result.json"
+
+
+@dataclass(frozen=True, slots=True)
+class HarborTrial:
+    """One Harbor trial, read from its `result.json`.
+
+    `task` is the Harbor task name. `agent` is the agent name that the trial reports.
+    `model` is the model name of the trial configuration, for example
+    `openai/gpt-6.1-sol`. `reward` is the `reward` value of the verifier. It is None when
+    the verifier gave no reward, for example after an infrastructure error. `rewards`
+    holds all named rewards. `error` is the exception type of a failed trial. The token
+    counts and `cost_usd` are None when the agent did not report them. `path` is the
+    `result.json` file.
+    """
+
+    job: str
+    trial: str
+    task: str
+    agent: str
+    agent_version: str | None
+    model: str
+    reward: float | None
+    rewards: Mapping[str, float]
+    error: str | None
+    input_tokens: int | None
+    cache_tokens: int | None
+    output_tokens: int | None
+    cost_usd: float | None
+    task_checksum: str | None
+    started_at: str | None
+    finished_at: str | None
+    path: Path
+
+    @property
+    def is_scored(self) -> bool:
+        """True when the verifier gave a reward."""
+        return self.reward is not None
+
+    @property
+    def is_passed(self) -> bool:
+        """True when the reward is 1 or more."""
+        return self.reward is not None and self.reward >= 1
+
+
+@dataclass(frozen=True, slots=True)
+class RunSummary:
+    """The counts, the pass rate and the cost of a set of trials.
+
+    A case is one task with one agent and one model. `pass_rate` is `passed / scored`,
+    and `pass_rate_ci95` is its 95% cluster interval over the cases. Both are
+    fractions, or None without a scored trial. `errors` counts the trials with an
+    exception. `cost_usd` is the sum of the reported costs, or None when no trial
+    reports a cost. `cost_per_pass_usd` divides that sum by the passed trials.
+    """
+
+    trials: int
+    scored: int
+    passed: int
+    errors: int
+    cases: int
+    pass_rate: float | None
+    pass_rate_ci95: tuple[float, float] | None
+    input_tokens: int
+    cache_tokens: int
+    output_tokens: int
+    cost_usd: float | None
+    cost_per_pass_usd: float | None
+    tasks: tuple[str, ...]
+    agents: tuple[str, ...]
+    models: tuple[str, ...]
+    started_at: str | None
+    finished_at: str | None
+
+
+def _integer(value: Any) -> int | None:
+    return value if isinstance(value, int) and not isinstance(value, bool) else None
+
+
+def _number(value: Any) -> float | None:
+    if isinstance(value, bool) or not isinstance(value, (int, float)):
+        return None
+    return float(value)
+
+
+def _text(value: Any) -> str | None:
+    return str(value) if value not in (None, "") else None
+
+
+def read_trial(path: Path | str) -> HarborTrial | None:
+    """Read one finished trial from its `result.json`, or return None for other files.
+
+    A job `result.json`, an unfinished trial and a file that is not JSON give None.
+    """
+    path = Path(path)
+    try:
+        data = json.loads(path.read_text())
+    except (OSError, ValueError):
+        return None
+    if not isinstance(data, dict):
+        return None
+    if not (
+        data.get("task_name") and data.get("trial_name") and data.get("finished_at")
+    ):
+        return None
+    agent = (data.get("config") or {}).get("agent") or {}
+    info = data.get("agent_info") or {}
+    usage = data.get("agent_result") or {}
+    rewards = {
+        str(name): number
+        for name, value in (
+            (data.get("verifier_result") or {}).get("rewards") or {}
+        ).items()
+        if (number := _number(value)) is not None
+    }
+    return HarborTrial(
+        job=path.parent.parent.name,
+        trial=str(data["trial_name"]),
+        task=str(data["task_name"]),
+        agent=str(
+            info.get("name") or agent.get("name") or agent.get("import_path") or ""
+        ),
+        agent_version=_text(info.get("version")),
+        model=str(agent.get("model_name") or ""),
+        reward=rewards.get("reward"),
+        rewards=MappingProxyType(rewards),
+        error=_text((data.get("exception_info") or {}).get("exception_type")),
+        input_tokens=_integer(usage.get("n_input_tokens")),
+        cache_tokens=_integer(usage.get("n_cache_tokens")),
+        output_tokens=_integer(usage.get("n_output_tokens")),
+        cost_usd=_number(usage.get("cost_usd")),
+        task_checksum=_text(data.get("task_checksum")),
+        started_at=_text(data.get("started_at")),
+        finished_at=_text(data.get("finished_at")),
+        path=path,
+    )
+
+
+def read_trials(directory: Path | str) -> tuple[HarborTrial, ...]:
+    """Read the finished trials of a Harbor job directory or of a round of jobs.
+
+    A trial `result.json` is one or two levels below `directory`. The trials come in
+    start order. A directory that does not exist raises `FileNotFoundError`.
+    """
+    root = Path(directory)
+    if not root.is_dir():
+        raise FileNotFoundError(f"no Harbor job or round directory at {root}")
+    paths = sorted({*root.glob(f"*/{RESULT_FILE}"), *root.glob(f"*/*/{RESULT_FILE}")})
+    trials = [trial for trial in map(read_trial, paths) if trial is not None]
+    return tuple(
+        sorted(trials, key=lambda trial: (trial.started_at or "", str(trial.path)))
+    )
+
+
+def case_results(
+    trials: Iterable[HarborTrial],
+) -> dict[tuple[str, str, str], tuple[int, int]]:
+    """Count the passed and the scored trials of each case: a task, an agent and a model.
+
+    The keys are `(task, agent, model)`. A trial without a reward does not count, so a
+    case without a scored trial is absent.
+    """
+    cases: dict[tuple[str, str, str], tuple[int, int]] = {}
+    for trial in trials:
+        if not trial.is_scored:
+            continue
+        key = (trial.task, trial.agent, trial.model)
+        passed, runs = cases.get(key, (0, 0))
+        cases[key] = (passed + trial.is_passed, runs + 1)
+    return cases
+
+
+def summarize(trials: Iterable[HarborTrial]) -> RunSummary:
+    """Summarize a set of trials: counts, the pass rate with its interval, and the cost."""
+    trials = tuple(trials)
+    cases = case_results(trials)
+    scored = sum(trial.is_scored for trial in trials)
+    passed = sum(trial.is_passed for trial in trials)
+    costs = [trial.cost_usd for trial in trials if trial.cost_usd is not None]
+    cost = sum(costs) if costs else None
+    starts = [trial.started_at for trial in trials if trial.started_at]
+    ends = [trial.finished_at for trial in trials if trial.finished_at]
+    return RunSummary(
+        trials=len(trials),
+        scored=scored,
+        passed=passed,
+        errors=sum(trial.error is not None for trial in trials),
+        cases=len(cases),
+        pass_rate=passed / scored if scored else None,
+        pass_rate_ci95=cluster_interval(list(cases.values())),
+        input_tokens=sum(trial.input_tokens or 0 for trial in trials),
+        cache_tokens=sum(trial.cache_tokens or 0 for trial in trials),
+        output_tokens=sum(trial.output_tokens or 0 for trial in trials),
+        cost_usd=cost,
+        cost_per_pass_usd=cost / passed if cost is not None and passed else None,
+        tasks=tuple(sorted({trial.task for trial in trials})),
+        agents=tuple(sorted({trial.agent for trial in trials})),
+        models=tuple(sorted({trial.model for trial in trials})),
+        started_at=min(starts) if starts else None,
+        finished_at=max(ends) if ends else None,
+    )
+
+
+def _file_sha256(path: Path) -> str:
+    digest = hashlib.sha256()
+    with path.open("rb") as stream:
+        for block in iter(lambda: stream.read(1 << 20), b""):
+            digest.update(block)
+    return digest.hexdigest()
+
+
+def seal_rows(trials: Iterable[HarborTrial], root: Path | str) -> list[dict[str, Any]]:
+    """Return one seal row for each trial, with the task checksum and the result digest.
+
+    `result` is the path of the trial `result.json`, relative to `root`, and
+    `result_sha256` is the sha256 of that file.
+    """
+    root = Path(root)
+    return [
+        {
+            "agent": trial.agent,
+            "model": trial.model,
+            "task": trial.task,
+            "reward": trial.reward,
+            "error": trial.error,
+            "input": trial.input_tokens,
+            "cache": trial.cache_tokens,
+            "output": trial.output_tokens,
+            "agent_version": trial.agent_version,
+            "task_checksum": trial.task_checksum,
+            "started_at": trial.started_at,
+            "finished_at": trial.finished_at,
+            "result": trial.path.relative_to(root).as_posix(),
+            "result_sha256": _file_sha256(trial.path),
+        }
+        for trial in sorted(
+            trials,
+            key=lambda trial: (trial.agent, trial.model, trial.task, str(trial.path)),
+        )
+    ]
+
+
+def seal_digest(body: Mapping[str, Any]) -> str:
+    """Return the sha256 of a seal body with sorted keys, without its `digest` key."""
+    content = {key: value for key, value in body.items() if key != "digest"}
+    return hashlib.sha256(json.dumps(content, sort_keys=True).encode()).hexdigest()
+
+
+def seal_round(
+    round_name: str,
+    runs: Sequence[Mapping[str, Any]],
+    trials: Iterable[HarborTrial],
+    root: Path | str,
+    sealed_at: str | None = None,
+) -> dict[str, Any]:
+    """Return the seal of a round: its run records, one row for each trial and a digest.
+
+    Each run record holds the commit, the pins, the profiles and the artifact sha256 of
+    one run. The rows hold the task checksums and the result digests. `sealed_at`
+    defaults to the current UTC time.
+    """
+    body: dict[str, Any] = {
+        "round": round_name,
+        "sealed_at": sealed_at or time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()),
+        "runs": [dict(run) for run in runs],
+        "rows": seal_rows(trials, root),
+    }
+    body["digest"] = seal_digest(body)
+    return body

vis_agent-0.2.38.dev7565/src/blockether/vis/evals/stats.py

@@ -0,0 +1,170 @@
+"""Statistics for evaluation results: intervals, power, paired tests and multiple tests.
+
+Each rate, interval and change is a fraction from 0 to 1. A unit is the
+`(passed, runs)` count of one case, for example one task on one route. The repeats of
+one unit are not independent, so the cluster interval and the power use an effective
+size between the unit count and the run count.
+"""
+
+from __future__ import annotations
+
+import math
+from collections import Counter
+from collections.abc import Hashable, Sequence
+
+__all__ = [
+    "Z95",
+    "Z_POWER",
+    "benjamini_hochberg",
+    "cluster_interval",
+    "cohen_kappa",
+    "detectable_change",
+    "effective_size",
+    "mcnemar_exact",
+    "paired_change",
+    "wilson_interval",
+]
+
+Z95 = 1.959964
+"""The normal quantile of a two-sided test at the 5% level."""
+
+Z_POWER = 0.841621
+"""The normal quantile of 80% power."""
+
+
+def wilson_interval(
+    passed: int, runs: int, size: float | None = None
+) -> tuple[float, float] | None:
+    """Return the 95% Wilson interval of a pass rate, or None without runs.
+
+    `size` replaces the run count when the runs are not independent.
+    """
+    size = runs if size is None else size
+    if not runs or size <= 0:
+        return None
+    rate = passed / runs
+    spread = Z95**2 / size
+    center = (rate + spread / 2) / (1 + spread)
+    half = Z95 * math.sqrt(rate * (1 - rate) / size + spread / size / 4) / (1 + spread)
+    return max(0.0, center - half), min(1.0, center + half)
+
+
+def effective_size(units: Sequence[tuple[int, int]]) -> float:
+    """Return the run count of clustered results, made smaller by their design effect.
+
+    Each unit is `(passed, runs)` of one case. The size stays between the unit count
+    and the run count. Without any variation, it is the unit count.
+    """
+    units = [(passed, runs) for passed, runs in units if runs]
+    count = len(units)
+    total = sum(runs for _, runs in units)
+    if count < 2 or total == count:
+        return float(total)
+    rate = sum(passed for passed, _ in units) / total
+    if rate in (0, 1):
+        return float(count)
+    spread = sum((passed - rate * runs) ** 2 for passed, runs in units)
+    variance = count / (count - 1) * spread / total**2
+    if not variance:
+        return float(total)
+    return min(float(total), max(float(count), rate * (1 - rate) / variance))
+
+
+def cluster_interval(units: Sequence[tuple[int, int]]) -> tuple[float, float] | None:
+    """Return the 95% interval of the pooled pass rate of clustered units, or None."""
+    passed = sum(passed for passed, _ in units)
+    runs = sum(runs for _, runs in units)
+    return wilson_interval(passed, runs, effective_size(units))
+
+
+def detectable_change(units: Sequence[tuple[int, int]]) -> float | None:
+    """Return the smallest change of the pass rate that a second run of this size detects.
+
+    The test compares two independent runs of the same size, two-sided at the 5% level,
+    with 80% power. The change has steps of 0.001. None means that there are no runs, or
+    that no change from 0 to 1 is detectable.
+    """
+    total = sum(runs for _, runs in units)
+    if not total:
+        return None
+    rate = sum(passed for passed, _ in units) / total
+    size = effective_size(units)
+
+    def detects(other: float) -> bool:
+        pooled = (rate + other) / 2
+        noise = Z95 * math.sqrt(2 * pooled * (1 - pooled) / size)
+        power = Z_POWER * math.sqrt((rate * (1 - rate) + other * (1 - other)) / size)
+        return abs(other - rate) >= noise + power
+
+    smallest = None
+    for sign in (1, -1):
+        for step in range(1, 1001):
+            other = rate + sign * step / 1000
+            if not 0 <= other <= 1:
+                break
+            if detects(other):
+                smallest = step if smallest is None else min(smallest, step)
+                break
+    return smallest / 1000 if smallest is not None else None
+
+
+def paired_change(changes: Sequence[float]) -> tuple[tuple[float, float], float]:
+    """Return the Agresti-Min 95% interval of a mean paired change, and its floor.
+
+    Each change is the candidate pass rate of one case minus its baseline pass rate.
+    The floor is the smallest true change that this many cases detect with 80% power.
+    """
+    count = len(changes)
+    mean = sum(changes) / (count + 2)
+    square = (sum(change * change for change in changes) + 1) / (count + 2)
+    error = math.sqrt(max(0.0, square - mean * mean) / (count + 2))
+    return (mean - Z95 * error, mean + Z95 * error), (Z95 + Z_POWER) * error
+
+
+def cohen_kappa(pairs: Sequence[tuple[Hashable, Hashable]]) -> float | None:
+    """Return Cohen's kappa of two raters over the same items, or None when undefined."""
+    if not pairs:
+        return None
+    count = len(pairs)
+    observed = sum(first == second for first, second in pairs) / count
+    firsts = Counter(first for first, _ in pairs)
+    seconds = Counter(second for _, second in pairs)
+    expected = sum(firsts[label] * seconds[label] for label in firsts) / count**2
+    if expected >= 1:
+        return None
+    return (observed - expected) / (1 - expected)
+
+
+def mcnemar_exact(worse: int, better: int) -> float:
+    """Return the two-sided exact McNemar p-value of paired pass and fail results.
+
+    `worse` counts the cases that pass only in the baseline. `better` counts the cases
+    that pass only in the candidate. Cases with the same result on both sides do not
+    change the value. Without a changed case, the value is 1.
+    """
+    if worse < 0 or better < 0:
+        raise ValueError("worse and better must not be negative")
+    total = worse + better
+    if not total:
+        return 1.0
+    tail = sum(math.comb(total, count) for count in range(min(worse, better) + 1))
+    return min(1.0, 2 * tail / 2**total)
+
+
+def benjamini_hochberg(p_values: Sequence[float]) -> list[float]:
+    """Return the Benjamini-Hochberg q-value of each p-value, in the input order.
+
+    A q-value is the smallest false discovery rate at which its test is a discovery.
+    A larger p-value never gets a smaller q-value, and each q-value is at most 1.
+    """
+    if any(not 0 <= value <= 1 for value in p_values):
+        raise ValueError("each p-value must be from 0 to 1")
+    count = len(p_values)
+    order = sorted(range(count), key=lambda index: p_values[index])
+    q_values = [1.0] * count
+    smallest = 1.0
+    for rank in range(count, 0, -1):
+        index = order[rank - 1]
+        smallest = min(smallest, p_values[index] * count / rank)
+        q_values[index] = smallest
+    return q_values