vis-agent 0.3.4.dev8047__tar.gz → 0.3.4.dev8055__tar.gz

{vis_agent-0.3.4.dev8047 → vis_agent-0.3.4.dev8055}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: vis-agent
-Version: 0.3.4.dev8047
+Version: 0.3.4.dev8055
 Summary: The Vis Python SDK — author extensions and control local or remote engines.
 Project-URL: Homepage, https://vis.blockether.com
 Project-URL: Documentation, https://vis.blockether.com

{vis_agent-0.3.4.dev8047 → vis_agent-0.3.4.dev8055}/contracts/schema/evals.json

@@ -149,6 +149,12 @@
         },
         "candidate_name": {
           "type": "string"
+        },
+        "margin_points": {
+          "description": "The smallest change in points that matters. It splits a result without a clear change into equivalent, not worse, not better or inconclusive.",
+          "type": "number",
+          "exclusiveMinimum": 0,
+          "maximum": 100
         }
       },
       "required": [
@@ -649,7 +655,16 @@
           "maxItems": 2
         },
         "verdict": {
-          "type": "string"
+          "type": "string",
+          "enum": [
+            "better",
+            "worse",
+            "no measurable difference",
+            "equivalent",
+            "not worse",
+            "not better",
+            "inconclusive"
+          ]
         },
         "detectable_change_points": {
           "type": "number"
@@ -683,6 +698,12 @@
           "items": {
             "type": "string"
           }
+        },
+        "margin_points": {
+          "type": [
+            "number",
+            "null"
+          ]
         }
       },
       "required": [
@@ -698,7 +719,8 @@
         "worse",
         "better",
         "unpaired",
-        "caveats"
+        "caveats",
+        "margin_points"
       ],
       "additionalProperties": false
     },

{vis_agent-0.3.4.dev8047 → vis_agent-0.3.4.dev8055}/pyproject.toml

@@ -8,7 +8,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "vis-agent"
-version = "0.3.4.dev8047"
+version = "0.3.4.dev8055"
 description = "The Vis Python SDK — author extensions and control local or remote engines."
 readme = "README.md"
 requires-python = ">=3.12"

vis_agent-0.3.4.dev8055/release-tag

@@ -0,0 +1 @@
+beta-6e5ce3880c1ae076ac0f06479a3186f14709b26e

{vis_agent-0.3.4.dev8047 → vis_agent-0.3.4.dev8055}/src/blockether/vis/evals/_gateway.py

@@ -68,6 +68,15 @@ def _required_text(body: Mapping, key: str) -> str:
     return value
 
 
+def _number(body: Mapping, key: str) -> float | None:
+    value = body.get(key)
+    if value is None:
+        return None
+    if isinstance(value, bool) or not isinstance(value, (int, float)):
+        raise ValueError(f"`{key}` must be a number.")
+    return float(value)
+
+
 def _cases(body: Mapping, key: str) -> dict[tuple[str, ...], tuple[int, int]]:
     rows = body.get(key)
     if not isinstance(rows, list):
@@ -114,6 +123,7 @@ def _compare(body: Mapping):
         _cases(body, "candidate"),
         baseline_name=_word(body, "baseline_name", "baseline"),
         candidate_name=_word(body, "candidate_name", "candidate"),
+        margin_points=_number(body, "margin_points"),
     )
 
 

{vis_agent-0.3.4.dev8047 → vis_agent-0.3.4.dev8055}/src/blockether/vis/evals/compare.py

@@ -3,7 +3,9 @@
 A case is a key such as `(task, agent, model)` from `case_results`, with its
 `(passed, runs)` count. A route is the agent with the model of a case. Call
 `compare_cases` with the cases of a baseline and a candidate. It pairs the cases,
-gives the paired difference with its 95% interval and tests each route.
+gives the paired difference with its 95% interval and tests each route. Give
+`margin_points`, the smallest change that matters to you, to tell an equivalent
+result from a result that needs more cases.
 """
 
 from __future__ import annotations
@@ -51,13 +53,25 @@ class CaseComparison:
 
     The rates are the mean pass rates of the paired cases, in percent.
     `difference_ci95` is the Agresti-Min 95% interval of the paired difference, in
-    points. `verdict` is `better` or `worse` only when that interval excludes zero,
-    else `no measurable difference`. At this case count, a true change of
-    `detectable_change_points` or more shows with 80% power. `p_value` is the two-sided
-    exact McNemar p-value of the changed cases. With repeats, it is the exact sign test
-    of the case changes. `routes` holds this test for each route. `worse` and `better`
-    list the changed cases, largest change first. `unpaired` counts cases on one side
-    only; the comparison leaves them out.
+    points. `verdict` is `better` or `worse` only when that interval excludes zero.
+    Without `margin_points`, every other result is `no measurable difference`. With
+    it, the position of the interval against the margin gives the verdict:
+
+    - `equivalent`: the interval is inside the margin on both sides.
+    - `not worse`: the interval is above the negative margin.
+    - `not better`: the interval is below the margin, but reaches the negative margin.
+    - `inconclusive`: the interval reaches both sides of the margin.
+
+    The verdict uses the 95% interval before rounding. With a margin, a `better` or
+    `worse` verdict gets a caveat when the change is, or can be, smaller than the margin.
+
+    At this case count, a true change of `detectable_change_points` or more shows with
+    about 80% power, by a normal approximation. `p_value` is the two-sided exact McNemar
+    p-value of the changed cases. With repeats, it is the exact sign test of the case
+    changes. `routes` holds this test for each route. `worse` and `better` list the
+    changed cases, largest change first. `unpaired` counts cases on one side only; the
+    comparison leaves them out. Caveats name unpaired cases, different routes, a change
+    smaller than the margin and a verdict that the exact test does not confirm.
     """
 
     paired_cases: int
@@ -73,6 +87,7 @@ class CaseComparison:
     better: tuple[CaseChange, ...]
     unpaired: int
     caveats: tuple[str, ...]
+    margin_points: float | None = None
 
 
 def _route(key: tuple[str, ...]) -> str:
@@ -87,6 +102,59 @@ def _points(change: float) -> float:
     return round(change * 100, 1)
 
 
+def _crossed_caveat(base: tuple[str, ...], other: tuple[str, ...]) -> str:
+    if base[1] != other[1] and base[2] != other[2]:
+        return (
+            f"The sides used different agents and models, {_route(base)} and "
+            f"{_route(other)}. The change can come from either one."
+        )
+    if base[1] != other[1]:
+        return (
+            f"The sides used different agents, {base[1]} and {other[1]}, "
+            f"with the same model {base[2]}."
+        )
+    return (
+        f"The sides used different models, {base[2]} and {other[2]}, "
+        f"with the same agent {base[1]}."
+    )
+
+
+def _margin_caveat(
+    verdict: str, interval: tuple[float, float], margin: float | None
+) -> str | None:
+    if not margin or verdict not in ("better", "worse"):
+        return None
+    low, high = interval if verdict == "better" else (-interval[1], -interval[0])
+    change = "gain" if verdict == "better" else "loss"
+    if high < margin:
+        return (
+            f"The {change} is real, but smaller than the margin of {margin:g} points."
+        )
+    if low < margin:
+        return (
+            f"The {change} is real, but it can be smaller than the margin of "
+            f"{margin:g} points."
+        )
+    return None
+
+
+def _verdict(interval: tuple[float, float], margin: float | None) -> str:
+    low, high = interval
+    if low > 0:
+        return "better"
+    if high < 0:
+        return "worse"
+    if not margin:
+        return "no measurable difference"
+    if -margin < low and high < margin:
+        return "equivalent"
+    if low > -margin:
+        return "not worse"
+    if high < margin:
+        return "not better"
+    return "inconclusive"
+
+
 def _route_tests(
     changes: Mapping[tuple[str, ...], float], crossed: str | None
 ) -> tuple[RouteTest, ...]:
@@ -117,13 +185,18 @@ def compare_cases(
     *,
     baseline_name: str = "baseline",
     candidate_name: str = "candidate",
+    margin_points: float | None = None,
 ) -> CaseComparison:
     """Compare the `(passed, runs)` cases of a baseline and a candidate.
 
     Each key is `(task, agent, model)`. When the sides share no key and each side has
     one route, the comparison pairs the cases by task and adds a caveat. The names
-    appear in error messages. Raise ValueError when no case pairs.
+    appear in error messages. `margin_points` is the smallest change in points that
+    matters, above 0 and at most 100. Raise ValueError when no case pairs, or for a
+    margin outside that range.
     """
+    if margin_points is not None and not 0 < margin_points <= 100:
+        raise ValueError("margin_points must be above 0 and at most 100")
     caveats = []
     crossed = None
     if not baseline.keys() & candidate.keys():
@@ -135,12 +208,10 @@ def compare_cases(
                 "has more than one route. Compare results with the same agent and model."
             )
         crossed = f"{min(base_routes)} against {min(other_routes)}"
+        if base_routes != other_routes:
+            caveats.append(_crossed_caveat(next(iter(baseline)), next(iter(candidate))))
         baseline = {key[:1]: value for key, value in baseline.items()}
         candidate = {key[:1]: value for key, value in candidate.items()}
-        caveats.append(
-            f"The sides used different routes, {min(base_routes)} and "
-            f"{min(other_routes)}. Model and harness differ together."
-        )
     paired = sorted(baseline.keys() & candidate.keys())
     if not paired:
         raise ValueError(f"{baseline_name} and {candidate_name} share no case")
@@ -156,20 +227,24 @@ def compare_cases(
             )
             changed.append((change, row))
     (low, high), floor = paired_change(list(changes.values()))
-    interval = (_points(low), _points(high))
-    verdict = (
-        "better"
-        if interval[0] > 0
-        else "worse"
-        if interval[1] < 0
-        else "no measurable difference"
-    )
+    verdict = _verdict((low * 100, high * 100), margin_points)
     unpaired = len(baseline.keys() ^ candidate.keys())
     if unpaired:
         cases = "1 case" if unpaired == 1 else f"{unpaired} cases"
         caveats.append(f"{cases} ran on one side only. The comparison leaves them out.")
     count = len(paired)
     values = list(changes.values())
+    p_value = mcnemar_exact(
+        sum(value < 0 for value in values), sum(value > 0 for value in values)
+    )
+    if verdict in ("better", "worse") and p_value >= 0.05:
+        caveats.append(
+            f"The interval says {verdict}, but the exact test gives p = "
+            f"{p_value:.3f}. Confirm the change with more cases."
+        )
+    margin_caveat = _margin_caveat(verdict, (low * 100, high * 100), margin_points)
+    if margin_caveat:
+        caveats.append(margin_caveat)
 
     def rate(cases: Cases) -> float:
         return round(
@@ -181,12 +256,10 @@ def compare_cases(
         baseline_rate_percent=rate(baseline),
         candidate_rate_percent=rate(candidate),
         difference_points=round(100 * sum(values) / count, 1),
-        difference_ci95=interval,
+        difference_ci95=(_points(low), _points(high)),
         verdict=verdict,
         detectable_change_points=_points(floor),
-        p_value=mcnemar_exact(
-            sum(value < 0 for value in values), sum(value > 0 for value in values)
-        ),
+        p_value=p_value,
         routes=_route_tests(changes, crossed),
         worse=tuple(
             row for change, row in sorted(changed, key=lambda i: i[0]) if change < 0
@@ -196,4 +269,5 @@ def compare_cases(
         ),
         unpaired=unpaired,
         caveats=tuple(caveats),
+        margin_points=margin_points,
     )

{vis_agent-0.3.4.dev8047 → vis_agent-0.3.4.dev8055}/src/blockether/vis/evals/stats.py

@@ -94,8 +94,10 @@ def detectable_change(units: Sequence[tuple[int, int]]) -> float | None:
     """Return the smallest change of the pass rate that a second run of this size detects.
 
     The test compares two independent runs of the same size, two-sided at the 5% level,
-    with 80% power. The change has steps of 0.001. None means that there are no runs, or
-    that no change from 0 to 1 is detectable.
+    with 80% power. The change holds for a rise and for a fall, so the larger of the two
+    applies. A direction without a detectable change before 0 or 1 does not count. The
+    change has steps of 0.001. None means that there are no runs, or that no change
+    from 0 to 1 is detectable.
     """
     total = sum(runs for _, runs in units)
     if not total:
@@ -109,16 +111,16 @@ def detectable_change(units: Sequence[tuple[int, int]]) -> float | None:
         power = Z_POWER * math.sqrt((rate * (1 - rate) + other * (1 - other)) / size)
         return abs(other - rate) >= noise + power
 
-    smallest = None
+    steps = []
     for sign in (1, -1):
         for step in range(1, 1001):
             other = rate + sign * step / 1000
             if not 0 <= other <= 1:
                 break
             if detects(other):
-                smallest = step if smallest is None else min(smallest, step)
+                steps.append(step)
                 break
-    return smallest / 1000 if smallest is not None else None
+    return max(steps) / 1000 if steps else None
 
 
 def paired_change(changes: Sequence[float]) -> tuple[tuple[float, float], float]:

vis_agent-0.3.4.dev8055/tests/test_evals_compare.py

@@ -0,0 +1,196 @@
+"""Tests for the paired comparison of `blockether.vis.evals`.
+
+The calibration tests run seeded simulations. They prove the claims of the
+evaluation guide on false verdicts and power.
+"""
+
+import random
+
+import pytest
+from blockether.vis.evals import CaseChange, RouteTest, compare_cases
+
+ROUTE = ("vis", "zai/glm")
+
+
+def _cases(passes):
+    return {
+        (f"case-{index:02}", *ROUTE): (passed, 2) for index, passed in enumerate(passes)
+    }
+
+
+def test_compare_cases_reports_a_measurable_drop():
+    found = compare_cases(_cases([2] * 12), _cases([0] * 8 + [2] * 4))
+    assert found.verdict == "worse"
+    assert (found.paired_cases, found.baseline_rate_percent) == (12, 100.0)
+    assert found.candidate_rate_percent == 33.3
+    assert found.worse[0] == CaseChange("case-00 (vis with zai/glm)", "2/2", "0/2")
+    assert found.better == ()
+    assert found.routes == (
+        RouteTest("vis with zai/glm", 12, 8, 0, 2 / 2**8, 2 / 2**8),
+    )
+
+
+def test_compare_cases_without_change_has_no_verdict():
+    found = compare_cases(_cases([1, 2]), _cases([1, 2]))
+    assert found.verdict == "no measurable difference"
+    assert found.difference_points == 0.0
+    assert found.p_value == 1.0
+
+
+def test_compare_cases_leaves_out_unpaired_cases():
+    found = compare_cases(_cases([2, 2]), _cases([2]))
+    assert (found.paired_cases, found.unpaired) == (1, 1)
+    assert found.caveats == (
+        "1 case ran on one side only. The comparison leaves them out.",
+    )
+
+
+def test_compare_cases_pairs_two_single_routes_by_task():
+    other = {("case-00", "vis", "zai/glm-next"): (0, 1)}
+    found = compare_cases({("case-00", *ROUTE): (1, 1)}, other)
+    assert found.worse == (CaseChange("case-00", "1/1", "0/1"),)
+    assert found.routes[0].route == "vis with zai/glm against vis with zai/glm-next"
+    assert found.caveats == (
+        "The sides used different models, zai/glm and zai/glm-next, "
+        "with the same agent vis.",
+    )
+
+
+@pytest.mark.parametrize(
+    ("other", "caveat"),
+    [
+        (
+            ("codex", "zai/glm"),
+            "The sides used different agents, vis and codex, with the same model "
+            "zai/glm.",
+        ),
+        (
+            ("codex", "openai/gpt"),
+            "The sides used different agents and models, vis with zai/glm and codex "
+            "with openai/gpt. The change can come from either one.",
+        ),
+    ],
+)
+def test_the_route_caveat_names_only_the_parts_that_differ(other, caveat):
+    found = compare_cases({("case-00", *ROUTE): (1, 1)}, {("case-00", *other): (1, 1)})
+    assert found.caveats == (caveat,)
+
+
+def test_compare_cases_needs_a_shared_case():
+    two_routes = {("a", *ROUTE): (1, 1), ("a", "vis", "other"): (1, 1)}
+    with pytest.raises(ValueError, match="more than one route"):
+        compare_cases(two_routes, {("b", "x", "y"): (1, 1)}, candidate_name="run-2")
+
+
+def _changes(changes):
+    baseline, candidate = {}, {}
+    for index, change in enumerate(changes):
+        key = (f"case-{index:03}", *ROUTE)
+        baseline[key] = (1 if change < 0 else 0, 1)
+        candidate[key] = (1 if change > 0 else 0, 1)
+    return baseline, candidate
+
+
+@pytest.mark.parametrize(
+    ("changes", "verdict"),
+    [
+        ([0] * 100, "equivalent"),
+        ([0] * 4, "inconclusive"),
+        ([1] * 3 + [0] * 57, "not worse"),
+        ([-1] * 3 + [0] * 57, "not better"),
+    ],
+)
+def test_a_margin_tells_an_equivalent_result_from_too_few_cases(changes, verdict):
+    found = compare_cases(*_changes(changes), margin_points=10)
+    assert (found.verdict, found.margin_points) == (verdict, 10)
+    assert compare_cases(*_changes(changes)).verdict == "no measurable difference"
+
+
+@pytest.mark.parametrize("margin", [-1, 0, 101])
+def test_compare_cases_refuses_a_margin_outside_0_to_100(margin):
+    with pytest.raises(ValueError, match="margin_points"):
+        compare_cases(*_changes([0]), margin_points=margin)
+
+
+@pytest.mark.parametrize(
+    ("change", "margin", "caveat"),
+    [
+        (1, 15, "The gain is real, but smaller than the margin of 15 points."),
+        (
+            1,
+            10,
+            "The gain is real, but it can be smaller than the margin of 10 points.",
+        ),
+        (-1, 15, "The loss is real, but smaller than the margin of 15 points."),
+    ],
+)
+def test_a_verdict_inside_the_margin_gets_a_caveat(change, margin, caveat):
+    # The 95% interval of 40 changed cases in 400 is 7.0 to 12.9 points.
+    found = compare_cases(*_changes([change] * 40 + [0] * 360), margin_points=margin)
+    verdict = "better" if change > 0 else "worse"
+    assert (found.verdict, found.caveats) == (verdict, (caveat,))
+
+
+def test_a_change_beyond_the_margin_gets_no_margin_caveat():
+    found = compare_cases(*_changes([1] * 120 + [0] * 280), margin_points=10)
+    assert (found.verdict, found.caveats) == ("better", ())
+
+
+def test_compare_cases_names_a_verdict_that_the_exact_test_does_not_confirm():
+    # 12 tasks: 6 tasks became better and 1 worse, as in a small agent comparison.
+    found = compare_cases(*_changes([1] * 6 + [-1] + [0] * 5))
+    assert (found.verdict, found.p_value) == ("better", 0.125)
+    assert found.caveats == (
+        "The interval says better, but the exact test gives p = 0.125. "
+        "Confirm the change with more cases.",
+    )
+
+
+def _simulate(rng, tasks, change, margin=None, runs=1):
+    rates = [rng.uniform(0.3, 0.7) for _ in range(tasks)]
+
+    def side(shift):
+        return {
+            (f"t{index}", *ROUTE): (
+                sum(rng.random() < rate + shift for _ in range(runs)),
+                runs,
+            )
+            for index, rate in enumerate(rates)
+        }
+
+    return compare_cases(side(0.0), side(change), margin_points=margin)
+
+
+@pytest.mark.parametrize(
+    ("tasks", "runs"), [(12, 1), (16, 1), (30, 1), (30, 3), (80, 1)]
+)
+def test_without_a_change_a_verdict_is_rare(tasks, runs):
+    rng = random.Random(tasks * runs)
+    trials = 1000
+    verdicts = sum(
+        _simulate(rng, tasks, 0.0, runs=runs).verdict != "no measurable difference"
+        for _ in range(trials)
+    )
+    assert verdicts / trials <= 0.06
+
+
+def test_the_detectable_change_shows_with_80_percent_power():
+    rng = random.Random(3)
+    floor = _simulate(random.Random(1), 40, 0.0).detectable_change_points
+    trials = 600
+    better = sum(
+        _simulate(rng, 40, floor / 100).verdict == "better" for _ in range(trials)
+    )
+    assert better / trials >= 0.80
+
+
+@pytest.mark.parametrize("tasks", [60, 200])
+def test_a_candidate_worse_by_the_margin_is_rarely_called_not_worse(tasks):
+    rng = random.Random(5)
+    trials = 600
+    wrong = sum(
+        _simulate(rng, tasks, -0.10, margin=10).verdict
+        in ("better", "equivalent", "not worse")
+        for _ in range(trials)
+    )
+    assert wrong / trials <= 0.04

{vis_agent-0.3.4.dev8047 → vis_agent-0.3.4.dev8055}/tests/test_evals_gateway.py

@@ -50,6 +50,10 @@ def test_compare_pairs_the_cases():
     )["result"]
     assert result["paired_cases"] == 20
     assert result["verdict"] == "better"
+    same = side([1] * 60)
+    body = {"baseline": same, "candidate": same, "margin_points": 5}
+    result = roundtrip("compare", body)["result"]
+    assert (result["verdict"], result["margin_points"]) == ("equivalent", 5.0)
 
 
 def test_agreement_gives_the_kappa_advice():
@@ -83,6 +87,7 @@ def test_leaderboard_reads_every_page(monkeypatch):
         ("report", [], "must be a JSON object"),
         ("report", {}, "`summary` must be an object"),
         ("compare", {"baseline": [{}], "candidate": []}, "needs task, agent"),
+        ("compare", {"baseline": [], "candidate": [], "margin_points": "5"}, "number"),
         ("agreement", {"items": [{"name": "x"}]}, "needs name, automatic"),
         ("leaderboard", {"name": "1-0-0"}, "`package` is required"),
         ("leaderboard", {"package": "org/bench"}, "`name` is required"),