benchflow 0.7.2.dev2000__tar.gz → 0.7.3__tar.gz

{benchflow-0.7.2.dev2000 → benchflow-0.7.3}/CHANGELOG.md

@@ -2,6 +2,27 @@
 
 ## [Unreleased]
 
+## 0.7.3 — 2026-08-16
+
+### Added
+- **`bench traj upload` waits out short rate-limit responses instead of
+  failing.** When the contribution service answers 429 with a short
+  `Retry-After`, the handshake now sleeps it out with jitter and retries up
+  to three times (two-minute cap per wait), so a crowd of simultaneous
+  contributors self-heals instead of surfacing errors. Longer waits still
+  fail fast with the actionable retry-after message. (#1027)
+
+### Changed
+- **The contribution service rate-limits per contributor, not per venue.**
+  Upload budgets are token buckets keyed on contributor identity with a
+  wide per-IP abuse backstop, refill continuously, and answer 429 with
+  seconds-until-next-token instead of the remainder of the clock hour, so
+  many contributors behind one NAT no longer starve each other. Contended
+  bucket updates back off with jitter rather than shedding simultaneous
+  crowds. (#1027, #1028)
+
+## 0.7.2 — 2026-08-16
+
 ### Added
 - **The upload preview itemizes what redaction masked, by kind.** The
   redactor now categorizes every replacement by the rule that fired — API
@@ -61,6 +82,14 @@
   `version` / `sessionId`, the filename stem as a session fallback) and
   hides any badge whose value is unknown — including the cost badge when no
   event carries cost data. Presentation only.
+- **Upload progress no longer claims the broker is waking up.** A warm
+  retry printed `Uploading… the first request can take a minute while the
+  service wakes up` even when the service was already up. The line is now
+  `Uploading… this can take up to a minute; retries are safe.`
+- **`bench traj setup --list` no longer wraps session paths mid-token.**
+  Each hit prints index/source/time, then the path on its own line, then
+  the snippet, via plain `print` so Rich does not split a long JSONL path
+  and break copy-paste.
 
 ### Changed
 - **Trajectory viewer tool calls are color-coded and backgrounds are light.**

{benchflow-0.7.2.dev2000 → benchflow-0.7.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: benchflow
-Version: 0.7.2.dev2000
+Version: 0.7.3
 Summary: Multi-turn agent benchmarking with ACP — run any agent, any model, any provider.
 Project-URL: Homepage, https://github.com/benchflow-ai/benchflow
 Project-URL: Repository, https://github.com/benchflow-ai/benchflow

{benchflow-0.7.2.dev2000 → benchflow-0.7.3}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "benchflow"
-version = "0.7.2.dev2000"
+version = "0.7.3"
 description = "Multi-turn agent benchmarking with ACP — run any agent, any model, any provider."
 readme = "README.md"
 requires-python = ">=3.12"

{benchflow-0.7.2.dev2000 → benchflow-0.7.3}/src/benchflow/cli/traj.py

@@ -351,10 +351,9 @@ def _run_upload(options: _UploadOptions) -> None:
             console.print("[yellow]Upload cancelled.[/yellow]")
             return
         if not destination.direct:
-            console.print(
-                "Uploading… the first request can take a minute "
-                "while the service wakes up."
-            )
+            # Honest on both cold and warm runs: the broker may already be
+            # up. Don't claim it's waking when a retry is just transferring.
+            console.print("Uploading… this can take up to a minute; retries are safe.")
         with upload_progress(staged.files, console=console) as hooks:
             result = _publish(staged, destination=destination, hooks=hooks)
         _print_upload_result(staged, result, direct=destination.direct)
@@ -717,11 +716,15 @@ def _print_session_hits() -> None:
 
     hits = list_recent_sessions()
     if not hits:
-        console.print("No recent Claude Code, Codex, or trial sessions found.")
+        print("No recent Claude Code, Codex, or trial sessions found.")
         return
     for index, hit in enumerate(hits, start=1):
         snippet = hit.snippet or "(no prompt yet)"
-        console.print(f"{index}. [{hit.source}] {hit.when}  {hit.path}\n   {snippet}")
+        # Plain print + path on its own line: Rich wrapping used to split
+        # long session paths mid-token and make them unselectable.
+        print(f"{index}. [{hit.source}] {hit.when}")
+        print(f"   {hit.path}")
+        print(f"   {snippet}")
 
 
 def _interactive_view() -> None:

{benchflow-0.7.2.dev2000 → benchflow-0.7.3}/src/benchflow/publish/broker.py

@@ -3,6 +3,8 @@
 from __future__ import annotations
 
 import logging
+import random
+import time
 from collections.abc import Callable
 from contextlib import contextmanager, nullcontext
 from dataclasses import dataclass
@@ -18,6 +20,11 @@ from benchflow.publish.traj_capture import StagedCapture, StagedFile
 # the first SAS grant. Retries are safe; keep this under two minutes.
 HANDSHAKE_TIMEOUT_SEC = 90.0
 PUT_TIMEOUT_SEC = 300.0
+# The broker's token buckets answer 429 with an honest, usually-short
+# Retry-After. Waiting it out inline turns a crowd burst into a smooth queue;
+# anything longer than this cap is surfaced to the contributor instead.
+HANDSHAKE_RETRY_LIMIT = 3
+HANDSHAKE_RETRY_MAX_WAIT_SEC = 120.0
 
 
 @dataclass(frozen=True)
@@ -57,9 +64,7 @@ def upload_capture_via_broker(
     manager = nullcontext(http_client) if http_client is not None else httpx.Client()
     try:
         with _quiet_httpx_request_logging(), manager as client:
-            response = client.post(
-                endpoint, json=request_body, timeout=HANDSHAKE_TIMEOUT_SEC
-            )
+            response = _post_handshake(client, endpoint, request_body)
             if response.status_code == 409:
                 return _already_uploaded_result(response, staged, broker_url)
             _raise_for_broker_response(response, operation="upload handshake")
@@ -107,6 +112,35 @@ def upload_capture_via_broker(
     )
 
 
+def _post_handshake(
+    client: httpx.Client, endpoint: str, request_body: dict[str, Any]
+) -> httpx.Response:
+    """POST the handshake, waiting out short broker 429s instead of failing."""
+    response = client.post(endpoint, json=request_body, timeout=HANDSHAKE_TIMEOUT_SEC)
+    for _retry in range(HANDSHAKE_RETRY_LIMIT):
+        if response.status_code != 429:
+            return response
+        retry_after = _retry_after_seconds(response)
+        if retry_after is None or retry_after > HANDSHAKE_RETRY_MAX_WAIT_SEC:
+            return response
+        time.sleep(retry_after + random.uniform(0.0, 1.0))
+        response = client.post(
+            endpoint, json=request_body, timeout=HANDSHAKE_TIMEOUT_SEC
+        )
+    return response
+
+
+def _retry_after_seconds(response: httpx.Response) -> float | None:
+    value = response.headers.get("Retry-After")
+    if value is None:
+        return None
+    try:
+        seconds = float(value.strip())
+    except ValueError:
+        return None
+    return seconds if seconds >= 0 else None
+
+
 def _is_create_only_conflict(response: httpx.Response) -> bool:
     """Treat create-only blob conflicts as an idempotent skip.
 

{benchflow-0.7.2.dev2000 → benchflow-0.7.3}/tests/test_traj_upload_cli.py

@@ -248,12 +248,18 @@ def test_broker_conflict_is_success_and_rate_limit_is_actionable(
             )
         )
     )
-    limited = httpx.Client(
-        transport=httpx.MockTransport(
-            lambda request: httpx.Response(
-                429, text="slow down", headers={"Retry-After": "60"}
-            )
-        )
+    throttles = 0
+
+    def throttle(_request: httpx.Request) -> httpx.Response:
+        nonlocal throttles
+        throttles += 1
+        return httpx.Response(429, text="slow down", headers={"Retry-After": "60"})
+
+    limited = httpx.Client(transport=httpx.MockTransport(throttle))
+    sleeps: list[float] = []
+    monkeypatch.setattr(
+        "benchflow.publish.broker.time",
+        SimpleNamespace(sleep=sleeps.append),
     )
     monkeypatch.setattr("benchflow.publish.broker.httpx.Client", lambda: conflict)
     result = runner.invoke(app, _upload_command(trial))
@@ -265,6 +271,41 @@ def test_broker_conflict_is_success_and_rate_limit_is_actionable(
     result = runner.invoke(app, _upload_command(trial))
     assert result.exit_code == 1
     assert "retry after 60" in result.output
+    assert throttles == 4  # the handshake waited out three short 429s first
+    assert all(60 <= wait <= 61 for wait in sleeps)
+
+
+def test_broker_handshake_recovers_from_transient_429(
+    tmp_path: Path, monkeypatch: pytest.MonkeyPatch
+) -> None:
+    """A crowd-burst 429 with a short Retry-After self-heals without failing."""
+    trial = _trial(tmp_path)
+    monkeypatch.setenv("BENCHFLOW_TRAJ_BROKER_URL", "https://broker.test")
+    posts = 0
+
+    def handler(request: httpx.Request) -> httpx.Response:
+        nonlocal posts
+        if request.method == "POST":
+            posts += 1
+            if posts <= 2:
+                return httpx.Response(
+                    429, text="slow down", headers={"Retry-After": "1"}
+                )
+            return httpx.Response(200, json=_broker_payload(request))
+        return httpx.Response(201)
+
+    client = httpx.Client(transport=httpx.MockTransport(handler))
+    sleeps: list[float] = []
+    monkeypatch.setattr(
+        "benchflow.publish.broker.time",
+        SimpleNamespace(sleep=sleeps.append),
+    )
+    monkeypatch.setattr("benchflow.publish.broker.httpx.Client", lambda: client)
+    result = runner.invoke(app, _upload_command(trial))
+
+    assert result.exit_code == 0, result.output
+    assert posts == 3
+    assert len(sleeps) == 2
 
 
 def test_missing_broker_names_both_available_modes(
@@ -813,6 +854,59 @@ def test_list_recent_sessions_scans_all_projects_most_recent_first(
     assert "prize session please" in hits[1].snippet
 
 
+def test_setup_list_prints_path_on_its_own_line(
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    """Guards the collector-audit friction fix from PR #1024: ``bench traj
+    setup --list`` used Rich wrapping on ``index. [source] when  /very/long/path``,
+    which split session paths mid-token and made them unselectable. The path
+    must sit on its own physical line after the index/source/time line."""
+    from benchflow.trajectories.sessions import SessionHit
+
+    long_path = Path(
+        "/Users/someone/.claude/projects/"
+        "-Users-someone-very-long-project-name-that-used-to-wrap/"
+        "01234567-89ab-cdef-0123-456789abcdef.jsonl"
+    )
+    monkeypatch.setattr(
+        "benchflow.trajectories.sessions.list_recent_sessions",
+        lambda: [
+            SessionHit(
+                path=long_path,
+                source="claude",
+                mtime=1_700_000_000,
+                snippet="fit a Michaelis-Menten curve",
+            )
+        ],
+    )
+    result = runner.invoke(app, ["traj", "setup", "--list"])
+
+    assert result.exit_code == 0, result.output
+    lines = [line.rstrip() for line in click.unstyle(result.output).splitlines()]
+    assert f"   {long_path}" in lines
+    path_line = next(line for line in lines if str(long_path) in line)
+    assert path_line.strip() == str(long_path)
+    assert "fit a Michaelis-Menten curve" in result.output
+
+
+def test_broker_upload_does_not_claim_the_service_is_waking(
+    tmp_path: Path, monkeypatch: pytest.MonkeyPatch
+) -> None:
+    """Guards the collector-audit friction fix from PR #1024: a warm rerun
+    still printed ``while the service wakes up``, which is stale once the
+    broker is up. The progress line must stay honest on both cold and warm
+    runs."""
+    captured: dict[str, str] = {}
+    _capture_broker_source_id(monkeypatch, captured)
+    monkeypatch.setenv("BENCHFLOW_TRAJ_BROKER_URL", "https://broker.test")
+    result = runner.invoke(app, _upload_command(_trial(tmp_path)))
+
+    assert result.exit_code == 0, result.output
+    output = click.unstyle(result.output)
+    assert "wakes up" not in output
+    assert "Uploading… this can take up to a minute; retries are safe." in output
+
+
 def _repo_line(session_cwd: Path) -> str:
     return (
         f"Repo: benchflow-ai/benchflow (from session cwd {session_cwd}; "

{benchflow-0.7.2.dev2000 → benchflow-0.7.3}/tests/test_trajectory_upload_service.py

@@ -44,6 +44,7 @@ from services.trajectory_upload.validation import (
 from services.trajectory_upload.validator import (
     AzureCaptureValidator,
     _capture_from_event,
+    drain,
 )
 
 
@@ -636,6 +637,18 @@ class FakeTable:
         self.entities.append(entity)
         self.version += 1
 
+    def create_entity(self, entity: dict) -> None:
+        from azure.core.exceptions import ResourceExistsError
+
+        for existing in self.entities:
+            if (
+                existing["PartitionKey"] == entity["PartitionKey"]
+                and existing["RowKey"] == entity["RowKey"]
+            ):
+                raise ResourceExistsError("entity already exists")
+        self.entities.append(entity)
+        self.version += 1
+
     def update_entity(self, *, entity: dict, etag: str, **_kwargs) -> None:
         assert etag == str(self.version)
         self.entities.append(entity)
@@ -695,7 +708,7 @@ def test_broker_regrant_does_not_downgrade_ledger_state(
         ),
         ip_hash_key=b"test",
     )
-    monkeypatch.setattr(backend, "_consume_rate_limit", lambda _client_ip: None)
+    monkeypatch.setattr(backend, "_consume_rate_limit", lambda *_args: None)
     monkeypatch.setattr(
         "azure.storage.blob.generate_blob_sas", lambda **_kwargs: "sp=c&sig=test"
     )
@@ -730,7 +743,7 @@ def test_broker_persists_declared_artifact_sizes(
         ),
         ip_hash_key=b"test",
     )
-    monkeypatch.setattr(backend, "_consume_rate_limit", lambda _client_ip: None)
+    monkeypatch.setattr(backend, "_consume_rate_limit", lambda *_args: None)
     monkeypatch.setattr(
         "azure.storage.blob.generate_blob_sas", lambda **_kwargs: "sp=c&sig=test"
     )
@@ -776,7 +789,7 @@ def test_broker_rejects_manifest_change_during_validation(
         blob_service=SimpleNamespace(),
         ip_hash_key=b"test",
     )
-    monkeypatch.setattr(backend, "_consume_rate_limit", lambda _client_ip: None)
+    monkeypatch.setattr(backend, "_consume_rate_limit", lambda *_args: None)
 
     with pytest.raises(UploadDeclarationConflict):
         backend.create_upload(request, client_ip="127.0.0.1")
@@ -820,7 +833,7 @@ def test_broker_supersedes_conflicting_incomplete_attempt(
         ),
         ip_hash_key=b"test",
     )
-    monkeypatch.setattr(backend, "_consume_rate_limit", lambda _client_ip: None)
+    monkeypatch.setattr(backend, "_consume_rate_limit", lambda *_args: None)
     monkeypatch.setattr(
         "azure.storage.blob.generate_blob_sas", lambda **_kwargs: "sp=c&sig=test"
     )
@@ -865,7 +878,7 @@ def test_broker_reopens_rejected_digest_in_isolated_attempt(
         ),
         ip_hash_key=b"test",
     )
-    monkeypatch.setattr(backend, "_consume_rate_limit", lambda _client_ip: None)
+    monkeypatch.setattr(backend, "_consume_rate_limit", lambda *_args: None)
     monkeypatch.setattr(
         "azure.storage.blob.generate_blob_sas", lambda **_kwargs: "sp=c&sig=test"
     )
@@ -909,6 +922,114 @@ def test_terminal_digest_handshakes_consume_rate_limit(
         backend.create_upload(request, client_ip="127.0.0.1")
 
 
+def _rate_backend(
+    table: FakeTable, *, rate_limit: int = 20, ip_rate_limit: int = 500
+) -> AzureUploadBroker:
+    return AzureUploadBroker(
+        account_name="account",
+        container="bronze",
+        table=table,
+        blob_service=SimpleNamespace(),
+        ip_hash_key=b"test",
+        rate_limit=rate_limit,
+        ip_rate_limit=ip_rate_limit,
+    )
+
+
+def test_shared_nat_contributors_have_independent_budgets() -> None:
+    """A venue NAT full of contributors must not share one hourly budget."""
+    backend = _rate_backend(FakeTable(), rate_limit=1, ip_rate_limit=400)
+    for index in range(300):
+        backend._consume_rate_limit("203.0.113.7", f"contributor-{index}")
+
+    with pytest.raises(RateLimited):
+        backend._consume_rate_limit("203.0.113.7", "contributor-0")
+
+
+def test_contributor_budget_refills_continuously() -> None:
+    """Retry-After reflects the next token, not the top of the clock hour."""
+    table = FakeTable()
+    backend = _rate_backend(table, rate_limit=2)
+    backend._consume_rate_limit("198.51.100.9", "octocat")
+    backend._consume_rate_limit("198.51.100.9", "octocat")
+
+    with pytest.raises(RateLimited) as excinfo:
+        backend._consume_rate_limit("198.51.100.9", "octocat")
+    assert 1 <= excinfo.value.retry_after <= 1800
+
+    bucket = next(
+        entity
+        for entity in reversed(table.entities)
+        if entity["PartitionKey"] == "ratebucket"
+        and entity["RowKey"].startswith("contributor-")
+    )
+    bucket["updated_at"] = (datetime.now(UTC) - timedelta(minutes=31)).isoformat()
+    backend._consume_rate_limit("198.51.100.9", "octocat")
+
+
+def test_token_bucket_survives_contended_conditional_updates(
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    """Guards PR #1027's burst fix: ETag races must not fail a crowd closed."""
+
+    class ContendedTable(FakeTable):
+        def __init__(self) -> None:
+            super().__init__()
+            self.conflicts = 0
+
+        def update_entity(self, *, entity: dict, etag: str, **kwargs) -> None:
+            from azure.core.exceptions import ResourceModifiedError
+
+            if self.conflicts:
+                self.conflicts -= 1
+                raise ResourceModifiedError("etag changed")
+            super().update_entity(entity=entity, etag=etag, **kwargs)
+
+    table = ContendedTable()
+    backend = _rate_backend(table, rate_limit=20, ip_rate_limit=500)
+    waits: list[float] = []
+    monkeypatch.setattr(
+        "services.trajectory_upload.azure_backend.time",
+        SimpleNamespace(sleep=waits.append),
+    )
+    backend._consume_rate_limit("203.0.113.9", "contended")
+
+    table.conflicts = 8
+    backend._consume_rate_limit("203.0.113.9", "contended")
+    assert waits  # the survivor backed off between optimistic retries
+
+
+def test_ip_backstop_caps_identity_rotation() -> None:
+    """One host rotating github ids is still bounded by the per-IP bucket."""
+    backend = _rate_backend(FakeTable(), rate_limit=20, ip_rate_limit=3)
+    for index in range(3):
+        backend._consume_rate_limit("192.0.2.4", f"rotating-{index}")
+
+    with pytest.raises(RateLimited):
+        backend._consume_rate_limit("192.0.2.4", "rotating-3")
+
+    backend._consume_rate_limit("192.0.2.5", "rotating-3")
+
+
+def test_validator_drain_stops_on_empty_queue_and_budget() -> None:
+    class CountingValidator:
+        def __init__(self, available: int) -> None:
+            self.available = available
+            self.calls = 0
+
+        def run_once(self) -> bool:
+            self.calls += 1
+            return self.calls <= self.available
+
+    counting = CountingValidator(available=3)
+    assert drain(counting, budget_seconds=30.0) == 3
+    assert counting.calls == 4
+
+    exhausted = CountingValidator(available=10)
+    assert drain(exhausted, budget_seconds=0.0) == 0
+    assert exhausted.calls == 0
+
+
 def _quarantine_capture(tmp_path: Path) -> tuple[str, dict[str, bytes]]:
     trial = _trial(tmp_path)
     with stage_trajectory_capture(