benchflow 0.7.2.dev1998__tar.gz → 0.7.2.dev2000__tar.gz

{benchflow-0.7.2.dev1998 → benchflow-0.7.2.dev2000}/CHANGELOG.md

@@ -34,13 +34,34 @@
 - **Trajectory uploads are tagged with the session's repository by
   default.** Unless `--source-id` is given, `bench traj upload` reads the
   session's recorded working directory (Claude `cwd` events, Codex
-  `session_meta`), resolves its git `origin` remote (invocation directory as
-  fallback), and stores `repo/<owner>/<name>` as the manifest source id,
-  printing `Repo: owner/name (use --no-repo to omit)`. `--no-repo` opts out
+  `session_meta`), resolves its git `origin` remote, and stores
+  `repo/<owner>/<name>` as the manifest source id, printing
+  `Repo: owner/name (from session cwd /path; use --no-repo to omit)` (the
+  local path is terminal output only, never uploaded). `--no-repo` opts out
   — the `benchflow-traj-upload` skill now surfaces the detected tag during
   the confirm step so contributors can decline it for private repos — and
   undetectable repos fall back silently to the path-derived source id.
 
+### Fixed
+- **The repo tag derives only from the session's own recorded cwd.** The
+  initial repo-tagging implementation (#1015) fell back to the upload
+  invocation directory's git remote when the session cwd yielded nothing; a
+  collector-side audit showed this mis-attributes provenance — a session
+  recorded in a non-repo directory, uploaded from the benchflow checkout,
+  was tagged `repo/benchflow-ai/benchflow` (two community-dataset entries
+  carry the mis-tag). The fallback is removed: no session cwd, a missing
+  directory, or no GitHub remote now mean no repo tag, exactly like
+  `--no-repo`.
+- **The trajectory viewer header no longer shows `?` badges on real Claude
+  Code sessions.** `bench eval view` on a `~/.claude` session JSONL rendered
+  `model: ?`, `session: ?...`, `claude code: ?`, and `total cost: $0.0000`
+  because the header only read a `type: system` event that real session
+  files don't contain. The header now derives its metadata from what the
+  file actually carries (first assistant event's `message.model`, per-event
+  `version` / `sessionId`, the filename stem as a session fallback) and
+  hides any badge whose value is unknown — including the cost badge when no
+  event carries cost data. Presentation only.
+
 ### Changed
 - **Trajectory viewer tool calls are color-coded and backgrounds are light.**
   Each tool kind now gets a muted GitHub-label-style accent on its name pill

{benchflow-0.7.2.dev1998 → benchflow-0.7.2.dev2000}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: benchflow
-Version: 0.7.2.dev1998
+Version: 0.7.2.dev2000
 Summary: Multi-turn agent benchmarking with ACP — run any agent, any model, any provider.
 Project-URL: Homepage, https://github.com/benchflow-ai/benchflow
 Project-URL: Repository, https://github.com/benchflow-ai/benchflow

{benchflow-0.7.2.dev1998 → benchflow-0.7.2.dev2000}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "benchflow"
-version = "0.7.2.dev1998"
+version = "0.7.2.dev2000"
 description = "Multi-turn agent benchmarking with ACP — run any agent, any model, any provider."
 readme = "README.md"
 requires-python = ">=3.12"

{benchflow-0.7.2.dev1998 → benchflow-0.7.2.dev2000}/src/benchflow/cli/traj.py

@@ -292,17 +292,23 @@ def register_traj(app: typer.Typer) -> None:
 def _run_upload(options: _UploadOptions) -> None:
     prompted = options.path is None
     path = options.path or _prompt_for_path()
-    repo_slug: str | None = None
+    detected: _DetectedRepo | None = None
     if options.source_id is not None:
         source_id = options.source_id
     else:
         if options.repo:
-            repo_slug = _detect_repo_slug(path)
-        source_id = f"repo/{repo_slug}" if repo_slug else default_source_id(path)
-    if repo_slug:
-        # Contributor-visible metadata: surface the tag so private-repo
-        # sessions can opt out before anything leaves the machine.
-        console.print(f"Repo: {repo_slug} (use --no-repo to omit)")
+            detected = _detect_repo_slug(path)
+        source_id = f"repo/{detected.slug}" if detected else default_source_id(path)
+    if detected:
+        # Contributor-visible metadata: surface the tag and where it came
+        # from so private-repo sessions can opt out before anything leaves
+        # the machine. The local path stays terminal-only; the uploaded
+        # source id is just repo/<owner>/<name>. Plain print keeps the line
+        # one physical line (no Rich wrapping) so the path stays selectable.
+        print(
+            f"Repo: {detected.slug} "
+            f"(from session cwd {detected.session_cwd}; use --no-repo to omit)"
+        )
     destination = _resolve_destination(options)
 
     with (
@@ -419,25 +425,35 @@ def _infer_email() -> str:
 _SESSION_CWD_SCAN_LINES = 50
 
 
-def _detect_repo_slug(path: Path) -> str | None:
+@dataclass(frozen=True)
+class _DetectedRepo:
+    slug: str
+    session_cwd: Path
+
+
+def _detect_repo_slug(path: Path) -> _DetectedRepo | None:
     """Best-effort ``owner/name`` for the repository the session was about.
 
     Reads the working directory the session recorded (Claude events carry a
-    ``cwd`` field; Codex ``session_meta`` payloads do too), asks that
-    directory's git for the ``origin`` remote, and falls back to the
-    invocation directory. Every failure is silent — repo tagging must never
-    break an upload — and local-path remotes never produce a tag, so no
-    local absolute path can leak into the manifest.
+    ``cwd`` field; Codex ``session_meta`` payloads do too) and asks that
+    directory's git for the ``origin`` remote. The trajectory's own recorded
+    cwd is the ONLY provenance source: there is deliberately no fallback to
+    the upload invocation directory, which mis-attributed sessions recorded
+    outside a repo to whatever checkout the contributor happened to upload
+    from. No session cwd, a missing directory, or no GitHub remote all mean
+    no tag. Every failure is silent — repo tagging must never break an
+    upload — and local-path remotes never produce a tag, so no local
+    absolute path can leak into the manifest.
     """
-    for candidate in (_session_cwd(path), Path.cwd()):
-        if candidate is None or not candidate.is_dir():
-            continue
-        remote = _command_stdout(
-            "git", "-C", str(candidate), "remote", "get-url", "origin"
-        )
-        slug = _repo_slug_from_remote(remote) if remote else None
-        if slug:
-            return slug
+    session_cwd = _session_cwd(path)
+    if session_cwd is None or not session_cwd.is_dir():
+        return None
+    remote = _command_stdout(
+        "git", "-C", str(session_cwd), "remote", "get-url", "origin"
+    )
+    slug = _repo_slug_from_remote(remote) if remote else None
+    if slug:
+        return _DetectedRepo(slug=slug, session_cwd=session_cwd)
     return None
 
 

{benchflow-0.7.2.dev1998 → benchflow-0.7.2.dev2000}/src/benchflow/trajectories/viewer.py

@@ -431,26 +431,18 @@ def render_rollout(rollout_dir: Path, prompts: list[str] | None = None) -> str:
     while len(prompts) < len(turn_files):
         prompts.append("")
 
-    first_events = _parse_jsonl(turn_files[0].read_text())
-    sys_event = next((e for e in first_events if e.get("type") == "system"), {})
-    total_cost = 0
-    total_turns_count = 0
-
+    all_events: list[dict] = []
     all_blocks = []
     for i, tf in enumerate(turn_files):
         events = _parse_jsonl(tf.read_text())
+        all_events.extend(events)
         all_blocks.append(render_turn(events, i + 1, prompts[i]))
-        for e in events:
-            if e.get("type") == "result":
-                total_cost += e.get("total_cost_usd", 0)
-                total_turns_count += e.get("num_turns", 0)
-
-    # `or "?"` (not just a .get default): a present-but-null value in the
-    # stream-json (e.g. "session_id": null) bypasses the default and would crash
-    # html.escape() / the [:16] slice below with a raw TypeError.
-    session_id = str(sys_event.get("session_id") or "?")
-    model = str(sys_event.get("model") or "?")
-    version = str(sys_event.get("claude_code_version") or "?")
+
+    badges = _stream_header_badges(all_events)
+    badges.append(("turns", str(len(turn_files))))
+    total_cost = _result_cost(all_events)
+    if total_cost is not None:
+        badges.append(("total cost", f"${total_cost:.4f}"))
 
     return f"""<!DOCTYPE html>
 <html>
@@ -464,14 +456,7 @@ def render_rollout(rollout_dir: Path, prompts: list[str] | None = None) -> str:
 <div class="header">
 {_WORDMARK_HTML}
 <h1>{html.escape(rollout_dir.name)}</h1>
-<div class="meta">
-<span>model: {html.escape(model)}</span>
-<span>session: {html.escape(session_id[:16])}...</span>
-<span>claude code: {html.escape(version)}</span>
-<span>turns: {len(turn_files)}</span>
-<span>total cost: ${total_cost:.4f}</span>
-</div>
-</div>
+{_meta_badges_html(badges)}</div>
 {_join_with_divider(all_blocks)}
 </body>
 </html>"""
@@ -672,18 +657,89 @@ def render_jsonl_file(path: Path) -> str:
     body = render_turn(events, 1, "")
     if not body.strip():
         return _NO_TRAJECTORIES_HTML
-    return _stream_json_page(path.name, events, [body])
+    return _stream_json_page(path.name, events, [body], session_fallback=path.stem)
+
+
+def _stream_header_badges(
+    events: list[dict], *, session_fallback: str | None = None
+) -> list[tuple[str, str]]:
+    """Header badges derivable from what the stream actually contains.
+
+    ``claude -p`` stream-json carries a ``type: system`` init event
+    (``session_id`` / ``model`` / ``claude_code_version``); real ``~/.claude``
+    session files don't — their metadata lives per event (``sessionId``,
+    ``version``) and on assistant events (``message.model``). Pull from
+    whichever is present and omit anything unknown: a header of ``?`` badges
+    at the approve moment reads as a broken viewer, not as missing data.
+    Truthiness (not ``.get`` defaults) also swallows present-but-null values
+    like ``"session_id": null``, which previously needed an explicit guard to
+    avoid a TypeError.
+    """
+    sys_event = next((e for e in events if e.get("type") == "system"), {})
 
+    def _first(values) -> object | None:
+        return next((value for value in values if value), None)
 
-def _stream_json_page(title: str, events: list[dict], turn_blocks: list[str]) -> str:
-    sys_event = next((e for e in events if e.get("type") == "system"), {})
-    total_cost = 0.0
+    model = sys_event.get("model") or _first(
+        event["message"].get("model")
+        for event in events
+        if event.get("type") == "assistant" and isinstance(event.get("message"), dict)
+    )
+    session_id = (
+        sys_event.get("session_id")
+        or _first(event.get("sessionId") for event in events)
+        or session_fallback
+    )
+    version = sys_event.get("claude_code_version") or _first(
+        event.get("version") for event in events
+    )
+
+    badges: list[tuple[str, str]] = []
+    if model:
+        badges.append(("model", str(model)))
+    if session_id:
+        sid = str(session_id)
+        badges.append(("session", sid[:16] + ("..." if len(sid) > 16 else "")))
+    if version:
+        badges.append(("claude code", str(version)))
+    return badges
+
+
+def _result_cost(events: list[dict]) -> float | None:
+    """Total cost summed from result events, or ``None`` when no event
+    carries cost data — so the header can hide the badge instead of
+    asserting a fictional ``$0.0000``."""
+    total = 0.0
+    seen = False
     for event in events:
-        if event.get("type") == "result":
-            total_cost += float(event.get("total_cost_usd") or 0)
-    session_id = str(sys_event.get("session_id") or "?")
-    model = str(sys_event.get("model") or "?")
-    version = str(sys_event.get("claude_code_version") or "?")
+        if event.get("type") == "result" and event.get("total_cost_usd") is not None:
+            total += float(event.get("total_cost_usd") or 0)
+            seen = True
+    return total if seen else None
+
+
+def _meta_badges_html(badges: list[tuple[str, str]]) -> str:
+    """The header's ``.meta`` badge row; empty string when nothing is known."""
+    if not badges:
+        return ""
+    spans = "\n".join(
+        f"<span>{html.escape(label)}: {html.escape(value)}</span>"
+        for label, value in badges
+    )
+    return f'<div class="meta">\n{spans}\n</div>\n'
+
+
+def _stream_json_page(
+    title: str,
+    events: list[dict],
+    turn_blocks: list[str],
+    *,
+    session_fallback: str | None = None,
+) -> str:
+    badges = _stream_header_badges(events, session_fallback=session_fallback)
+    total_cost = _result_cost(events)
+    if total_cost is not None:
+        badges.append(("total cost", f"${total_cost:.4f}"))
     return f"""<!DOCTYPE html>
 <html>
 <head>
@@ -696,13 +752,7 @@ def _stream_json_page(title: str, events: list[dict], turn_blocks: list[str]) ->
 <div class="header">
 {_WORDMARK_HTML}
 <h1>{html.escape(title)}</h1>
-<div class="meta">
-<span>model: {html.escape(model)}</span>
-<span>session: {html.escape(session_id[:16])}...</span>
-<span>claude code: {html.escape(version)}</span>
-<span>total cost: ${total_cost:.4f}</span>
-</div>
-</div>
+{_meta_badges_html(badges)}</div>
 {_join_with_divider(turn_blocks)}
 </body>
 </html>"""

{benchflow-0.7.2.dev1998 → benchflow-0.7.2.dev2000}/tests/test_cli_edge_case_hardening.py

@@ -732,6 +732,66 @@ def test_render_jsonl_file_claude_and_codex_sessions(tmp_path):
     assert "looking now" in codex_html
 
 
+def test_viewer_header_reads_real_session_metadata_and_hides_unknown_badges(
+    tmp_path,
+):
+    """Guards the viewer-header fix from the collector-side audit (follow-up
+    to PR #1021's approve flow): real ``~/.claude`` session JSONL carries no
+    ``type: system`` event, so the header rendered ``model: ?``,
+    ``session: ?...``, ``claude code: ?``, and ``total cost: $0.0000`` right
+    at the approve moment. The header must derive model from the first
+    assistant event's ``message.model``, session/version from the per-event
+    ``sessionId``/``version`` fields, and hide any badge — including the cost
+    badge — whose value the file does not contain."""
+    from benchflow.trajectories.viewer import render_jsonl_file
+
+    session = tmp_path / "real-session.jsonl"
+    session.write_text(
+        json.dumps(
+            {
+                "type": "user",
+                "sessionId": "f684cfbe-1051-4d94-92ad-f8b558dd98cd",
+                "version": "2.1.233",
+                "cwd": "/private/tmp/bio-dogfood",
+                "message": {"role": "user", "content": "run the analysis"},
+            }
+        )
+        + "\n"
+        + json.dumps(
+            {
+                "type": "assistant",
+                "sessionId": "f684cfbe-1051-4d94-92ad-f8b558dd98cd",
+                "version": "2.1.233",
+                "message": {
+                    "role": "assistant",
+                    "model": "claude-opus-5",
+                    "content": [{"type": "text", "text": "on it"}],
+                },
+            }
+        )
+        + "\n"
+    )
+    html_out = render_jsonl_file(session)
+    assert "model: claude-opus-5" in html_out
+    assert "session: f684cfbe-1051-4d..." in html_out
+    assert "claude code: 2.1.233" in html_out
+    assert ": ?" not in html_out  # no unknown badge ever renders a "?"
+    assert "total cost" not in html_out  # no cost data → no $0.0000 badge
+
+    # A session with no metadata at all: only the filename-stem session badge
+    # remains; nothing renders as "?" or as a fictional zero cost.
+    bare = tmp_path / "bare.jsonl"
+    bare.write_text(
+        json.dumps({"type": "user", "message": {"content": "review me"}}) + "\n"
+    )
+    bare_html = render_jsonl_file(bare)
+    assert "session: bare" in bare_html
+    assert "model:" not in bare_html
+    assert "claude code:" not in bare_html
+    assert "total cost" not in bare_html
+    assert ": ?" not in bare_html
+
+
 def test_serve_jsonl_does_not_write_sidecar(tmp_path, monkeypatch):
     """Guards skill-first traj review: viewing ~/.claude sessions must not write HTML beside them."""
     from benchflow.trajectories.viewer import serve

{benchflow-0.7.2.dev1998 → benchflow-0.7.2.dev2000}/tests/test_traj_upload_cli.py

@@ -813,7 +813,11 @@ def test_list_recent_sessions_scans_all_projects_most_recent_first(
     assert "prize session please" in hits[1].snippet
 
 
-REPO_LINE = "Repo: benchflow-ai/benchflow (use --no-repo to omit)"
+def _repo_line(session_cwd: Path) -> str:
+    return (
+        f"Repo: benchflow-ai/benchflow (from session cwd {session_cwd}; "
+        "use --no-repo to omit)"
+    )
 
 
 def _session_with_cwd(tmp_path: Path, event: dict) -> Path:
@@ -866,7 +870,9 @@ def test_upload_tags_source_id_with_the_session_repo(
 ) -> None:
     """Guards PR #1015 (repo tagging follow-up to PR #1013): by default the
     upload's source id becomes ``repo/<owner>/<name>``, resolved from the
-    session's recorded cwd git remote, and the CLI prints the repo line."""
+    session's recorded cwd git remote, and the CLI prints the repo line
+    naming the session cwd it came from (terminal output only — the local
+    path never enters the uploaded source id)."""
     workdir = tmp_path / "workdir"
     workdir.mkdir()
     session = _session_with_cwd(tmp_path, event_for_cwd(str(workdir)))
@@ -880,7 +886,7 @@ def test_upload_tags_source_id_with_the_session_repo(
 
     assert result.exit_code == 0, result.output
     assert captured["source_id"] == "repo/benchflow-ai/benchflow"
-    assert REPO_LINE in click.unstyle(result.output)
+    assert _repo_line(workdir) in click.unstyle(result.output)
     assert str(workdir) not in captured["source_id"]
 
 
@@ -946,3 +952,51 @@ def test_undetectable_repo_falls_back_silently(
     assert result.exit_code == 0, result.output
     assert captured["source_id"] == "session"
     assert "Repo:" not in click.unstyle(result.output)
+
+
+@pytest.mark.parametrize(
+    "event",
+    [
+        {"type": "message", "text": "demo"},
+        None,  # replaced with a cwd pointing at a non-repo directory below
+    ],
+    ids=["no-recorded-cwd", "cwd-is-not-a-repo"],
+)
+def test_session_without_usable_cwd_never_tags_from_invocation_directory(
+    tmp_path: Path, monkeypatch: pytest.MonkeyPatch, event: dict | None
+) -> None:
+    """Guards this PR's fix to PR #1015 against the invocation-directory
+    fallback the collector-side audit flagged: a session recorded outside any
+    git repo (e.g. /tmp/collector-walkthrough), uploaded from inside the
+    benchflow checkout, was mis-tagged ``repo/benchflow-ai/benchflow`` — two
+    community-dataset entries carry that mis-attribution. The repo tag must
+    derive ONLY from the trajectory's own recorded cwd, so even when the
+    upload invocation directory resolves to a GitHub remote, a session with
+    no usable cwd uploads untagged with the default source id."""
+    not_a_repo = tmp_path / "not-a-repo"
+    not_a_repo.mkdir()
+    if event is None:
+        event = {"type": "user", "cwd": str(not_a_repo), "message": {"content": "hi"}}
+    session = _session_with_cwd(tmp_path, event)
+
+    def invocation_dir_has_a_remote(*args: str) -> str | None:
+        # Answer every origin-remote lookup EXCEPT the session's recorded cwd:
+        # the invocation directory (and anything else) looks like a checkout
+        # of benchflow, exactly the audit's mis-attribution scenario.
+        if args[:2] == ("git", "-C") and args[2] == str(not_a_repo):
+            return None
+        if args[3:] == ("remote", "get-url", "origin"):
+            return "https://github.com/benchflow-ai/benchflow.git"
+        return None
+
+    monkeypatch.setattr(
+        "benchflow.cli.traj._command_stdout", invocation_dir_has_a_remote
+    )
+    captured: dict[str, str] = {}
+    _capture_broker_source_id(monkeypatch, captured)
+
+    result = runner.invoke(app, _upload_command(session))
+
+    assert result.exit_code == 0, result.output
+    assert captured["source_id"] == "session"
+    assert "Repo:" not in click.unstyle(result.output)