benchflow 0.7.2.dev1982__tar.gz → 0.7.2.dev1986__tar.gz

{benchflow-0.7.2.dev1982 → benchflow-0.7.2.dev1986}/CHANGELOG.md

@@ -2,6 +2,17 @@
 
 ## [Unreleased]
 
+### Added
+- **Trajectory uploads are tagged with the session's repository by
+  default.** Unless `--source-id` is given, `bench traj upload` reads the
+  session's recorded working directory (Claude `cwd` events, Codex
+  `session_meta`), resolves its git `origin` remote (invocation directory as
+  fallback), and stores `repo/<owner>/<name>` as the manifest source id,
+  printing `Repo: owner/name (use --no-repo to omit)`. `--no-repo` opts out
+  — the `benchflow-traj-upload` skill now surfaces the detected tag during
+  the confirm step so contributors can decline it for private repos — and
+  undetectable repos fall back silently to the path-derived source id.
+
 ### Changed
 - **Trajectory viewer restyled to match www.benchflow.ai.** All three
   `bench eval view` pages (stream-json/JSONL, ACP events, multi-turn trial)

{benchflow-0.7.2.dev1982 → benchflow-0.7.2.dev1986}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: benchflow
-Version: 0.7.2.dev1982
+Version: 0.7.2.dev1986
 Summary: Multi-turn agent benchmarking with ACP — run any agent, any model, any provider.
 Project-URL: Homepage, https://github.com/benchflow-ai/benchflow
 Project-URL: Repository, https://github.com/benchflow-ai/benchflow

{benchflow-0.7.2.dev1982 → benchflow-0.7.2.dev1986}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "benchflow"
-version = "0.7.2.dev1982"
+version = "0.7.2.dev1986"
 description = "Multi-turn agent benchmarking with ACP — run any agent, any model, any provider."
 readme = "README.md"
 requires-python = ">=3.12"

{benchflow-0.7.2.dev1982 → benchflow-0.7.2.dev1986}/src/benchflow/cli/traj.py

@@ -2,6 +2,7 @@
 
 from __future__ import annotations
 
+import json
 import os
 import shutil
 import subprocess
@@ -29,6 +30,7 @@ from benchflow.publish.traj_capture import (
     stage_trajectory_artifacts,
     validate_email,
     validate_github_id,
+    validate_source_id,
 )
 from benchflow.publish.traj_report import (
     DEFAULT_PREVIEW_STEPS,
@@ -127,6 +129,7 @@ class _UploadOptions:
     github_id: str | None
     email: str | None
     source_id: str | None
+    repo: bool
     direct: bool
     container_url: str | None
     dry_run: bool
@@ -223,6 +226,14 @@ def register_traj(app: typer.Typer) -> None:
             str | None,
             typer.Option("--source-id", help="Stable contributor source identifier"),
         ] = None,
+        repo: Annotated[
+            bool,
+            typer.Option(
+                "--repo/--no-repo",
+                help="Tag the upload with the session's repository "
+                "(owner/name from its git remote) as the source id",
+            ),
+        ] = True,
         direct: Annotated[
             bool,
             typer.Option("--direct", help="Upload with local Azure credentials"),
@@ -254,6 +265,7 @@ def register_traj(app: typer.Typer) -> None:
                     github_id=github_id,
                     email=email,
                     source_id=source_id,
+                    repo=repo,
                     direct=direct,
                     container_url=container_url,
                     dry_run=dry_run,
@@ -268,7 +280,17 @@ def register_traj(app: typer.Typer) -> None:
 def _run_upload(options: _UploadOptions) -> None:
     prompted = options.path is None
     path = options.path or _prompt_for_path()
-    source_id = options.source_id or default_source_id(path)
+    repo_slug: str | None = None
+    if options.source_id is not None:
+        source_id = options.source_id
+    else:
+        if options.repo:
+            repo_slug = _detect_repo_slug(path)
+        source_id = f"repo/{repo_slug}" if repo_slug else default_source_id(path)
+    if repo_slug:
+        # Contributor-visible metadata: surface the tag so private-repo
+        # sessions can opt out before anything leaves the machine.
+        console.print(f"Repo: {repo_slug} (use --no-repo to omit)")
     destination = _resolve_destination(options)
 
     with (
@@ -381,6 +403,92 @@ def _infer_email() -> str:
     return ""
 
 
+_SESSION_CWD_SCAN_LINES = 50
+
+
+def _detect_repo_slug(path: Path) -> str | None:
+    """Best-effort ``owner/name`` for the repository the session was about.
+
+    Reads the working directory the session recorded (Claude events carry a
+    ``cwd`` field; Codex ``session_meta`` payloads do too), asks that
+    directory's git for the ``origin`` remote, and falls back to the
+    invocation directory. Every failure is silent — repo tagging must never
+    break an upload — and local-path remotes never produce a tag, so no
+    local absolute path can leak into the manifest.
+    """
+    for candidate in (_session_cwd(path), Path.cwd()):
+        if candidate is None or not candidate.is_dir():
+            continue
+        remote = _command_stdout(
+            "git", "-C", str(candidate), "remote", "get-url", "origin"
+        )
+        slug = _repo_slug_from_remote(remote) if remote else None
+        if slug:
+            return slug
+    return None
+
+
+def _session_cwd(path: Path) -> Path | None:
+    """Working directory recorded by the first session event that has one."""
+    session = path.expanduser()
+    if not session.is_file():
+        return None
+    try:
+        with session.open(encoding="utf-8", errors="replace") as stream:
+            for line_number, line in enumerate(stream):
+                if line_number >= _SESSION_CWD_SCAN_LINES:
+                    break
+                body = line.strip()
+                if not body:
+                    continue
+                try:
+                    event = json.loads(body)
+                except json.JSONDecodeError:
+                    continue
+                if not isinstance(event, dict):
+                    continue
+                cwd = event.get("cwd")
+                if isinstance(cwd, str) and cwd:
+                    return Path(cwd)
+                payload = event.get("payload")
+                if (
+                    event.get("type") == "session_meta"
+                    and isinstance(payload, dict)
+                    and isinstance(payload.get("cwd"), str)
+                    and payload["cwd"]
+                ):
+                    return Path(payload["cwd"])
+    except OSError:
+        return None
+    return None
+
+
+def _repo_slug_from_remote(remote: str) -> str | None:
+    """Normalize an https/ssh git remote URL to ``owner/name``.
+
+    Only URL-shaped remotes qualify; a filesystem-path remote returns
+    ``None`` so local paths never enter the uploaded source id.
+    """
+    value = remote.strip()
+    if "://" in value:
+        _, _, rest = value.partition("://")
+        _, _, repo_path = rest.partition("/")
+    elif ":" in value and "@" in value.partition(":")[0]:
+        repo_path = value.partition(":")[2]
+    else:
+        return None
+    repo_path = repo_path.strip("/").removesuffix(".git")
+    segments = [segment for segment in repo_path.split("/") if segment]
+    if len(segments) < 2:
+        return None
+    slug = f"{segments[-2]}/{segments[-1]}"
+    try:
+        validate_source_id(f"repo/{slug}")
+    except ValueError:
+        return None
+    return slug
+
+
 def _git_config(key: str) -> str | None:
     return _command_stdout("git", "config", "--get", key)
 

{benchflow-0.7.2.dev1982 → benchflow-0.7.2.dev1986}/tests/test_traj_upload_cli.py

@@ -780,3 +780,138 @@ def test_list_recent_sessions_scans_all_projects_most_recent_first(
     assert all(hit.source == "claude" for hit in hits)
     assert "GC content" in hits[0].snippet
     assert "prize session please" in hits[1].snippet
+
+
+REPO_LINE = "Repo: benchflow-ai/benchflow (use --no-repo to omit)"
+
+
+def _session_with_cwd(tmp_path: Path, event: dict) -> Path:
+    session = tmp_path / "session.jsonl"
+    session.write_text(json.dumps(event) + "\n", encoding="utf-8")
+    return session
+
+
+def _git_remote_stub(url: str | None, workdir: Path):
+    """A ``_command_stdout`` stub answering only the origin-remote lookup."""
+
+    def fake(*args: str) -> str | None:
+        if args == ("git", "-C", str(workdir), "remote", "get-url", "origin"):
+            return url
+        return None
+
+    return fake
+
+
+def _capture_broker_source_id(
+    monkeypatch: pytest.MonkeyPatch, captured: dict[str, str]
+) -> None:
+    def fake_upload(staged, *, broker_url, on_file_complete, on_bytes):
+        captured["source_id"] = staged.manifest["source_id"]
+        for staged_file in staged.files:
+            on_file_complete(staged_file)
+        return SimpleNamespace(url=broker_url, uploaded=("payload",), skipped=())
+
+    monkeypatch.setattr(
+        "benchflow.publish.broker.upload_capture_via_broker", fake_upload
+    )
+
+
+@pytest.mark.parametrize(
+    ("event_for_cwd", "remote"),
+    [
+        (
+            lambda cwd: {"type": "user", "cwd": cwd, "message": {"content": "hi"}},
+            "https://github.com/benchflow-ai/benchflow.git",
+        ),
+        (
+            lambda cwd: {"type": "session_meta", "payload": {"cwd": cwd}},
+            "git@github.com:benchflow-ai/benchflow.git",
+        ),
+    ],
+    ids=["claude-https", "codex-ssh"],
+)
+def test_upload_tags_source_id_with_the_session_repo(
+    tmp_path: Path, monkeypatch: pytest.MonkeyPatch, event_for_cwd, remote: str
+) -> None:
+    """Guards PR #1015 (repo tagging follow-up to PR #1013): by default the
+    upload's source id becomes ``repo/<owner>/<name>``, resolved from the
+    session's recorded cwd git remote, and the CLI prints the repo line."""
+    workdir = tmp_path / "workdir"
+    workdir.mkdir()
+    session = _session_with_cwd(tmp_path, event_for_cwd(str(workdir)))
+    monkeypatch.setattr(
+        "benchflow.cli.traj._command_stdout", _git_remote_stub(remote, workdir)
+    )
+    captured: dict[str, str] = {}
+    _capture_broker_source_id(monkeypatch, captured)
+
+    result = runner.invoke(app, _upload_command(session))
+
+    assert result.exit_code == 0, result.output
+    assert captured["source_id"] == "repo/benchflow-ai/benchflow"
+    assert REPO_LINE in click.unstyle(result.output)
+    assert str(workdir) not in captured["source_id"]
+
+
+def test_no_repo_keeps_the_default_source_id(
+    tmp_path: Path, monkeypatch: pytest.MonkeyPatch
+) -> None:
+    """Guards PR #1015 (repo tagging follow-up to PR #1013): ``--no-repo``
+    opts out even when a repo is detectable, keeping the path-derived id."""
+    workdir = tmp_path / "workdir"
+    workdir.mkdir()
+    session = _session_with_cwd(
+        tmp_path, {"type": "user", "cwd": str(workdir), "message": {"content": "hi"}}
+    )
+    monkeypatch.setattr(
+        "benchflow.cli.traj._command_stdout",
+        _git_remote_stub("https://github.com/benchflow-ai/benchflow.git", workdir),
+    )
+    captured: dict[str, str] = {}
+    _capture_broker_source_id(monkeypatch, captured)
+
+    result = runner.invoke(app, _upload_command(session, "--no-repo"))
+
+    assert result.exit_code == 0, result.output
+    assert captured["source_id"] == "session"
+    assert "Repo:" not in click.unstyle(result.output)
+
+
+def test_explicit_source_id_wins_over_repo_tagging(
+    tmp_path: Path, monkeypatch: pytest.MonkeyPatch
+) -> None:
+    """Guards PR #1015 (repo tagging follow-up to PR #1013): an explicit
+    ``--source-id`` always wins and repo detection never runs."""
+
+    def fail_detection(path: Path) -> str | None:
+        raise AssertionError("repo detection ran despite explicit --source-id")
+
+    monkeypatch.setattr("benchflow.cli.traj._detect_repo_slug", fail_detection)
+    captured: dict[str, str] = {}
+    _capture_broker_source_id(monkeypatch, captured)
+
+    result = runner.invoke(
+        app, _upload_command(_trial(tmp_path), "--source-id", "my-project/run-42")
+    )
+
+    assert result.exit_code == 0, result.output
+    assert captured["source_id"] == "my-project/run-42"
+    assert "Repo:" not in click.unstyle(result.output)
+
+
+def test_undetectable_repo_falls_back_silently(
+    tmp_path: Path, monkeypatch: pytest.MonkeyPatch
+) -> None:
+    """Guards PR #1015 (repo tagging follow-up to PR #1013): a session with
+    no recorded cwd and no reachable git remote uploads with the default
+    source id, no repo line, and no error."""
+    session = _session_with_cwd(tmp_path, {"type": "message", "text": "demo"})
+    monkeypatch.setattr("benchflow.cli.traj._command_stdout", lambda *_args: None)
+    captured: dict[str, str] = {}
+    _capture_broker_source_id(monkeypatch, captured)
+
+    result = runner.invoke(app, _upload_command(session))
+
+    assert result.exit_code == 0, result.output
+    assert captured["source_id"] == "session"
+    assert "Repo:" not in click.unstyle(result.output)