benchflow 0.6.10.dev1954__tar.gz → 0.6.10.dev1961__tar.gz

{benchflow-0.6.10.dev1954 → benchflow-0.6.10.dev1961}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: benchflow
-Version: 0.6.10.dev1954
+Version: 0.6.10.dev1961
 Summary: Multi-turn agent benchmarking with ACP — run any agent, any model, any provider.
 Project-URL: Homepage, https://github.com/benchflow-ai/benchflow
 Project-URL: Repository, https://github.com/benchflow-ai/benchflow
@@ -95,14 +95,24 @@ For local Claude Code or Codex jsonl formatted trajectory files that you are pro
 # Install or upgrade BenchFlow
 uv tool install --python 3.12 --upgrade benchflow
 
-# Upload one capture
+# Interactive upload (prompts for path, GitHub ID, then email)
+bench traj upload
+
+# Or provide every input up front
 bench traj upload /absolute/path/to/trial \
   --github-id YOUR_GITHUB_ID \
   --email YOU@example.com
 ```
 
 The path may be a trial directory, a directory of JSONL files, or one JSONL
-file. Both contributor fields are required and are stored in `manifest.json`.
+file. Path and contributor details are required inputs; any that are omitted are
+prompted for interactively. Detected secret values are replaced locally with
+`<XXX-benchflow-key-values-XXX>` before upload. Contributor details are stored
+in `manifest.json`. After the path is entered, the CLI shows a redacted preview
+and a trajectory report with partitioned human, thinking, and tool-call counts,
+creation time, file size, and the number of masked key values. Interactive users
+confirm the report before a byte-progress display starts the upload. The full
+redacted report is retained in the uploaded `manifest.json`.
 See the concise [upload skill](./.agents/skills/benchflow-traj-upload/SKILL.md)
 or the [trajectory upload guide](./docs/traj-upload.md).
 

{benchflow-0.6.10.dev1954 → benchflow-0.6.10.dev1961}/README.md

@@ -21,14 +21,24 @@ For local Claude Code or Codex jsonl formatted trajectory files that you are pro
 # Install or upgrade BenchFlow
 uv tool install --python 3.12 --upgrade benchflow
 
-# Upload one capture
+# Interactive upload (prompts for path, GitHub ID, then email)
+bench traj upload
+
+# Or provide every input up front
 bench traj upload /absolute/path/to/trial \
   --github-id YOUR_GITHUB_ID \
   --email YOU@example.com
 ```
 
 The path may be a trial directory, a directory of JSONL files, or one JSONL
-file. Both contributor fields are required and are stored in `manifest.json`.
+file. Path and contributor details are required inputs; any that are omitted are
+prompted for interactively. Detected secret values are replaced locally with
+`<XXX-benchflow-key-values-XXX>` before upload. Contributor details are stored
+in `manifest.json`. After the path is entered, the CLI shows a redacted preview
+and a trajectory report with partitioned human, thinking, and tool-call counts,
+creation time, file size, and the number of masked key values. Interactive users
+confirm the report before a byte-progress display starts the upload. The full
+redacted report is retained in the uploaded `manifest.json`.
 See the concise [upload skill](./.agents/skills/benchflow-traj-upload/SKILL.md)
 or the [trajectory upload guide](./docs/traj-upload.md).
 

{benchflow-0.6.10.dev1954 → benchflow-0.6.10.dev1961}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "benchflow"
-version = "0.6.10.dev1954"
+version = "0.6.10.dev1961"
 description = "Multi-turn agent benchmarking with ACP — run any agent, any model, any provider."
 readme = "README.md"
 requires-python = ">=3.12"

benchflow-0.6.10.dev1961/src/benchflow/cli/_traj_upload_ui.py

@@ -0,0 +1,138 @@
+"""Rich terminal rendering for trajectory inspection and upload progress."""
+
+from __future__ import annotations
+
+from collections.abc import Callable, Iterator
+from contextlib import contextmanager
+from dataclasses import dataclass
+
+from rich.console import Console
+from rich.panel import Panel
+from rich.progress import (
+    BarColumn,
+    DownloadColumn,
+    Progress,
+    SpinnerColumn,
+    TaskProgressColumn,
+    TextColumn,
+    TimeElapsedColumn,
+)
+from rich.table import Table
+from rich.text import Text
+
+from benchflow.publish.redact import REDACTED
+from benchflow.publish.traj_capture import StagedFile
+from benchflow.publish.traj_report import PREVIEW_WORD_LIMIT, TrajectoryReport
+
+
+def render_trajectory_report(report: TrajectoryReport, *, console: Console) -> None:
+    """Render upload facts and a bounded, already-redacted trajectory preview."""
+    facts = Table.grid(padding=(0, 2))
+    facts.add_column(style="bold cyan", no_wrap=True)
+    facts.add_column()
+    facts.add_row("Primary trajectory", Text(report.primary_file))
+    facts.add_row("Format", report.format.value)
+    facts.add_row("Created", _format_created_at(report))
+    facts.add_row("JSONL files", str(report.file_count))
+    facts.add_row("Trajectory size", format_bytes(report.size_bytes))
+    facts.add_row("Total steps", str(report.total_steps))
+    facts.add_row("Thinking steps", str(report.thinking_steps))
+    facts.add_row("Tool-call steps", str(report.tool_call_steps))
+    facts.add_row("Human steps", str(report.human_steps))
+    facts.add_row(
+        "API keys / secrets masked",
+        Text(str(report.masked_values), style="bold green"),
+    )
+    facts.add_row("Safe replacement", Text(REDACTED))
+    console.print(Panel(facts, title="Trajectory report", border_style="cyan"))
+
+    if not report.preview:
+        return
+    preview = Table(
+        title=(
+            f"First {len(report.preview)} trajectory steps "
+            f"(up to {PREVIEW_WORD_LIMIT} words each)"
+        ),
+        show_lines=True,
+        header_style="bold cyan",
+    )
+    preview.add_column("#", justify="right", style="dim", no_wrap=True)
+    preview.add_column("Kind", no_wrap=True)
+    preview.add_column("Preview", overflow="fold")
+    for step in report.preview:
+        preview.add_row(str(step.number), step.kind, Text(step.summary))
+    console.print(preview)
+    if len(report.preview) < report.total_steps:
+        console.print(
+            f"[dim]Showing {len(report.preview)} of {report.total_steps} steps. "
+            "Use --preview-steps to change the preview.[/dim]"
+        )
+
+
+@dataclass(frozen=True)
+class UploadProgressHooks:
+    """Transport callbacks that feed one shared progress bar."""
+
+    on_bytes: Callable[[int], None]
+    on_file_complete: Callable[[StagedFile], None]
+
+
+@contextmanager
+def upload_progress(
+    files: tuple[StagedFile, ...],
+    *,
+    console: Console,
+) -> Iterator[UploadProgressHooks]:
+    """Display streamed byte progress and yield the transport callbacks."""
+    total_bytes = sum(item.size_bytes for item in files)
+    streamed_bytes = 0
+    boundary_bytes = 0
+    progress = Progress(
+        SpinnerColumn(style="cyan"),
+        TextColumn("[bold cyan]{task.description}"),
+        BarColumn(),
+        TaskProgressColumn(),
+        DownloadColumn(binary_units=True),
+        TimeElapsedColumn(),
+        console=console,
+    )
+    with progress:
+        task_id = progress.add_task("Uploading trajectory", total=total_bytes)
+
+        def on_bytes(count: int) -> None:
+            nonlocal streamed_bytes
+            streamed_bytes = min(streamed_bytes + count, total_bytes)
+            progress.update(task_id, completed=streamed_bytes)
+
+        def on_file_complete(staged_file: StagedFile) -> None:
+            # Resync at each file boundary: a transport retry may re-read
+            # bytes, so the boundary is authoritative for completed work.
+            nonlocal boundary_bytes, streamed_bytes
+            boundary_bytes = min(boundary_bytes + staged_file.size_bytes, total_bytes)
+            streamed_bytes = boundary_bytes
+            progress.update(
+                task_id,
+                completed=boundary_bytes,
+                description=f"Uploaded {staged_file.relname}",
+            )
+
+        yield UploadProgressHooks(on_bytes=on_bytes, on_file_complete=on_file_complete)
+        progress.update(
+            task_id,
+            completed=total_bytes,
+            description="Upload complete",
+        )
+
+
+def _format_created_at(report: TrajectoryReport) -> str:
+    local = report.created_at.astimezone()
+    return f"{local:%Y-%m-%d %H:%M:%S %Z} ({report.created_at_source})"
+
+
+def format_bytes(size: int) -> str:
+    value = float(size)
+    for unit in ("B", "KB", "MB", "GB"):
+        if value < 1024 or unit == "GB":
+            return f"{value:.1f} {unit}"
+        value /= 1024
+    raise AssertionError("unreachable")

benchflow-0.6.10.dev1961/src/benchflow/cli/traj.py

@@ -0,0 +1,278 @@
+"""``bench traj upload`` — contribute validated trajectory captures."""
+
+from __future__ import annotations
+
+import os
+from collections.abc import Callable
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Annotated, Protocol
+
+import typer
+from rich.markup import escape
+
+from benchflow.cli._shared import console, print_error
+from benchflow.cli._traj_upload_ui import (
+    UploadProgressHooks,
+    format_bytes,
+    render_trajectory_report,
+    upload_progress,
+)
+from benchflow.publish.traj_capture import (
+    StagedCapture,
+    default_source_id,
+    finalize_trajectory_capture,
+    stage_trajectory_artifacts,
+    validate_email,
+    validate_github_id,
+)
+from benchflow.publish.traj_report import (
+    DEFAULT_PREVIEW_STEPS,
+    MAX_PREVIEW_STEPS,
+    build_trajectory_report,
+)
+
+# The environment variable remains an override for development and disaster
+# recovery.
+DEFAULT_TRAJ_BROKER_URL: str | None = (
+    "https://tasksminer-traj-broker.nicewave-c3abaecf.westus2.azurecontainerapps.io"
+)
+
+
+@dataclass(frozen=True)
+class _UploadOptions:
+    path: Path | None
+    github_id: str | None
+    email: str | None
+    source_id: str | None
+    direct: bool
+    container_url: str | None
+    dry_run: bool
+    preview_steps: int
+
+
+@dataclass(frozen=True)
+class _UploadDestination:
+    url: str
+    direct: bool
+
+
+class _PublishResult(Protocol):
+    @property
+    def url(self) -> str: ...
+
+    @property
+    def uploaded(self) -> tuple[str, ...]: ...
+
+    @property
+    def skipped(self) -> tuple[str, ...]: ...
+
+
+def register_traj(app: typer.Typer) -> None:
+    """Attach the trajectory contribution group to the top-level app."""
+    traj_app = typer.Typer(help="Trajectory commands.")
+    app.add_typer(traj_app, name="traj", rich_help_panel="Core")
+
+    @traj_app.command("upload")
+    def upload(
+        path: Annotated[
+            Path | None,
+            typer.Argument(help="Trajectory JSONL file, directory, or trial directory"),
+        ] = None,
+        github_id: Annotated[
+            str | None,
+            typer.Option("--github-id", help="Contributor GitHub username"),
+        ] = None,
+        email: Annotated[
+            str | None,
+            typer.Option("--email", help="Contributor email stored in the manifest"),
+        ] = None,
+        source_id: Annotated[
+            str | None,
+            typer.Option("--source-id", help="Stable contributor source identifier"),
+        ] = None,
+        direct: Annotated[
+            bool,
+            typer.Option("--direct", help="Upload with local Azure credentials"),
+        ] = False,
+        container_url: Annotated[
+            str | None,
+            typer.Option("--container-url", help="Azure container URL for --direct"),
+        ] = None,
+        dry_run: Annotated[
+            bool,
+            typer.Option("--dry-run", help="Validate and stage without uploading"),
+        ] = False,
+        preview_steps: Annotated[
+            int,
+            typer.Option(
+                "--preview-steps",
+                min=0,
+                max=MAX_PREVIEW_STEPS,
+                help="Number of redacted trajectory steps to preview",
+            ),
+        ] = DEFAULT_PREVIEW_STEPS,
+    ) -> None:
+        """Inspect, redact, confirm, and upload trajectory JSONL."""
+        try:
+            _run_upload(
+                _UploadOptions(
+                    path=path,
+                    github_id=github_id,
+                    email=email,
+                    source_id=source_id,
+                    direct=direct,
+                    container_url=container_url,
+                    dry_run=dry_run,
+                    preview_steps=preview_steps,
+                )
+            )
+        except ValueError as exc:
+            print_error(str(exc))
+            raise typer.Exit(1) from None
+
+
+def _run_upload(options: _UploadOptions) -> None:
+    interactive = any(
+        value is None for value in (options.path, options.github_id, options.email)
+    )
+    path = options.path or _prompt_for_path()
+    source_id = options.source_id or default_source_id(path)
+    destination = _resolve_destination(options)
+
+    with (
+        console.status(
+            "[bold cyan]Inspecting trajectory and masking key values…"
+        ) as status,
+        stage_trajectory_artifacts(path, source_id=source_id) as artifacts,
+    ):
+        report = build_trajectory_report(
+            artifacts.files,
+            masked_values=artifacts.redaction_replacements,
+            preview_steps=options.preview_steps,
+        )
+        status.stop()
+        render_trajectory_report(report, console=console)
+
+        github_id, email = _resolve_contributor(options.github_id, options.email)
+        staged = finalize_trajectory_capture(
+            artifacts,
+            uploaded_by=os.environ.get("BENCHFLOW_TRAJ_UPLOADED_BY"),
+            github_id=github_id,
+            email=email,
+            trajectory_report=report.as_manifest_metadata(),
+        )
+        if options.dry_run:
+            _print_dry_run(staged)
+            return
+        if interactive and not typer.confirm(
+            "Upload this trajectory?",
+            default=False,
+        ):
+            console.print("[yellow]Upload cancelled.[/yellow]")
+            return
+        with upload_progress(staged.files, console=console) as hooks:
+            result = _publish(staged, destination=destination, hooks=hooks)
+        _print_upload_result(staged, result)
+
+
+def _resolve_contributor(
+    github_id: str | None,
+    email: str | None,
+) -> tuple[str, str]:
+    return (
+        github_id or _prompt_valid("GitHub ID", validate_github_id),
+        email or _prompt_valid("Email", validate_email),
+    )
+
+
+def _prompt_for_path() -> Path:
+    while True:
+        raw = typer.prompt("Trajectory JSONL file or trial directory").strip()
+        # Shells wrap dragged-in paths in quotes; accept them as typed.
+        if len(raw) >= 2 and raw[0] == raw[-1] and raw[0] in {"'", '"'}:
+            raw = raw[1:-1]
+        path = Path(raw).expanduser()
+        if path.exists():
+            return path
+        print_error(f"path not found: {path}")
+
+
+def _prompt_valid(label: str, validate: Callable[[str], str]) -> str:
+    while True:
+        try:
+            return validate(typer.prompt(label))
+        except ValueError as exc:
+            print_error(str(exc))
+
+
+def _resolve_destination(options: _UploadOptions) -> _UploadDestination:
+    if options.direct:
+        destination = options.container_url or os.environ.get(
+            "BENCHFLOW_AZURE_CONTAINER_URL"
+        )
+        if not destination:
+            raise ValueError(
+                "--direct requires --container-url or BENCHFLOW_AZURE_CONTAINER_URL"
+            )
+        return _UploadDestination(url=destination, direct=True)
+    if options.container_url:
+        raise ValueError("--container-url is only valid with --direct")
+    destination = os.environ.get("BENCHFLOW_TRAJ_BROKER_URL") or DEFAULT_TRAJ_BROKER_URL
+    if not destination:
+        raise ValueError(
+            "no trajectory broker is configured; set BENCHFLOW_TRAJ_BROKER_URL, "
+            "or use --direct with --container-url/BENCHFLOW_AZURE_CONTAINER_URL "
+            "if you have Azure credentials"
+        )
+    return _UploadDestination(url=destination, direct=False)
+
+
+def _publish(
+    staged: StagedCapture,
+    *,
+    destination: _UploadDestination,
+    hooks: UploadProgressHooks,
+) -> _PublishResult:
+    if destination.direct:
+        from benchflow.publish.azure_blob import upload_capture_direct
+
+        return upload_capture_direct(
+            staged,
+            container_url=destination.url,
+            on_file_complete=hooks.on_file_complete,
+            on_bytes=hooks.on_bytes,
+        )
+    from benchflow.publish.broker import upload_capture_via_broker
+
+    return upload_capture_via_broker(
+        staged,
+        broker_url=destination.url,
+        on_file_complete=hooks.on_file_complete,
+        on_bytes=hooks.on_bytes,
+    )
+
+
+def _print_upload_result(staged: StagedCapture, result: _PublishResult) -> None:
+    if not result.uploaded:
+        console.print(f"[green]Already uploaded:[/green] {escape(result.url)} (no-op)")
+        return
+    size = sum(item.size_bytes for item in staged.files)
+    console.print(
+        "[green]Uploaded trajectory:[/green] "
+        f"{escape(result.url)} "
+        f"({len(result.uploaded)} uploaded, {len(result.skipped)} skipped, "
+        f"{format_bytes(size)}, {staged.redaction_replacements} redactions)"
+    )
+
+
+def _print_dry_run(staged: StagedCapture) -> None:
+    console.print("[bold]Dry run[/bold] — no files uploaded")
+    console.print(f"Digest: sha256:{staged.traj_digest}")
+    for staged_file in staged.files:
+        console.print(
+            f"  {escape(staged_file.relname)} ({format_bytes(staged_file.size_bytes)})"
+        )
+    if staged.ignored:
+        console.print(f"Ignored: {escape(', '.join(staged.ignored))}")
+    console.print(f"Redactions: {staged.redaction_replacements}")

benchflow-0.6.10.dev1961/src/benchflow/publish/_progress.py

@@ -0,0 +1,38 @@
+"""Byte-count reporting for streamed uploads."""
+
+from __future__ import annotations
+
+from collections.abc import Callable, Iterator
+from typing import BinaryIO
+
+_CHUNK_BYTES = 64 * 1024
+
+
+class ProgressReader:
+    """File-like proxy that reports bytes consumed to a callback.
+
+    Supports both transport styles: httpx drains iterables, the Azure SDK
+    drains ``read``. ``seek``/``tell`` are delegated so either consumer can
+    still derive an exact Content-Length without emitting progress.
+    """
+
+    def __init__(self, stream: BinaryIO, on_bytes: Callable[[int], None]) -> None:
+        self._stream = stream
+        self._on_bytes = on_bytes
+
+    def read(self, size: int = -1) -> bytes:
+        chunk = self._stream.read(size)
+        if chunk:
+            self._on_bytes(len(chunk))
+        return chunk
+
+    def __iter__(self) -> Iterator[bytes]:
+        while chunk := self._stream.read(_CHUNK_BYTES):
+            self._on_bytes(len(chunk))
+            yield chunk
+
+    def seek(self, offset: int, whence: int = 0) -> int:
+        return self._stream.seek(offset, whence)
+
+    def tell(self) -> int:
+        return self._stream.tell()

{benchflow-0.6.10.dev1954 → benchflow-0.6.10.dev1961}/src/benchflow/publish/azure_blob.py

@@ -3,13 +3,15 @@
 from __future__ import annotations
 
 import logging
+from collections.abc import Callable
 from contextlib import contextmanager
 from dataclasses import dataclass
 from typing import Any
 from urllib.parse import urlparse
 
 from benchflow import __version__
-from benchflow.publish.traj_capture import StagedCapture
+from benchflow.publish._progress import ProgressReader
+from benchflow.publish.traj_capture import StagedCapture, StagedFile
 
 
 @dataclass(frozen=True)
@@ -33,6 +35,8 @@ def upload_capture_direct(
     *,
     container_url: str,
     credential: Any = None,
+    on_file_complete: Callable[[StagedFile], None] | None = None,
+    on_bytes: Callable[[int], None] | None = None,
 ) -> BlobPublishResult:
     """Create staged files in an Azure container without overwriting blobs."""
     try:
@@ -68,9 +72,12 @@ def upload_capture_direct(
             blob_name = prefix + staged_file.relname
             try:
                 with staged_file.local_path.open("rb") as stream:
+                    data = (
+                        stream if on_bytes is None else ProgressReader(stream, on_bytes)
+                    )
                     client.upload_blob(
                         name=blob_name,
-                        data=stream,
+                        data=data,
                         overwrite=False,
                         content_settings=ContentSettings(
                             content_type=staged_file.content_type
@@ -100,6 +107,8 @@ def upload_capture_direct(
                 ) from exc
             else:
                 uploaded.append(blob_name)
+            if on_file_complete is not None:
+                on_file_complete(staged_file)
     return BlobPublishResult(
         account=account,
         container=container,

{benchflow-0.6.10.dev1954 → benchflow-0.6.10.dev1961}/src/benchflow/publish/broker.py

@@ -3,6 +3,7 @@
 from __future__ import annotations
 
 import logging
+from collections.abc import Callable
 from contextlib import contextmanager, nullcontext
 from dataclasses import dataclass
 from typing import Any
@@ -10,7 +11,8 @@ from urllib.parse import urlparse
 
 import httpx
 
-from benchflow.publish.traj_capture import StagedCapture
+from benchflow.publish._progress import ProgressReader
+from benchflow.publish.traj_capture import StagedCapture, StagedFile
 
 
 @dataclass(frozen=True)
@@ -30,6 +32,8 @@ def upload_capture_via_broker(
     *,
     broker_url: str,
     http_client: httpx.Client | None = None,
+    on_file_complete: Callable[[StagedFile], None] | None = None,
+    on_bytes: Callable[[int], None] | None = None,
 ) -> BrokerPublishResult:
     """Request scoped upload URLs and PUT every staged file in server order."""
     endpoint = f"{broker_url.rstrip('/')}/v1/uploads"
@@ -41,6 +45,7 @@ def upload_capture_via_broker(
         "traj_digest": staged.manifest["traj_digest"],
         "uploaded_by": staged.manifest["uploaded_by"],
         "artifacts": artifacts,
+        "manifest_sha256": staged.files[-1].sha256,
     }
     if contributor := staged.manifest.get("contributor"):
         request_body["contributor"] = contributor
@@ -60,20 +65,25 @@ def upload_capture_via_broker(
             for staged_file, upload in zip(staged.files, objects, strict=True):
                 object_name = prefix + staged_file.relname
                 with staged_file.local_path.open("rb") as stream:
+                    content = (
+                        stream if on_bytes is None else ProgressReader(stream, on_bytes)
+                    )
                     put_response = client.put(
                         upload["put_url"],
                         headers=upload["headers"],
-                        content=stream,
+                        content=content,
                         timeout=300,
                     )
                 if put_response.status_code in {409, 412}:
                     skipped.append(object_name)
-                    continue
-                _raise_for_broker_response(
-                    put_response,
-                    operation=f"upload of {staged_file.relname}",
-                )
-                uploaded.append(object_name)
+                else:
+                    _raise_for_broker_response(
+                        put_response,
+                        operation=f"upload of {staged_file.relname}",
+                    )
+                    uploaded.append(object_name)
+                if on_file_complete is not None:
+                    on_file_complete(staged_file)
     except httpx.HTTPError as exc:
         raise ValueError(f"trajectory broker request failed: {exc}") from exc