benchflow 0.7.2.dev1994__tar.gz → 0.7.2.dev1998__tar.gz

{benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev1998}/CHANGELOG.md

@@ -3,6 +3,23 @@
 ## [Unreleased]
 
 ### Added
+- **The upload preview itemizes what redaction masked, by kind.** The
+  redactor now categorizes every replacement by the rule that fired — API
+  keys, bearer tokens, private key blocks, passwords, URL credentials, and
+  credential-bearing field values — and the terminal trajectory report shows
+  a `Masked for you: 2 API keys, 1 bearer token — originals never leave this
+  machine` breakdown under the masked-count row, plus a reassurance that
+  redaction ran locally and the server independently rescans staged
+  artifacts (or `No secrets or personal identifiers detected — nothing
+  needed masking.` when nothing matched). `bench traj upload --dry-run`
+  prints the same breakdown as a plain `Masked for you:` line, and
+  `bench eval view --confirm` gains a display-only `--redaction-summary`
+  flag that renders it in the confirm bar next to the Approve button; the
+  `benchflow-traj-upload` skill stages a dry run first and passes the line
+  through. The total `redaction_replacements` count and the manifest
+  `trajectory_report` contract are unchanged (the server validates the
+  report with a closed schema and exact recompute equality, so per-category
+  counts stay display-only).
 - **The trajectory viewer can collect the eval-prize confirmation in the
   browser.** `bench eval view PATH --confirm` renders the normal page plus a
   sticky site-styled bottom bar ("Submit this trajectory to the BenchFlow

{benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev1998}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: benchflow
-Version: 0.7.2.dev1994
+Version: 0.7.2.dev1998
 Summary: Multi-turn agent benchmarking with ACP — run any agent, any model, any provider.
 Project-URL: Homepage, https://github.com/benchflow-ai/benchflow
 Project-URL: Repository, https://github.com/benchflow-ai/benchflow

{benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev1998}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "benchflow"
-version = "0.7.2.dev1994"
+version = "0.7.2.dev1998"
 description = "Multi-turn agent benchmarking with ACP — run any agent, any model, any provider."
 readme = "README.md"
 requires-python = ">=3.12"

{benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev1998}/src/benchflow/cli/_traj_upload_ui.py

@@ -20,10 +20,29 @@ from rich.progress import (
 from rich.table import Table
 from rich.text import Text
 
-from benchflow.publish.redact import REDACTED
+from benchflow.publish.redact import REDACTED, format_redaction_breakdown
 from benchflow.publish.traj_capture import StagedFile
 from benchflow.publish.traj_report import PREVIEW_WORD_LIMIT, TrajectoryReport
 
+# Redaction-transparency copy shown with the masked-value count. The wording is
+# deliberately honest: redaction is a local structural pass (no agents, no
+# token spend), and the server independently rescans staged artifacts.
+NO_MASKING_NEEDED_COPY = (
+    "No secrets or personal identifiers detected — nothing needed masking."
+)
+REDACTION_REASSURANCE_COPY = (
+    "Redaction ran locally before anything was staged; "
+    "the server independently rescans and rejects any survivor."
+)
+
+
+def masked_for_you_line(report: TrajectoryReport) -> str:
+    """Itemized ``Masked for you`` copy for a report with masked values."""
+    breakdown = format_redaction_breakdown(dict(report.masked_categories))
+    if not breakdown:  # category data unavailable — fall back to the total
+        breakdown = f"{report.masked_values} secret-like values"
+    return f"{breakdown} — originals never leave this machine"
+
 
 def render_trajectory_report(report: TrajectoryReport, *, console: Console) -> None:
     """Render upload facts and a bounded, already-redacted trajectory preview."""
@@ -43,8 +62,17 @@ def render_trajectory_report(report: TrajectoryReport, *, console: Console) -> N
         "API keys / secrets masked",
         Text(str(report.masked_values), style="bold green"),
     )
+    if report.masked_values:
+        facts.add_row(
+            "Masked for you",
+            Text(masked_for_you_line(report), style="green"),
+        )
+    else:
+        facts.add_row("Masked for you", Text(NO_MASKING_NEEDED_COPY, style="dim"))
     facts.add_row("Safe replacement", Text(REDACTED))
     console.print(Panel(facts, title="Trajectory report", border_style="cyan"))
+    if report.masked_values:
+        console.print(f"[dim]{REDACTION_REASSURANCE_COPY}[/dim]")
 
     if not report.preview:
         return

{benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev1998}/src/benchflow/cli/main.py

@@ -1239,11 +1239,25 @@ def eval_view(
             ),
         ),
     ] = False,
+    redaction_summary: Annotated[
+        str | None,
+        typer.Option(
+            "--redaction-summary",
+            help=(
+                "Masked-secret breakdown shown in the --confirm bar, e.g. "
+                "'2 API keys, 1 bearer token' (lift it from "
+                "`bench traj upload PATH --dry-run`). Display-only; no effect "
+                "without --confirm."
+            ),
+        ),
+    ] = None,
 ) -> None:
     """View a trial or session trajectory in the browser."""
     from benchflow.trajectories import viewer
 
-    decision = viewer.serve(str(rollout_dir), port, confirm=confirm)
+    decision = viewer.serve(
+        str(rollout_dir), port, confirm=confirm, redaction_summary=redaction_summary
+    )
     # Exit-code contract for --confirm: 0 approved, 3 rejected. 3 is chosen
     # so a rejection never collides with the CLI's existing error exits
     # (1 = errors, 2 = Typer usage errors).

{benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev1998}/src/benchflow/cli/traj.py

@@ -23,6 +23,7 @@ from benchflow.cli._traj_upload_ui import (
     render_trajectory_report,
     upload_progress,
 )
+from benchflow.publish.redact import format_redaction_breakdown
 from benchflow.publish.traj_capture import (
     StagedCapture,
     default_source_id,
@@ -314,6 +315,7 @@ def _run_upload(options: _UploadOptions) -> None:
             artifacts.files,
             masked_values=artifacts.redaction_replacements,
             preview_steps=options.preview_steps,
+            masked_categories=artifacts.redaction_categories,
         )
         status.stop()
         render_trajectory_report(report, console=console)
@@ -633,6 +635,13 @@ def _print_dry_run(staged: StagedCapture) -> None:
     if staged.ignored:
         console.print(f"Ignored: {escape(', '.join(staged.ignored))}")
     console.print(f"Redactions: {staged.redaction_replacements}")
+    # One plain, greppable line so the upload skill can lift the breakdown into
+    # the viewer's confirm bar (bench eval view --redaction-summary "...").
+    breakdown = format_redaction_breakdown(dict(staged.artifact_redaction_categories))
+    if breakdown:
+        print(f"Masked for you: {breakdown}")
+    else:
+        print("Masked for you: nothing — no secrets detected")
 
 
 def _print_contributor_prompt() -> None:

{benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev1998}/src/benchflow/publish/redact.py

@@ -4,10 +4,19 @@ from __future__ import annotations
 
 import re
 import shlex
+from collections import Counter
 from collections.abc import Mapping, Sequence
 from typing import Any, NamedTuple
 
-from benchflow.trajectories.types import redact_trajectory_text_with_count
+from benchflow.trajectories.types import (
+    REDACTION_CATEGORY_API_KEY,
+    REDACTION_CATEGORY_BEARER_TOKEN,
+    REDACTION_CATEGORY_CREDENTIAL_FIELD,
+    REDACTION_CATEGORY_PASSWORD,
+    REDACTION_CATEGORY_PRIVATE_KEY,
+    REDACTION_CATEGORY_URL_CREDENTIAL,
+    redact_trajectory_text_with_categories,
+)
 
 REDACTED = "<XXX-benchflow-key-values-XXX>"
 _CANONICAL_REDACTED = "***REDACTED***"
@@ -67,23 +76,81 @@ COMPACT_SENSITIVE_KEY_SUFFIXES = tuple(
 class RedactionPattern(NamedTuple):
     pattern: re.Pattern[str]
     replacement: str
+    category: str
 
 
 VALUE_PATTERNS = (
     # Google AI Studio's newer token format is not covered by the canonical
     # trajectory redactor yet.
-    RedactionPattern(re.compile(r"AQ\.[0-9A-Za-z_-]{20,}"), REDACTED),
+    RedactionPattern(
+        re.compile(r"AQ\.[0-9A-Za-z_-]{20,}"), REDACTED, REDACTION_CATEGORY_API_KEY
+    ),
     RedactionPattern(
         re.compile(r"Bearer\s+[A-Za-z0-9._~+/=-]{16,}", re.IGNORECASE),
         f"Bearer {REDACTED}",
+        REDACTION_CATEGORY_BEARER_TOKEN,
     ),
 )
 
+# Display order and pluralized labels for the per-category breakdown shown to
+# contributors ("Masked for you: 2 API keys, 1 bearer token"). Keys are the
+# canonical category names from ``benchflow.trajectories.types``; the taxonomy
+# names what the redaction rules actually detect — nothing more.
+REDACTION_CATEGORY_LABELS: tuple[tuple[str, str], ...] = (
+    (REDACTION_CATEGORY_API_KEY, "API keys"),
+    (REDACTION_CATEGORY_BEARER_TOKEN, "bearer tokens"),
+    (REDACTION_CATEGORY_PRIVATE_KEY, "private key blocks"),
+    (REDACTION_CATEGORY_PASSWORD, "passwords"),
+    (REDACTION_CATEGORY_URL_CREDENTIAL, "URL credentials"),
+    (REDACTION_CATEGORY_CREDENTIAL_FIELD, "credential-bearing field values"),
+)
+
+
+def redaction_breakdown(counts: Mapping[str, int]) -> tuple[tuple[str, int], ...]:
+    """Return non-zero category counts in canonical display order."""
+    ordered = [
+        (category, counts[category])
+        for category, _ in REDACTION_CATEGORY_LABELS
+        if counts.get(category)
+    ]
+    # Unknown categories cannot arise from this module's own accumulation, but
+    # persisted/forwarded counts must never be dropped silently.
+    ordered.extend(
+        (category, count)
+        for category, count in counts.items()
+        if count and category not in dict(REDACTION_CATEGORY_LABELS)
+    )
+    return tuple(ordered)
+
+
+def format_redaction_breakdown(counts: Mapping[str, int]) -> str:
+    """Render category counts as prose, e.g. ``2 API keys, 1 bearer token``."""
+    plurals = dict(REDACTION_CATEGORY_LABELS)
+    parts = []
+    for category, count in redaction_breakdown(counts):
+        label = category if count == 1 else plurals.get(category, f"{category}s")
+        parts.append(f"{count} {label}")
+    return ", ".join(parts)
 
-def redact_value(value: Any, *, field_name: str | None = None) -> tuple[Any, int]:
-    """Return a structurally redacted JSON value and replacement count."""
+
+def redact_value(
+    value: Any,
+    *,
+    field_name: str | None = None,
+    categories: Counter[str] | None = None,
+) -> tuple[Any, int]:
+    """Return a structurally redacted JSON value and replacement count.
+
+    When *categories* is given, every replacement also increments the matching
+    per-category counter (see ``REDACTION_CATEGORY_LABELS``); the counter's
+    total always equals the returned replacement count.
+    """
     if field_name is not None and _is_sensitive_key(field_name):
-        return (value, 0) if _is_redacted_value(value) else (REDACTED, 1)
+        if _is_redacted_value(value):
+            return value, 0
+        if categories is not None:
+            categories[_field_category(field_name)] += 1
+        return REDACTED, 1
 
     if isinstance(value, Mapping):
         redacted: dict[Any, Any] = {}
@@ -104,7 +171,7 @@ def redact_value(value: Any, *, field_name: str | None = None) -> tuple[Any, int
         for key, item in value.items():
             clean_key = key
             if isinstance(key, str):
-                _, key_replacements = _redact_text(key)
+                _, key_replacements = _redact_text(key, categories=categories)
                 if key_replacements:
                     secret_key_index += 1
                     clean_key = f"***REDACTED_KEY_{secret_key_index}***"
@@ -122,73 +189,102 @@ def redact_value(value: Any, *, field_name: str | None = None) -> tuple[Any, int
                     if isinstance(key, str)
                     else None
                 ),
+                categories=categories,
             )
             redacted[clean_key] = clean
             replacements += count
         return redacted, replacements
 
     if isinstance(value, Sequence) and not isinstance(value, (str, bytes, bytearray)):
-        return _redact_argv(value, redact_other_values=True)
+        return _redact_argv(value, redact_other_values=True, categories=categories)
 
     if not isinstance(value, str):
         return value, 0
 
-    return _redact_text(value)
+    return _redact_text(value, categories=categories)
 
 
-def redact_value_to_stability(value: Any) -> tuple[Any, int]:
+def redact_value_to_stability(
+    value: Any, *, categories: Counter[str] | None = None
+) -> tuple[Any, int]:
     """Redact until another server-side pass would make no replacements."""
     redacted = value
     replacements = 0
     for _ in range(_MAX_REDACTION_PASSES):
-        redacted, count = redact_value(redacted)
+        redacted, count = redact_value(redacted, categories=categories)
         replacements += count
         if count == 0:
             return redacted, replacements
     raise ValueError("trajectory secret redaction did not converge")
 
 
-def _redact_text(value: str) -> tuple[str, int]:
+def _redact_text(
+    value: str, *, categories: Counter[str] | None = None
+) -> tuple[str, int]:
     # The canonical carrier patterns intentionally ignore their asterisk marker.
     # Protect our public-upload marker with that value during the scan so a second
     # client/server pass is idempotent even inside text such as ``API_KEY=...``.
     protected = value.replace(REDACTED, _CANONICAL_REDACTED)
-    redacted_text, replacements = redact_trajectory_text_with_count(protected)
+    redacted_text, text_categories = redact_trajectory_text_with_categories(protected)
+    replacements = sum(text_categories.values())
+    if categories is not None:
+        categories.update(text_categories)
     redacted_text = redacted_text.replace(_CANONICAL_REDACTED, REDACTED)
-    for pattern, replacement in VALUE_PATTERNS:
+    for pattern, replacement, category in VALUE_PATTERNS:
         redacted_text, count = pattern.subn(replacement, redacted_text)
         replacements += count
-    cli_redacted, cli_replacements = _redact_cli_text(redacted_text)
+        if count and categories is not None:
+            categories[category] += count
+    cli_redacted, cli_replacements = _redact_cli_text(
+        redacted_text, categories=categories
+    )
     redacted_text = cli_redacted
     replacements += cli_replacements
     return redacted_text, replacements
 
 
-def _redact_cli_text(value: str) -> tuple[str, int]:
+def _redact_cli_text(
+    value: str, *, categories: Counter[str] | None = None
+) -> tuple[str, int]:
     if "--" not in value:
         return value, 0
     try:
         argv = shlex.split(value)
     except ValueError:
         return value, 0
-    redacted, replacements = _redact_argv(argv, redact_other_values=False)
-    return (shlex.join(redacted), replacements) if replacements else (value, 0)
+    # Count categories only for the accepted result: a zero-replacement parse
+    # returns the original text, so nothing may be attributed for it.
+    argv_categories: Counter[str] | None = None if categories is None else Counter()
+    redacted, replacements = _redact_argv(
+        argv, redact_other_values=False, categories=argv_categories
+    )
+    if not replacements:
+        return value, 0
+    if categories is not None and argv_categories is not None:
+        categories.update(argv_categories)
+    return shlex.join(redacted), replacements
 
 
 def _redact_argv(
-    values: Sequence[Any], *, redact_other_values: bool
+    values: Sequence[Any],
+    *,
+    redact_other_values: bool,
+    categories: Counter[str] | None = None,
 ) -> tuple[list[Any], int]:
     redacted: list[Any] = []
     replacements = 0
-    redact_next = False
+    sensitive_option: str | None = None
     for item in values:
-        if redact_next:
+        if sensitive_option is not None:
+            option = sensitive_option
+            sensitive_option = None
             if _is_redacted_value(item):
                 redacted.append(item)
             else:
                 redacted.append(REDACTED)
                 replacements += 1
-            redact_next = False
+                if categories is not None:
+                    categories[_field_category(option)] += 1
             continue
 
         if isinstance(item, str) and item.startswith("-"):
@@ -200,13 +296,16 @@ def _redact_argv(
                     else:
                         redacted.append(f"{option}={REDACTED}")
                         replacements += 1
+                        if categories is not None:
+                            categories[_field_category(option)] += 1
                 else:
                     redacted.append(item)
-                    redact_next = not separator
+                    if not separator:
+                        sensitive_option = option
                 continue
 
         if redact_other_values:
-            clean, count = redact_value(item)
+            clean, count = redact_value(item, categories=categories)
             redacted.append(clean)
             replacements += count
         else:
@@ -220,6 +319,22 @@ def _is_redacted_value(value: Any) -> bool:
     )
 
 
+def _field_category(field_name: str) -> str:
+    """Categorize a masked value by the credential-bearing name that matched.
+
+    Structural redaction fires on the field/option NAME, so the name is the
+    only honest signal for what kind of secret the value was.
+    """
+    compact = _normalize_key(field_name).replace("_", "")
+    if compact.endswith(("password", "passwd")):
+        return REDACTION_CATEGORY_PASSWORD
+    if compact.endswith("apikey"):
+        return REDACTION_CATEGORY_API_KEY
+    if compact.endswith("authorization") or "bearertoken" in compact:
+        return REDACTION_CATEGORY_BEARER_TOKEN
+    return REDACTION_CATEGORY_CREDENTIAL_FIELD
+
+
 def _is_sensitive_key(field_name: str) -> bool:
     normalized = _normalize_key(field_name)
     return (

{benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev1998}/src/benchflow/publish/traj_capture.py

@@ -7,6 +7,7 @@ import json
 import re
 import shutil
 import tempfile
+from collections import Counter
 from collections.abc import Iterator, Mapping
 from contextlib import contextmanager
 from dataclasses import dataclass
@@ -15,7 +16,11 @@ from pathlib import Path
 from typing import Any, Never
 
 from benchflow import __version__
-from benchflow.publish.redact import redact_value, redact_value_to_stability
+from benchflow.publish.redact import (
+    redact_value,
+    redact_value_to_stability,
+    redaction_breakdown,
+)
 
 MAX_FILE_BYTES = 128 * 1024**2
 MAX_ARTIFACTS = 8
@@ -81,6 +86,10 @@ class StagedTrajectoryArtifacts:
     _metadata_dir: Path | None
     _staging_dir: Path
     _redact: bool
+    # Per-category artifact replacement counts in display order (display-only:
+    # the manifest/server contract stays untouched). Sums to
+    # ``redaction_replacements``.
+    redaction_categories: tuple[tuple[str, int], ...] = ()
 
 
 @dataclass(frozen=True)
@@ -92,6 +101,8 @@ class StagedCapture:
     ignored: tuple[str, ...]
     artifact_redaction_replacements: int
     redaction_replacements: int
+    # Display-only per-category counts for the artifact replacements.
+    artifact_redaction_categories: tuple[tuple[str, int], ...] = ()
 
 
 @dataclass(frozen=True)
@@ -199,12 +210,15 @@ def stage_trajectory_artifacts(
         staging_dir = Path(temp_name)
         payloads: list[StagedFile] = []
         replacement_count = 0
+        replacement_categories: Counter[str] = Counter()
         for source in resolved.files:
             relname = validate_artifact_name(f"trajectory/{source.name}")
             target = staging_dir / relname
             target.parent.mkdir(parents=True, exist_ok=True)
             if redact:
-                replacements = _redact_jsonl(source, target)
+                replacements = _redact_jsonl(
+                    source, target, categories=replacement_categories
+                )
                 replacement_count += replacements
             else:
                 shutil.copyfile(source, target)
@@ -240,6 +254,7 @@ def stage_trajectory_artifacts(
             _metadata_dir=resolved.metadata_dir,
             _staging_dir=staging_dir,
             _redact=redact,
+            redaction_categories=redaction_breakdown(replacement_categories),
         )
 
 
@@ -305,6 +320,7 @@ def finalize_trajectory_capture(
         ignored=artifacts.ignored,
         artifact_redaction_replacements=artifacts.redaction_replacements,
         redaction_replacements=replacement_count,
+        artifact_redaction_categories=artifacts.redaction_categories,
     )
 
 
@@ -430,7 +446,9 @@ def _validate_jsonl(path: Path) -> None:
         raise ValueError(f"trajectory JSONL has no records: {path}")
 
 
-def _redact_jsonl(source: Path, target: Path) -> int:
+def _redact_jsonl(
+    source: Path, target: Path, *, categories: Counter[str] | None = None
+) -> int:
     replacements = 0
     with (
         source.open(encoding="utf-8", newline="") as input_stream,
@@ -443,7 +461,9 @@ def _redact_jsonl(source: Path, target: Path) -> int:
                 continue
             value = strict_json_loads(body)
             try:
-                redacted, count = redact_value_to_stability(value)
+                redacted, count = redact_value_to_stability(
+                    value, categories=categories
+                )
             except RecursionError as exc:  # defense in depth after complexity gate
                 raise ValueError(
                     f"{source}: line {line_number}: trajectory JSONL nesting "

{benchflow-0.7.2.dev1994 → benchflow-0.7.2.dev1998}/src/benchflow/publish/traj_report.py

@@ -54,9 +54,20 @@ class TrajectoryReport:
     created_at_source: str
     masked_values: int
     preview: tuple[TrajectoryPreviewStep, ...]
+    # Per-category masked-value counts, display order, summing to
+    # ``masked_values``. Display-only: the manifest ``trajectory_report``
+    # contract (``TrajectoryReportInfo``, ``extra="forbid"``, exact
+    # recompute-equality on the server) must not gain fields, so
+    # ``as_manifest_metadata`` deliberately excludes this.
+    masked_categories: tuple[tuple[str, int], ...] = ()
 
     def as_manifest_metadata(self) -> dict[str, Any]:
-        """Serialize every displayed report field for the upload manifest."""
+        """Serialize every displayed report field for the upload manifest.
+
+        ``masked_categories`` is intentionally absent: the server validates
+        ``trajectory_report`` with ``extra="forbid"`` and an exact equality
+        check against its own recomputation, so new fields would be rejected.
+        """
         return {
             "primary_file": self.primary_file,
             "format": self.format.value,
@@ -133,6 +144,7 @@ def build_trajectory_report(
     *,
     masked_values: int,
     preview_steps: int = DEFAULT_PREVIEW_STEPS,
+    masked_categories: tuple[tuple[str, int], ...] = (),
 ) -> TrajectoryReport:
     """Summarize one canonical trajectory view from staged, redacted artifacts."""
     if not artifacts:
@@ -183,6 +195,7 @@ def build_trajectory_report(
         created_at_source=created_at_source,
         masked_values=masked_values,
         preview=tuple(analysis.preview),
+        masked_categories=masked_categories,
     )