benchflow 0.7.8.dev2321__tar.gz → 0.7.8.dev2329__tar.gz

{benchflow-0.7.8.dev2321 → benchflow-0.7.8.dev2329}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: benchflow
-Version: 0.7.8.dev2321
+Version: 0.7.8.dev2329
 Summary: Multi-turn agent benchmarking with ACP — run any agent, any model, any provider.
 Project-URL: Homepage, https://github.com/benchflow-ai/benchflow
 Project-URL: Repository, https://github.com/benchflow-ai/benchflow

{benchflow-0.7.8.dev2321 → benchflow-0.7.8.dev2329}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "benchflow"
-version = "0.7.8.dev2321"
+version = "0.7.8.dev2329"
 description = "Multi-turn agent benchmarking with ACP — run any agent, any model, any provider."
 readme = "README.md"
 requires-python = ">=3.12"

{benchflow-0.7.8.dev2321 → benchflow-0.7.8.dev2329}/src/benchflow/acp/session.py

@@ -162,6 +162,10 @@ class ToolCallRecord:
         self.kind = kind
         self.status = ToolCallStatus.PENDING
         self.content: list[dict] = []
+        # ACP ``rawInput`` / ``rawOutput``: the agent's own view of the call.
+        # codex-acp puts the command and its output here and nowhere else.
+        self.raw_input: object | None = None
+        self.raw_output: object | None = None
         self.started_at = datetime.now()
         self.finished_at: datetime | None = None
 
@@ -178,6 +182,13 @@ class ToolCallRecord:
         ):
             self.finished_at = datetime.now()
 
+    def absorb_raw_io(self, update: dict) -> None:
+        """Keep the latest ``rawInput`` / ``rawOutput`` an update carries."""
+        if update.get("rawInput") is not None:
+            self.raw_input = update["rawInput"]
+        if update.get("rawOutput") is not None:
+            self.raw_output = update["rawOutput"]
+
 
 class ACPSession:
     """Tracks mutable state for one ACP session.
@@ -417,6 +428,20 @@ class ACPSession:
                     update.get("kind", "other"), update.get("title", "")
                 ),
             )
+            # The opening notification may already carry content (codex-acp
+            # sends file-change diffs this way) and raw I/O.
+            initial_content = update.get("content")
+            if isinstance(initial_content, list) and initial_content:
+                record.content.extend(initial_content)
+            record.absorb_raw_io(update)
+            # An opening call may already be terminal (codex-acp file edits
+            # arrive completed with no later update); honor its status so it
+            # never lingers in pending_tool_call_ids().
+            if update.get("status") is not None:
+                try:
+                    record.update_status(ToolCallStatus(update["status"]))
+                except ValueError:
+                    logger.warning(f"Unknown tool call status: {update.get('status')}")
             self._record_tool_call(record)
 
         elif update_type == "tool_call_update":
@@ -440,6 +465,7 @@ class ACPSession:
                 status = ToolCallStatus.IN_PROGRESS
             content = update.get("content")
             record.update_status(status, content)
+            record.absorb_raw_io(update)
             if status in (ToolCallStatus.PENDING, ToolCallStatus.IN_PROGRESS):
                 self._pending_tool_call_update_counts[tc_id] = (
                     self._pending_tool_call_update_counts.get(tc_id, 0) + 1

{benchflow-0.7.8.dev2321 → benchflow-0.7.8.dev2329}/src/benchflow/trajectories/_capture.py

@@ -70,16 +70,21 @@ def _events_to_trajectory(events: list[dict]) -> list[dict]:
     for event in events:
         if event["type"] == "tool_call":
             tc = event["record"]
-            out.append(
-                {
-                    "type": "tool_call",
-                    "tool_call_id": tc.tool_call_id,
-                    "kind": tc.kind,
-                    "title": tc.title,
-                    "status": tc.status.value,
-                    "content": tc.content,
-                }
-            )
+            record = {
+                "type": "tool_call",
+                "tool_call_id": tc.tool_call_id,
+                "kind": tc.kind,
+                "title": tc.title,
+                "status": tc.status.value,
+                "content": tc.content,
+            }
+            # Only present when the agent sent them, so trajectories of agents
+            # without raw I/O keep their exact shape.
+            if getattr(tc, "raw_input", None) is not None:
+                record["raw_input"] = tc.raw_input
+            if getattr(tc, "raw_output", None) is not None:
+                record["raw_output"] = tc.raw_output
+            out.append(record)
         elif event["type"] in ("user_message", "agent_message", "agent_thought"):
             out.append({"type": event["type"], "text": event["text"]})
         elif event["type"] == "agent_timeout":

{benchflow-0.7.8.dev2321 → benchflow-0.7.8.dev2329}/src/benchflow/trajectories/viewer/assets/boot.js

@@ -114,6 +114,7 @@ BF.navigation = (() => {
     bootError("The embedded viewer data must be a JSON object.");
     return;
   }
+  BF.theme.init();
   if (boot.mode === "single") {
     BF.navigation.startSingle(boot.payload);
     return;

{benchflow-0.7.8.dev2321 → benchflow-0.7.8.dev2329}/src/benchflow/trajectories/viewer/assets/detail.js

@@ -14,6 +14,7 @@ BF.detail = (() => {
     ["trace", "view-trace"],
     ["verifier", "view-verifier"],
     ["metrics", "view-metrics"],
+    ["rubric", "view-rubric"],
   ]);
   const STEP_KINDS = new Set(["prompt", "message", "thought", "tool", "timeout", "unknown"]);
   const TOOL_HUES = new Set(["read", "edit", "execute", "fetch", "search", "think", "skill", "other"]);
@@ -178,6 +179,9 @@ BF.detail = (() => {
     ) {
       available.push(["metrics", "Metrics"]);
     }
+    if (payload.rubric && typeof payload.rubric === "object") {
+      available.push(["rubric", "Rubric"]);
+    }
 
     const tabs = document.getElementById("tabs");
     tabs.textContent = "";
@@ -331,12 +335,13 @@ BF.detail = (() => {
     });
   }
 
-  function collapsibleText(container, text, className) {
+  function collapsibleText(container, text, className, language = null) {
     const value = String(text);
     const lines = value.split("\n");
     const body = el("div", className);
     if (value.length <= COLLAPSE_CHARS && lines.length <= COLLAPSE_LINES) {
       textWithRedaction(body, value);
+      BF.highlight.paint(body, value, language);
       container.appendChild(body);
       return;
     }
@@ -354,6 +359,7 @@ BF.detail = (() => {
       const open = button.getAttribute("aria-expanded") === "true";
       body.textContent = "";
       textWithRedaction(body, open ? preview + " ..." : value);
+      if (!open) BF.highlight.paint(body, value, language);
       button.setAttribute("aria-expanded", open ? "false" : "true");
       button.textContent = open
         ? "\u25b8 expand (" + value.length.toLocaleString() + " chars)"
@@ -413,9 +419,10 @@ BF.detail = (() => {
       const tool = record.tool && typeof record.tool === "object" ? record.tool : {};
       const outputClass = "tout" + (toolHue === "execute" ? " term" : "");
       const content = Array.isArray(tool.content) ? tool.content : [];
-      content.forEach((item) => {
+      content.forEach((item, position) => {
         const wrapper = el("div");
-        collapsibleText(wrapper, item, outputClass);
+        const language = BF.highlight.languageFor(String(item), toolHue, tool.title, position);
+        collapsibleText(wrapper, item, outputClass, language);
         card.appendChild(wrapper);
       });
     } else if (kind === "timeout") {
@@ -581,6 +588,72 @@ BF.detail = (() => {
     if (!main.children.length) main.appendChild(el("div", "panelblock", "No verifier artifacts in this rollout."));
   }
 
+  function renderRubric(payload) {
+    const rubric = payload.rubric && typeof payload.rubric === "object" ? payload.rubric : null;
+    const main = document.getElementById("view-rubric");
+    main.textContent = "";
+    if (!rubric) {
+      main.appendChild(el("div", "panelblock", "No review report found for this rollout."));
+      return;
+    }
+    const scoring = rubric.scoring && typeof rubric.scoring === "object" ? rubric.scoring : {};
+    const head = el("div", "panelblock");
+    head.appendChild(el("h2", null, "Review"));
+    const facts = el("table", "plain");
+    const rows = [
+      ["Reviewer", rubric.reviewer_model || ""],
+      ["Valid", rubric.review_valid ? "yes" : "no"],
+      ["Verifier pass", scoring.deterministic_pass === undefined ? "" : (scoring.deterministic_pass ? "yes" : "no")],
+      ["Blockers", scoring.all_blockers_pass === undefined ? ""
+        : (scoring.all_blockers_pass ? "all pass" : "failed: " + (scoring.failed_blockers || []).join(", "))],
+      ["Raw quality", scoring.raw_quality === undefined ? "" : String(scoring.raw_quality)],
+      ["Gated quality", scoring.gated_quality === undefined ? "" : String(scoring.gated_quality)],
+      ["Decision", scoring.decision || ""],
+    ];
+    rows.forEach(([label, value]) => {
+      const row = el("tr");
+      row.appendChild(el("th", null, label));
+      const cell = el("td", null, value);
+      if (label === "Blockers" || label === "Verifier pass") {
+        cell.style.color = /fail|^no$/.test(value) ? "var(--bad-ink)" : (value ? "var(--ok-ink)" : "");
+      }
+      row.appendChild(cell);
+      facts.appendChild(row);
+    });
+    head.appendChild(facts);
+    if (rubric.summary) collapsibleText(head, rubric.summary, "body");
+    main.appendChild(head);
+
+    const criteria = Array.isArray(rubric.criteria) ? rubric.criteria : [];
+    if (criteria.length) {
+      const block = el("div", "panelblock");
+      block.appendChild(el("h2", null, "Criteria"));
+      const table = el("table", "plain");
+      const heading = el("tr");
+      ["Criterion", "Kind", "Verdict", "Reviewer note"].forEach((label) => heading.appendChild(el("th", null, label)));
+      table.appendChild(heading);
+      criteria.forEach((criterion) => {
+        const record = criterion && typeof criterion === "object" ? criterion : {};
+        const row = el("tr");
+        row.appendChild(el("td", null, record.name));
+        // blocker true: v0.2 blocker; false: v0.2 scored; null: legacy v0.1 (outcome only)
+        const scored = record.blocker === false;
+        const kind = record.blocker === true ? "blocker" : (scored ? "weight " + (record.weight ?? "?") : "criterion");
+        row.appendChild(el("td", null, kind));
+        const verdict = el("td", null, scored ? ((record.score ?? "?") + " / 2") : (record.outcome || ""));
+        if (!scored && record.outcome) {
+          verdict.style.color = record.outcome === "pass" ? "var(--ok-ink)"
+            : (record.outcome === "fail" ? "var(--bad-ink)" : "var(--muted-foreground)");
+        }
+        row.appendChild(verdict);
+        row.appendChild(el("td", null, record.explanation || ""));
+        table.appendChild(row);
+      });
+      block.appendChild(table);
+      main.appendChild(block);
+    }
+  }
+
   function renderMetrics(payload) {
     const meta = payload.meta || {};
     const main = document.getElementById("view-metrics");
@@ -662,7 +735,7 @@ BF.detail = (() => {
   }
 
   function clearDetail() {
-    ["hdr", "tabs", "view-trace", "view-verifier", "view-metrics"].forEach((id) => {
+    ["hdr", "tabs", "view-trace", "view-verifier", "view-metrics", "view-rubric"].forEach((id) => {
       document.getElementById(id).textContent = "";
     });
     document.getElementById("toolbar").textContent = "";
@@ -702,6 +775,7 @@ BF.detail = (() => {
     renderTrace(payload);
     renderVerifier(payload);
     renderMetrics(payload);
+    renderRubric(payload);
     document.title = (payload.meta && payload.meta.task_name) || payload.rollout_name || "benchflow trajectory";
     if (options.focusHeading) requestAnimationFrame(() => heading.focus());
   }

{benchflow-0.7.8.dev2321 → benchflow-0.7.8.dev2329}/src/benchflow/trajectories/viewer/assets/template.html

@@ -5,6 +5,15 @@
 <meta name="viewport" content="width=device-width, initial-scale=1">
 <title>__BENCHFLOW_TITLE__</title>
 <link rel="icon" href="data:image/svg+xml,%3Csvg xmlns='http://www.w3.org/2000/svg' viewBox='0 0 16 16'%3E%3Ccircle cx='8' cy='8' r='6' fill='%230a0a0a'/%3E%3C/svg%3E">
+<script>
+(function () {
+  var stored = null;
+  try { stored = localStorage.getItem("bf-theme"); } catch (e) {}
+  var dark = stored ? stored === "dark"
+    : window.matchMedia && window.matchMedia("(prefers-color-scheme: dark)").matches;
+  if (dark) document.documentElement.dataset.theme = "dark";
+})();
+</script>
 <style>
 /*__BENCHFLOW_THEME_CSS__*/
 /*__BENCHFLOW_VIEWER_CSS__*/
@@ -16,6 +25,7 @@
       <svg viewBox="0 0 514 512" fill="currentColor" aria-hidden="true"><path fill-rule="evenodd" clip-rule="evenodd" d="M445.422 66.4597L511.882 0L389.022 293.965L295.129 387.859L0 511.882L69.3042 442.577L81.0101 454.283L89.0554 446.238L77.3493 434.532L130.65 381.232L162.469 413.051L170.514 405.006L138.695 373.187L191.995 319.887L203.701 331.593L211.746 323.547L200.04 311.841L253.34 258.541L285.16 290.36L293.205 282.315L261.386 250.496L314.686 197.196L326.392 208.902L334.437 200.856L322.731 189.15L376.031 135.85L407.851 167.669L415.896 159.624L384.077 127.805L437.377 74.5049L449.083 86.2108L457.128 78.1656L445.422 66.4597ZM399.127 389.865V299.369L513.197 26.4333V503.935L399.127 389.865ZM391.061 397.931L505.132 512.001H29.1594L300.605 397.931H391.061Z"/></svg>
       <span class="brand">BenchFlow</span>
       <span class="app">trajectory viewer</span>
+      <button id="theme-toggle" type="button" aria-label="toggle color theme">theme</button>
   </div>
   <main id="view-index" class="hidden" aria-label="run catalog"></main>
   <div id="content">
@@ -26,10 +36,14 @@
     <main id="view-trace" role="tabpanel"></main>
     <main id="view-verifier" role="tabpanel" class="hidden"></main>
     <main id="view-metrics" role="tabpanel" class="hidden"></main>
+    <main id="view-rubric" role="tabpanel" class="hidden"></main>
   </div>
 </div>
 <script id="bf-payload" type="application/json">__BENCHFLOW_PAYLOAD__</script>
 <script>
+//__BENCHFLOW_VENDOR_JS__
+</script>
+<script>
 //__BENCHFLOW_VIEWER_JS__
 </script>
 </body>

{benchflow-0.7.8.dev2321 → benchflow-0.7.8.dev2329}/src/benchflow/trajectories/viewer/assets/theme.css

@@ -126,4 +126,34 @@
   --warn-bg: color-mix(in srgb, var(--warning) 12%, var(--card));
   --warn-line: color-mix(in srgb, var(--warning) 28%, var(--card));
   --warn-strip: color-mix(in srgb, var(--warning) 60%, var(--card));
+
+  /* syntax highlighting palette (GitHub light); terminal blocks and the dark
+     theme swap in the dark palette below */
+  --hl-fg: #24292e; --hl-kw: #d73a49; --hl-str: #032f62; --hl-num: #005cc5; --hl-cmt: #6a737d;
+  --hl-fn: #6f42c1; --hl-var: #e36209; --hl-add: #22863a; --hl-add-bg: #e6ffed;
+  --hl-del: #b31d28; --hl-del-bg: #ffeef0;
+}
+
+/* dark theme: an override of the same tokens, applied by data-theme="dark" on
+   <html> (chosen from prefers-color-scheme by default, toggled from the
+   wordmark). Pages without the attribute render exactly as before. */
+html[data-theme="dark"] {
+  color-scheme: dark;
+  --background: #0f1216; --foreground: #e6e6e6;
+  --card: #171b21; --card-foreground: #e6e6e6;
+  --primary: #e6e6e6; --primary-foreground: #111111;
+  --secondary: #222831; --secondary-foreground: #e6e6e6;
+  --muted: #222831; --muted-foreground: #9aa4b2;
+  --accent: #222831; --accent-foreground: #e6e6e6;
+  --border: #2b323c;
+  --ink-secondary: #c3c9d1; --faint: #6b7480; --rule-strong: #3a424d;
+  --code-bg: #0b0e12; --code-ink: #e6e6e6;
+  --kind-execute-bg: #2a2410; --kind-execute-line: #4a3c14; --kind-execute-ink: #e2b84f; --kind-execute-strip: #b58a2c;
+  --kind-edit-bg: #14213a;    --kind-edit-line: #1f3a63;    --kind-edit-ink: #8fbcf0;    --kind-edit-strip: #4f83c4;
+  --kind-read-bg: #10281f;    --kind-read-line: #1c4a37;    --kind-read-ink: #7ed3ae;    --kind-read-strip: #3e9c74;
+  --kind-agent-bg: #241c3a;   --kind-agent-line: #3b2d61;   --kind-agent-ink: #c6b0ee;   --kind-agent-strip: #8a6bd0;
+  --kind-web-bg: #0f2a33;     --kind-web-line: #1c4a58;     --kind-web-ink: #7ecbe0;     --kind-web-strip: #3f9fb8;
+  --hl-fg: #e6edf3; --hl-kw: #ff7b72; --hl-str: #a5d6ff; --hl-num: #79c0ff; --hl-cmt: #8b949e;
+  --hl-fn: #d2a8ff; --hl-var: #ffa657; --hl-add: #7ee787; --hl-add-bg: #12261e;
+  --hl-del: #ffa198; --hl-del-bg: #3a1618;
 }

benchflow-0.7.8.dev2329/src/benchflow/trajectories/viewer/assets/theme.js

@@ -0,0 +1,103 @@
+// Color theme (light/dark) and syntax highlighting of tool content.
+//
+// The theme is a data attribute on <html>; template.html applies the stored
+// or system preference before first paint, this module owns the toggle.
+// Highlighting is deliberately conservative: a block is colored only when its
+// language is known from a markdown fence, a diff signature, or the tool kind,
+// never from auto-detection, so plain logs stay plain.
+// Extends the BF namespace declared in viewer.js, like detail.js and catalog.js.
+
+BF.theme = (() => {
+  const KEY = "bf-theme";
+
+  function current() {
+    return document.documentElement.dataset.theme === "dark" ? "dark" : "light";
+  }
+
+  function apply(theme) {
+    if (theme === "dark") document.documentElement.dataset.theme = "dark";
+    else delete document.documentElement.dataset.theme;
+    const button = document.getElementById("theme-toggle");
+    if (button) button.textContent = theme === "dark" ? "light" : "dark";
+  }
+
+  function toggle() {
+    const next = current() === "dark" ? "light" : "dark";
+    try { localStorage.setItem(KEY, next); } catch (e) { /* storage may be disabled */ }
+    apply(next);
+  }
+
+  function init() {
+    apply(current());
+    const button = document.getElementById("theme-toggle");
+    if (button) button.addEventListener("click", toggle);
+  }
+
+  return { init, toggle, current };
+})();
+
+BF.highlight = (() => {
+  const FENCE = /^```([A-Za-z0-9_+-]*)\s*\n/;
+  const HEREDOC = /^(.*?\b(?:python3?|python -)\b[^\n]*<<-?\s*['"]?(\w+)['"]?\s*\n)([\s\S]*?)(\n\2\b[\s\S]*)$/;
+  const EXTENSIONS = {
+    py: "python", sh: "bash", bash: "bash", json: "json", yaml: "yaml", yml: "yaml",
+    toml: "ini", ini: "ini", md: "markdown", c: "c", h: "c", cpp: "cpp", hpp: "cpp",
+    cc: "cpp", js: "javascript", mjs: "javascript", ts: "typescript", rs: "rust",
+    diff: "diff", patch: "diff",
+  };
+  const FENCE_ALIASES = { console: "shell", sh: "bash", shell: "shell", text: null, "": null };
+
+  function known(language) {
+    return language && window.hljs && hljs.getLanguage(language) ? language : null;
+  }
+
+  function fromFence(text) {
+    const match = text.match(FENCE);
+    if (!match) return null;
+    const name = match[1].toLowerCase();
+    return known(name in FENCE_ALIASES ? FENCE_ALIASES[name] : name);
+  }
+
+  function isDiff(text) {
+    const head = text.slice(0, 400);
+    return /^\*\*\* Begin Patch|^diff |^--- [^\n]*\n\+\+\+ |^@@ /m.test(head);
+  }
+
+  // Language for one content block of a tool card, or null to leave it plain.
+  function languageFor(text, hue, title, position) {
+    if (!window.hljs) return null;
+    const fenced = fromFence(text);
+    if (fenced) return fenced;
+    if (isDiff(text)) return "diff";
+    if (text.startsWith("```")) return null;
+    if (hue === "execute" && position === 0) return "bash";
+    if ((hue === "read" || hue === "edit") && typeof title === "string") {
+      const match = title.match(/\.([A-Za-z0-9]+)\s*$/);
+      if (match) return known(EXTENSIONS[match[1].toLowerCase()]);
+    }
+    return null;
+  }
+
+  function render(text, language) {
+    const options = { language, ignoreIllegals: true };
+    const heredoc = language === "bash" ? text.match(HEREDOC) : null;
+    if (heredoc && hljs.getLanguage("python")) {
+      return hljs.highlight(heredoc[1], options).value
+        + hljs.highlight(heredoc[3], { language: "python", ignoreIllegals: true }).value
+        + hljs.highlight(heredoc[4], options).value;
+    }
+    return hljs.highlight(text, options).value;
+  }
+
+  // Replace the block's text nodes with highlighted markup. Blocks that carry
+  // redaction markers keep their spans and are left uncolored.
+  function paint(block, text, language) {
+    if (!language || !window.hljs || block.querySelector(".redacted")) return;
+    if (block.textContent !== text) return;
+    block.innerHTML = render(text, language);
+    block.classList.add("hljs");
+    block.dataset.language = language;
+  }
+
+  return { languageFor, paint };
+})();

benchflow-0.7.8.dev2329/src/benchflow/trajectories/viewer/assets/vendor/LICENSE.highlight.js

@@ -0,0 +1,29 @@
+BSD 3-Clause License
+
+Copyright (c) 2006, Ivan Sagalaev.
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+* Redistributions of source code must retain the above copyright notice, this
+  list of conditions and the following disclaimer.
+
+* Redistributions in binary form must reproduce the above copyright notice,
+  this list of conditions and the following disclaimer in the documentation
+  and/or other materials provided with the distribution.
+
+* Neither the name of the copyright holder nor the names of its
+  contributors may be used to endorse or promote products derived from
+  this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.