pi-nebius 0.3.2 → 0.4.0

package/CHANGELOG.md

@@ -3,6 +3,14 @@
 Versions use Semantic Versioning. Entries describe changes included in the named version;
 a version is released only when its matching Git tag and GitHub release are published.
 
+## [0.4.0](https://github.com/PeterHdd/pi-nebius/compare/v0.3.2...v0.4.0) (2026-09-23)
+
+
+### Features
+
+* add request-level benchmark token observability ([33385c2](https://github.com/PeterHdd/pi-nebius/commit/33385c2364bf958607d0c8d0a2ac8d6bf6899663))
+* add request-level benchmark token observability ([5ecdaea](https://github.com/PeterHdd/pi-nebius/commit/5ecdaea0bda25b4df3fcfe50d5b9e0e15883350c))
+
 ## [0.3.2](https://github.com/PeterHdd/pi-nebius/compare/v0.3.1...v0.3.2) (2026-09-17)
 
 

package/README.md

@@ -109,6 +109,10 @@ middle values for an even number of runs). With one run, both show that run's me
 Failed runs contribute to the performance metrics; missing measurements appear as `n/a`.
 Custom prompts have no automatic correctness check, so a completed run does not establish success.
 
+After the summary, a per-request trace shows token usage, context sizes in bytes, tool-output sizes,
+and repeated operations. These observations help identify where context grows without changing
+the agent. Byte counts are not token estimates, and repeated calls are not necessarily wasteful.
+
 Detailed results, request settings, and resulting files are saved under `benchmark-results/`.
 Benchmarks use your saved model settings at the start; later changes do not affect an active comparison.
 

package/dist/benchmark/context-observation.js

@@ -0,0 +1,59 @@
+import { isRecord } from "../models.js";
+export function textSize(content) {
+    const text = typeof content === "string"
+        ? content
+        : Array.isArray(content)
+            ? content
+                .filter((block) => isRecord(block) && block.type === "text" && typeof block.text === "string")
+                .map((block) => block.text)
+                .join("\n")
+            : "";
+    return {
+        bytes: Buffer.byteLength(text, "utf8"),
+        lines: text ? text.split("\n").length - Number(text.endsWith("\n")) : 0,
+    };
+}
+/** Counts serialized JSON bytes, not model tokens. Never retains request content. */
+export function observeContext(body) {
+    if (typeof body !== "string")
+        return null; // Do not consume Request/stream bodies for observation.
+    try {
+        const payload = JSON.parse(body);
+        if (!isRecord(payload) || !Array.isArray(payload.messages))
+            return null;
+        const bytes = { system: 0, tools: 0, user: 0, assistant: 0, toolResults: 0, other: 0 };
+        const toolResults = [];
+        for (const message of payload.messages) {
+            const role = isRecord(message) ? message.role : undefined;
+            const category = role === "system" || role === "developer"
+                ? "system"
+                : role === "user"
+                    ? "user"
+                    : role === "assistant"
+                        ? "assistant"
+                        : role === "tool"
+                            ? "toolResults"
+                            : "other";
+            bytes[category] += Buffer.byteLength(JSON.stringify(message), "utf8");
+            if (isRecord(message) && role === "tool" && typeof message.tool_call_id === "string")
+                toolResults.push({ id: message.tool_call_id, ...textSize(message.content) });
+        }
+        if (payload.tools !== undefined)
+            bytes.tools = Buffer.byteLength(JSON.stringify(payload.tools), "utf8");
+        return {
+            measurement: "serialized-json-bytes",
+            bytes,
+            messageCount: payload.messages.length,
+            toolResults,
+        };
+    }
+    catch {
+        return null;
+    }
+}
+/** Stable object-key order lets equivalent argument objects compare within a run. */
+export function canonical(value) {
+    return (JSON.stringify(value, (_key, item) => isRecord(item)
+        ? Object.fromEntries(Object.entries(item).sort(([a], [b]) => a.localeCompare(b)))
+        : item) ?? "null");
+}

package/dist/benchmark/instrumentation.js

@@ -1,5 +1,6 @@
-import { createHash } from "node:crypto";
+import { createHash, createHmac, randomBytes } from "node:crypto";
 import { isRecord } from "../models.js";
+import { canonical, observeContext, textSize } from "./context-observation.js";
 export const hash = (text) => createHash("sha256").update(text).digest("hex");
 const count = (value) => typeof value === "number" && Number.isSafeInteger(value) && value >= 0 ? value : null;
 export function reportedUsage(value) {
@@ -46,6 +47,11 @@ export function redactor(secrets) {
 export class Instrumentation {
     state = emptyObservation();
     compacting = false;
+    // Per-run HMAC keys are never persisted: compare values without retaining commands or file text.
+    fingerprintKey = randomBytes(32);
+    fingerprint(value) {
+        return createHmac("sha256", this.fingerprintKey).update(canonical(value)).digest("hex");
+    }
     update;
     now;
     redact;
@@ -83,8 +89,14 @@ export class Instrumentation {
             case "auto_retry_start":
                 this.state.retries++;
                 break;
-            case "tool_execution_start":
+            case "tool_execution_start": {
+                const fingerprint = this.fingerprint(event.args);
+                const previous = this.state.tools.find((tool) => tool.name === event.toolName && tool.argumentsFingerprint === fingerprint);
+                const origin = this.state.requests.find((request) => request.generatedToolCalls?.some((tool) => tool.id === event.toolCallId));
                 this.state.tools.push({
+                    request: origin?.request ?? null,
+                    argumentsFingerprint: fingerprint,
+                    repeatedArgumentsOf: previous?.id ?? null,
                     id: event.toolCallId,
                     name: event.toolName,
                     startedAtMs: at,
@@ -93,6 +105,7 @@ export class Instrumentation {
                     error: null,
                 });
                 break;
+            }
             case "tool_execution_end": {
                 const tool = this.state.tools.find((item) => item.id === event.toolCallId);
                 if (tool) {
@@ -104,6 +117,11 @@ export class Instrumentation {
             case "message_end": {
                 const message = event.message;
                 if (message.role === "assistant") {
+                    const request = this.state.requests.filter((item) => item.purpose === "agent").at(-1);
+                    if (request)
+                        request.generatedToolCalls = message.content
+                            .filter((block) => block.type === "toolCall")
+                            .map((block) => ({ id: block.id, name: block.name }));
                     this.state.assistantMessages++;
                     this.state.lastAssistantStopReason = message.stopReason;
                     if (message.errorMessage)
@@ -129,6 +147,14 @@ export class Instrumentation {
                         this.state.tools.push(tool);
                     }
                     tool.isError = message.isError;
+                    const fingerprint = this.fingerprint(message.content);
+                    tool.output = { ...textSize(message.content), fingerprint };
+                    const previous = this.state.tools.find((item) => item !== tool &&
+                        item.name === tool.name &&
+                        item.argumentsFingerprint !== undefined &&
+                        item.argumentsFingerprint === tool.argumentsFingerprint &&
+                        item.output?.fingerprint === fingerprint);
+                    tool.repeatedOutputOf = previous?.id ?? null;
                     if (message.isError)
                         tool.error = this.redact(message.content
                             .filter((item) => item.type === "text")
@@ -149,6 +175,8 @@ export class Instrumentation {
                 return fetcher(input, init);
             const trace = {
                 request: this.state.requests.length + 1,
+                context: observeContext(init?.body),
+                generatedToolCalls: [],
                 purpose: this.compacting ? "compaction" : "agent",
                 startedAtMs: this.now(),
                 endedAtMs: null,

package/dist/benchmark/report.js

@@ -1,4 +1,5 @@
 import { distribution } from "./metrics.js";
+import { requestReport } from "./request-report.js";
 /** Client-observed first generated content on the first agent request, not worker startup. */
 export function observedTtftMs(run) {
     const first = run.observation.requests.find((request) => request.purpose === "agent");
@@ -89,6 +90,13 @@ export function terminalReport(results) {
             ]
             : []),
         "",
+        ...results.runs.flatMap(requestReport),
+        "",
+        "Request tokens are provider-reported per HTTP attempt, including retries and compaction when observed.",
+        "Δ input is the change from the preceding request, not tokens attributable to its tools.",
+        "Context sizes are measured UTF-8 JSON bytes including message wrappers; category token counts are unavailable.",
+        "Tool text sizes exclude images and may already be truncated by Pi. Commands, arguments, and successful output text are not stored.",
+        "Input amplification = cumulative input / last agent request input; not a waste score or final context size.",
         "Detailed per-run measurements and validator output: results.json.",
     ].join("\n");
 }

package/dist/benchmark/request-report.js

@@ -0,0 +1,118 @@
+const number = (value) => value == null ? "n/a" : value.toLocaleString("en-US");
+// biome-ignore lint/suspicious/noControlCharactersInRegex: remove terminal control characters from labels
+const label = (value) => value.replace(/[\x00-\x1f\x7f-\x9f]/g, " ").slice(0, 80);
+const table = (rows) => {
+    const widths = rows[0]?.map((_, index) => Math.max(...rows.map((row) => row[index]?.length ?? 0))) ?? [];
+    return rows.map((row) => row
+        .map((cell, index) => cell.padEnd(widths[index] ?? 0))
+        .join(" │ ")
+        .trimEnd());
+};
+export function requestReport(run) {
+    const requests = run.observation.requests;
+    const tools = run.observation.tools ?? [];
+    const rows = [["Request", "Kind", "Input", "Δ input", "Output", "Cached", "Tools produced"]];
+    const sizes = [
+        ["Request", "System B", "Schemas B", "User B", "Assistant B", "Tool results B", "Other B"],
+    ];
+    for (const [index, request] of requests.entries()) {
+        const input = request.usage?.inputTokens;
+        const previous = requests[index - 1]?.usage?.inputTokens;
+        const delta = input != null && previous != null ? input - previous : null;
+        rows.push([
+            String(request.request ?? index + 1),
+            request.purpose,
+            number(input),
+            delta !== null && delta > 0 ? `+${number(delta)}` : number(delta),
+            number(request.usage?.outputTokens),
+            number(request.usage?.cachedInputTokens),
+            request.generatedToolCalls?.length
+                ? request.generatedToolCalls.map((tool) => label(tool.name)).join(", ")
+                : "—",
+        ]);
+        const bytes = request.context?.bytes;
+        sizes.push([
+            String(request.request ?? index + 1),
+            ...[
+                bytes?.system,
+                bytes?.tools,
+                bytes?.user,
+                bytes?.assistant,
+                bytes?.toolResults,
+                bytes?.other,
+            ].map(number),
+        ]);
+    }
+    const input = run.tokens.cumulativeInputTokens;
+    const output = run.tokens.cumulativeOutputTokens;
+    rows.push([
+        "TOTAL",
+        "all attempts",
+        number(input),
+        "—",
+        number(output),
+        number(run.tokens.cachedInputTokens),
+        "",
+    ]);
+    const last = run.tokens.lastRequestInputTokens;
+    const amplification = run.tokens.inputAmplificationVsLastRequest;
+    const total = input != null && output != null ? input + output : null;
+    const consumption = run.success && run.validation.checked
+        ? "Tokens to validated solution"
+        : run.failure !== null
+            ? "Tokens consumed before failure"
+            : "Tokens consumed (correctness not checked)";
+    const lines = [
+        "",
+        `Request trace: ${label(run.model)} #${run.run}`,
+        ...table(rows),
+        `${consumption}: ${number(total)} (input ${number(input)} + output ${number(output)}).`,
+        `Last agent request input: ${number(last)}; input amplification: ${amplification == null ? "n/a" : `${amplification.toFixed(2)}x`}.`,
+        "",
+        "Context composition — measured JSON bytes (B), NOT token attribution:",
+        ...table(sizes),
+    ];
+    if (tools.length) {
+        lines.push("", "Tool results — model-facing text after Pi processing:");
+        const toolRows = [
+            [
+                "Tool",
+                "From request",
+                "Text bytes",
+                "Lines",
+                "Included in requests",
+                "Repeated arguments",
+                "Same result",
+            ],
+        ];
+        for (const [index, tool] of tools.entries()) {
+            const included = requests
+                .filter((request) => request.context?.toolResults.some((result) => result.id === tool.id))
+                .map((request) => request.request);
+            const reference = (id) => {
+                const at = tools.findIndex((item) => item.id === id);
+                return at < 0 ? "—" : `T${at + 1}`;
+            };
+            toolRows.push([
+                `T${index + 1} ${label(tool.name)}`,
+                number(tool.request),
+                number(tool.output?.bytes),
+                number(tool.output?.lines),
+                requests.every((request) => request.context != null)
+                    ? included.join(", ") || "none"
+                    : `${included.join(", ") || "none observed"} (partial)`,
+                reference(tool.repeatedArgumentsOf),
+                reference(tool.repeatedOutputOf),
+            ]);
+        }
+        lines.push(...table(toolRows));
+        const large = tools
+            .map((tool, index) => ({ tool, index }))
+            .filter(({ tool }) => (tool.output?.bytes ?? 0) >= 16384);
+        for (const { tool, index } of large)
+            lines.push(`Observation: T${index + 1} returned ${number(tool.output?.bytes)} text bytes (large-output threshold: 16 KiB).`);
+        if (tools.some((tool) => tool.repeatedArgumentsOf))
+            lines.push("Repeated arguments/results are candidates to inspect, not proof of unnecessary work or unchanged files.");
+    }
+    return lines;
+}

package/docs/benchmarking.md

@@ -251,3 +251,44 @@ No live benchmark has been run in this environment: `NEBIUS_API_KEY` was unavail
 See [benchmark-research.md](benchmark-research.md) for inspected source APIs and Nebius methodology.
 
 Result schema version 2 removes monetary fields from version 1 (per-run estimates, pricing snapshots, aggregate costs, and the pricing hash). Existing saved results are not rewritten. Task definitions still use schema version 1.
+
+
+## Request-level observability
+
+After the comparison summary, each run shows its individual HTTP requests: provider-reported
+input/output tokens, cached input tokens when available, change in input from the preceding request,
+and tool calls produced by the response. A request can produce zero, one, or multiple tool calls.
+Retries and compaction requests remain separate rows; a request is not necessarily an agent turn.
+Missing usage stays `n/a`, including totals when any required usage is missing.
+
+The context-composition table measures **UTF-8 serialized JSON bytes**, not tokens. Categories are
+system/developer messages, tool schemas, user messages, assistant messages (including tool arguments),
+tool-result messages, and other messages. Message wrapper fields are included; array separators and
+other HTTP payload fields are not. File reads and shell output are subtypes of tool results, not
+additional categories. Category token counts are unavailable; no tokenizer estimates are substituted.
+Input-token differences are net changes, not causal attribution to the most recent tool.
+
+Tool rows use local labels such as `T1 read` and `T2 bash`. They show model-facing text bytes/lines,
+the originating request, and the subsequent request numbers containing that tool-call ID. Repeated
+HTTP attempts count as repeated inclusions. These are observations after Pi's tool processing;
+original shell stdout/stderr may already have been truncated. Text sizes exclude images. If a request
+body cannot be inspected without consuming it, context measurements are unavailable and inclusion
+lists are labelled partial.
+
+Outputs of at least 16 KiB receive a large-output observation. Matching tool names and canonicalized
+arguments flag repeated operations; matching result content also flags repeated results. This does
+not establish that files or external state were unchanged, or that a call was avoidable. No tool calls
+are blocked and no output or context is reduced. Only sizes, IDs, names, and per-run keyed fingerprints
+are added to traces; successful output, commands, paths in arguments, and prompts are not retained.
+Fingerprint keys are ephemeral and are not saved; fingerprints are not comparable across runs.
+
+Input amplification is cumulative input over **all observed HTTP attempts** divided by the final
+agent request's input tokens. It is unavailable with incomplete usage or a zero/missing denominator.
+Compaction can decrease the denominator, so the ratio is not a waste score. Final-request input also
+is not the context size after the response. Successful validated runs show input + output tokens as
+**tokens to validated solution**; failures and unvalidated runs are labelled separately. Cached and
+reasoning token counters are details of usage, not additional tokens to add to that sum.
+
+These are additive optional fields in result schema 2 (`RequestTrace.context`,
+`RequestTrace.generatedToolCalls`, and tool fingerprints/output sizes). Old reports remain readable;
+absent observations are unavailable. Individual data remains in `results.json` and the live journals.

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "pi-nebius",
-  "version": "0.3.2",
+  "version": "0.4.0",
   "description": "Nebius Token Factory provider and agentic coding benchmarks for Pi",
   "repository": {
     "type": "git",

package/src/benchmark/context-observation.ts

@@ -0,0 +1,68 @@
+import { isRecord } from "../models.ts";
+import type { RequestContext } from "./types.ts";
+
+export function textSize(content: unknown): { bytes: number; lines: number } {
+  const text =
+    typeof content === "string"
+      ? content
+      : Array.isArray(content)
+        ? content
+            .filter(
+              (block) => isRecord(block) && block.type === "text" && typeof block.text === "string",
+            )
+            .map((block) => block.text)
+            .join("\n")
+        : "";
+  return {
+    bytes: Buffer.byteLength(text, "utf8"),
+    lines: text ? text.split("\n").length - Number(text.endsWith("\n")) : 0,
+  };
+}
+
+/** Counts serialized JSON bytes, not model tokens. Never retains request content. */
+export function observeContext(body: unknown): RequestContext | null {
+  if (typeof body !== "string") return null; // Do not consume Request/stream bodies for observation.
+  try {
+    const payload: unknown = JSON.parse(body);
+    if (!isRecord(payload) || !Array.isArray(payload.messages)) return null;
+    const bytes = { system: 0, tools: 0, user: 0, assistant: 0, toolResults: 0, other: 0 };
+    const toolResults: RequestContext["toolResults"] = [];
+    for (const message of payload.messages) {
+      const role = isRecord(message) ? message.role : undefined;
+      const category =
+        role === "system" || role === "developer"
+          ? "system"
+          : role === "user"
+            ? "user"
+            : role === "assistant"
+              ? "assistant"
+              : role === "tool"
+                ? "toolResults"
+                : "other";
+      bytes[category] += Buffer.byteLength(JSON.stringify(message), "utf8");
+      if (isRecord(message) && role === "tool" && typeof message.tool_call_id === "string")
+        toolResults.push({ id: message.tool_call_id, ...textSize(message.content) });
+    }
+    if (payload.tools !== undefined)
+      bytes.tools = Buffer.byteLength(JSON.stringify(payload.tools), "utf8");
+    return {
+      measurement: "serialized-json-bytes",
+      bytes,
+      messageCount: payload.messages.length,
+      toolResults,
+    };
+  } catch {
+    return null;
+  }
+}
+
+/** Stable object-key order lets equivalent argument objects compare within a run. */
+export function canonical(value: unknown): string {
+  return (
+    JSON.stringify(value, (_key, item) =>
+      isRecord(item)
+        ? Object.fromEntries(Object.entries(item).sort(([a], [b]) => a.localeCompare(b)))
+        : item,
+    ) ?? "null"
+  );
+}

package/src/benchmark/instrumentation.ts

@@ -1,6 +1,7 @@
-import { createHash } from "node:crypto";
+import { createHash, createHmac, randomBytes } from "node:crypto";
 import type { AgentSessionEvent } from "@earendil-works/pi-coding-agent";
 import { isRecord } from "../models.ts";
+import { canonical, observeContext, textSize } from "./context-observation.ts";
 import type { Observation, ReportedUsage, RequestTrace } from "./types.ts";
 
 export const hash = (text: string) => createHash("sha256").update(text).digest("hex");
@@ -51,6 +52,11 @@ export function redactor(secrets: string[]) {
 export class Instrumentation {
   readonly state = emptyObservation();
   private compacting = false;
+  // Per-run HMAC keys are never persisted: compare values without retaining commands or file text.
+  private readonly fingerprintKey = randomBytes(32);
+  private fingerprint(value: unknown) {
+    return createHmac("sha256", this.fingerprintKey).update(canonical(value)).digest("hex");
+  }
   private readonly update: (state: Observation) => void;
   private readonly now: () => number;
   private readonly redact: (text: string) => string;
@@ -93,8 +99,18 @@ export class Instrumentation {
       case "auto_retry_start":
         this.state.retries++;
         break;
-      case "tool_execution_start":
+      case "tool_execution_start": {
+        const fingerprint = this.fingerprint(event.args);
+        const previous = this.state.tools.find(
+          (tool) => tool.name === event.toolName && tool.argumentsFingerprint === fingerprint,
+        );
+        const origin = this.state.requests.find((request) =>
+          request.generatedToolCalls?.some((tool) => tool.id === event.toolCallId),
+        );
         this.state.tools.push({
+          request: origin?.request ?? null,
+          argumentsFingerprint: fingerprint,
+          repeatedArgumentsOf: previous?.id ?? null,
           id: event.toolCallId,
           name: event.toolName,
           startedAtMs: at,
@@ -103,6 +119,7 @@ export class Instrumentation {
           error: null,
         });
         break;
+      }
       case "tool_execution_end": {
         const tool = this.state.tools.find((item) => item.id === event.toolCallId);
         if (tool) {
@@ -114,6 +131,11 @@ export class Instrumentation {
       case "message_end": {
         const message = event.message;
         if (message.role === "assistant") {
+          const request = this.state.requests.filter((item) => item.purpose === "agent").at(-1);
+          if (request)
+            request.generatedToolCalls = message.content
+              .filter((block) => block.type === "toolCall")
+              .map((block) => ({ id: block.id, name: block.name }));
           this.state.assistantMessages++;
           this.state.lastAssistantStopReason = message.stopReason;
           if (message.errorMessage)
@@ -138,6 +160,17 @@ export class Instrumentation {
             this.state.tools.push(tool);
           }
           tool.isError = message.isError;
+          const fingerprint = this.fingerprint(message.content);
+          tool.output = { ...textSize(message.content), fingerprint };
+          const previous = this.state.tools.find(
+            (item) =>
+              item !== tool &&
+              item.name === tool.name &&
+              item.argumentsFingerprint !== undefined &&
+              item.argumentsFingerprint === tool.argumentsFingerprint &&
+              item.output?.fingerprint === fingerprint,
+          );
+          tool.repeatedOutputOf = previous?.id ?? null;
           if (message.isError)
             tool.error = this.redact(
               message.content
@@ -160,6 +193,8 @@ export class Instrumentation {
       if (!url.pathname.endsWith("/chat/completions")) return fetcher(input, init);
       const trace: RequestTrace = {
         request: this.state.requests.length + 1,
+        context: observeContext(init?.body),
+        generatedToolCalls: [],
         purpose: this.compacting ? "compaction" : "agent",
         startedAtMs: this.now(),
         endedAtMs: null,

package/src/benchmark/report.ts

@@ -1,4 +1,5 @@
 import { distribution } from "./metrics.ts";
+import { requestReport } from "./request-report.ts";
 import type { Results, RunResult } from "./types.ts";
 
 /** Client-observed first generated content on the first agent request, not worker startup. */
@@ -106,6 +107,13 @@ export function terminalReport(results: Results): string {
         ]
       : []),
     "",
+    ...results.runs.flatMap(requestReport),
+    "",
+    "Request tokens are provider-reported per HTTP attempt, including retries and compaction when observed.",
+    "Δ input is the change from the preceding request, not tokens attributable to its tools.",
+    "Context sizes are measured UTF-8 JSON bytes including message wrappers; category token counts are unavailable.",
+    "Tool text sizes exclude images and may already be truncated by Pi. Commands, arguments, and successful output text are not stored.",
+    "Input amplification = cumulative input / last agent request input; not a waste score or final context size.",
     "Detailed per-run measurements and validator output: results.json.",
   ].join("\n");
 }

package/src/benchmark/request-report.ts

@@ -0,0 +1,130 @@
+import type { RunResult } from "./types.ts";
+
+const number = (value: number | null | undefined) =>
+  value == null ? "n/a" : value.toLocaleString("en-US");
+// biome-ignore lint/suspicious/noControlCharactersInRegex: remove terminal control characters from labels
+const label = (value: string) => value.replace(/[\x00-\x1f\x7f-\x9f]/g, " ").slice(0, 80);
+const table = (rows: string[][]) => {
+  const widths =
+    rows[0]?.map((_, index) => Math.max(...rows.map((row) => row[index]?.length ?? 0))) ?? [];
+  return rows.map((row) =>
+    row
+      .map((cell, index) => cell.padEnd(widths[index] ?? 0))
+      .join(" │ ")
+      .trimEnd(),
+  );
+};
+
+export function requestReport(run: RunResult): string[] {
+  const requests = run.observation.requests;
+  const tools = run.observation.tools ?? [];
+  const rows = [["Request", "Kind", "Input", "Δ input", "Output", "Cached", "Tools produced"]];
+  const sizes = [
+    ["Request", "System B", "Schemas B", "User B", "Assistant B", "Tool results B", "Other B"],
+  ];
+  for (const [index, request] of requests.entries()) {
+    const input = request.usage?.inputTokens;
+    const previous = requests[index - 1]?.usage?.inputTokens;
+    const delta = input != null && previous != null ? input - previous : null;
+    rows.push([
+      String(request.request ?? index + 1),
+      request.purpose,
+      number(input),
+      delta !== null && delta > 0 ? `+${number(delta)}` : number(delta),
+      number(request.usage?.outputTokens),
+      number(request.usage?.cachedInputTokens),
+      request.generatedToolCalls?.length
+        ? request.generatedToolCalls.map((tool) => label(tool.name)).join(", ")
+        : "—",
+    ]);
+    const bytes = request.context?.bytes;
+    sizes.push([
+      String(request.request ?? index + 1),
+      ...[
+        bytes?.system,
+        bytes?.tools,
+        bytes?.user,
+        bytes?.assistant,
+        bytes?.toolResults,
+        bytes?.other,
+      ].map(number),
+    ]);
+  }
+  const input = run.tokens.cumulativeInputTokens;
+  const output = run.tokens.cumulativeOutputTokens;
+  rows.push([
+    "TOTAL",
+    "all attempts",
+    number(input),
+    "—",
+    number(output),
+    number(run.tokens.cachedInputTokens),
+    "",
+  ]);
+  const last = run.tokens.lastRequestInputTokens;
+  const amplification = run.tokens.inputAmplificationVsLastRequest;
+  const total = input != null && output != null ? input + output : null;
+  const consumption =
+    run.success && run.validation.checked
+      ? "Tokens to validated solution"
+      : run.failure !== null
+        ? "Tokens consumed before failure"
+        : "Tokens consumed (correctness not checked)";
+  const lines = [
+    "",
+    `Request trace: ${label(run.model)} #${run.run}`,
+    ...table(rows),
+    `${consumption}: ${number(total)} (input ${number(input)} + output ${number(output)}).`,
+    `Last agent request input: ${number(last)}; input amplification: ${amplification == null ? "n/a" : `${amplification.toFixed(2)}x`}.`,
+    "",
+    "Context composition — measured JSON bytes (B), NOT token attribution:",
+    ...table(sizes),
+  ];
+  if (tools.length) {
+    lines.push("", "Tool results — model-facing text after Pi processing:");
+    const toolRows = [
+      [
+        "Tool",
+        "From request",
+        "Text bytes",
+        "Lines",
+        "Included in requests",
+        "Repeated arguments",
+        "Same result",
+      ],
+    ];
+    for (const [index, tool] of tools.entries()) {
+      const included = requests
+        .filter((request) => request.context?.toolResults.some((result) => result.id === tool.id))
+        .map((request) => request.request);
+      const reference = (id: string | null | undefined) => {
+        const at = tools.findIndex((item) => item.id === id);
+        return at < 0 ? "—" : `T${at + 1}`;
+      };
+      toolRows.push([
+        `T${index + 1} ${label(tool.name)}`,
+        number(tool.request),
+        number(tool.output?.bytes),
+        number(tool.output?.lines),
+        requests.every((request) => request.context != null)
+          ? included.join(", ") || "none"
+          : `${included.join(", ") || "none observed"} (partial)`,
+        reference(tool.repeatedArgumentsOf),
+        reference(tool.repeatedOutputOf),
+      ]);
+    }
+    lines.push(...table(toolRows));
+    const large = tools
+      .map((tool, index) => ({ tool, index }))
+      .filter(({ tool }) => (tool.output?.bytes ?? 0) >= 16384);
+    for (const { tool, index } of large)
+      lines.push(
+        `Observation: T${index + 1} returned ${number(tool.output?.bytes)} text bytes (large-output threshold: 16 KiB).`,
+      );
+    if (tools.some((tool) => tool.repeatedArgumentsOf))
+      lines.push(
+        "Repeated arguments/results are candidates to inspect, not proof of unnecessary work or unchanged files.",
+      );
+  }
+  return lines;
+}

package/src/benchmark/types.ts

@@ -26,7 +26,22 @@ export interface ReportedUsage {
   reasoningTokens: number | null;
   totalTokens: number | null;
 }
+export interface RequestContext {
+  measurement: "serialized-json-bytes";
+  bytes: {
+    system: number;
+    tools: number;
+    user: number;
+    assistant: number;
+    toolResults: number;
+    other: number;
+  };
+  messageCount: number;
+  toolResults: Array<{ id: string; bytes: number; lines: number }>;
+}
 export interface RequestTrace {
+  context?: RequestContext | null;
+  generatedToolCalls?: Array<{ id: string; name: string }>;
   request: number;
   purpose: "agent" | "compaction";
   startedAtMs: number;
@@ -42,6 +57,11 @@ export interface RequestTrace {
   streamComplete: boolean;
 }
 export interface ToolTrace {
+  request?: number | null;
+  argumentsFingerprint?: string;
+  repeatedArgumentsOf?: string | null;
+  output?: { bytes: number; lines: number; fingerprint: string };
+  repeatedOutputOf?: string | null;
   id: string;
   name: string;
   startedAtMs: number | null;