whatbroke-cli 0.2.0

package/dist/chunk-PXR57ZGT.js

@@ -0,0 +1,1028 @@
+// src/diff.ts
+function stats(run) {
+  const models = [...new Set(run.llmCalls.map((c) => c.model))];
+  let inputTokens = 0;
+  let outputTokens = 0;
+  let costUsd = 0;
+  let latencyMs = 0;
+  for (const c of run.llmCalls) {
+    inputTokens += c.tokens?.input ?? 0;
+    outputTokens += c.tokens?.output ?? 0;
+    costUsd += c.cost_usd ?? 0;
+    latencyMs += c.latency_ms ?? 0;
+  }
+  for (const t of run.toolCalls) {
+    latencyMs += t.latency_ms ?? 0;
+  }
+  return {
+    llmCalls: run.llmCalls.length,
+    toolCalls: run.toolCalls.length,
+    inputTokens,
+    outputTokens,
+    costUsd,
+    latencyMs,
+    models
+  };
+}
+function alignTools(a, b) {
+  const n = a.length;
+  const m = b.length;
+  const dp = Array.from({ length: n + 1 }, () => new Array(m + 1).fill(0));
+  for (let i2 = n - 1; i2 >= 0; i2--) {
+    for (let j2 = m - 1; j2 >= 0; j2--) {
+      dp[i2][j2] = a[i2].name === b[j2].name ? dp[i2 + 1][j2 + 1] + 1 : Math.max(dp[i2 + 1][j2], dp[i2][j2 + 1]);
+    }
+  }
+  const pairs = [];
+  let i = 0;
+  let j = 0;
+  while (i < n && j < m) {
+    if (a[i].name === b[j].name) {
+      pairs.push([i, j]);
+      i++;
+      j++;
+    } else if (dp[i + 1][j] >= dp[i][j + 1]) {
+      i++;
+    } else {
+      j++;
+    }
+  }
+  return pairs;
+}
+function changedKeys(before = {}, after = {}) {
+  const keys = /* @__PURE__ */ new Set([...Object.keys(before), ...Object.keys(after)]);
+  const changed = [];
+  for (const key of keys) {
+    if (JSON.stringify(before[key]) !== JSON.stringify(after[key])) changed.push(key);
+  }
+  return changed;
+}
+function diffRun(runId, before, after, opts) {
+  const findings = [];
+  const add = (f) => findings.push(f);
+  if (before.status === "ok" && after.status === "error") {
+    add({
+      severity: "breaking",
+      kind: "status_changed",
+      subject: "status",
+      run: runId,
+      message: `run now fails${after.error ? `: ${after.error}` : ""}`
+    });
+  } else if (before.status === "error" && after.status === "ok") {
+    add({
+      severity: "info",
+      kind: "status_changed",
+      subject: "status",
+      run: runId,
+      message: "run now succeeds (was failing)"
+    });
+  }
+  const sBefore = stats(before);
+  const sAfter = stats(after);
+  if (sBefore.models.length && sAfter.models.length && sBefore.models.join(",") !== sAfter.models.join(",")) {
+    add({
+      severity: "info",
+      kind: "model_changed",
+      subject: "model",
+      run: runId,
+      message: `model: ${sBefore.models.join(", ")} -> ${sAfter.models.join(", ")}`
+    });
+  }
+  const beforeErrors = new Set(before.toolCalls.filter((t) => t.error).map((t) => t.name));
+  for (const t of after.toolCalls) {
+    if (t.error && !beforeErrors.has(t.name)) {
+      add({
+        severity: "breaking",
+        kind: "tool_error",
+        subject: t.name,
+        run: runId,
+        message: `tool ${t.name} now errors: ${t.error}`
+      });
+    }
+  }
+  const orderBefore = before.toolCalls.map((t) => t.name).join(" > ");
+  const orderAfter = after.toolCalls.map((t) => t.name).join(" > ");
+  const sameToolSet = [...before.toolCalls.map((t) => t.name)].sort().join("\n") === [...after.toolCalls.map((t) => t.name)].sort().join("\n");
+  let pairs;
+  if (sameToolSet) {
+    if (orderBefore !== orderAfter) {
+      add({
+        severity: "warning",
+        kind: "tool_reordered",
+        subject: "tool-order",
+        run: runId,
+        message: `tool order changed: ${orderBefore} -> ${orderAfter}`
+      });
+    }
+    pairs = [];
+    const seen = /* @__PURE__ */ new Map();
+    for (let i = 0; i < before.toolCalls.length; i++) {
+      const name = before.toolCalls[i].name;
+      const nth = seen.get(name) ?? 0;
+      seen.set(name, nth + 1);
+      let count = 0;
+      for (let j = 0; j < after.toolCalls.length; j++) {
+        if (after.toolCalls[j].name === name && count++ === nth) {
+          pairs.push([i, j]);
+          break;
+        }
+      }
+    }
+  } else {
+    pairs = alignTools(before.toolCalls, after.toolCalls);
+    const matchedBefore = new Set(pairs.map(([i]) => i));
+    const matchedAfter = new Set(pairs.map(([, j]) => j));
+    for (let i = 0; i < before.toolCalls.length; i++) {
+      if (!matchedBefore.has(i)) {
+        add({
+          severity: "breaking",
+          kind: "tool_removed",
+          subject: before.toolCalls[i].name,
+          run: runId,
+          message: `tool call dropped: ${before.toolCalls[i].name}`,
+          detail: { args: before.toolCalls[i].args }
+        });
+      }
+    }
+    for (let j = 0; j < after.toolCalls.length; j++) {
+      if (!matchedAfter.has(j)) {
+        add({
+          severity: "warning",
+          kind: "tool_added",
+          subject: after.toolCalls[j].name,
+          run: runId,
+          message: `new tool call: ${after.toolCalls[j].name}`,
+          detail: { args: after.toolCalls[j].args }
+        });
+      }
+    }
+  }
+  for (const [i, j] of pairs) {
+    const keys = changedKeys(before.toolCalls[i].args, after.toolCalls[j].args);
+    if (keys.length) {
+      add({
+        severity: "warning",
+        kind: "tool_args_changed",
+        subject: before.toolCalls[i].name,
+        run: runId,
+        message: `${before.toolCalls[i].name} called with different args (${keys.join(", ")})`,
+        detail: {
+          tool: before.toolCalls[i].name,
+          before: pick(before.toolCalls[i].args, keys),
+          after: pick(after.toolCalls[j].args, keys)
+        }
+      });
+    }
+  }
+  if (opts.compareOutputs) {
+    const outBefore = before.outputs.map((o) => o.content).join("\n");
+    const outAfter = after.outputs.map((o) => o.content).join("\n");
+    if (outBefore && !outAfter) {
+      add({
+        severity: "breaking",
+        kind: "output_missing",
+        subject: "output",
+        run: runId,
+        message: "run no longer produces an output"
+      });
+    } else if (outBefore !== outAfter) {
+      add({
+        severity: "warning",
+        kind: "output_changed",
+        subject: "output",
+        run: runId,
+        message: "final output changed",
+        detail: { before: truncate(outBefore), after: truncate(outAfter) }
+      });
+    }
+  }
+  if (sBefore.latencyMs > 0 && sAfter.latencyMs > sBefore.latencyMs * opts.latencyThreshold) {
+    const pct = Math.round((sAfter.latencyMs / sBefore.latencyMs - 1) * 100);
+    add({
+      severity: "warning",
+      kind: "latency_regression",
+      subject: "latency",
+      run: runId,
+      message: `latency up ${pct}% (${fmtMs(sBefore.latencyMs)} -> ${fmtMs(sAfter.latencyMs)})`
+    });
+  }
+  if (sBefore.costUsd > 0 && sAfter.costUsd > sBefore.costUsd * opts.costThreshold) {
+    const pct = Math.round((sAfter.costUsd / sBefore.costUsd - 1) * 100);
+    add({
+      severity: "warning",
+      kind: "cost_increase",
+      subject: "cost",
+      run: runId,
+      message: `cost up ${pct}% ($${sBefore.costUsd.toFixed(4)} -> $${sAfter.costUsd.toFixed(4)})`
+    });
+  }
+  const tokBefore = sBefore.inputTokens + sBefore.outputTokens;
+  const tokAfter = sAfter.inputTokens + sAfter.outputTokens;
+  if (tokBefore > 0 && (tokAfter > tokBefore * 2 || tokAfter < tokBefore * 0.5)) {
+    add({
+      severity: "info",
+      kind: "token_change",
+      subject: "tokens",
+      run: runId,
+      message: `total tokens ${tokBefore} -> ${tokAfter}`
+    });
+  }
+  return { run: runId, findings, before: sBefore, after: sAfter };
+}
+function pick(obj = {}, keys) {
+  const out = {};
+  for (const k of keys) if (k in obj) out[k] = obj[k];
+  return out;
+}
+function truncate(s, max = 300) {
+  return s.length > max ? s.slice(0, max) + "\u2026" : s;
+}
+function fmtMs(ms) {
+  return ms >= 1e3 ? `${(ms / 1e3).toFixed(1)}s` : `${Math.round(ms)}ms`;
+}
+function diffTraces(before, after, options = {}) {
+  const opts = {
+    latencyThreshold: options.latencyThreshold ?? 1.5,
+    costThreshold: options.costThreshold ?? 1.25,
+    compareOutputs: options.compareOutputs ?? true
+  };
+  const runDiffs = [];
+  const allRuns = /* @__PURE__ */ new Set([...before.keys(), ...after.keys()]);
+  for (const id of allRuns) {
+    const a = before.get(id);
+    const b = after.get(id);
+    if (a && !b) {
+      runDiffs.push({
+        run: id,
+        findings: [
+          {
+            severity: "breaking",
+            kind: "run_missing",
+            subject: "run",
+            run: id,
+            message: "run missing from the new trace"
+          }
+        ],
+        before: stats(a)
+      });
+    } else if (!a && b) {
+      runDiffs.push({
+        run: id,
+        findings: [
+          {
+            severity: "info",
+            kind: "run_added",
+            subject: "run",
+            run: id,
+            message: "new run, nothing to compare against"
+          }
+        ],
+        after: stats(b)
+      });
+    } else if (a && b) {
+      runDiffs.push(diffRun(id, a, b, opts));
+    }
+  }
+  const findings = runDiffs.flatMap((r) => r.findings);
+  return {
+    runs: runDiffs,
+    findings,
+    breaking: findings.filter((f) => f.severity === "breaking").length,
+    warnings: findings.filter((f) => f.severity === "warning").length,
+    info: findings.filter((f) => f.severity === "info").length
+  };
+}
+
+// src/samples.ts
+var SAMPLE_RE = /^(.+)#(\d+)$/;
+function hasSamples(runs) {
+  for (const id of runs.keys()) if (SAMPLE_RE.test(id)) return true;
+  return false;
+}
+function groupSamples(runs) {
+  const groups = /* @__PURE__ */ new Map();
+  for (const [id, run] of runs) {
+    const match = SAMPLE_RE.exec(id);
+    const base = match ? match[1] : id;
+    const list = groups.get(base) ?? [];
+    list.push(run);
+    groups.set(base, list);
+  }
+  return groups;
+}
+function findingKey(f) {
+  return `${f.kind}:${f.subject ?? f.message}`;
+}
+function pairFindings(a, b, base, options) {
+  const result = diffTraces(/* @__PURE__ */ new Map([[base, a]]), /* @__PURE__ */ new Map([[base, b]]), options);
+  return result.findings;
+}
+function meanStats(samples) {
+  if (!samples.length) return void 0;
+  const n = samples.length;
+  const sum = (fn) => samples.reduce((t, s) => t + fn(s), 0);
+  return {
+    llmCalls: Math.round(sum((s) => s.llmCalls) / n),
+    toolCalls: Math.round(sum((s) => s.toolCalls) / n),
+    inputTokens: Math.round(sum((s) => s.inputTokens) / n),
+    outputTokens: Math.round(sum((s) => s.outputTokens) / n),
+    costUsd: sum((s) => s.costUsd) / n,
+    latencyMs: sum((s) => s.latencyMs) / n,
+    models: [...new Set(samples.flatMap((s) => s.models))]
+  };
+}
+function statsOf(run, options) {
+  const result = diffTraces(/* @__PURE__ */ new Map([["x", run]]), /* @__PURE__ */ new Map([["x", run]]), options);
+  return result.runs[0].before;
+}
+function diffSampledRun(base, beforeSamples, afterSamples, options) {
+  const baselineKeys = /* @__PURE__ */ new Set();
+  for (let i = 0; i < beforeSamples.length; i++) {
+    for (let j = i + 1; j < beforeSamples.length; j++) {
+      for (const f of pairFindings(beforeSamples[i], beforeSamples[j], base, options)) {
+        baselineKeys.add(findingKey(f));
+      }
+    }
+  }
+  const totalPairs = beforeSamples.length * afterSamples.length;
+  const counts = /* @__PURE__ */ new Map();
+  for (const a of beforeSamples) {
+    for (const b of afterSamples) {
+      const seen = /* @__PURE__ */ new Set();
+      for (const f of pairFindings(a, b, base, options)) {
+        const key = findingKey(f);
+        if (seen.has(key)) continue;
+        seen.add(key);
+        const entry = counts.get(key);
+        if (entry) entry.count++;
+        else counts.set(key, { finding: f, count: 1 });
+      }
+    }
+  }
+  const findings = [];
+  for (const [key, { finding, count }] of counts) {
+    const f = { ...finding, rate: `${count}/${totalPairs}` };
+    if (baselineKeys.has(key)) {
+      f.flaky = true;
+      if (f.severity !== "info") f.severity = "info";
+    } else if (count / totalPairs < 0.5 && f.severity === "breaking") {
+      f.severity = "warning";
+    }
+    findings.push(f);
+  }
+  const order = { breaking: 0, warning: 1, info: 2 };
+  findings.sort((a, b) => order[a.severity] - order[b.severity]);
+  return {
+    run: base,
+    findings,
+    before: meanStats(beforeSamples.map((r) => statsOf(r, options))),
+    after: meanStats(afterSamples.map((r) => statsOf(r, options)))
+  };
+}
+function diffTracesSampled(before, after, options = {}) {
+  const beforeGroups = groupSamples(before);
+  const afterGroups = groupSamples(after);
+  const runDiffs = [];
+  const bases = /* @__PURE__ */ new Set([...beforeGroups.keys(), ...afterGroups.keys()]);
+  for (const base of bases) {
+    const beforeSamples = beforeGroups.get(base) ?? [];
+    const afterSamples = afterGroups.get(base) ?? [];
+    if (beforeSamples.length && !afterSamples.length) {
+      runDiffs.push({
+        run: base,
+        findings: [
+          {
+            severity: "breaking",
+            kind: "run_missing",
+            subject: "run",
+            run: base,
+            message: "run missing from the new trace"
+          }
+        ],
+        before: meanStats(beforeSamples.map((r) => statsOf(r, options)))
+      });
+    } else if (!beforeSamples.length && afterSamples.length) {
+      runDiffs.push({
+        run: base,
+        findings: [
+          {
+            severity: "info",
+            kind: "run_added",
+            subject: "run",
+            run: base,
+            message: "new run, nothing to compare against"
+          }
+        ],
+        after: meanStats(afterSamples.map((r) => statsOf(r, options)))
+      });
+    } else {
+      runDiffs.push(diffSampledRun(base, beforeSamples, afterSamples, options));
+    }
+  }
+  const findings = runDiffs.flatMap((r) => r.findings);
+  return {
+    runs: runDiffs,
+    findings,
+    breaking: findings.filter((f) => f.severity === "breaking").length,
+    warnings: findings.filter((f) => f.severity === "warning").length,
+    info: findings.filter((f) => f.severity === "info").length
+  };
+}
+
+// src/parse.ts
+import { readFileSync } from "fs";
+function parseTrace(text) {
+  const runs = /* @__PURE__ */ new Map();
+  const get = (id) => {
+    let run = runs.get(id);
+    if (!run) {
+      run = {
+        id,
+        meta: {},
+        llmCalls: [],
+        toolCalls: [],
+        outputs: [],
+        status: "unknown"
+      };
+      runs.set(id, run);
+    }
+    return run;
+  };
+  const lines = text.split("\n");
+  for (let i = 0; i < lines.length; i++) {
+    const line = lines[i].trim();
+    if (!line) continue;
+    let event;
+    try {
+      event = JSON.parse(line);
+    } catch {
+      throw new Error(`invalid JSON on line ${i + 1}`);
+    }
+    if (!event || typeof event !== "object" || !("type" in event)) {
+      throw new Error(`line ${i + 1} is not a trace event (missing "type")`);
+    }
+    const runId = "run" in event && event.run ? String(event.run) : "default";
+    const run = get(runId);
+    switch (event.type) {
+      case "run_start":
+        run.meta = { ...run.meta, ...event.meta };
+        break;
+      case "llm_call":
+        run.llmCalls.push(event);
+        break;
+      case "tool_call":
+        run.toolCalls.push(event);
+        break;
+      case "output":
+        run.outputs.push(event);
+        break;
+      case "run_end":
+        run.status = event.status;
+        if (event.error) run.error = event.error;
+        break;
+      default:
+        break;
+    }
+  }
+  for (const run of runs.values()) {
+    if (run.status === "unknown" && (run.llmCalls.length || run.toolCalls.length || run.outputs.length)) {
+      run.status = "ok";
+    }
+  }
+  return runs;
+}
+function loadTrace(path) {
+  let text;
+  try {
+    text = readFileSync(path, "utf8");
+  } catch {
+    throw new Error(`could not read trace file: ${path}`);
+  }
+  try {
+    return parseTrace(text);
+  } catch (err) {
+    throw new Error(`${path}: ${err instanceof Error ? err.message : String(err)}`);
+  }
+}
+
+// src/record.ts
+import { appendFileSync, mkdirSync } from "fs";
+import { dirname } from "path";
+var Recorder = class {
+  file;
+  run;
+  constructor(options) {
+    this.file = options.file;
+    this.run = options.run ?? "default";
+    mkdirSync(dirname(this.file) || ".", { recursive: true });
+    this.write({ type: "run_start", run: this.run, ts: Date.now(), meta: options.meta });
+  }
+  write(event) {
+    appendFileSync(this.file, JSON.stringify(event) + "\n");
+  }
+  llmCall(data) {
+    this.write({
+      type: "llm_call",
+      run: this.run,
+      ts: Date.now(),
+      model: data.model,
+      latency_ms: data.latencyMs,
+      tokens: { input: data.inputTokens, output: data.outputTokens },
+      cost_usd: data.costUsd,
+      stop_reason: data.stopReason,
+      error: data.error
+    });
+  }
+  toolCall(name, args, data) {
+    this.write({
+      type: "tool_call",
+      run: this.run,
+      ts: Date.now(),
+      name,
+      args,
+      latency_ms: data?.latencyMs,
+      error: data?.error
+    });
+  }
+  output(content) {
+    this.write({ type: "output", run: this.run, ts: Date.now(), content });
+  }
+  end(status = "ok", error) {
+    this.write({ type: "run_end", run: this.run, ts: Date.now(), status, error });
+  }
+  /**
+   * Wraps an OpenAI client (openai npm package) so every
+   * chat.completions.create call is recorded, including tool calls the
+   * model requested. Returns the same client.
+   */
+  wrapOpenAI(client) {
+    const anyClient = client;
+    const completions = anyClient?.chat?.completions;
+    if (!completions?.create) {
+      throw new Error("wrapOpenAI expects an OpenAI client with chat.completions.create");
+    }
+    const original = completions.create.bind(completions);
+    const recorder = this;
+    completions.create = async function(params, ...rest) {
+      const started = Date.now();
+      try {
+        const response = await original(params, ...rest);
+        recorder.llmCall({
+          model: response?.model ?? params?.model ?? "unknown",
+          latencyMs: Date.now() - started,
+          inputTokens: response?.usage?.prompt_tokens,
+          outputTokens: response?.usage?.completion_tokens,
+          stopReason: response?.choices?.[0]?.finish_reason
+        });
+        const toolCalls = response?.choices?.[0]?.message?.tool_calls ?? [];
+        for (const call of toolCalls) {
+          recorder.toolCall(call?.function?.name ?? "unknown", safeParse(call?.function?.arguments));
+        }
+        return response;
+      } catch (err) {
+        recorder.llmCall({
+          model: params?.model ?? "unknown",
+          latencyMs: Date.now() - started,
+          error: err instanceof Error ? err.message : String(err)
+        });
+        throw err;
+      }
+    };
+    return client;
+  }
+  /**
+   * Wraps an Anthropic client (@anthropic-ai/sdk) so every messages.create
+   * call is recorded, including tool_use blocks. Returns the same client.
+   */
+  wrapAnthropic(client) {
+    const anyClient = client;
+    const messages = anyClient?.messages;
+    if (!messages?.create) {
+      throw new Error("wrapAnthropic expects an Anthropic client with messages.create");
+    }
+    const original = messages.create.bind(messages);
+    const recorder = this;
+    messages.create = async function(params, ...rest) {
+      const started = Date.now();
+      try {
+        const response = await original(params, ...rest);
+        recorder.llmCall({
+          model: response?.model ?? params?.model ?? "unknown",
+          latencyMs: Date.now() - started,
+          inputTokens: response?.usage?.input_tokens,
+          outputTokens: response?.usage?.output_tokens,
+          stopReason: response?.stop_reason
+        });
+        const blocks = Array.isArray(response?.content) ? response.content : [];
+        for (const block of blocks) {
+          if (block?.type === "tool_use") {
+            recorder.toolCall(block.name ?? "unknown", block.input ?? {});
+          }
+        }
+        return response;
+      } catch (err) {
+        recorder.llmCall({
+          model: params?.model ?? "unknown",
+          latencyMs: Date.now() - started,
+          error: err instanceof Error ? err.message : String(err)
+        });
+        throw err;
+      }
+    };
+    return client;
+  }
+};
+function safeParse(json) {
+  if (typeof json !== "string") return void 0;
+  try {
+    const parsed = JSON.parse(json);
+    return typeof parsed === "object" && parsed !== null ? parsed : void 0;
+  } catch {
+    return void 0;
+  }
+}
+
+// src/proxy.ts
+import http from "http";
+var STRIP_REQUEST_HEADERS = /* @__PURE__ */ new Set([
+  "host",
+  "connection",
+  "content-length",
+  "accept-encoding",
+  "x-whatbroke-run"
+]);
+var STRIP_RESPONSE_HEADERS = /* @__PURE__ */ new Set([
+  "content-length",
+  "content-encoding",
+  "transfer-encoding",
+  "connection"
+]);
+function startProxy(options) {
+  const recorders = /* @__PURE__ */ new Map();
+  const recorderFor = (run) => {
+    let rec = recorders.get(run);
+    if (!rec) {
+      rec = new Recorder({ file: options.file, run });
+      recorders.set(run, rec);
+    }
+    return rec;
+  };
+  const server = http.createServer(async (req, res) => {
+    const chunks = [];
+    for await (const chunk of req) chunks.push(chunk);
+    const body = Buffer.concat(chunks);
+    const path = req.url ?? "/";
+    const anthropic = path.startsWith("/v1/messages");
+    const origin = options.target ?? (anthropic ? "https://api.anthropic.com" : "https://api.openai.com");
+    const headers = {};
+    for (const [key, value] of Object.entries(req.headers)) {
+      if (typeof value === "string" && !STRIP_REQUEST_HEADERS.has(key.toLowerCase())) {
+        headers[key] = value;
+      }
+    }
+    headers["accept-encoding"] = "identity";
+    const runHeader = req.headers["x-whatbroke-run"];
+    const run = typeof runHeader === "string" && runHeader || options.run || "default";
+    const started = Date.now();
+    let upstream;
+    try {
+      upstream = await fetch(origin + path, {
+        method: req.method,
+        headers,
+        body: body.length ? body : void 0
+      });
+    } catch (err) {
+      const message = err instanceof Error ? err.message : String(err);
+      recorderFor(run).llmCall({ model: "unknown", latencyMs: Date.now() - started, error: message });
+      res.writeHead(502, { "content-type": "application/json" });
+      res.end(JSON.stringify({ error: { message: `whatbroke proxy: upstream unreachable: ${message}` } }));
+      return;
+    }
+    const responseHeaders = {};
+    upstream.headers.forEach((value, key) => {
+      if (!STRIP_RESPONSE_HEADERS.has(key.toLowerCase())) responseHeaders[key] = value;
+    });
+    const contentType = upstream.headers.get("content-type") ?? "";
+    const streaming = contentType.includes("text/event-stream");
+    let text = "";
+    if (streaming && upstream.body) {
+      res.writeHead(upstream.status, responseHeaders);
+      const decoder = new TextDecoder();
+      for await (const chunk of upstream.body) {
+        res.write(chunk);
+        text += decoder.decode(chunk, { stream: true });
+      }
+      text += decoder.decode();
+      res.end();
+    } else {
+      text = await upstream.text();
+      res.writeHead(upstream.status, responseHeaders);
+      res.end(text);
+    }
+    if (upstream.status >= 400) {
+      recorderFor(run).llmCall({
+        model: "unknown",
+        latencyMs: Date.now() - started,
+        error: `upstream ${upstream.status}: ${truncate2(text)}`
+      });
+      return;
+    }
+    let summary = null;
+    try {
+      if (streaming) {
+        const events = parseSse(text);
+        summary = anthropic ? summarizeAnthropicStream(events) : summarizeOpenAIStream(events);
+      } else {
+        const parsed = JSON.parse(text);
+        summary = anthropic ? summarizeAnthropic(parsed) : summarizeOpenAI(parsed);
+      }
+    } catch {
+    }
+    if (!summary) return;
+    const rec = recorderFor(run);
+    rec.llmCall({
+      model: summary.model,
+      latencyMs: Date.now() - started,
+      inputTokens: summary.inputTokens,
+      outputTokens: summary.outputTokens,
+      stopReason: summary.stopReason
+    });
+    for (const call of summary.toolCalls) rec.toolCall(call.name, call.args);
+    if (!summary.toolCalls.length && summary.text.trim()) rec.output(summary.text);
+    options.onRecord?.(run, summary.model, summary.toolCalls.map((c) => c.name));
+  });
+  let closing;
+  return new Promise((resolve, reject) => {
+    server.once("error", reject);
+    server.listen(options.port ?? 4141, "127.0.0.1", () => {
+      const port = server.address().port;
+      resolve({
+        port,
+        url: `http://127.0.0.1:${port}`,
+        close: () => {
+          closing ??= new Promise((done, fail) => {
+            for (const rec of recorders.values()) rec.end("ok");
+            server.close((err) => err ? fail(err) : done());
+          });
+          return closing;
+        }
+      });
+    });
+  });
+}
+function parseSse(text) {
+  const events = [];
+  for (const line of text.split(/\r?\n/)) {
+    if (!line.startsWith("data:")) continue;
+    const data = line.slice(5).trim();
+    if (!data || data === "[DONE]") continue;
+    try {
+      events.push(JSON.parse(data));
+    } catch {
+    }
+  }
+  return events;
+}
+function summarizeOpenAI(body) {
+  const choice = body?.choices?.[0];
+  if (!choice) return null;
+  const message = choice.message ?? {};
+  const toolCalls = (message.tool_calls ?? []).map((c) => ({
+    name: c?.function?.name ?? "unknown",
+    args: safeParse2(c?.function?.arguments)
+  }));
+  return {
+    model: body.model ?? "unknown",
+    inputTokens: body.usage?.prompt_tokens,
+    outputTokens: body.usage?.completion_tokens,
+    stopReason: choice.finish_reason ?? void 0,
+    toolCalls,
+    text: typeof message.content === "string" ? message.content : ""
+  };
+}
+function summarizeOpenAIStream(events) {
+  if (!events.length) return null;
+  let model = "unknown";
+  let text = "";
+  let stopReason;
+  let inputTokens;
+  let outputTokens;
+  const tools = /* @__PURE__ */ new Map();
+  for (const event of events) {
+    if (event?.model) model = event.model;
+    if (event?.usage) {
+      inputTokens = event.usage.prompt_tokens ?? inputTokens;
+      outputTokens = event.usage.completion_tokens ?? outputTokens;
+    }
+    const choice = event?.choices?.[0];
+    if (!choice) continue;
+    if (choice.finish_reason) stopReason = choice.finish_reason;
+    const delta = choice.delta ?? {};
+    if (typeof delta.content === "string") text += delta.content;
+    for (const call of delta.tool_calls ?? []) {
+      const index = call?.index ?? 0;
+      const entry = tools.get(index) ?? { name: "unknown", args: "" };
+      if (call?.function?.name) entry.name = call.function.name;
+      if (call?.function?.arguments) entry.args += call.function.arguments;
+      tools.set(index, entry);
+    }
+  }
+  return {
+    model,
+    inputTokens,
+    outputTokens,
+    stopReason,
+    toolCalls: [...tools.values()].map((t) => ({ name: t.name, args: safeParse2(t.args) })),
+    text
+  };
+}
+function summarizeAnthropic(body) {
+  if (body?.type !== "message") return null;
+  const blocks = Array.isArray(body.content) ? body.content : [];
+  const toolCalls = blocks.filter((b) => b?.type === "tool_use").map((b) => ({ name: b.name ?? "unknown", args: b.input ?? {} }));
+  const text = blocks.filter((b) => b?.type === "text").map((b) => b.text ?? "").join("");
+  return {
+    model: body.model ?? "unknown",
+    inputTokens: body.usage?.input_tokens,
+    outputTokens: body.usage?.output_tokens,
+    stopReason: body.stop_reason ?? void 0,
+    toolCalls,
+    text
+  };
+}
+function summarizeAnthropicStream(events) {
+  if (!events.length) return null;
+  let model = "unknown";
+  let text = "";
+  let stopReason;
+  let inputTokens;
+  let outputTokens;
+  const tools = /* @__PURE__ */ new Map();
+  for (const event of events) {
+    switch (event?.type) {
+      case "message_start":
+        model = event.message?.model ?? model;
+        inputTokens = event.message?.usage?.input_tokens ?? inputTokens;
+        break;
+      case "content_block_start":
+        if (event.content_block?.type === "tool_use") {
+          tools.set(event.index ?? 0, { name: event.content_block.name ?? "unknown", json: "" });
+        }
+        break;
+      case "content_block_delta":
+        if (event.delta?.type === "text_delta") text += event.delta.text ?? "";
+        if (event.delta?.type === "input_json_delta") {
+          const entry = tools.get(event.index ?? 0);
+          if (entry) entry.json += event.delta.partial_json ?? "";
+        }
+        break;
+      case "message_delta":
+        stopReason = event.delta?.stop_reason ?? stopReason;
+        outputTokens = event.usage?.output_tokens ?? outputTokens;
+        break;
+    }
+  }
+  return {
+    model,
+    inputTokens,
+    outputTokens,
+    stopReason,
+    toolCalls: [...tools.values()].map((t) => ({ name: t.name, args: safeParse2(t.json) ?? {} })),
+    text
+  };
+}
+function safeParse2(json) {
+  if (typeof json !== "string" || !json) return void 0;
+  try {
+    const parsed = JSON.parse(json);
+    return typeof parsed === "object" && parsed !== null ? parsed : void 0;
+  } catch {
+    return void 0;
+  }
+}
+function truncate2(s, max = 200) {
+  return s.length > max ? s.slice(0, max) + "\u2026" : s;
+}
+
+// src/report.ts
+import pc from "picocolors";
+var ICONS = {
+  breaking: "x",
+  warning: "!",
+  info: "i"
+};
+function color(severity, text) {
+  if (severity === "breaking") return pc.red(text);
+  if (severity === "warning") return pc.yellow(text);
+  return pc.dim(text);
+}
+function statLine(diff) {
+  const b = diff.before;
+  const a = diff.after;
+  if (!b || !a) return null;
+  const parts = [];
+  parts.push(`${b.toolCalls} -> ${a.toolCalls} tool calls`);
+  parts.push(`${b.llmCalls} -> ${a.llmCalls} llm calls`);
+  if (b.latencyMs || a.latencyMs) parts.push(`${fmtMs(b.latencyMs)} -> ${fmtMs(a.latencyMs)}`);
+  if (b.costUsd || a.costUsd)
+    parts.push(`$${b.costUsd.toFixed(4)} -> $${a.costUsd.toFixed(4)}`);
+  return parts.join("  \xB7  ");
+}
+function renderTerminal(result) {
+  const lines = [];
+  lines.push("");
+  for (const run of result.runs) {
+    const worst = worstSeverity(run.findings);
+    const badge = worst === "breaking" ? pc.bgRed(pc.white(" BROKE ")) : worst === "warning" ? pc.bgYellow(pc.black(" CHANGED ")) : run.findings.length ? pc.bgBlue(pc.white(" INFO ")) : pc.bgGreen(pc.black(" OK "));
+    lines.push(`${badge} ${pc.bold(run.run)}`);
+    const stat = statLine(run);
+    if (stat) lines.push(pc.dim(`  ${stat}`));
+    for (const f of run.findings) {
+      lines.push(color(f.severity, `  ${ICONS[f.severity]} ${f.message}${annotation(f)}`));
+      if (f.detail?.before !== void 0 || f.detail?.after !== void 0) {
+        if (f.detail.before !== void 0)
+          lines.push(pc.red(`      - ${compact(f.detail.before)}`));
+        if (f.detail.after !== void 0)
+          lines.push(pc.green(`      + ${compact(f.detail.after)}`));
+      }
+    }
+    lines.push("");
+  }
+  lines.push(summaryLine(result));
+  lines.push("");
+  return lines.join("\n");
+}
+function renderMarkdown(result) {
+  const lines = [];
+  lines.push("## whatbroke report");
+  lines.push("");
+  lines.push(summaryPlain(result));
+  lines.push("");
+  for (const run of result.runs) {
+    if (!run.findings.length) continue;
+    lines.push(`### ${run.run}`);
+    lines.push("");
+    for (const f of run.findings) {
+      const tag = f.severity === "breaking" ? "**BROKE**" : f.severity === "warning" ? "changed" : "info";
+      lines.push(`- ${tag}: ${f.message}${annotationPlain(f)}`);
+      if (f.detail?.before !== void 0) lines.push(`  - before: \`${compact(f.detail.before)}\``);
+      if (f.detail?.after !== void 0) lines.push(`  - after: \`${compact(f.detail.after)}\``);
+    }
+    lines.push("");
+  }
+  if (result.runs.every((r) => !r.findings.length)) {
+    lines.push("No behavioral changes detected.");
+    lines.push("");
+  }
+  return lines.join("\n");
+}
+function annotationPlain(f) {
+  if (!f.rate) return "";
+  return f.flaky ? ` (${f.rate} run pairs, also flaps in the baseline)` : ` (${f.rate} run pairs)`;
+}
+function annotation(f) {
+  const plain = annotationPlain(f);
+  return plain ? pc.dim(plain) : "";
+}
+function worstSeverity(findings) {
+  if (findings.some((f) => f.severity === "breaking")) return "breaking";
+  if (findings.some((f) => f.severity === "warning")) return "warning";
+  if (findings.length) return "info";
+  return null;
+}
+function compact(value) {
+  const s = typeof value === "string" ? value : JSON.stringify(value);
+  return s.length > 120 ? s.slice(0, 120) + "\u2026" : s;
+}
+function summaryPlain(result) {
+  if (!result.breaking && !result.warnings && !result.info) {
+    return "No behavioral changes detected.";
+  }
+  const parts = [];
+  if (result.breaking) parts.push(`${result.breaking} breaking`);
+  if (result.warnings) parts.push(`${result.warnings} changed`);
+  if (result.info) parts.push(`${result.info} info`);
+  return parts.join(", ");
+}
+function summaryLine(result) {
+  if (!result.breaking && !result.warnings && !result.info) {
+    return pc.green("nothing broke.");
+  }
+  const parts = [];
+  if (result.breaking) parts.push(pc.red(`${result.breaking} breaking`));
+  if (result.warnings) parts.push(pc.yellow(`${result.warnings} changed`));
+  if (result.info) parts.push(pc.dim(`${result.info} info`));
+  return parts.join(pc.dim("  \xB7  "));
+}
+
+export {
+  diffTraces,
+  hasSamples,
+  diffTracesSampled,
+  parseTrace,
+  loadTrace,
+  Recorder,
+  startProxy,
+  renderTerminal,
+  renderMarkdown
+};