whatbroke-cli 0.2.0

package/dist/cli.d.ts

@@ -0,0 +1 @@
+#!/usr/bin/env node

package/dist/cli.js

@@ -0,0 +1,186 @@
+#!/usr/bin/env node
+import {
+  diffTraces,
+  diffTracesSampled,
+  hasSamples,
+  loadTrace,
+  renderMarkdown,
+  renderTerminal,
+  startProxy
+} from "./chunk-PXR57ZGT.js";
+
+// src/cli.ts
+import pc from "picocolors";
+var HELP = `whatbroke - diff your AI agent's behavior between two runs
+
+usage:
+  whatbroke diff <before.jsonl> <after.jsonl> [options]
+  whatbroke record --out <trace.jsonl> [options]
+
+diff options:
+  --json              machine-readable output
+  --md                markdown output (drop it in a PR comment)
+  --fail-on <level>   exit 1 on: breaking (default), warning, never
+  --latency <ratio>   flag latency regressions above this ratio (default 1.5)
+  --cost <ratio>      flag cost increases above this ratio (default 1.25)
+  --no-outputs        skip comparing final outputs
+
+record options:
+  --out <file>        trace file to write (required)
+  --port <n>          port to listen on (default 4141)
+  --run <name>        run id when no x-whatbroke-run header is sent
+  --target <url>      forward everything to this origin instead
+
+record starts a local proxy. point your agent at it and run it unchanged:
+  OPENAI_BASE_URL=http://127.0.0.1:4141/v1     (openai sdk)
+  ANTHROPIC_BASE_URL=http://127.0.0.1:4141     (anthropic sdk)
+stop it with ctrl-c, then diff the trace against a baseline.
+
+nondeterministic agent? record each scenario a few times as name#1, name#2, ...
+and diff will report a flap rate per finding instead of noise.
+
+trace format: one JSON event per line
+  {"type":"run_start","run":"refund-flow","meta":{"model":"gpt-4o"}}
+  {"type":"llm_call","run":"refund-flow","model":"gpt-4o","latency_ms":900,"tokens":{"input":512,"output":128}}
+  {"type":"tool_call","run":"refund-flow","name":"lookup_order","args":{"order_id":"A-1042"}}
+  {"type":"output","run":"refund-flow","content":"Refund issued."}
+  {"type":"run_end","run":"refund-flow","status":"ok"}
+
+you can also record with the SDK (import { Recorder } from "whatbroke") or write
+the JSONL yourself from any language. Full docs: https://github.com/arthi-arumugam-git/whatbroke
+`;
+function fail(message) {
+  console.error(pc.red(`whatbroke: ${message}`));
+  console.error(pc.dim(`try: whatbroke --help`));
+  process.exit(2);
+}
+function main() {
+  const argv = process.argv.slice(2);
+  if (!argv.length || argv.includes("-h") || argv.includes("--help")) {
+    console.log(HELP);
+    process.exit(0);
+  }
+  const command = argv[0];
+  if (command === "record") {
+    runRecord(argv.slice(1));
+    return;
+  }
+  if (command !== "diff") fail(`unknown command: ${command}`);
+  const positional = [];
+  let format = "terminal";
+  let failOn = "breaking";
+  let latencyThreshold = 1.5;
+  let costThreshold = 1.25;
+  let compareOutputs = true;
+  for (let i = 1; i < argv.length; i++) {
+    const arg = argv[i];
+    switch (arg) {
+      case "--json":
+        format = "json";
+        break;
+      case "--md":
+        format = "md";
+        break;
+      case "--no-outputs":
+        compareOutputs = false;
+        break;
+      case "--fail-on": {
+        const value = argv[++i];
+        if (value !== "breaking" && value !== "warning" && value !== "never") {
+          fail(`--fail-on must be breaking, warning, or never`);
+        }
+        failOn = value;
+        break;
+      }
+      case "--latency": {
+        const value = Number(argv[++i]);
+        if (!Number.isFinite(value) || value <= 0) fail(`--latency needs a positive number`);
+        latencyThreshold = value;
+        break;
+      }
+      case "--cost": {
+        const value = Number(argv[++i]);
+        if (!Number.isFinite(value) || value <= 0) fail(`--cost needs a positive number`);
+        costThreshold = value;
+        break;
+      }
+      default:
+        if (arg.startsWith("-")) fail(`unknown option: ${arg}`);
+        positional.push(arg);
+    }
+  }
+  if (positional.length !== 2) {
+    fail("diff needs exactly two trace files: whatbroke diff before.jsonl after.jsonl");
+  }
+  let result;
+  try {
+    const before = loadTrace(positional[0]);
+    const after = loadTrace(positional[1]);
+    const options = { latencyThreshold, costThreshold, compareOutputs };
+    result = hasSamples(before) || hasSamples(after) ? diffTracesSampled(before, after, options) : diffTraces(before, after, options);
+  } catch (err) {
+    fail(err instanceof Error ? err.message : String(err));
+  }
+  if (format === "json") {
+    console.log(JSON.stringify(result, null, 2));
+  } else if (format === "md") {
+    console.log(renderMarkdown(result));
+  } else {
+    console.log(renderTerminal(result));
+  }
+  const shouldFail = failOn === "breaking" && result.breaking > 0 || failOn === "warning" && (result.breaking > 0 || result.warnings > 0);
+  process.exit(shouldFail ? 1 : 0);
+}
+function runRecord(argv) {
+  let out = "";
+  let port = 4141;
+  let run;
+  let target;
+  for (let i = 0; i < argv.length; i++) {
+    const arg = argv[i];
+    switch (arg) {
+      case "--out":
+        out = argv[++i] ?? "";
+        break;
+      case "--port": {
+        const value = Number(argv[++i]);
+        if (!Number.isInteger(value) || value < 0 || value > 65535) {
+          fail(`--port needs a number between 0 and 65535`);
+        }
+        port = value;
+        break;
+      }
+      case "--run":
+        run = argv[++i];
+        break;
+      case "--target":
+        target = argv[++i];
+        break;
+      default:
+        fail(`unknown option: ${arg}`);
+    }
+  }
+  if (!out) fail("record needs --out <trace.jsonl>");
+  startProxy({
+    file: out,
+    port,
+    run,
+    target,
+    onRecord: (runId, model, toolNames) => {
+      const tools = toolNames.length ? ` -> ${toolNames.join(", ")}` : "";
+      console.log(pc.dim(`  [${runId}] ${model}${tools}`));
+    }
+  }).then((proxy) => {
+    console.log(`recording to ${out}`);
+    console.log(pc.dim(`  OPENAI_BASE_URL=${proxy.url}/v1`));
+    console.log(pc.dim(`  ANTHROPIC_BASE_URL=${proxy.url}`));
+    console.log(pc.dim("  ctrl-c to stop"));
+    console.log("");
+    process.on("SIGINT", () => {
+      proxy.close().finally(() => process.exit(0));
+    });
+  }).catch((err) => {
+    fail(err instanceof Error ? err.message : String(err));
+  });
+}
+main();

package/dist/index.d.ts

@@ -0,0 +1,187 @@
+interface RunStartEvent {
+    type: "run_start";
+    run: string;
+    ts?: number;
+    meta?: Record<string, unknown>;
+}
+interface LlmCallEvent {
+    type: "llm_call";
+    run: string;
+    ts?: number;
+    model: string;
+    latency_ms?: number;
+    tokens?: {
+        input?: number;
+        output?: number;
+    };
+    cost_usd?: number;
+    stop_reason?: string;
+    error?: string;
+}
+interface ToolCallEvent {
+    type: "tool_call";
+    run: string;
+    ts?: number;
+    name: string;
+    args?: Record<string, unknown>;
+    latency_ms?: number;
+    error?: string;
+}
+interface OutputEvent {
+    type: "output";
+    run: string;
+    ts?: number;
+    content: string;
+}
+interface RunEndEvent {
+    type: "run_end";
+    run: string;
+    ts?: number;
+    status: "ok" | "error";
+    error?: string;
+}
+type TraceEvent = RunStartEvent | LlmCallEvent | ToolCallEvent | OutputEvent | RunEndEvent;
+interface Run {
+    id: string;
+    meta: Record<string, unknown>;
+    llmCalls: LlmCallEvent[];
+    toolCalls: ToolCallEvent[];
+    outputs: OutputEvent[];
+    status: "ok" | "error" | "unknown";
+    error?: string;
+}
+type Severity = "breaking" | "warning" | "info";
+interface Finding {
+    severity: Severity;
+    /** stable identity for the thing that changed (tool name, "output", ...), used to correlate findings across sampled runs */
+    subject?: string;
+    /** how often this finding appeared across sampled run pairs, e.g. "4/9" */
+    rate?: string;
+    /** true when the same finding also shows up between baseline samples, i.e. the agent was already flaky here */
+    flaky?: boolean;
+    kind: "run_missing" | "run_added" | "status_changed" | "tool_removed" | "tool_added" | "tool_args_changed" | "tool_reordered" | "tool_error" | "output_changed" | "output_missing" | "latency_regression" | "cost_increase" | "token_change" | "model_changed";
+    run: string;
+    message: string;
+    detail?: Record<string, unknown>;
+}
+interface RunStats {
+    llmCalls: number;
+    toolCalls: number;
+    inputTokens: number;
+    outputTokens: number;
+    costUsd: number;
+    latencyMs: number;
+    models: string[];
+}
+interface RunDiff {
+    run: string;
+    findings: Finding[];
+    before?: RunStats;
+    after?: RunStats;
+}
+interface DiffResult {
+    runs: RunDiff[];
+    findings: Finding[];
+    breaking: number;
+    warnings: number;
+    info: number;
+}
+interface DiffOptions {
+    /** flag latency regressions above this ratio, e.g. 1.5 = 50% slower (default 1.5) */
+    latencyThreshold?: number;
+    /** flag cost increases above this ratio (default 1.25) */
+    costThreshold?: number;
+    /** compare final outputs (default true) */
+    compareOutputs?: boolean;
+}
+
+declare function diffTraces(before: Map<string, Run>, after: Map<string, Run>, options?: DiffOptions): DiffResult;
+
+declare function hasSamples(runs: Map<string, Run>): boolean;
+/**
+ * Like diffTraces, but runs named `name#1`, `name#2`, ... are treated as
+ * repeated samples of the same scenario. Every before sample is compared to
+ * every after sample, findings carry an occurrence rate, and findings that
+ * also appear between baseline samples are demoted to flaky info, because the
+ * agent already behaved that way before the change.
+ */
+declare function diffTracesSampled(before: Map<string, Run>, after: Map<string, Run>, options?: DiffOptions): DiffResult;
+
+declare function parseTrace(text: string): Map<string, Run>;
+declare function loadTrace(path: string): Map<string, Run>;
+
+interface ProxyOptions {
+    /** trace file path, e.g. traces/current.jsonl */
+    file: string;
+    /** port to listen on, 0 picks a free one (default 4141) */
+    port?: number;
+    /** upstream origin override, e.g. http://localhost:8080 for a mock */
+    target?: string;
+    /** run id when the client sends no x-whatbroke-run header */
+    run?: string;
+    /** called once per recorded llm call, used by the CLI for progress output */
+    onRecord?: (run: string, model: string, toolNames: string[]) => void;
+}
+interface ProxyHandle {
+    port: number;
+    url: string;
+    close: () => Promise<void>;
+}
+/**
+ * Starts a local HTTP proxy that forwards requests to the OpenAI or
+ * Anthropic API and records every LLM call, tool call, and final text
+ * output to a JSONL trace. Point OPENAI_BASE_URL or ANTHROPIC_BASE_URL at
+ * it and run your agent unchanged. Requests to /v1/messages go to
+ * api.anthropic.com, everything else to api.openai.com, unless `target`
+ * says otherwise. Group runs with an x-whatbroke-run request header.
+ */
+declare function startProxy(options: ProxyOptions): Promise<ProxyHandle>;
+
+interface RecorderOptions {
+    /** trace file path, e.g. traces/before.jsonl */
+    file: string;
+    /** run id for this scenario, defaults to "default" */
+    run?: string;
+    meta?: Record<string, unknown>;
+}
+/**
+ * Writes trace events to a JSONL file. Use one recorder per scenario run,
+ * or pass a run id per call if you drive many scenarios from one place.
+ */
+declare class Recorder {
+    private file;
+    private run;
+    constructor(options: RecorderOptions);
+    private write;
+    llmCall(data: {
+        model: string;
+        latencyMs?: number;
+        inputTokens?: number;
+        outputTokens?: number;
+        costUsd?: number;
+        stopReason?: string;
+        error?: string;
+    }): void;
+    toolCall(name: string, args?: Record<string, unknown>, data?: {
+        latencyMs?: number;
+        error?: string;
+    }): void;
+    output(content: string): void;
+    end(status?: "ok" | "error", error?: string): void;
+    /**
+     * Wraps an OpenAI client (openai npm package) so every
+     * chat.completions.create call is recorded, including tool calls the
+     * model requested. Returns the same client.
+     */
+    wrapOpenAI<T extends object>(client: T): T;
+    /**
+     * Wraps an Anthropic client (@anthropic-ai/sdk) so every messages.create
+     * call is recorded, including tool_use blocks. Returns the same client.
+     */
+    wrapAnthropic<T extends object>(client: T): T;
+}
+
+declare function renderTerminal(result: DiffResult): string;
+declare function renderMarkdown(result: DiffResult): string;
+
+export { type DiffOptions, type DiffResult, type Finding, Recorder, type Run, type RunDiff, type RunStats, type Severity, type TraceEvent, diffTraces, diffTracesSampled, hasSamples, loadTrace, parseTrace, renderMarkdown, renderTerminal, startProxy };

package/dist/index.js

@@ -0,0 +1,22 @@
+import {
+  Recorder,
+  diffTraces,
+  diffTracesSampled,
+  hasSamples,
+  loadTrace,
+  parseTrace,
+  renderMarkdown,
+  renderTerminal,
+  startProxy
+} from "./chunk-PXR57ZGT.js";
+export {
+  Recorder,
+  diffTraces,
+  diffTracesSampled,
+  hasSamples,
+  loadTrace,
+  parseTrace,
+  renderMarkdown,
+  renderTerminal,
+  startProxy
+};

package/examples/support-agent-gpt4o.jsonl

@@ -0,0 +1,28 @@
+{"type":"run_start","run":"refund-flow","meta":{"model":"gpt-4o","scenario":"customer asks for a refund on order A-1042"}}
+{"type":"llm_call","run":"refund-flow","model":"gpt-4o","latency_ms":840,"tokens":{"input":1240,"output":86},"cost_usd":0.0044}
+{"type":"tool_call","run":"refund-flow","name":"lookup_order","args":{"order_id":"A-1042"},"latency_ms":120}
+{"type":"llm_call","run":"refund-flow","model":"gpt-4o","latency_ms":920,"tokens":{"input":1410,"output":64},"cost_usd":0.0048}
+{"type":"tool_call","run":"refund-flow","name":"issue_refund","args":{"order_id":"A-1042","amount":42.5,"reason":"damaged item"},"latency_ms":340}
+{"type":"llm_call","run":"refund-flow","model":"gpt-4o","latency_ms":610,"tokens":{"input":1520,"output":58},"cost_usd":0.0049}
+{"type":"output","run":"refund-flow","content":"Your refund of $42.50 for order A-1042 has been issued. It should appear in 3-5 business days."}
+{"type":"run_end","run":"refund-flow","status":"ok"}
+{"type":"run_start","run":"cancel-subscription","meta":{"model":"gpt-4o","scenario":"customer wants to cancel their subscription"}}
+{"type":"llm_call","run":"cancel-subscription","model":"gpt-4o","latency_ms":780,"tokens":{"input":1180,"output":92},"cost_usd":0.0042}
+{"type":"tool_call","run":"cancel-subscription","name":"get_subscription","args":{"customer_id":"C-889"},"latency_ms":95}
+{"type":"llm_call","run":"cancel-subscription","model":"gpt-4o","latency_ms":850,"tokens":{"input":1350,"output":71},"cost_usd":0.0046}
+{"type":"tool_call","run":"cancel-subscription","name":"cancel_subscription","args":{"customer_id":"C-889","effective":"end_of_period"},"latency_ms":210}
+{"type":"llm_call","run":"cancel-subscription","model":"gpt-4o","latency_ms":590,"tokens":{"input":1460,"output":55},"cost_usd":0.0047}
+{"type":"output","run":"cancel-subscription","content":"Your subscription is cancelled effective at the end of the current billing period. You keep access until then."}
+{"type":"run_end","run":"cancel-subscription","status":"ok"}
+{"type":"run_start","run":"order-status","meta":{"model":"gpt-4o","scenario":"customer asks where their order is"}}
+{"type":"llm_call","run":"order-status","model":"gpt-4o","latency_ms":720,"tokens":{"input":1100,"output":78},"cost_usd":0.0039}
+{"type":"tool_call","run":"order-status","name":"lookup_order","args":{"order_id":"B-2210"},"latency_ms":110}
+{"type":"llm_call","run":"order-status","model":"gpt-4o","latency_ms":640,"tokens":{"input":1290,"output":66},"cost_usd":0.0043}
+{"type":"output","run":"order-status","content":"Order B-2210 shipped yesterday and is expected to arrive Friday."}
+{"type":"run_end","run":"order-status","status":"ok"}
+{"type":"run_start","run":"angry-escalation","meta":{"model":"gpt-4o","scenario":"furious customer demands a manager"}}
+{"type":"llm_call","run":"angry-escalation","model":"gpt-4o","latency_ms":880,"tokens":{"input":1330,"output":104},"cost_usd":0.0047}
+{"type":"tool_call","run":"angry-escalation","name":"create_ticket","args":{"priority":"high","team":"support-escalations"},"latency_ms":150}
+{"type":"llm_call","run":"angry-escalation","model":"gpt-4o","latency_ms":700,"tokens":{"input":1480,"output":89},"cost_usd":0.0049}
+{"type":"output","run":"angry-escalation","content":"I understand this has been frustrating. I've escalated your case to our support team with high priority. Ticket #4471, someone will reach out within 2 hours."}
+{"type":"run_end","run":"angry-escalation","status":"ok"}

package/examples/support-agent-gpt5mini.jsonl

@@ -0,0 +1,26 @@
+{"type":"run_start","run":"refund-flow","meta":{"model":"gpt-5-mini","scenario":"customer asks for a refund on order A-1042"}}
+{"type":"llm_call","run":"refund-flow","model":"gpt-5-mini","latency_ms":610,"tokens":{"input":1240,"output":71},"cost_usd":0.0011}
+{"type":"tool_call","run":"refund-flow","name":"lookup_order","args":{"order_id":"A-1042"},"latency_ms":120}
+{"type":"llm_call","run":"refund-flow","model":"gpt-5-mini","latency_ms":680,"tokens":{"input":1410,"output":52},"cost_usd":0.0012}
+{"type":"tool_call","run":"refund-flow","name":"issue_refund","args":{"order_id":"A-1042","amount":425,"reason":"damaged item"},"latency_ms":340}
+{"type":"llm_call","run":"refund-flow","model":"gpt-5-mini","latency_ms":450,"tokens":{"input":1520,"output":49},"cost_usd":0.0012}
+{"type":"output","run":"refund-flow","content":"Your refund of $425.00 for order A-1042 has been issued. It should appear in 3-5 business days."}
+{"type":"run_end","run":"refund-flow","status":"ok"}
+{"type":"run_start","run":"cancel-subscription","meta":{"model":"gpt-5-mini","scenario":"customer wants to cancel their subscription"}}
+{"type":"llm_call","run":"cancel-subscription","model":"gpt-5-mini","latency_ms":590,"tokens":{"input":1180,"output":88},"cost_usd":0.001}
+{"type":"tool_call","run":"cancel-subscription","name":"get_subscription","args":{"customer_id":"C-889"},"latency_ms":95}
+{"type":"llm_call","run":"cancel-subscription","model":"gpt-5-mini","latency_ms":620,"tokens":{"input":1350,"output":112},"cost_usd":0.0011}
+{"type":"output","run":"cancel-subscription","content":"I can help with that. To confirm the cancellation, please reply CONFIRM and I will process it right away."}
+{"type":"run_end","run":"cancel-subscription","status":"ok"}
+{"type":"run_start","run":"order-status","meta":{"model":"gpt-5-mini","scenario":"customer asks where their order is"}}
+{"type":"llm_call","run":"order-status","model":"gpt-5-mini","latency_ms":540,"tokens":{"input":1100,"output":74},"cost_usd":0.001}
+{"type":"tool_call","run":"order-status","name":"lookup_order","args":{"order_id":"B-2210"},"latency_ms":110}
+{"type":"llm_call","run":"order-status","model":"gpt-5-mini","latency_ms":480,"tokens":{"input":1290,"output":66},"cost_usd":0.001}
+{"type":"output","run":"order-status","content":"Order B-2210 shipped yesterday and is expected to arrive Friday."}
+{"type":"run_end","run":"order-status","status":"ok"}
+{"type":"run_start","run":"angry-escalation","meta":{"model":"gpt-5-mini","scenario":"furious customer demands a manager"}}
+{"type":"llm_call","run":"angry-escalation","model":"gpt-5-mini","latency_ms":650,"tokens":{"input":1330,"output":96},"cost_usd":0.001}
+{"type":"tool_call","run":"angry-escalation","name":"create_ticket","args":{"priority":"normal","team":"support-general"},"latency_ms":150}
+{"type":"llm_call","run":"angry-escalation","model":"gpt-5-mini","latency_ms":510,"tokens":{"input":1480,"output":81},"cost_usd":0.001}
+{"type":"output","run":"angry-escalation","content":"I'm sorry to hear that. I've created a ticket for your issue and our team will get back to you soon."}
+{"type":"run_end","run":"angry-escalation","status":"ok"}

package/package.json

@@ -0,0 +1,61 @@
+{
+  "name": "whatbroke-cli",
+  "version": "0.2.0",
+  "description": "Diff your AI agent's behavior between two runs. Swap a model, change a prompt, run whatbroke, see exactly what changed.",
+  "keywords": [
+    "ai-agents",
+    "llm",
+    "evals",
+    "regression-testing",
+    "diff",
+    "tracing",
+    "openai",
+    "anthropic"
+  ],
+  "license": "MIT",
+  "author": "Arthi Arumugam",
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/arthi-arumugam-git/whatbroke.git"
+  },
+  "homepage": "https://github.com/arthi-arumugam-git/whatbroke#readme",
+  "bugs": {
+    "url": "https://github.com/arthi-arumugam-git/whatbroke/issues"
+  },
+  "sideEffects": false,
+  "type": "module",
+  "bin": {
+    "whatbroke": "dist/cli.js"
+  },
+  "main": "./dist/index.js",
+  "types": "./dist/index.d.ts",
+  "exports": {
+    ".": {
+      "types": "./dist/index.d.ts",
+      "import": "./dist/index.js"
+    }
+  },
+  "files": [
+    "dist",
+    "examples"
+  ],
+  "scripts": {
+    "build": "tsup src/index.ts src/cli.ts --format esm --dts --clean",
+    "test": "vitest run",
+    "dev": "tsx src/cli.ts",
+    "prepublishOnly": "npm run build && npm test"
+  },
+  "devDependencies": {
+    "@types/node": "^22.10.0",
+    "tsup": "^8.3.0",
+    "tsx": "^4.19.0",
+    "typescript": "^5.7.0",
+    "vitest": "^2.1.0"
+  },
+  "dependencies": {
+    "picocolors": "^1.1.0"
+  },
+  "engines": {
+    "node": ">=18"
+  }
+}