skilldiff 0.1.0

package/dist/src/report.js

@@ -0,0 +1,52 @@
+// skilldiff v0.1 — behavior diff report formatting.
+// Same text is used for CLI output and (later) the GitHub PR comment body.
+import { summarize } from "./assertions.js";
+function sideLines(side) {
+    const lines = [];
+    lines.push(`${side.label}: ${side.trace.toolCalls.length} tool calls, ${side.trace.filesChanged.length} file(s) changed, ${side.trace.commandsRun.length} command(s) run`);
+    if (side.trace.filesChanged.length > 0) {
+        lines.push(`  files: ${side.trace.filesChanged.join(", ")}`);
+    }
+    if (side.trace.commandsRun.length > 0) {
+        lines.push(`  commands: ${side.trace.commandsRun.join(" | ")}`);
+    }
+    if (side.trace.warnings.length > 0) {
+        lines.push(`  warnings: ${side.trace.warnings.join("; ")}`);
+    }
+    return lines;
+}
+export function formatReport(scenarioName, sides) {
+    const out = [];
+    out.push(`skilldiff behavior report — ${scenarioName}`);
+    out.push("");
+    for (const side of sides) {
+        out.push(...sideLines(side));
+    }
+    // Assertion results are expected to be identical for old and new (the scenario
+    // describes desired behavior); the interesting signal is WHICH side fails.
+    out.push("");
+    out.push("Assertions:");
+    const newSide = sides[sides.length - 1];
+    for (const r of newSide.assertions) {
+        const mark = r.pass ? "✓" : "✗";
+        out.push(`  ${mark} [${r.kind}] ${r.expected}`);
+        if (!r.pass) {
+            out.push(`      actual (new): ${r.actual}`);
+            const oldSide = sides[0];
+            if (oldSide && oldSide !== newSide) {
+                const oldMatching = oldSide.assertions.find((o) => o.kind === r.kind && o.expected === r.expected);
+                if (oldMatching && !oldMatching.pass) {
+                    out.push(`      note: old skill also failed this assertion`);
+                }
+                else if (oldMatching?.pass) {
+                    out.push(`      actual (old): ${oldMatching.actual}`);
+                    out.push(`      note: this is a REGRESSION — old skill passed, new skill fails`);
+                }
+            }
+        }
+    }
+    const { passed, failed } = summarize(newSide.assertions);
+    out.push("");
+    out.push(`Result (new skill): ${passed} passed, ${failed} failed`);
+    return out.join("\n");
+}

package/dist/src/runner.d.ts

@@ -0,0 +1,29 @@
+import type { Scenario } from "./scenario.js";
+import type { SideResult } from "./report.js";
+/** A recorded trace fixture — captured once from a live run, replayed deterministically. */
+export interface RecordedTrace {
+    toolCalls: Array<{
+        tool: string;
+        input: Record<string, unknown>;
+    }>;
+    output?: string;
+    filesChanged?: string[];
+    warnings?: string[];
+}
+export declare function loadRecordedTrace(path: string): Promise<RecordedTrace>;
+export interface RunOptions {
+    /** Recorded trace for the old skill (required in recorded mode). */
+    oldTrace?: string;
+    /** Recorded trace for the new skill (optional; defaults to old-only mode). */
+    newTrace?: string;
+    /** Force live mode even if recorded traces are given. */
+    live?: boolean;
+    /** Git ref to fetch the OLD skill version from (e.g. base branch). Enables the full pipeline. */
+    base?: string;
+}
+export interface RunResult {
+    report: string;
+    passed: boolean;
+    sides: SideResult[];
+}
+export declare function runScenario(scenario: Scenario, opts: RunOptions): Promise<RunResult>;

package/dist/src/runner.js

@@ -0,0 +1,118 @@
+// skilldiff v0.1 — scenario runner skeleton.
+// Two modes:
+//   recorded: replays a captured trace JSON (deterministic, for CI without quota)
+//   freebuff: live run via the Freebuff/Codebuff harness (scripts/freebuff-adapter.ts)
+import { readFile, cp, rm } from "node:fs/promises";
+import { resolve, join } from "node:path";
+import { mkdtemp } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { fetchSkillVersion, cleanupSkillVersion } from "./baseline.js";
+import { buildTrace } from "./trace-utils.js";
+import { evaluateAssertions } from "./assertions.js";
+import { formatReport } from "./report.js";
+export async function loadRecordedTrace(path) {
+    const raw = JSON.parse(await readFile(path, "utf8"));
+    if (!Array.isArray(raw.toolCalls)) {
+        throw new Error(`${path}: recorded trace must have a toolCalls array`);
+    }
+    return raw;
+}
+function runRecordedSide(label, trace, scenario, fixtureRoot) {
+    const runTrace = buildTrace({
+        toolCalls: trace.toolCalls,
+        output: trace.output ?? "",
+        fixtureRoot,
+        filesChanged: trace.filesChanged,
+        warnings: trace.warnings,
+    });
+    return { label, trace: runTrace, assertions: evaluateAssertions(runTrace, scenario.expect) };
+}
+async function runFreebuffSide(label, scenario, fixtureRoot, skillVersionDir) {
+    // Dynamic import: the SDK is only needed for live runs.
+    const { runFreebuffHarness } = await import("../scripts/freebuff-adapter.js");
+    // If a skill version dir is given, copy those files INTO the fixture so the
+    // agent reads the version under test from the usual .claude/skills location.
+    if (skillVersionDir) {
+        for (const skillPath of scenario.skillPaths) {
+            await cp(join(skillVersionDir, skillPath), join(fixtureRoot, skillPath), { force: true });
+        }
+    }
+    // Also pass them inline for harnesses that support direct skill injection.
+    const skillFiles = skillVersionDir
+        ? await Promise.all(scenario.skillPaths.map(async (p) => ({
+            name: p.split("/").slice(-2, -1)[0] ?? "skill",
+            path: join(skillVersionDir, p),
+        })))
+        : undefined;
+    const { traces, error } = await runFreebuffHarness({
+        cwd: fixtureRoot,
+        prompt: scenario.prompt,
+        maxTurns: scenario.maxTurns,
+        skillFiles,
+    });
+    const warnings = [];
+    if (error)
+        warnings.push(`harness error: ${error}`);
+    const output = traces
+        .filter((t) => t.tool === "message" || t.tool === "text")
+        .map((t) => String(t.input?.text ?? ""))
+        .join("\n");
+    const runTrace = buildTrace({
+        toolCalls: traces,
+        output,
+        fixtureRoot,
+        warnings,
+    });
+    return { label, trace: runTrace, assertions: evaluateAssertions(runTrace, scenario.expect) };
+}
+export async function runScenario(scenario, opts) {
+    const fixtureRoot = resolve(scenario.fixture);
+    if (!opts.live && opts.oldTrace) {
+        // Deterministic recorded mode
+        const old = await loadRecordedTrace(opts.oldTrace);
+        const sides = [runRecordedSide("old skill", old, scenario, fixtureRoot)];
+        if (opts.newTrace) {
+            const neu = await loadRecordedTrace(opts.newTrace);
+            sides.push(runRecordedSide("new skill", neu, scenario, fixtureRoot));
+        }
+        const report = formatReport(scenario.name, sides);
+        const lastAssertions = sides[sides.length - 1].assertions;
+        return { report, passed: lastAssertions.every((a) => a.pass), sides };
+    }
+    // Live mode — currently Freebuff only; other harnesses come online in later v0.1 cuts.
+    const harness = scenario.harness ?? "freebuff";
+    if (harness !== "freebuff") {
+        throw new Error(`live harness '${harness}' not wired yet — use recorded mode or harness: freebuff`);
+    }
+    if (opts.base) {
+        // Full pipeline: fetch old skill from base ref -> snapshot the new skill's
+        // current fixture files -> run old, restore, run new, diff.
+        const oldVersion = await fetchSkillVersion(fixtureRoot, opts.base, scenario.skillPaths);
+        // Snapshot current (new) skill files so we can restore after the old-skill run.
+        const snapshotDir = await mkdtemp(join(tmpdir(), "skilldiff-current-"));
+        for (const skillPath of scenario.skillPaths) {
+            await cp(join(fixtureRoot, skillPath), join(snapshotDir, skillPath), { force: true });
+        }
+        try {
+            const oldSide = await runFreebuffSide("old skill", scenario, fixtureRoot, oldVersion.dir);
+            // Restore new skill files before the new run.
+            for (const skillPath of scenario.skillPaths) {
+                await cp(join(snapshotDir, skillPath), join(fixtureRoot, skillPath), { force: true });
+            }
+            const newSide = await runFreebuffSide("new skill", scenario, fixtureRoot);
+            const report = formatReport(scenario.name, [oldSide, newSide]);
+            return { report, passed: newSide.assertions.every((a) => a.pass), sides: [oldSide, newSide] };
+        }
+        finally {
+            // Restore new skill files even on failure so the fixture isn't left with old versions.
+            for (const skillPath of scenario.skillPaths) {
+                await cp(join(snapshotDir, skillPath), join(fixtureRoot, skillPath), { force: true }).catch(() => { });
+            }
+            await cleanupSkillVersion(oldVersion);
+            await rm(snapshotDir, { recursive: true, force: true }).catch(() => { });
+        }
+    }
+    const newSide = await runFreebuffSide("new skill", scenario, fixtureRoot);
+    const report = formatReport(scenario.name, [newSide]);
+    return { report, passed: newSide.assertions.every((a) => a.pass), sides: [newSide] };
+}

package/dist/src/scenario.d.ts

@@ -0,0 +1,36 @@
+/** The 5 assertion kinds, per design-decisions.md. */
+export type AssertionKind = "files_changed" | "commands_run" | "tool_calls" | "must_not" | "output_contains";
+export interface Expectations {
+    /** Paths that must have been changed/created relative to fixture root. Partial match. */
+    files_changed?: string[];
+    /** Commands that must have been run. Partial match against full command strings. */
+    commands_run?: string[];
+    /** Tool names that must have been invoked. Partial match against normalized tool names. */
+    tool_calls?: string[];
+    /** Things that must NOT have happened. Same shape as the positive assertions. */
+    must_not?: {
+        files_changed?: string[];
+        commands_run?: string[];
+        tool_calls?: string[];
+    };
+    /** Substring(s) that must appear in the agent's final text output. */
+    output_contains?: string[];
+}
+export interface Scenario {
+    name: string;
+    /** Skill paths (repo-relative) whose behavior is under test. */
+    skillPaths: string[];
+    /** Path to the fixture repo (absolute or relative to the scenario file). */
+    fixture: string;
+    /** Prompt given to the agent. */
+    prompt: string;
+    /** Harness to use: freebuff | cursor-agent | codex | claude-code. Default: auto-detect. */
+    harness?: string;
+    /** Per-scenario turn cap. */
+    maxTurns?: number;
+    expect: Expectations;
+}
+/** Parse and validate a scenario YAML string. */
+export declare function parseScenarioYaml(yamlText: string, sourceName?: string): Scenario;
+/** Load a scenario from a YAML file path. */
+export declare function loadScenario(path: string): Promise<Scenario>;

package/dist/src/scenario.js

@@ -0,0 +1,71 @@
+// skilldiff v0.1 — scenario types + YAML loader.
+// Scenario YAML describes: which skill files, which fixture repo, the prompt to run,
+// and the 5 assertion kinds (files_changed, commands_run, tool_calls, must_not, output_contains).
+import { parse } from "yaml";
+import { readFile } from "node:fs/promises";
+function expectStringArray(value, field, scenarioName) {
+    if (value === undefined)
+        return undefined;
+    if (!Array.isArray(value) || value.some((v) => typeof v !== "string")) {
+        throw new Error(`${scenarioName}: ${field} must be a list of strings`);
+    }
+    return value;
+}
+/** Parse and validate a scenario YAML string. */
+export function parseScenarioYaml(yamlText, sourceName = "(inline)") {
+    let raw;
+    try {
+        raw = parse(yamlText);
+    }
+    catch (err) {
+        throw new Error(`${sourceName}: invalid YAML — ${err.message}`);
+    }
+    if (!raw || typeof raw !== "object") {
+        throw new Error(`${sourceName}: scenario must be a YAML mapping`);
+    }
+    const name = typeof raw.name === "string" ? raw.name : "";
+    if (!name)
+        throw new Error(`${sourceName}: missing required field 'name'`);
+    const prompt = typeof raw.prompt === "string" ? raw.prompt : "";
+    if (!prompt)
+        throw new Error(`${sourceName}: missing required field 'prompt'`);
+    const fixture = typeof raw.fixture === "string" ? raw.fixture : "";
+    if (!fixture)
+        throw new Error(`${sourceName}: missing required field 'fixture'`);
+    const skillPaths = expectStringArray(raw.skillPaths, "skillPaths", name) ?? [];
+    if (skillPaths.length === 0) {
+        throw new Error(`${sourceName}: 'skillPaths' must list at least one skill file under test`);
+    }
+    const harness = typeof raw.harness === "string" ? raw.harness : undefined;
+    if (harness && !["freebuff", "cursor-agent", "codex", "claude-code"].includes(harness)) {
+        throw new Error(`${sourceName}: unknown harness '${harness}' (expected freebuff | cursor-agent | codex | claude-code)`);
+    }
+    const maxTurns = typeof raw.maxTurns === "number" && Number.isInteger(raw.maxTurns) && raw.maxTurns > 0
+        ? raw.maxTurns
+        : undefined;
+    const expectRaw = (raw.expect ?? {});
+    if (typeof raw.expect !== "object" || raw.expect === null) {
+        throw new Error(`${sourceName}: 'expect' must be a mapping`);
+    }
+    const mustNotRaw = (expectRaw.must_not ?? {});
+    if (typeof expectRaw.must_not !== "undefined" && (typeof expectRaw.must_not !== "object" || expectRaw.must_not === null)) {
+        throw new Error(`${name}: expect.must_not must be a mapping`);
+    }
+    const expect = {
+        files_changed: expectStringArray(expectRaw.files_changed, "expect.files_changed", name),
+        commands_run: expectStringArray(expectRaw.commands_run, "expect.commands_run", name),
+        tool_calls: expectStringArray(expectRaw.tool_calls, "expect.tool_calls", name),
+        must_not: {
+            files_changed: expectStringArray(mustNotRaw.files_changed, "expect.must_not.files_changed", name),
+            commands_run: expectStringArray(mustNotRaw.commands_run, "expect.must_not.commands_run", name),
+            tool_calls: expectStringArray(mustNotRaw.tool_calls, "expect.must_not.tool_calls", name),
+        },
+        output_contains: expectStringArray(expectRaw.output_contains, "expect.output_contains", name),
+    };
+    return { name, skillPaths, fixture, prompt, harness, maxTurns, expect };
+}
+/** Load a scenario from a YAML file path. */
+export async function loadScenario(path) {
+    const text = await readFile(path, "utf8");
+    return parseScenarioYaml(text, path);
+}

package/dist/src/trace-utils.d.ts

@@ -0,0 +1,27 @@
+import { type RunTrace } from "./assertions.js";
+interface ToolUse {
+    tool: string;
+    input: Record<string, unknown>;
+}
+/**
+ * Derive commandsRun from bash-like tool calls.
+ * Handles both `command: "..."` (string) and `command: ["..."]` (argv array) shapes.
+ */
+export declare function extractCommands(toolCalls: ToolUse[]): string[];
+/**
+ * Diff the fixture working tree against the harness's starting point using git.
+ * Falls back to an empty list if the fixture is not a git repo.
+ */
+export declare function detectFilesChanged(fixtureRoot: string): string[];
+/**
+ * Build a complete RunTrace from raw tool calls + final output.
+ * Optionally takes filesChanged if the caller tracked it differently (e.g. pre/post snapshot).
+ */
+export declare function buildTrace(opts: {
+    toolCalls: ToolUse[];
+    output: string;
+    fixtureRoot?: string;
+    filesChanged?: string[];
+    warnings?: string[];
+}): RunTrace;
+export {};

package/dist/src/trace-utils.js

@@ -0,0 +1,54 @@
+// skilldiff v0.1 — helpers to build a RunTrace from harness output.
+import { execFileSync } from "node:child_process";
+import { normalizeToolName } from "./assertions.js";
+/**
+ * Derive commandsRun from bash-like tool calls.
+ * Handles both `command: "..."` (string) and `command: ["..."]` (argv array) shapes.
+ */
+export function extractCommands(toolCalls) {
+    const commands = [];
+    for (const tc of toolCalls) {
+        if (normalizeToolName(tc.tool) !== "bash")
+            continue;
+        const cmd = tc.input.command ?? tc.input.cmd ?? tc.input.script;
+        if (typeof cmd === "string") {
+            commands.push(cmd);
+        }
+        else if (Array.isArray(cmd)) {
+            commands.push(cmd.join(" "));
+        }
+    }
+    return commands;
+}
+/**
+ * Diff the fixture working tree against the harness's starting point using git.
+ * Falls back to an empty list if the fixture is not a git repo.
+ */
+export function detectFilesChanged(fixtureRoot) {
+    try {
+        const out = execFileSync("git", ["status", "--porcelain"], { cwd: fixtureRoot, encoding: "utf8" });
+        return out
+            .split("\n")
+            .map((line) => line.trim())
+            .filter((line) => line.length > 0)
+            .map((line) => line.replace(/^\S+\s+/, "")) // strip status code, keep path
+            .map((line) => line.replace(/^"(.*)"$/, "$1")); // unquote paths with spaces
+    }
+    catch {
+        return [];
+    }
+}
+/**
+ * Build a complete RunTrace from raw tool calls + final output.
+ * Optionally takes filesChanged if the caller tracked it differently (e.g. pre/post snapshot).
+ */
+export function buildTrace(opts) {
+    const filesChanged = opts.filesChanged ?? (opts.fixtureRoot ? detectFilesChanged(opts.fixtureRoot) : []);
+    return {
+        toolCalls: opts.toolCalls,
+        filesChanged,
+        commandsRun: extractCommands(opts.toolCalls),
+        output: opts.output,
+        warnings: opts.warnings ?? [],
+    };
+}

package/package.json

@@ -0,0 +1,56 @@
+{
+  "name": "skilldiff",
+  "version": "0.1.0",
+  "description": "Behavioral regression testing for agent skills. Runs your skill in a real agent harness and diffs the behavior on every PR.",
+  "type": "module",
+  "license": "MIT",
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/scs0209/skilldiff.git"
+  },
+  "homepage": "https://github.com/scs0209/skilldiff#readme",
+  "bugs": "https://github.com/scs0209/skilldiff/issues",
+  "keywords": [
+    "agent",
+    "skills",
+    "testing",
+    "regression",
+    "claude",
+    "cursor",
+    "codex",
+    "codebuff",
+    "ci"
+  ],
+  "files": [
+    "dist/",
+    "scripts/",
+    "README.md",
+    "LICENSE",
+    "CHANGELOG.md"
+  ],
+  "bin": {
+    "skilldiff": "./dist/src/cli.js"
+  },
+  "scripts": {
+    "build": "tsc",
+    "prepublishOnly": "npm run build && npm test",
+    "spike": "tsx scripts/spike.ts",
+    "spike:recorded": "tsx scripts/recorded-spike.ts",
+    "spike:freebuff": "tsx scripts/freebuff-spike.ts",
+    "test": "vitest run"
+  },
+  "dependencies": {
+    "@anthropic-ai/claude-agent-sdk": "^0.1.0",
+    "@codebuff/sdk": "^0.10.7",
+    "yaml": "^2.9.1"
+  },
+  "devDependencies": {
+    "@types/node": "^22.0.0",
+    "tsx": "^4.19.0",
+    "typescript": "^5.6.0",
+    "vitest": "^3.0.0"
+  },
+  "engines": {
+    "node": ">=20"
+  }
+}

package/scripts/adapters.ts

@@ -0,0 +1,77 @@
+// Harness CLI adapters — normalize each harness's JSON event stream into tool traces.
+// Shared between the live spike (scripts/spike.ts) and the recorded spike (scripts/recorded-spike.ts).
+
+export interface ToolUse {
+  tool: string;
+  input: Record<string, unknown>;
+}
+
+export interface HarnessAdapter {
+  name: string;
+  cmd: string;
+  baseArgs: string[]; // headless/print flags, prompt appended last
+  extractToolUses: (line: unknown) => ToolUse[];
+}
+
+// ---- Claude Code: stream-json assistant events -------------------------
+export const claudeAdapter: HarnessAdapter = {
+  name: "claude-code",
+  cmd: "claude",
+  baseArgs: ["-p", "--output-format", "stream-json", "--verbose", "--max-turns", "5"],
+  extractToolUses: (line) => {
+    const msg = line as { type?: string; message?: { content?: Array<{ type: string; name?: string; input?: Record<string, unknown> }> } };
+    if (msg.type !== "assistant" || !msg.message?.content) return [];
+    return msg.message.content
+      .filter((b) => b.type === "tool_use" && b.name)
+      .map((b) => ({ tool: b.name!, input: b.input ?? {} }));
+  },
+};
+
+// ---- Cursor agent: stream-json events ----------------------------------
+export const cursorAdapter: HarnessAdapter = {
+  name: "cursor-agent",
+  cmd: "cursor-agent",
+  baseArgs: ["-p", "--output-format", "stream-json", "--force"],
+  extractToolUses: (line) => {
+    const msg = line as { type?: string; tool_call?: { name?: string; args?: Record<string, unknown> }; result?: { tool_calls?: Array<{ type?: string; name?: string; input?: Record<string, unknown>; arguments?: unknown }> } };
+    if (msg.type === "tool_call" && msg.tool_call?.name) {
+      return [{ tool: msg.tool_call.name, input: msg.tool_call.args ?? {} }];
+    }
+    if (msg.type === "result" && msg.result?.tool_calls) {
+      return msg.result.tool_calls
+        .filter((tc) => tc.name)
+        .map((tc) => ({ tool: tc.name!, input: tc.input ?? {} }));
+    }
+    return [];
+  },
+};
+
+// ---- Codex: exec --json headless events --------------------------------
+// Handles both codex exec --json output (item.started/item.completed)
+// and rollout transcript shape (response_item.function_call with exec_command).
+export const codexAdapter: HarnessAdapter = {
+  name: "codex",
+  cmd: "codex",
+  baseArgs: ["exec", "--json", "--skip-git-repo-check", "--full-auto"],
+  extractToolUses: (line) => {
+    const msg = line as { type?: string; item?: { type?: string; name?: string; command?: string[] } };
+    if ((msg.type === "item.started" || msg.type === "item.completed") && msg.item?.type === "command_execution") {
+      return [{ tool: "Bash", input: { command: (msg.item.command ?? []).join(" ") } }];
+    }
+    const item = line as { type?: string; payload?: { type?: string; name?: string; arguments?: string } };
+    if (item.type === "response_item" && item.payload?.type === "function_call" && item.payload.name) {
+      let args: Record<string, unknown> = {};
+      try {
+        args = JSON.parse(item.payload.arguments ?? "{}") as Record<string, unknown>;
+      } catch {
+        // keep empty args if arguments is not valid JSON
+      }
+      const name = item.payload.name === "exec_command" ? "Bash" : item.payload.name;
+      return [{ tool: name, input: args }];
+    }
+    return [];
+  },
+};
+
+// Prefer cursor/codex; claude CLI often runs on a console account with no credits.
+export const ADAPTERS = [cursorAdapter, codexAdapter, claudeAdapter];

package/scripts/freebuff-adapter.ts

@@ -0,0 +1,109 @@
+// Freebuff/Codebuff adapter — runs skills headless via @codebuff/sdk using the
+// Freebuff desktop auth token (they share the same codebuff.com account system).
+//
+// Token resolution order:
+//   1. CODEBUFF_API_KEY env var
+//   2. ~/.config/freebuff-desktop/state.json -> authSessions["https://www.codebuff.com"].token
+
+import { CodebuffClient } from "@codebuff/sdk";
+import { readFile } from "node:fs/promises";
+import { homedir } from "node:os";
+import { join } from "node:path";
+
+export interface FreebuffToolUse {
+  tool: string;
+  input: Record<string, unknown>;
+}
+
+export async function resolveFreebuffToken(explicit?: string): Promise<string | null> {
+  if (process.env.CODEBUFF_API_KEY) return process.env.CODEBUFF_API_KEY;
+  if (explicit) return explicit;
+  try {
+    const statePath = join(homedir(), ".config", "freebuff-desktop", "state.json");
+    const state = JSON.parse(await readFile(statePath, "utf8"));
+    const token = state?.authSessions?.["https://www.codebuff.com"]?.token;
+    return typeof token === "string" && token.length > 0 ? token : null;
+  } catch {
+    return null;
+  }
+}
+
+export async function runFreebuffHarness(opts: {
+  cwd: string;
+  prompt: string;
+  maxTurns?: number;
+  token?: string;
+  /** Skill versions to inject into the prompt (old or new skill under test). */
+  skillFiles?: Array<{ name: string; path: string }>;
+}): Promise<{ traces: FreebuffToolUse[]; error?: string }> {
+  const token = await resolveFreebuffToken(opts.token);
+  if (!token) {
+    return { traces: [], error: "No Codebuff/Freebuff token. Login to Freebuff desktop or set CODEBUFF_API_KEY." };
+  }
+
+  const client = new CodebuffClient({ apiKey: token, cwd: opts.cwd });
+  const traces: FreebuffToolUse[] = [];
+  let error: string | undefined;
+
+  // Skill injection: inline the skill file contents into the prompt so the agent
+  // is tested ON this skill version (old vs new), not on whatever is in .claude/.
+  let prompt = opts.prompt;
+  if (opts.skillFiles && opts.skillFiles.length > 0) {
+    const skillsBlock = await Promise.all(
+      opts.skillFiles.map(async (sf) => {
+        const content = await readFile(sf.path, "utf8").catch(() => `<<unreadable: ${sf.path}>>`);
+        return `<skill name="${sf.name}">\n${content}\n</skill>`;
+      }),
+    );
+    prompt = `${prompt}\n\nYou have these skills loaded. Follow the matching skill's instructions exactly:\n\n${skillsBlock.join("\n\n")}`;
+  }
+
+  const runState = await client.run({
+    agent: "codebuff/base@latest",
+    prompt,
+    handleEvent: (event: { type: string; [k: string]: unknown }) => {
+      // Tool call events carry the observable behavior we assert on.
+      if (event.type === "tool_call" || event.type === "toolCall") {
+        const e = event as { type: string; toolName?: string; tool?: string; input?: Record<string, unknown>; args?: Record<string, unknown> };
+        traces.push({ tool: e.toolName ?? e.tool ?? "unknown", input: e.input ?? e.args ?? {} });
+      } else if (event.type === "error") {
+        const e = event as { message?: string };
+        error = e.message ?? "unknown error";
+      }
+    },
+  });
+
+  // Fall back to harvesting tool calls from the message history if events were sparse
+  if (traces.length === 0 && runState) {
+    try {
+      const steps = (runState as unknown as { steps?: Array<{ toolCall?: { toolName?: string; input?: Record<string, unknown> } }> }).steps ?? [];
+      for (const step of steps) {
+        if (step.toolCall?.toolName) {
+          traces.push({ tool: step.toolCall.toolName, input: step.toolCall.input ?? {} });
+        }
+      }
+    } catch {
+      // best-effort fallback only
+    }
+  }
+
+  void opts.maxTurns;
+  return { traces, error };
+}
+
+// convenience used by the spike to verify the token works at all
+export async function checkFreebuffAuth(token?: string): Promise<{ ok: boolean; user?: string; error?: string }> {
+  const t = await resolveFreebuffToken(token);
+  if (!t) return { ok: false, error: "no token" };
+  try {
+    const res = await fetch("https://www.codebuff.com/api/v1/me?fields=id,email", {
+      headers: { Authorization: `Bearer ${t}` },
+      signal: AbortSignal.timeout(10000),
+    });
+    if (!res.ok) return { ok: false, error: `HTTP ${res.status}` };
+    const body = (await res.json()) as { id?: string; email?: string };
+    return { ok: true, user: body.email ?? body.id };
+  } catch (err) {
+    return { ok: false, error: (err as Error).message };
+  }
+}