snapeval 1.0.1

package/dist/src/adapters/inference/copilot.js

@@ -0,0 +1,25 @@
+import { execFileSync } from 'node:child_process';
+import { AdapterNotAvailableError } from '../../errors.js';
+export class CopilotInference {
+    fallback;
+    name = 'copilot';
+    constructor(fallback) {
+        this.fallback = fallback;
+    }
+    async chat(messages, _options) {
+        // Concatenate messages into a single prompt string
+        const prompt = messages.map((m) => m.content).join('\n');
+        const result = execFileSync('gh', ['copilot', '-p', prompt], { encoding: 'utf-8' });
+        return result.trim();
+    }
+    async embed(text) {
+        if (this.fallback) {
+            return this.fallback.embed(text);
+        }
+        throw new AdapterNotAvailableError('copilot-embed', 'Copilot CLI does not support embeddings. Provide a fallback InferenceAdapter (e.g. GitHubModelsInference).');
+    }
+    estimateCost(_tokens) {
+        return 0;
+    }
+}
+//# sourceMappingURL=copilot.js.map

package/dist/src/adapters/inference/copilot.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"copilot.js","sourceRoot":"","sources":["../../../../src/adapters/inference/copilot.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAElD,OAAO,EAAE,wBAAwB,EAAE,MAAM,iBAAiB,CAAC;AAE3D,MAAM,OAAO,gBAAgB;IAGE;IAFpB,IAAI,GAAG,SAAS,CAAC;IAE1B,YAA6B,QAA2B;QAA3B,aAAQ,GAAR,QAAQ,CAAmB;IAAG,CAAC;IAE5D,KAAK,CAAC,IAAI,CAAC,QAAmB,EAAE,QAAsB;QACpD,mDAAmD;QACnD,MAAM,MAAM,GAAG,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACzD,MAAM,MAAM,GAAG,YAAY,CAAC,IAAI,EAAE,CAAC,SAAS,EAAE,IAAI,EAAE,MAAM,CAAC,EAAE,EAAE,QAAQ,EAAE,OAAO,EAAE,CAAC,CAAC;QACpF,OAAO,MAAM,CAAC,IAAI,EAAE,CAAC;IACvB,CAAC;IAED,KAAK,CAAC,KAAK,CAAC,IAAY;QACtB,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC;YAClB,OAAO,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QACnC,CAAC;QACD,MAAM,IAAI,wBAAwB,CAChC,eAAe,EACf,4GAA4G,CAC7G,CAAC;IACJ,CAAC;IAED,YAAY,CAAC,OAAe;QAC1B,OAAO,CAAC,CAAC;IACX,CAAC;CACF"}

package/dist/src/adapters/inference/github-models.d.ts

@@ -0,0 +1,9 @@
+import type { InferenceAdapter, Message, ChatOptions } from '../../types.js';
+export declare class GitHubModelsInference implements InferenceAdapter {
+    private readonly token;
+    readonly name = "github-models";
+    constructor(token?: string);
+    chat(messages: Message[], options?: ChatOptions): Promise<string>;
+    embed(text: string): Promise<number[]>;
+    estimateCost(_tokens: number): number;
+}

package/dist/src/adapters/inference/github-models.js

@@ -0,0 +1,62 @@
+import { RateLimitError } from '../../errors.js';
+const API_BASE = 'https://models.github.ai/inference';
+const CHAT_MODEL = 'openai/gpt-4o-mini';
+const EMBEDDING_MODEL = 'openai/text-embedding-3-small';
+export class GitHubModelsInference {
+    token;
+    name = 'github-models';
+    constructor(token = process.env.GITHUB_TOKEN ?? '') {
+        this.token = token;
+    }
+    async chat(messages, options) {
+        const body = {
+            model: CHAT_MODEL,
+            messages,
+        };
+        if (options?.temperature !== undefined)
+            body.temperature = options.temperature;
+        if (options?.maxTokens !== undefined)
+            body.max_tokens = options.maxTokens;
+        if (options?.responseFormat === 'json') {
+            body.response_format = { type: 'json_object' };
+        }
+        const response = await fetch(`${API_BASE}/chat/completions`, {
+            method: 'POST',
+            headers: {
+                Authorization: `Bearer ${this.token}`,
+                'Content-Type': 'application/json',
+            },
+            body: JSON.stringify(body),
+        });
+        if (response.status === 429) {
+            throw new RateLimitError(this.name);
+        }
+        if (!response.ok) {
+            throw new Error(`GitHub Models API error: ${response.status} ${response.statusText}`);
+        }
+        const data = (await response.json());
+        return data.choices[0].message.content;
+    }
+    async embed(text) {
+        const response = await fetch(`${API_BASE}/embeddings`, {
+            method: 'POST',
+            headers: {
+                Authorization: `Bearer ${this.token}`,
+                'Content-Type': 'application/json',
+            },
+            body: JSON.stringify({ model: EMBEDDING_MODEL, input: text }),
+        });
+        if (response.status === 429) {
+            throw new RateLimitError(this.name);
+        }
+        if (!response.ok) {
+            throw new Error(`GitHub Models API error: ${response.status} ${response.statusText}`);
+        }
+        const data = (await response.json());
+        return data.data[0].embedding;
+    }
+    estimateCost(_tokens) {
+        return 0;
+    }
+}
+//# sourceMappingURL=github-models.js.map

package/dist/src/adapters/inference/github-models.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"github-models.js","sourceRoot":"","sources":["../../../../src/adapters/inference/github-models.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,cAAc,EAAE,MAAM,iBAAiB,CAAC;AAEjD,MAAM,QAAQ,GAAG,oCAAoC,CAAC;AACtD,MAAM,UAAU,GAAG,oBAAoB,CAAC;AACxC,MAAM,eAAe,GAAG,+BAA+B,CAAC;AAExD,MAAM,OAAO,qBAAqB;IAGH;IAFpB,IAAI,GAAG,eAAe,CAAC;IAEhC,YAA6B,QAAgB,OAAO,CAAC,GAAG,CAAC,YAAY,IAAI,EAAE;QAA9C,UAAK,GAAL,KAAK,CAAyC;IAAG,CAAC;IAE/E,KAAK,CAAC,IAAI,CAAC,QAAmB,EAAE,OAAqB;QACnD,MAAM,IAAI,GAA4B;YACpC,KAAK,EAAE,UAAU;YACjB,QAAQ;SACT,CAAC;QACF,IAAI,OAAO,EAAE,WAAW,KAAK,SAAS;YAAE,IAAI,CAAC,WAAW,GAAG,OAAO,CAAC,WAAW,CAAC;QAC/E,IAAI,OAAO,EAAE,SAAS,KAAK,SAAS;YAAE,IAAI,CAAC,UAAU,GAAG,OAAO,CAAC,SAAS,CAAC;QAC1E,IAAI,OAAO,EAAE,cAAc,KAAK,MAAM,EAAE,CAAC;YACvC,IAAI,CAAC,eAAe,GAAG,EAAE,IAAI,EAAE,aAAa,EAAE,CAAC;QACjD,CAAC;QAED,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,GAAG,QAAQ,mBAAmB,EAAE;YAC3D,MAAM,EAAE,MAAM;YACd,OAAO,EAAE;gBACP,aAAa,EAAE,UAAU,IAAI,CAAC,KAAK,EAAE;gBACrC,cAAc,EAAE,kBAAkB;aACnC;YACD,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC;SAC3B,CAAC,CAAC;QAEH,IAAI,QAAQ,CAAC,MAAM,KAAK,GAAG,EAAE,CAAC;YAC5B,MAAM,IAAI,cAAc,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACtC,CAAC;QAED,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;YACjB,MAAM,IAAI,KAAK,CAAC,4BAA4B,QAAQ,CAAC,MAAM,IAAI,QAAQ,CAAC,UAAU,EAAE,CAAC,CAAC;QACxF,CAAC;QAED,MAAM,IAAI,GAAG,CAAC,MAAM,QAAQ,CAAC,IAAI,EAAE,CAElC,CAAC;QACF,OAAO,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC;IACzC,CAAC;IAED,KAAK,CAAC,KAAK,CAAC,IAAY;QACtB,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,GAAG,QAAQ,aAAa,EAAE;YACrD,MAAM,EAAE,MAAM;YACd,OAAO,EAAE;gBACP,aAAa,EAAE,UAAU,IAAI,CAAC,KAAK,EAAE;gBACrC,cAAc,EAAE,kBAAkB;aACnC;YACD,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,EAAE,KAAK,EAAE,eAAe,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC;SAC9D,CAAC,CAAC;QAEH,IAAI,QAAQ,CAAC,MAAM,KAAK,GAAG,EAAE,CAAC;YAC5B,MAAM,IAAI,cAAc,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACtC,CAAC;QAED,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;YACjB,MAAM,IAAI,KAAK,CAAC,4BAA4B,QAAQ,CAAC,MAAM,IAAI,QAAQ,CAAC,UAAU,EAAE,CAAC,CAAC;QACxF,CAAC;QAED,MAAM,IAAI,GAAG,CAAC,MAAM,QAAQ,CAAC,IAAI,EAAE,CAElC,CAAC;QACF,OAAO,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;IAChC,CAAC;IAED,YAAY,CAAC,OAAe;QAC1B,OAAO,CAAC,CAAC;IACX,CAAC;CACF"}

package/dist/src/adapters/inference/resolve.d.ts

@@ -0,0 +1,2 @@
+import type { InferenceAdapter } from '../../types.js';
+export declare function resolveInference(preference: string): InferenceAdapter;

package/dist/src/adapters/inference/resolve.js

@@ -0,0 +1,49 @@
+import { execFileSync } from 'node:child_process';
+import { AdapterNotAvailableError } from '../../errors.js';
+import { GitHubModelsInference } from './github-models.js';
+import { CopilotInference } from './copilot.js';
+function isCopilotAvailable() {
+    try {
+        execFileSync('gh', ['copilot', '--version'], { encoding: 'utf-8', stdio: 'pipe' });
+        return true;
+    }
+    catch {
+        return false;
+    }
+}
+function isGitHubTokenAvailable() {
+    return Boolean(process.env.GITHUB_TOKEN);
+}
+export function resolveInference(preference) {
+    if (preference === 'auto') {
+        const copilotAvailable = isCopilotAvailable();
+        const tokenAvailable = isGitHubTokenAvailable();
+        if (copilotAvailable && tokenAvailable) {
+            // Copilot for chat, GitHubModels as embedding fallback
+            const githubModels = new GitHubModelsInference();
+            return new CopilotInference(githubModels);
+        }
+        if (copilotAvailable) {
+            return new CopilotInference();
+        }
+        if (tokenAvailable) {
+            return new GitHubModelsInference();
+        }
+        throw new AdapterNotAvailableError('inference', 'No inference adapter available. Install GitHub Copilot CLI (`gh extension install github/gh-copilot`) or set GITHUB_TOKEN.');
+    }
+    if (preference === 'copilot') {
+        if (!isCopilotAvailable()) {
+            throw new AdapterNotAvailableError('copilot', 'GitHub Copilot CLI is not available. Install with: gh extension install github/gh-copilot');
+        }
+        const fallback = isGitHubTokenAvailable() ? new GitHubModelsInference() : undefined;
+        return new CopilotInference(fallback);
+    }
+    if (preference === 'github-models') {
+        if (!isGitHubTokenAvailable()) {
+            throw new AdapterNotAvailableError('github-models', 'GITHUB_TOKEN environment variable is not set.');
+        }
+        return new GitHubModelsInference();
+    }
+    throw new AdapterNotAvailableError(preference, `Unknown inference adapter "${preference}". Valid options: auto, copilot, github-models.`);
+}
+//# sourceMappingURL=resolve.js.map

package/dist/src/adapters/inference/resolve.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"resolve.js","sourceRoot":"","sources":["../../../../src/adapters/inference/resolve.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAElD,OAAO,EAAE,wBAAwB,EAAE,MAAM,iBAAiB,CAAC;AAC3D,OAAO,EAAE,qBAAqB,EAAE,MAAM,oBAAoB,CAAC;AAC3D,OAAO,EAAE,gBAAgB,EAAE,MAAM,cAAc,CAAC;AAEhD,SAAS,kBAAkB;IACzB,IAAI,CAAC;QACH,YAAY,CAAC,IAAI,EAAE,CAAC,SAAS,EAAE,WAAW,CAAC,EAAE,EAAE,QAAQ,EAAE,OAAO,EAAE,KAAK,EAAE,MAAM,EAAE,CAAC,CAAC;QACnF,OAAO,IAAI,CAAC;IACd,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,KAAK,CAAC;IACf,CAAC;AACH,CAAC;AAED,SAAS,sBAAsB;IAC7B,OAAO,OAAO,CAAC,OAAO,CAAC,GAAG,CAAC,YAAY,CAAC,CAAC;AAC3C,CAAC;AAED,MAAM,UAAU,gBAAgB,CAAC,UAAkB;IACjD,IAAI,UAAU,KAAK,MAAM,EAAE,CAAC;QAC1B,MAAM,gBAAgB,GAAG,kBAAkB,EAAE,CAAC;QAC9C,MAAM,cAAc,GAAG,sBAAsB,EAAE,CAAC;QAEhD,IAAI,gBAAgB,IAAI,cAAc,EAAE,CAAC;YACvC,uDAAuD;YACvD,MAAM,YAAY,GAAG,IAAI,qBAAqB,EAAE,CAAC;YACjD,OAAO,IAAI,gBAAgB,CAAC,YAAY,CAAC,CAAC;QAC5C,CAAC;QAED,IAAI,gBAAgB,EAAE,CAAC;YACrB,OAAO,IAAI,gBAAgB,EAAE,CAAC;QAChC,CAAC;QAED,IAAI,cAAc,EAAE,CAAC;YACnB,OAAO,IAAI,qBAAqB,EAAE,CAAC;QACrC,CAAC;QAED,MAAM,IAAI,wBAAwB,CAChC,WAAW,EACX,4HAA4H,CAC7H,CAAC;IACJ,CAAC;IAED,IAAI,UAAU,KAAK,SAAS,EAAE,CAAC;QAC7B,IAAI,CAAC,kBAAkB,EAAE,EAAE,CAAC;YAC1B,MAAM,IAAI,wBAAwB,CAChC,SAAS,EACT,2FAA2F,CAC5F,CAAC;QACJ,CAAC;QACD,MAAM,QAAQ,GAAG,sBAAsB,EAAE,CAAC,CAAC,CAAC,IAAI,qBAAqB,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC;QACpF,OAAO,IAAI,gBAAgB,CAAC,QAAQ,CAAC,CAAC;IACxC,CAAC;IAED,IAAI,UAAU,KAAK,eAAe,EAAE,CAAC;QACnC,IAAI,CAAC,sBAAsB,EAAE,EAAE,CAAC;YAC9B,MAAM,IAAI,wBAAwB,CAChC,eAAe,EACf,+CAA+C,CAChD,CAAC;QACJ,CAAC;QACD,OAAO,IAAI,qBAAqB,EAAE,CAAC;IACrC,CAAC;IAED,MAAM,IAAI,wBAAwB,CAChC,UAAU,EACV,8BAA8B,UAAU,iDAAiD,CAC1F,CAAC;AACJ,CAAC"}

package/dist/src/adapters/report/json.d.ts

@@ -0,0 +1,7 @@
+import type { ReportAdapter, EvalResults } from '../../types.js';
+export declare class JSONReporter implements ReportAdapter {
+    private readonly outputDir;
+    readonly name = "json";
+    constructor(outputDir: string);
+    report(results: EvalResults): Promise<void>;
+}

package/dist/src/adapters/report/json.js

@@ -0,0 +1,39 @@
+import * as fs from 'node:fs';
+import * as path from 'node:path';
+function buildGradingFile(results) {
+    const allAssertions = [];
+    for (const scenario of results.scenarios) {
+        if (scenario.grading) {
+            allAssertions.push(...scenario.grading.assertion_results);
+        }
+    }
+    const passed = allAssertions.filter((a) => a.passed).length;
+    const failed = allAssertions.filter((a) => !a.passed).length;
+    const total = allAssertions.length;
+    const pass_rate = total > 0 ? passed / total : 0;
+    const summary = { passed, failed, total, pass_rate };
+    return { assertion_results: allAssertions, summary };
+}
+export class JSONReporter {
+    outputDir;
+    name = 'json';
+    constructor(outputDir) {
+        this.outputDir = outputDir;
+    }
+    async report(results) {
+        fs.mkdirSync(this.outputDir, { recursive: true });
+        // grading.json
+        const gradingFile = buildGradingFile(results);
+        fs.writeFileSync(path.join(this.outputDir, 'grading.json'), JSON.stringify(gradingFile, null, 2), 'utf-8');
+        // timing.json
+        const timingData = {
+            total_tokens: results.timing.total_tokens,
+            duration_ms: results.timing.duration_ms,
+        };
+        fs.writeFileSync(path.join(this.outputDir, 'timing.json'), JSON.stringify(timingData, null, 2), 'utf-8');
+        // benchmark.json
+        const benchmarkData = { run_summary: results.summary };
+        fs.writeFileSync(path.join(this.outputDir, 'benchmark.json'), JSON.stringify(benchmarkData, null, 2), 'utf-8');
+    }
+}
+//# sourceMappingURL=json.js.map

package/dist/src/adapters/report/json.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"json.js","sourceRoot":"","sources":["../../../../src/adapters/report/json.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,MAAM,SAAS,CAAC;AAC9B,OAAO,KAAK,IAAI,MAAM,WAAW,CAAC;AASlC,SAAS,gBAAgB,CAAC,OAAoB;IAC5C,MAAM,aAAa,GAAsB,EAAE,CAAC;IAE5C,KAAK,MAAM,QAAQ,IAAI,OAAO,CAAC,SAAS,EAAE,CAAC;QACzC,IAAI,QAAQ,CAAC,OAAO,EAAE,CAAC;YACrB,aAAa,CAAC,IAAI,CAAC,GAAG,QAAQ,CAAC,OAAO,CAAC,iBAAiB,CAAC,CAAC;QAC5D,CAAC;IACH,CAAC;IAED,MAAM,MAAM,GAAG,aAAa,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,MAAM,CAAC;IAC5D,MAAM,MAAM,GAAG,aAAa,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,MAAM,CAAC;IAC7D,MAAM,KAAK,GAAG,aAAa,CAAC,MAAM,CAAC;IACnC,MAAM,SAAS,GAAG,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,MAAM,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;IAEjD,MAAM,OAAO,GAAmB,EAAE,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,SAAS,EAAE,CAAC;IACrE,OAAO,EAAE,iBAAiB,EAAE,aAAa,EAAE,OAAO,EAAE,CAAC;AACvD,CAAC;AAED,MAAM,OAAO,YAAY;IAGM;IAFpB,IAAI,GAAG,MAAM,CAAC;IAEvB,YAA6B,SAAiB;QAAjB,cAAS,GAAT,SAAS,CAAQ;IAAG,CAAC;IAElD,KAAK,CAAC,MAAM,CAAC,OAAoB;QAC/B,EAAE,CAAC,SAAS,CAAC,IAAI,CAAC,SAAS,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QAElD,eAAe;QACf,MAAM,WAAW,GAAG,gBAAgB,CAAC,OAAO,CAAC,CAAC;QAC9C,EAAE,CAAC,aAAa,CACd,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,cAAc,CAAC,EACzC,IAAI,CAAC,SAAS,CAAC,WAAW,EAAE,IAAI,EAAE,CAAC,CAAC,EACpC,OAAO,CACR,CAAC;QAEF,cAAc;QACd,MAAM,UAAU,GAAG;YACjB,YAAY,EAAE,OAAO,CAAC,MAAM,CAAC,YAAY;YACzC,WAAW,EAAE,OAAO,CAAC,MAAM,CAAC,WAAW;SACxC,CAAC;QACF,EAAE,CAAC,aAAa,CACd,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,aAAa,CAAC,EACxC,IAAI,CAAC,SAAS,CAAC,UAAU,EAAE,IAAI,EAAE,CAAC,CAAC,EACnC,OAAO,CACR,CAAC;QAEF,iBAAiB;QACjB,MAAM,aAAa,GAAG,EAAE,WAAW,EAAE,OAAO,CAAC,OAAO,EAAE,CAAC;QACvD,EAAE,CAAC,aAAa,CACd,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,gBAAgB,CAAC,EAC3C,IAAI,CAAC,SAAS,CAAC,aAAa,EAAE,IAAI,EAAE,CAAC,CAAC,EACtC,OAAO,CACR,CAAC;IACJ,CAAC;CACF"}

package/dist/src/adapters/report/terminal.d.ts

@@ -0,0 +1,5 @@
+import type { ReportAdapter, EvalResults } from '../../types.js';
+export declare class TerminalReporter implements ReportAdapter {
+    readonly name = "terminal";
+    report(results: EvalResults): Promise<void>;
+}

package/dist/src/adapters/report/terminal.js

@@ -0,0 +1,42 @@
+import chalk from 'chalk';
+function verdictIcon(verdict) {
+    switch (verdict) {
+        case 'pass':
+            return chalk.green('✓');
+        case 'regressed':
+            return chalk.red('✗');
+        default:
+            return chalk.yellow('?');
+    }
+}
+function formatScenario(scenario) {
+    const icon = verdictIcon(scenario.comparison.verdict);
+    const tier = `tier${scenario.comparison.tier}`;
+    const tokens = scenario.timing.total_tokens;
+    const durationS = (scenario.timing.duration_ms / 1000).toFixed(2);
+    const cost = scenario.newOutput.metadata.adapter;
+    return `  ${icon} Scenario ${scenario.scenarioId} [${tier}] — ${tokens} tokens, ${durationS}s (${cost})`;
+}
+export class TerminalReporter {
+    name = 'terminal';
+    async report(results) {
+        const { skillName, scenarios, summary, timing } = results;
+        console.log(chalk.bold(`\nSnapeval — ${skillName}`));
+        console.log(chalk.dim('─'.repeat(50)));
+        for (const scenario of scenarios) {
+            console.log(formatScenario(scenario));
+        }
+        console.log(chalk.dim('─'.repeat(50)));
+        const passedStr = chalk.green(`${summary.passed} passed`);
+        const regressedCount = summary.regressed;
+        const regressedStr = regressedCount > 0
+            ? chalk.red(`${regressedCount} regressed`)
+            : chalk.dim(`${regressedCount} regressed`);
+        const totalStr = `${summary.total_scenarios} total`;
+        const passRate = (summary.pass_rate * 100).toFixed(0);
+        console.log(`${passedStr}, ${regressedStr}, ${totalStr} (${passRate}%)`);
+        console.log(chalk.dim(`Tokens: ${timing.total_tokens} | Duration: ${(timing.duration_ms / 1000).toFixed(2)}s | Cost: $${summary.total_cost_usd.toFixed(4)}`));
+        console.log(chalk.dim(`Tier breakdown — schema: ${summary.tier_breakdown.tier1_schema}, embedding: ${summary.tier_breakdown.tier2_embedding}, llm: ${summary.tier_breakdown.tier3_llm_judge}`));
+    }
+}
+//# sourceMappingURL=terminal.js.map

package/dist/src/adapters/report/terminal.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"terminal.js","sourceRoot":"","sources":["../../../../src/adapters/report/terminal.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,MAAM,OAAO,CAAC;AAG1B,SAAS,WAAW,CAAC,OAA0B;IAC7C,QAAQ,OAAO,EAAE,CAAC;QAChB,KAAK,MAAM;YACT,OAAO,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;QAC1B,KAAK,WAAW;YACd,OAAO,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QACxB;YACE,OAAO,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;IAC7B,CAAC;AACH,CAAC;AAED,SAAS,cAAc,CAAC,QAAwB;IAC9C,MAAM,IAAI,GAAG,WAAW,CAAC,QAAQ,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC;IACtD,MAAM,IAAI,GAAG,OAAO,QAAQ,CAAC,UAAU,CAAC,IAAI,EAAE,CAAC;IAC/C,MAAM,MAAM,GAAG,QAAQ,CAAC,MAAM,CAAC,YAAY,CAAC;IAC5C,MAAM,SAAS,GAAG,CAAC,QAAQ,CAAC,MAAM,CAAC,WAAW,GAAG,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;IAClE,MAAM,IAAI,GAAG,QAAQ,CAAC,SAAS,CAAC,QAAQ,CAAC,OAAO,CAAC;IACjD,OAAO,KAAK,IAAI,aAAa,QAAQ,CAAC,UAAU,KAAK,IAAI,OAAO,MAAM,YAAY,SAAS,MAAM,IAAI,GAAG,CAAC;AAC3G,CAAC;AAED,MAAM,OAAO,gBAAgB;IAClB,IAAI,GAAG,UAAU,CAAC;IAE3B,KAAK,CAAC,MAAM,CAAC,OAAoB;QAC/B,MAAM,EAAE,SAAS,EAAE,SAAS,EAAE,OAAO,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC;QAE1D,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,gBAAgB,SAAS,EAAE,CAAC,CAAC,CAAC;QACrD,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;QAEvC,KAAK,MAAM,QAAQ,IAAI,SAAS,EAAE,CAAC;YACjC,OAAO,CAAC,GAAG,CAAC,cAAc,CAAC,QAAQ,CAAC,CAAC,CAAC;QACxC,CAAC;QAED,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;QAEvC,MAAM,SAAS,GAAG,KAAK,CAAC,KAAK,CAAC,GAAG,OAAO,CAAC,MAAM,SAAS,CAAC,CAAC;QAC1D,MAAM,cAAc,GAAG,OAAO,CAAC,SAAS,CAAC;QACzC,MAAM,YAAY,GAAG,cAAc,GAAG,CAAC;YACrC,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,cAAc,YAAY,CAAC;YAC1C,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,cAAc,YAAY,CAAC,CAAC;QAC7C,MAAM,QAAQ,GAAG,GAAG,OAAO,CAAC,eAAe,QAAQ,CAAC;QACpD,MAAM,QAAQ,GAAG,CAAC,OAAO,CAAC,SAAS,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;QAEtD,OAAO,CAAC,GAAG,CAAC,GAAG,SAAS,KAAK,YAAY,KAAK,QAAQ,KAAK,QAAQ,IAAI,CAAC,CAAC;QACzE,OAAO,CAAC,GAAG,CACT,KAAK,CAAC,GAAG,CACP,WAAW,MAAM,CAAC,YAAY,gBAAgB,CAAC,MAAM,CAAC,WAAW,GAAG,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,cAAc,OAAO,CAAC,cAAc,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CACtI,CACF,CAAC;QACF,OAAO,CAAC,GAAG,CACT,KAAK,CAAC,GAAG,CACP,4BAA4B,OAAO,CAAC,cAAc,CAAC,YAAY,gBAAgB,OAAO,CAAC,cAAc,CAAC,eAAe,UAAU,OAAO,CAAC,cAAc,CAAC,eAAe,EAAE,CACxK,CACF,CAAC;IACJ,CAAC;CACF"}

package/dist/src/adapters/skill/copilot-cli.d.ts

@@ -0,0 +1,6 @@
+import type { SkillAdapter, SkillOutput } from '../../types.js';
+export declare class CopilotCLIAdapter implements SkillAdapter {
+    readonly name = "copilot-cli";
+    invoke(skillPath: string, prompt: string, _files?: string[]): Promise<SkillOutput>;
+    isAvailable(): Promise<boolean>;
+}

package/dist/src/adapters/skill/copilot-cli.js

@@ -0,0 +1,51 @@
+import { execFile, execFileSync } from 'node:child_process';
+import { readFile } from 'node:fs/promises';
+import * as path from 'node:path';
+export class CopilotCLIAdapter {
+    name = 'copilot-cli';
+    async invoke(skillPath, prompt, _files) {
+        const startMs = Date.now();
+        // Try to include SKILL.md content as context if present
+        let skillMd = '';
+        try {
+            const skillFile = path.join(skillPath, 'SKILL.md');
+            skillMd = await readFile(skillFile, { encoding: 'utf-8' });
+        }
+        catch {
+            // ignore missing SKILL.md
+        }
+        const finalPrompt = skillMd ? `${skillMd}\n\n${prompt}` : prompt;
+        return new Promise((resolve, reject) => {
+            // Use gh copilot and pass flags after `--` so gh doesn't consume them.
+            // Use --silent to limit output to the model's response only.
+            execFile('gh', ['copilot', '--', '-p', finalPrompt, '--silent'], { encoding: 'utf-8', maxBuffer: 10 * 1024 * 1024 }, (error, stdout, _stderr) => {
+                if (error) {
+                    reject(error);
+                    return;
+                }
+                const durationMs = Date.now() - startMs;
+                const raw = stdout.trim();
+                resolve({
+                    raw,
+                    metadata: {
+                        tokens: 0,
+                        durationMs,
+                        model: 'copilot',
+                        adapter: this.name,
+                    },
+                });
+            });
+        });
+    }
+    async isAvailable() {
+        try {
+            // Use gh copilot --help as a lightweight availability check
+            execFileSync('gh', ['copilot', '--help'], { encoding: 'utf-8', stdio: 'pipe' });
+            return true;
+        }
+        catch {
+            return false;
+        }
+    }
+}
+//# sourceMappingURL=copilot-cli.js.map

package/dist/src/adapters/skill/copilot-cli.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"copilot-cli.js","sourceRoot":"","sources":["../../../../src/adapters/skill/copilot-cli.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAC5D,OAAO,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAC;AAC5C,OAAO,KAAK,IAAI,MAAM,WAAW,CAAC;AAGlC,MAAM,OAAO,iBAAiB;IACnB,IAAI,GAAG,aAAa,CAAC;IAE9B,KAAK,CAAC,MAAM,CAAC,SAAiB,EAAE,MAAc,EAAE,MAAiB;QAC/D,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAE3B,wDAAwD;QACxD,IAAI,OAAO,GAAG,EAAE,CAAC;QACjB,IAAI,CAAC;YACH,MAAM,SAAS,GAAG,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,UAAU,CAAC,CAAC;YACnD,OAAO,GAAG,MAAM,QAAQ,CAAC,SAAS,EAAE,EAAE,QAAQ,EAAE,OAAO,EAAE,CAAC,CAAC;QAC7D,CAAC;QAAC,MAAM,CAAC;YACP,0BAA0B;QAC5B,CAAC;QAED,MAAM,WAAW,GAAG,OAAO,CAAC,CAAC,CAAC,GAAG,OAAO,OAAO,MAAM,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC;QAEjE,OAAO,IAAI,OAAO,CAAc,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;YAClD,uEAAuE;YACvE,6DAA6D;YAC7D,QAAQ,CACN,IAAI,EACJ,CAAC,SAAS,EAAE,IAAI,EAAE,IAAI,EAAE,WAAW,EAAE,UAAU,CAAC,EAChD,EAAE,QAAQ,EAAE,OAAO,EAAE,SAAS,EAAE,EAAE,GAAG,IAAI,GAAG,IAAI,EAAE,EAClD,CAAC,KAAK,EAAE,MAAM,EAAE,OAAO,EAAE,EAAE;gBACzB,IAAI,KAAK,EAAE,CAAC;oBACV,MAAM,CAAC,KAAK,CAAC,CAAC;oBACd,OAAO;gBACT,CAAC;gBACD,MAAM,UAAU,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,OAAO,CAAC;gBACxC,MAAM,GAAG,GAAG,MAAM,CAAC,IAAI,EAAE,CAAC;gBAC1B,OAAO,CAAC;oBACN,GAAG;oBACH,QAAQ,EAAE;wBACR,MAAM,EAAE,CAAC;wBACT,UAAU;wBACV,KAAK,EAAE,SAAS;wBAChB,OAAO,EAAE,IAAI,CAAC,IAAI;qBACnB;iBACF,CAAC,CAAC;YACL,CAAC,CACF,CAAC;QACJ,CAAC,CAAC,CAAC;IACL,CAAC;IAED,KAAK,CAAC,WAAW;QACf,IAAI,CAAC;YACH,4DAA4D;YAC5D,YAAY,CAAC,IAAI,EAAE,CAAC,SAAS,EAAE,QAAQ,CAAC,EAAE,EAAE,QAAQ,EAAE,OAAO,EAAE,KAAK,EAAE,MAAM,EAAE,CAAC,CAAC;YAChF,OAAO,IAAI,CAAC;QACd,CAAC;QAAC,MAAM,CAAC;YACP,OAAO,KAAK,CAAC;QACf,CAAC;IACH,CAAC;CACF"}

package/dist/src/commands/approve.d.ts

@@ -0,0 +1,5 @@
+import type { SkillAdapter, EvalResults } from '../types.js';
+export declare function approveCommand(skillPath: string, skillAdapter: SkillAdapter, options?: {
+    scenarioIds?: number[];
+}): Promise<void>;
+export declare function approveFromResults(skillPath: string, results: EvalResults, scenarioIds?: number[]): void;

package/dist/src/commands/approve.js

@@ -0,0 +1,40 @@
+import * as fs from 'node:fs';
+import * as path from 'node:path';
+import { SnapshotManager } from '../engine/snapshot.js';
+import { SnapevalError } from '../errors.js';
+export async function approveCommand(skillPath, skillAdapter, options = {}) {
+    const evalsPath = path.join(skillPath, 'evals', 'evals.json');
+    if (!fs.existsSync(evalsPath)) {
+        throw new SnapevalError(`No evals.json found at ${evalsPath}. Run \`snapeval init\` first.`);
+    }
+    const evalsFile = JSON.parse(fs.readFileSync(evalsPath, 'utf-8'));
+    const manager = new SnapshotManager(path.join(skillPath, 'evals'));
+    // Determine which scenarios to approve
+    const targetCases = options.scenarioIds && options.scenarioIds.length > 0
+        ? evalsFile.evals.filter((e) => options.scenarioIds.includes(e.id))
+        : evalsFile.evals;
+    for (const evalCase of targetCases) {
+        const newOutput = await skillAdapter.invoke(skillPath, evalCase.prompt, evalCase.files);
+        manager.approve(evalCase.id, evalCase.prompt, newOutput);
+    }
+}
+export function approveFromResults(skillPath, results, scenarioIds) {
+    const evalsPath = path.join(skillPath, 'evals', 'evals.json');
+    if (!fs.existsSync(evalsPath)) {
+        throw new SnapevalError(`No evals.json found at ${evalsPath}.`);
+    }
+    const evalsFile = JSON.parse(fs.readFileSync(evalsPath, 'utf-8'));
+    const manager = new SnapshotManager(path.join(skillPath, 'evals'));
+    // Find regressed scenarios from results
+    const regressedResults = results.scenarios.filter((s) => s.comparison.verdict === 'regressed');
+    const toApprove = scenarioIds && scenarioIds.length > 0
+        ? regressedResults.filter((s) => scenarioIds.includes(s.scenarioId))
+        : regressedResults;
+    for (const scenario of toApprove) {
+        const evalCase = evalsFile.evals.find((e) => e.id === scenario.scenarioId);
+        if (!evalCase)
+            continue;
+        manager.approve(scenario.scenarioId, evalCase.prompt, scenario.newOutput);
+    }
+}
+//# sourceMappingURL=approve.js.map

package/dist/src/commands/approve.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"approve.js","sourceRoot":"","sources":["../../../src/commands/approve.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,MAAM,SAAS,CAAC;AAC9B,OAAO,KAAK,IAAI,MAAM,WAAW,CAAC;AAElC,OAAO,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AACxD,OAAO,EAAE,aAAa,EAAE,MAAM,cAAc,CAAC;AAE7C,MAAM,CAAC,KAAK,UAAU,cAAc,CAClC,SAAiB,EACjB,YAA0B,EAC1B,UAAsC,EAAE;IAExC,MAAM,SAAS,GAAG,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,OAAO,EAAE,YAAY,CAAC,CAAC;IAC9D,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,SAAS,CAAC,EAAE,CAAC;QAC9B,MAAM,IAAI,aAAa,CAAC,0BAA0B,SAAS,gCAAgC,CAAC,CAAC;IAC/F,CAAC;IAED,MAAM,SAAS,GAAc,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,YAAY,CAAC,SAAS,EAAE,OAAO,CAAC,CAAC,CAAC;IAC7E,MAAM,OAAO,GAAG,IAAI,eAAe,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,OAAO,CAAC,CAAC,CAAC;IAEnE,uCAAuC;IACvC,MAAM,WAAW,GAAG,OAAO,CAAC,WAAW,IAAI,OAAO,CAAC,WAAW,CAAC,MAAM,GAAG,CAAC;QACvE,CAAC,CAAC,SAAS,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,OAAO,CAAC,WAAY,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;QACpE,CAAC,CAAC,SAAS,CAAC,KAAK,CAAC;IAEpB,KAAK,MAAM,QAAQ,IAAI,WAAW,EAAE,CAAC;QACnC,MAAM,SAAS,GAAG,MAAM,YAAY,CAAC,MAAM,CAAC,SAAS,EAAE,QAAQ,CAAC,MAAM,EAAE,QAAQ,CAAC,KAAK,CAAC,CAAC;QACxF,OAAO,CAAC,OAAO,CAAC,QAAQ,CAAC,EAAE,EAAE,QAAQ,CAAC,MAAM,EAAE,SAAS,CAAC,CAAC;IAC3D,CAAC;AACH,CAAC;AAED,MAAM,UAAU,kBAAkB,CAChC,SAAiB,EACjB,OAAoB,EACpB,WAAsB;IAEtB,MAAM,SAAS,GAAG,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,OAAO,EAAE,YAAY,CAAC,CAAC;IAC9D,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,SAAS,CAAC,EAAE,CAAC;QAC9B,MAAM,IAAI,aAAa,CAAC,0BAA0B,SAAS,GAAG,CAAC,CAAC;IAClE,CAAC;IAED,MAAM,SAAS,GAAc,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,YAAY,CAAC,SAAS,EAAE,OAAO,CAAC,CAAC,CAAC;IAC7E,MAAM,OAAO,GAAG,IAAI,eAAe,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,OAAO,CAAC,CAAC,CAAC;IAEnE,wCAAwC;IACxC,MAAM,gBAAgB,GAAG,OAAO,CAAC,SAAS,CAAC,MAAM,CAC/C,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,UAAU,CAAC,OAAO,KAAK,WAAW,CAC5C,CAAC;IAEF,MAAM,SAAS,GAAG,WAAW,IAAI,WAAW,CAAC,MAAM,GAAG,CAAC;QACrD,CAAC,CAAC,gBAAgB,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,WAAW,CAAC,QAAQ,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC;QACpE,CAAC,CAAC,gBAAgB,CAAC;IAErB,KAAK,MAAM,QAAQ,IAAI,SAAS,EAAE,CAAC;QACjC,MAAM,QAAQ,GAAG,SAAS,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,KAAK,QAAQ,CAAC,UAAU,CAAC,CAAC;QAC3E,IAAI,CAAC,QAAQ;YAAE,SAAS;QACxB,OAAO,CAAC,OAAO,CAAC,QAAQ,CAAC,UAAU,EAAE,QAAQ,CAAC,MAAM,EAAE,QAAQ,CAAC,SAAS,CAAC,CAAC;IAC5E,CAAC;AACH,CAAC"}

package/dist/src/commands/capture.d.ts

@@ -0,0 +1,4 @@
+import type { SkillAdapter } from '../types.js';
+export declare function captureCommand(skillPath: string, skillAdapter: SkillAdapter, options?: {
+    runs?: number;
+}): Promise<void>;

package/dist/src/commands/capture.js

@@ -0,0 +1,18 @@
+import * as fs from 'node:fs';
+import * as path from 'node:path';
+import { SnapshotManager } from '../engine/snapshot.js';
+import { SnapevalError } from '../errors.js';
+export async function captureCommand(skillPath, skillAdapter, options = {}) {
+    const evalsPath = path.join(skillPath, 'evals', 'evals.json');
+    if (!fs.existsSync(evalsPath)) {
+        throw new SnapevalError(`No evals.json found at ${evalsPath}. Run \`snapeval init\` first.`);
+    }
+    const evalsFile = JSON.parse(fs.readFileSync(evalsPath, 'utf-8'));
+    const manager = new SnapshotManager(path.join(skillPath, 'evals'));
+    const runs = options.runs ?? 1;
+    for (const evalCase of evalsFile.evals) {
+        const output = await skillAdapter.invoke(skillPath, evalCase.prompt, evalCase.files);
+        manager.saveSnapshot(evalCase.id, evalCase.prompt, output, runs);
+    }
+}
+//# sourceMappingURL=capture.js.map

package/dist/src/commands/capture.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"capture.js","sourceRoot":"","sources":["../../../src/commands/capture.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,MAAM,SAAS,CAAC;AAC9B,OAAO,KAAK,IAAI,MAAM,WAAW,CAAC;AAElC,OAAO,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AACxD,OAAO,EAAE,aAAa,EAAE,MAAM,cAAc,CAAC;AAE7C,MAAM,CAAC,KAAK,UAAU,cAAc,CAClC,SAAiB,EACjB,YAA0B,EAC1B,UAA6B,EAAE;IAE/B,MAAM,SAAS,GAAG,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,OAAO,EAAE,YAAY,CAAC,CAAC;IAC9D,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,SAAS,CAAC,EAAE,CAAC;QAC9B,MAAM,IAAI,aAAa,CAAC,0BAA0B,SAAS,gCAAgC,CAAC,CAAC;IAC/F,CAAC;IAED,MAAM,SAAS,GAAc,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,YAAY,CAAC,SAAS,EAAE,OAAO,CAAC,CAAC,CAAC;IAC7E,MAAM,OAAO,GAAG,IAAI,eAAe,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,OAAO,CAAC,CAAC,CAAC;IACnE,MAAM,IAAI,GAAG,OAAO,CAAC,IAAI,IAAI,CAAC,CAAC;IAE/B,KAAK,MAAM,QAAQ,IAAI,SAAS,CAAC,KAAK,EAAE,CAAC;QACvC,MAAM,MAAM,GAAG,MAAM,YAAY,CAAC,MAAM,CAAC,SAAS,EAAE,QAAQ,CAAC,MAAM,EAAE,QAAQ,CAAC,KAAK,CAAC,CAAC;QACrF,OAAO,CAAC,YAAY,CAAC,QAAQ,CAAC,EAAE,EAAE,QAAQ,CAAC,MAAM,EAAE,MAAM,EAAE,IAAI,CAAC,CAAC;IACnE,CAAC;AACH,CAAC"}

package/dist/src/commands/check.d.ts

@@ -0,0 +1,6 @@
+import type { SkillAdapter, InferenceAdapter, EvalResults } from '../types.js';
+export declare function checkCommand(skillPath: string, skillAdapter: SkillAdapter, inference: InferenceAdapter, options: {
+    threshold: number;
+    budget: string;
+    skipEmbedding?: boolean;
+}): Promise<EvalResults>;

package/dist/src/commands/check.js

@@ -0,0 +1,68 @@
+import * as fs from 'node:fs';
+import * as path from 'node:path';
+import { SnapshotManager } from '../engine/snapshot.js';
+import { comparePipeline } from '../engine/comparison/pipeline.js';
+import { NoBaselineError, SnapevalError } from '../errors.js';
+import { BudgetEngine } from '../engine/budget.js';
+export async function checkCommand(skillPath, skillAdapter, inference, options) {
+    const evalsPath = path.join(skillPath, 'evals', 'evals.json');
+    if (!fs.existsSync(evalsPath)) {
+        throw new SnapevalError(`No evals.json found at ${evalsPath}`);
+    }
+    const evalsFile = JSON.parse(fs.readFileSync(evalsPath, 'utf-8'));
+    const manager = new SnapshotManager(path.join(skillPath, 'evals'));
+    const budget = new BudgetEngine(options.budget);
+    const startTime = Date.now();
+    if (manager.listSnapshotIds().length === 0) {
+        throw new NoBaselineError(skillPath);
+    }
+    const scenarios = [];
+    const tierBreakdown = { tier1_schema: 0, tier2_embedding: 0, tier3_llm_judge: 0 };
+    for (const evalCase of evalsFile.evals) {
+        const baseline = manager.loadSnapshot(evalCase.id);
+        if (!baseline)
+            continue;
+        const newOutput = await skillAdapter.invoke(skillPath, evalCase.prompt, evalCase.files);
+        const comparison = await comparePipeline(baseline.output.raw, newOutput.raw, inference, { threshold: options.threshold, skipEmbedding: options.skipEmbedding });
+        comparison.scenarioId = evalCase.id;
+        if (comparison.tier === 1)
+            tierBreakdown.tier1_schema++;
+        else if (comparison.tier === 2)
+            tierBreakdown.tier2_embedding++;
+        else
+            tierBreakdown.tier3_llm_judge++;
+        budget.addCost(inference.estimateCost(newOutput.metadata.tokens));
+        scenarios.push({
+            scenarioId: evalCase.id,
+            prompt: evalCase.prompt,
+            comparison,
+            timing: {
+                total_tokens: newOutput.metadata.tokens,
+                duration_ms: newOutput.metadata.durationMs,
+            },
+            newOutput,
+        });
+    }
+    const passed = scenarios.filter((s) => s.comparison.verdict === 'pass').length;
+    const regressed = scenarios.filter((s) => s.comparison.verdict === 'regressed').length;
+    const summary = {
+        total_scenarios: scenarios.length,
+        passed,
+        regressed,
+        pass_rate: scenarios.length > 0 ? passed / scenarios.length : 1.0,
+        total_tokens: scenarios.reduce((sum, s) => sum + s.timing.total_tokens, 0),
+        total_cost_usd: budget.totalCost,
+        total_duration_ms: Date.now() - startTime,
+        tier_breakdown: tierBreakdown,
+    };
+    return {
+        skillName: evalsFile.skill_name,
+        scenarios,
+        summary,
+        timing: {
+            total_tokens: summary.total_tokens,
+            duration_ms: summary.total_duration_ms,
+        },
+    };
+}
+//# sourceMappingURL=check.js.map

package/dist/src/commands/check.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"check.js","sourceRoot":"","sources":["../../../src/commands/check.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,MAAM,SAAS,CAAC;AAC9B,OAAO,KAAK,IAAI,MAAM,WAAW,CAAC;AASlC,OAAO,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AACxD,OAAO,EAAE,eAAe,EAAE,MAAM,kCAAkC,CAAC;AACnE,OAAO,EAAE,eAAe,EAAE,aAAa,EAAE,MAAM,cAAc,CAAC;AAC9D,OAAO,EAAE,YAAY,EAAE,MAAM,qBAAqB,CAAC;AAEnD,MAAM,CAAC,KAAK,UAAU,YAAY,CAChC,SAAiB,EACjB,YAA0B,EAC1B,SAA2B,EAC3B,OAAuE;IAEvE,MAAM,SAAS,GAAG,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,OAAO,EAAE,YAAY,CAAC,CAAC;IAC9D,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,SAAS,CAAC,EAAE,CAAC;QAC9B,MAAM,IAAI,aAAa,CAAC,0BAA0B,SAAS,EAAE,CAAC,CAAC;IACjE,CAAC;IACD,MAAM,SAAS,GAAc,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,YAAY,CAAC,SAAS,EAAE,OAAO,CAAC,CAAC,CAAC;IAC7E,MAAM,OAAO,GAAG,IAAI,eAAe,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,OAAO,CAAC,CAAC,CAAC;IACnE,MAAM,MAAM,GAAG,IAAI,YAAY,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;IAChD,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAE7B,IAAI,OAAO,CAAC,eAAe,EAAE,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC3C,MAAM,IAAI,eAAe,CAAC,SAAS,CAAC,CAAC;IACvC,CAAC;IAED,MAAM,SAAS,GAAqB,EAAE,CAAC;IACvC,MAAM,aAAa,GAAG,EAAE,YAAY,EAAE,CAAC,EAAE,eAAe,EAAE,CAAC,EAAE,eAAe,EAAE,CAAC,EAAE,CAAC;IAElF,KAAK,MAAM,QAAQ,IAAI,SAAS,CAAC,KAAK,EAAE,CAAC;QACvC,MAAM,QAAQ,GAAG,OAAO,CAAC,YAAY,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC;QACnD,IAAI,CAAC,QAAQ;YAAE,SAAS;QACxB,MAAM,SAAS,GAAG,MAAM,YAAY,CAAC,MAAM,CAAC,SAAS,EAAE,QAAQ,CAAC,MAAM,EAAE,QAAQ,CAAC,KAAK,CAAC,CAAC;QACxF,MAAM,UAAU,GAAG,MAAM,eAAe,CACtC,QAAQ,CAAC,MAAM,CAAC,GAAG,EACnB,SAAS,CAAC,GAAG,EACb,SAAS,EACT,EAAE,SAAS,EAAE,OAAO,CAAC,SAAS,EAAE,aAAa,EAAE,OAAO,CAAC,aAAa,EAAE,CACvE,CAAC;QACF,UAAU,CAAC,UAAU,GAAG,QAAQ,CAAC,EAAE,CAAC;QACpC,IAAI,UAAU,CAAC,IAAI,KAAK,CAAC;YAAE,aAAa,CAAC,YAAY,EAAE,CAAC;aACnD,IAAI,UAAU,CAAC,IAAI,KAAK,CAAC;YAAE,aAAa,CAAC,eAAe,EAAE,CAAC;;YAC3D,aAAa,CAAC,eAAe,EAAE,CAAC;QACrC,MAAM,CAAC,OAAO,CAAC,SAAS,CAAC,YAAY,CAAC,SAAS,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC;QAClE,SAAS,CAAC,IAAI,CAAC;YACb,UAAU,EAAE,QAAQ,CAAC,EAAE;YACvB,MAAM,EAAE,QAAQ,CAAC,MAAM;YACvB,UAAU;YACV,MAAM,EAAE;gBACN,YAAY,EAAE,SAAS,CAAC,QAAQ,CAAC,MAAM;gBACvC,WAAW,EAAE,SAAS,CAAC,QAAQ,CAAC,UAAU;aAC3C;YACD,SAAS;SACV,CAAC,CAAC;IACL,CAAC;IAED,MAAM,MAAM,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,UAAU,CAAC,OAAO,KAAK,MAAM,CAAC,CAAC,MAAM,CAAC;IAC/E,MAAM,SAAS,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,UAAU,CAAC,OAAO,KAAK,WAAW,CAAC,CAAC,MAAM,CAAC;IACvF,MAAM,OAAO,GAAqB;QAChC,eAAe,EAAE,SAAS,CAAC,MAAM;QACjC,MAAM;QACN,SAAS;QACT,SAAS,EAAE,SAAS,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,MAAM,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC,GAAG;QACjE,YAAY,EAAE,SAAS,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,MAAM,CAAC,YAAY,EAAE,CAAC,CAAC;QAC1E,cAAc,EAAE,MAAM,CAAC,SAAS;QAChC,iBAAiB,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS;QACzC,cAAc,EAAE,aAAa;KAC9B,CAAC;IACF,OAAO;QACL,SAAS,EAAE,SAAS,CAAC,UAAU;QAC/B,SAAS;QACT,OAAO;QACP,MAAM,EAAE;YACN,YAAY,EAAE,OAAO,CAAC,YAAY;YAClC,WAAW,EAAE,OAAO,CAAC,iBAAiB;SACvC;KACF,CAAC;AACJ,CAAC"}

package/dist/src/commands/init.d.ts

@@ -0,0 +1,2 @@
+import type { InferenceAdapter } from '../types.js';
+export declare function initCommand(skillPath: string, inference: InferenceAdapter): Promise<void>;

package/dist/src/commands/init.js

@@ -0,0 +1,27 @@
+import * as fs from 'node:fs';
+import * as path from 'node:path';
+import { generateEvals } from '../engine/generator.js';
+import { SnapevalError } from '../errors.js';
+export async function initCommand(skillPath, inference) {
+    // Locate the skill definition file (SKILL.md or skill.md)
+    const candidates = ['SKILL.md', 'skill.md'];
+    let skillFilePath = null;
+    for (const name of candidates) {
+        const candidate = path.join(skillPath, name);
+        if (fs.existsSync(candidate)) {
+            skillFilePath = candidate;
+            break;
+        }
+    }
+    if (!skillFilePath) {
+        throw new SnapevalError(`No SKILL.md found at ${skillPath}. Create a SKILL.md file to describe your skill.`);
+    }
+    const skillContent = fs.readFileSync(skillFilePath, 'utf-8');
+    const skillName = path.basename(skillPath);
+    const evalsFile = await generateEvals(skillContent, skillName, inference);
+    const evalsDir = path.join(skillPath, 'evals');
+    fs.mkdirSync(evalsDir, { recursive: true });
+    const evalsPath = path.join(evalsDir, 'evals.json');
+    fs.writeFileSync(evalsPath, JSON.stringify(evalsFile, null, 2), 'utf-8');
+}
+//# sourceMappingURL=init.js.map

package/dist/src/commands/init.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"init.js","sourceRoot":"","sources":["../../../src/commands/init.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,MAAM,SAAS,CAAC;AAC9B,OAAO,KAAK,IAAI,MAAM,WAAW,CAAC;AAElC,OAAO,EAAE,aAAa,EAAE,MAAM,wBAAwB,CAAC;AACvD,OAAO,EAAE,aAAa,EAAE,MAAM,cAAc,CAAC;AAE7C,MAAM,CAAC,KAAK,UAAU,WAAW,CAC/B,SAAiB,EACjB,SAA2B;IAE3B,0DAA0D;IAC1D,MAAM,UAAU,GAAG,CAAC,UAAU,EAAE,UAAU,CAAC,CAAC;IAC5C,IAAI,aAAa,GAAkB,IAAI,CAAC;IACxC,KAAK,MAAM,IAAI,IAAI,UAAU,EAAE,CAAC;QAC9B,MAAM,SAAS,GAAG,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,IAAI,CAAC,CAAC;QAC7C,IAAI,EAAE,CAAC,UAAU,CAAC,SAAS,CAAC,EAAE,CAAC;YAC7B,aAAa,GAAG,SAAS,CAAC;YAC1B,MAAM;QACR,CAAC;IACH,CAAC;IAED,IAAI,CAAC,aAAa,EAAE,CAAC;QACnB,MAAM,IAAI,aAAa,CACrB,wBAAwB,SAAS,kDAAkD,CACpF,CAAC;IACJ,CAAC;IAED,MAAM,YAAY,GAAG,EAAE,CAAC,YAAY,CAAC,aAAa,EAAE,OAAO,CAAC,CAAC;IAC7D,MAAM,SAAS,GAAG,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC;IAE3C,MAAM,SAAS,GAAG,MAAM,aAAa,CAAC,YAAY,EAAE,SAAS,EAAE,SAAS,CAAC,CAAC;IAE1E,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,OAAO,CAAC,CAAC;IAC/C,EAAE,CAAC,SAAS,CAAC,QAAQ,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAE5C,MAAM,SAAS,GAAG,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,YAAY,CAAC,CAAC;IACpD,EAAE,CAAC,aAAa,CAAC,SAAS,EAAE,IAAI,CAAC,SAAS,CAAC,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC,EAAE,OAAO,CAAC,CAAC;AAC3E,CAAC"}

package/dist/src/commands/report.d.ts

@@ -0,0 +1,4 @@
+import type { EvalResults } from '../types.js';
+export declare function reportCommand(skillPath: string, results: EvalResults, options?: {
+    verbose?: boolean;
+}): Promise<void>;

package/dist/src/commands/report.js

@@ -0,0 +1,26 @@
+import * as fs from 'node:fs';
+import * as path from 'node:path';
+import { JSONReporter } from '../adapters/report/json.js';
+import { TerminalReporter } from '../adapters/report/terminal.js';
+export async function reportCommand(skillPath, results, options = {}) {
+    // Determine next iteration number
+    const resultsBaseDir = path.join(skillPath, 'evals', 'results');
+    fs.mkdirSync(resultsBaseDir, { recursive: true });
+    const existingIterations = fs.readdirSync(resultsBaseDir)
+        .filter((d) => /^iteration-\d+$/.test(d))
+        .map((d) => parseInt(d.replace('iteration-', ''), 10))
+        .sort((a, b) => a - b);
+    const nextIteration = existingIterations.length > 0
+        ? existingIterations[existingIterations.length - 1] + 1
+        : 1;
+    const iterationDir = path.join(resultsBaseDir, `iteration-${nextIteration}`);
+    // Write JSON report
+    const jsonReporter = new JSONReporter(iterationDir);
+    await jsonReporter.report(results);
+    // Print terminal report
+    if (options.verbose !== false) {
+        const terminalReporter = new TerminalReporter();
+        await terminalReporter.report(results);
+    }
+}
+//# sourceMappingURL=report.js.map

package/dist/src/commands/report.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"report.js","sourceRoot":"","sources":["../../../src/commands/report.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,MAAM,SAAS,CAAC;AAC9B,OAAO,KAAK,IAAI,MAAM,WAAW,CAAC;AAElC,OAAO,EAAE,YAAY,EAAE,MAAM,4BAA4B,CAAC;AAC1D,OAAO,EAAE,gBAAgB,EAAE,MAAM,gCAAgC,CAAC;AAElE,MAAM,CAAC,KAAK,UAAU,aAAa,CACjC,SAAiB,EACjB,OAAoB,EACpB,UAAiC,EAAE;IAEnC,kCAAkC;IAClC,MAAM,cAAc,GAAG,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,OAAO,EAAE,SAAS,CAAC,CAAC;IAChE,EAAE,CAAC,SAAS,CAAC,cAAc,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAElD,MAAM,kBAAkB,GAAG,EAAE,CAAC,WAAW,CAAC,cAAc,CAAC;SACtD,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,iBAAiB,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;SACxC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,QAAQ,CAAC,CAAC,CAAC,OAAO,CAAC,YAAY,EAAE,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC;SACrD,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;IAEzB,MAAM,aAAa,GAAG,kBAAkB,CAAC,MAAM,GAAG,CAAC;QACjD,CAAC,CAAC,kBAAkB,CAAC,kBAAkB,CAAC,MAAM,GAAG,CAAC,CAAC,GAAG,CAAC;QACvD,CAAC,CAAC,CAAC,CAAC;IAEN,MAAM,YAAY,GAAG,IAAI,CAAC,IAAI,CAAC,cAAc,EAAE,aAAa,aAAa,EAAE,CAAC,CAAC;IAE7E,oBAAoB;IACpB,MAAM,YAAY,GAAG,IAAI,YAAY,CAAC,YAAY,CAAC,CAAC;IACpD,MAAM,YAAY,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;IAEnC,wBAAwB;IACxB,IAAI,OAAO,CAAC,OAAO,KAAK,KAAK,EAAE,CAAC;QAC9B,MAAM,gBAAgB,GAAG,IAAI,gBAAgB,EAAE,CAAC;QAChD,MAAM,gBAAgB,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;IACzC,CAAC;AACH,CAAC"}

package/dist/src/config.d.ts

@@ -0,0 +1,3 @@
+import type { SnapevalConfig } from './types.js';
+export declare const DEFAULT_CONFIG: SnapevalConfig;
+export declare function resolveConfig(cliFlags: Partial<SnapevalConfig>, projectRoot: string, skillDir?: string): SnapevalConfig;

package/dist/src/config.js

@@ -0,0 +1,30 @@
+import * as fs from 'node:fs';
+import * as path from 'node:path';
+export const DEFAULT_CONFIG = {
+    adapter: 'copilot-cli',
+    inference: 'auto',
+    threshold: 0.85,
+    runs: 3,
+    budget: 'unlimited',
+};
+function loadConfigFile(dirPath) {
+    const configPath = path.join(dirPath, 'snapeval.config.json');
+    if (!fs.existsSync(configPath))
+        return null;
+    const raw = fs.readFileSync(configPath, 'utf-8');
+    return JSON.parse(raw);
+}
+export function resolveConfig(cliFlags, projectRoot, skillDir) {
+    const skillDirConfig = skillDir ? loadConfigFile(skillDir) : null;
+    const projectConfig = loadConfigFile(projectRoot);
+    return {
+        ...DEFAULT_CONFIG,
+        ...(projectConfig ?? {}),
+        ...(skillDirConfig ?? {}),
+        ...stripUndefined(cliFlags),
+    };
+}
+function stripUndefined(obj) {
+    return Object.fromEntries(Object.entries(obj).filter(([, v]) => v !== undefined));
+}
+//# sourceMappingURL=config.js.map