snapeval 2.0.0 → 2.1.1

package/dist/src/adapters/harness/copilot-sdk.js

@@ -0,0 +1,101 @@
+import * as fs from 'node:fs';
+import * as path from 'node:path';
+import { getClient, isSDKInstalled } from '../copilot-sdk-client.js';
+export class CopilotSDKHarness {
+    name = 'copilot-sdk';
+    async run(options) {
+        const startMs = Date.now();
+        const client = await getClient();
+        fs.mkdirSync(options.outputDir, { recursive: true });
+        // Dynamically import SDK for approveAll
+        // @ts-ignore — module may not be installed (optional dep)
+        const { approveAll } = await import('@github/copilot-sdk');
+        // Build session config
+        const sessionConfig = {
+            model: 'gpt-4.1',
+            onPermissionRequest: approveAll,
+            workingDirectory: options.outputDir,
+            infiniteSessions: { enabled: false },
+        };
+        // Native skill loading: point skillDirectories at the skill's parent
+        if (options.skillPath) {
+            sessionConfig.skillDirectories = [options.skillPath];
+        }
+        const session = await client.createSession(sessionConfig);
+        try {
+            // Attach input files if provided
+            const attachments = [];
+            if (options.files) {
+                for (const file of options.files) {
+                    // Copy to outputDir for script assertions, and attach for the model
+                    const dest = path.join(options.outputDir, path.basename(file));
+                    fs.copyFileSync(file, dest);
+                    attachments.push({ type: 'file', path: dest, displayName: path.basename(file) });
+                }
+            }
+            const response = await session.sendAndWait({
+                prompt: options.prompt,
+                ...(attachments.length > 0 ? { attachments } : {}),
+            }, 300_000);
+            const raw = response?.data?.content ?? '';
+            // Collect full transcript from session events
+            const events = await session.getMessages();
+            const transcript = buildTranscript(events);
+            // Extract token count from events if available
+            const totalTokens = extractTokenCount(events);
+            const durationMs = Date.now() - startMs;
+            return {
+                raw: raw.trim(),
+                transcript,
+                files: [],
+                total_tokens: totalTokens,
+                duration_ms: durationMs,
+            };
+        }
+        finally {
+            await session.disconnect();
+        }
+    }
+    async isAvailable() {
+        return isSDKInstalled();
+    }
+}
+function buildTranscript(events) {
+    const lines = [];
+    for (const event of events) {
+        switch (event.type) {
+            case 'user.message':
+                lines.push(`[user] ${event.data?.content ?? ''}`);
+                break;
+            case 'assistant.message':
+                lines.push(`[assistant] ${event.data?.content ?? ''}`);
+                break;
+            case 'tool.execution_start':
+                lines.push(`[tool:start] ${event.data?.toolName ?? 'unknown'}(${JSON.stringify(event.data?.arguments ?? {})})`);
+                break;
+            case 'tool.execution_complete':
+                lines.push(`[tool:done] ${event.data?.toolName ?? 'unknown'} → ${truncate(event.data?.result ?? '', 200)}`);
+                break;
+            case 'skill.invoked':
+                lines.push(`[skill] ${event.data?.name ?? 'unknown'} (${event.data?.path ?? ''})`);
+                break;
+            case 'session.error':
+                lines.push(`[error] ${event.data?.message ?? ''}`);
+                break;
+        }
+    }
+    return lines.join('\n');
+}
+function extractTokenCount(events) {
+    let total = 0;
+    for (const event of events) {
+        if (event.type === 'assistant.usage') {
+            total += (event.data?.inputTokens ?? 0) + (event.data?.outputTokens ?? 0);
+        }
+    }
+    return total;
+}
+function truncate(str, max) {
+    return str.length > max ? str.slice(0, max) + '...' : str;
+}
+//# sourceMappingURL=copilot-sdk.js.map

package/dist/src/adapters/harness/copilot-sdk.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"copilot-sdk.js","sourceRoot":"","sources":["../../../../src/adapters/harness/copilot-sdk.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,MAAM,SAAS,CAAC;AAC9B,OAAO,KAAK,IAAI,MAAM,WAAW,CAAC;AAElC,OAAO,EAAE,SAAS,EAAE,cAAc,EAAE,MAAM,0BAA0B,CAAC;AAErE,MAAM,OAAO,iBAAiB;IACnB,IAAI,GAAG,aAAa,CAAC;IAE9B,KAAK,CAAC,GAAG,CAAC,OAKT;QACC,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAC3B,MAAM,MAAM,GAAG,MAAM,SAAS,EAAE,CAAC;QAEjC,EAAE,CAAC,SAAS,CAAC,OAAO,CAAC,SAAS,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QAErD,wCAAwC;QACxC,0DAA0D;QAC1D,MAAM,EAAE,UAAU,EAAE,GAAG,MAAM,MAAM,CAAC,qBAAqB,CAAC,CAAC;QAE3D,uBAAuB;QACvB,MAAM,aAAa,GAA4B;YAC7C,KAAK,EAAE,SAAS;YAChB,mBAAmB,EAAE,UAAU;YAC/B,gBAAgB,EAAE,OAAO,CAAC,SAAS;YACnC,gBAAgB,EAAE,EAAE,OAAO,EAAE,KAAK,EAAE;SACrC,CAAC;QAEF,qEAAqE;QACrE,IAAI,OAAO,CAAC,SAAS,EAAE,CAAC;YACtB,aAAa,CAAC,gBAAgB,GAAG,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;QACvD,CAAC;QAED,MAAM,OAAO,GAAG,MAAM,MAAM,CAAC,aAAa,CAAC,aAAa,CAAC,CAAC;QAE1D,IAAI,CAAC;YACH,iCAAiC;YACjC,MAAM,WAAW,GAAgE,EAAE,CAAC;YACpF,IAAI,OAAO,CAAC,KAAK,EAAE,CAAC;gBAClB,KAAK,MAAM,IAAI,IAAI,OAAO,CAAC,KAAK,EAAE,CAAC;oBACjC,oEAAoE;oBACpE,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,SAAS,EAAE,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC;oBAC/D,EAAE,CAAC,YAAY,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;oBAC5B,WAAW,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,WAAW,EAAE,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;gBACnF,CAAC;YACH,CAAC;YAED,MAAM,QAAQ,GAAG,MAAM,OAAO,CAAC,WAAW,CACxC;gBACE,MAAM,EAAE,OAAO,CAAC,MAAM;gBACtB,GAAG,CAAC,WAAW,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,WAAW,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;aACnD,EACD,OAAO,CACR,CAAC;YAEF,MAAM,GAAG,GAAG,QAAQ,EAAE,IAAI,EAAE,OAAO,IAAI,EAAE,CAAC;YAE1C,8CAA8C;YAC9C,MAAM,MAAM,GAAG,MAAM,OAAO,CAAC,WAAW,EAAE,CAAC;YAC3C,MAAM,UAAU,GAAG,eAAe,CAAC,MAAM,CAAC,CAAC;YAE3C,+CAA+C;YAC/C,MAAM,WAAW,GAAG,iBAAiB,CAAC,MAAM,CAAC,CAAC;YAE9C,MAAM,UAAU,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,OAAO,CAAC;YAExC,OAAO;gBACL,GAAG,EAAE,GAAG,CAAC,IAAI,EAAE;gBACf,UAAU;gBACV,KAAK,EAAE,EAAE;gBACT,YAAY,EAAE,WAAW;gBACzB,WAAW,EAAE,UAAU;aACxB,CAAC;QACJ,CAAC;gBAAS,CAAC;YACT,MAAM,OAAO,CAAC,UAAU,EAAE,CAAC;QAC7B,CAAC;IACH,CAAC;IAED,KAAK,CAAC,WAAW;QACf,OAAO,cAAc,EAAE,CAAC;IAC1B,CAAC;CACF;AAED,SAAS,eAAe,CAAC,MAAa;IACpC,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;QAC3B,QAAQ,KAAK,CAAC,IAAI,EAAE,CAAC;YACnB,KAAK,cAAc;gBACjB,KAAK,CAAC,IAAI,CAAC,UAAU,KAAK,CAAC,IAAI,EAAE,OAAO,IAAI,EAAE,EAAE,CAAC,CAAC;gBAClD,MAAM;YACR,KAAK,mBAAmB;gBACtB,KAAK,CAAC,IAAI,CAAC,eAAe,KAAK,CAAC,IAAI,EAAE,OAAO,IAAI,EAAE,EAAE,CAAC,CAAC;gBACvD,MAAM;YACR,KAAK,sBAAsB;gBACzB,KAAK,CAAC,IAAI,CAAC,gBAAgB,KAAK,CAAC,IAAI,EAAE,QAAQ,IAAI,SAAS,IAAI,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,IAAI,EAAE,SAAS,IAAI,EAAE,CAAC,GAAG,CAAC,CAAC;gBAChH,MAAM;YACR,KAAK,yBAAyB;gBAC5B,KAAK,CAAC,IAAI,CAAC,eAAe,KAAK,CAAC,IAAI,EAAE,QAAQ,IAAI,SAAS,MAAM,QAAQ,CAAC,KAAK,CAAC,IAAI,EAAE,MAAM,IAAI,EAAE,EAAE,GAAG,CAAC,EAAE,CAAC,CAAC;gBAC5G,MAAM;YACR,KAAK,eAAe;gBAClB,KAAK,CAAC,IAAI,CAAC,WAAW,KAAK,CAAC,IAAI,EAAE,IAAI,IAAI,SAAS,KAAK,KAAK,CAAC,IAAI,EAAE,IAAI,IAAI,EAAE,GAAG,CAAC,CAAC;gBACnF,MAAM;YACR,KAAK,eAAe;gBAClB,KAAK,CAAC,IAAI,CAAC,WAAW,KAAK,CAAC,IAAI,EAAE,OAAO,IAAI,EAAE,EAAE,CAAC,CAAC;gBACnD,MAAM;QACV,CAAC;IACH,CAAC;IACD,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAC1B,CAAC;AAED,SAAS,iBAAiB,CAAC,MAAa;IACtC,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;QAC3B,IAAI,KAAK,CAAC,IAAI,KAAK,iBAAiB,EAAE,CAAC;YACrC,KAAK,IAAI,CAAC,KAAK,CAAC,IAAI,EAAE,WAAW,IAAI,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,EAAE,YAAY,IAAI,CAAC,CAAC,CAAC;QAC5E,CAAC;IACH,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAED,SAAS,QAAQ,CAAC,GAAW,EAAE,GAAW;IACxC,OAAO,GAAG,CAAC,MAAM,GAAG,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,GAAG,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC;AAC5D,CAAC"}

package/dist/src/adapters/harness/resolve.js

@@ -1,9 +1,17 @@
 import { CopilotCLIHarness } from './copilot-cli.js';
-import { SnapevalError } from '../../errors.js';
+import { CopilotSDKHarness } from './copilot-sdk.js';
+import { AdapterNotAvailableError, SnapevalError } from '../../errors.js';
+import { isSDKInstalled } from '../copilot-sdk-client.js';
 export function resolveHarness(name) {
+    if (name === 'copilot-sdk') {
+        if (!isSDKInstalled()) {
+            throw new AdapterNotAvailableError('copilot-sdk', '@github/copilot-sdk is not installed. Install with: npm install @github/copilot-sdk');
+        }
+        return new CopilotSDKHarness();
+    }
     if (name === 'copilot-cli') {
         return new CopilotCLIHarness();
     }
-    throw new SnapevalError(`Unknown harness "${name}". Built-in options: copilot-cli.`);
+    throw new SnapevalError(`Unknown harness "${name}". Built-in options: copilot-sdk, copilot-cli.`);
 }
 //# sourceMappingURL=resolve.js.map

package/dist/src/adapters/harness/resolve.js.map

@@ -1 +1 @@
-{"version":3,"file":"resolve.js","sourceRoot":"","sources":["../../../../src/adapters/harness/resolve.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,iBAAiB,EAAE,MAAM,kBAAkB,CAAC;AACrD,OAAO,EAAE,aAAa,EAAE,MAAM,iBAAiB,CAAC;AAEhD,MAAM,UAAU,cAAc,CAAC,IAAY;IACzC,IAAI,IAAI,KAAK,aAAa,EAAE,CAAC;QAC3B,OAAO,IAAI,iBAAiB,EAAE,CAAC;IACjC,CAAC;IACD,MAAM,IAAI,aAAa,CAAC,oBAAoB,IAAI,mCAAmC,CAAC,CAAC;AACvF,CAAC"}
+{"version":3,"file":"resolve.js","sourceRoot":"","sources":["../../../../src/adapters/harness/resolve.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,iBAAiB,EAAE,MAAM,kBAAkB,CAAC;AACrD,OAAO,EAAE,iBAAiB,EAAE,MAAM,kBAAkB,CAAC;AACrD,OAAO,EAAE,wBAAwB,EAAE,aAAa,EAAE,MAAM,iBAAiB,CAAC;AAC1E,OAAO,EAAE,cAAc,EAAE,MAAM,0BAA0B,CAAC;AAE1D,MAAM,UAAU,cAAc,CAAC,IAAY;IACzC,IAAI,IAAI,KAAK,aAAa,EAAE,CAAC;QAC3B,IAAI,CAAC,cAAc,EAAE,EAAE,CAAC;YACtB,MAAM,IAAI,wBAAwB,CAChC,aAAa,EACb,qFAAqF,CACtF,CAAC;QACJ,CAAC;QACD,OAAO,IAAI,iBAAiB,EAAE,CAAC;IACjC,CAAC;IACD,IAAI,IAAI,KAAK,aAAa,EAAE,CAAC;QAC3B,OAAO,IAAI,iBAAiB,EAAE,CAAC;IACjC,CAAC;IACD,MAAM,IAAI,aAAa,CAAC,oBAAoB,IAAI,gDAAgD,CAAC,CAAC;AACpG,CAAC"}

package/dist/src/adapters/inference/copilot-sdk.js

@@ -3,6 +3,8 @@ export class CopilotSDKInference {
     name = 'copilot-sdk';
     async chat(messages, _options) {
         const client = await getClient();
+        // @ts-ignore — module may not be installed (optional dep)
+        const { approveAll } = await import('@github/copilot-sdk');
         const systemMessages = messages.filter((m) => m.role === 'system');
         const nonSystemMessages = messages.filter((m) => m.role !== 'system');
         const systemContent = systemMessages.map((m) => m.content).join('\n');
@@ -12,7 +14,8 @@ export class CopilotSDKInference {
             ...(systemContent
                 ? { systemMessage: { content: systemContent } }
                 : {}),
-            onPermissionRequest: async () => ({ kind: 'approved' }),
+            onPermissionRequest: approveAll,
+            infiniteSessions: { enabled: false },
         });
         try {
             const response = await session.sendAndWait({ prompt: userPrompt });

package/dist/src/adapters/inference/copilot-sdk.js.map

@@ -1 +1 @@
-{"version":3,"file":"copilot-sdk.js","sourceRoot":"","sources":["../../../../src/adapters/inference/copilot-sdk.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,SAAS,EAAE,MAAM,0BAA0B,CAAC;AAErD,MAAM,OAAO,mBAAmB;IACrB,IAAI,GAAG,aAAa,CAAC;IAE9B,KAAK,CAAC,IAAI,CAAC,QAAmB,EAAE,QAAsB;QACpD,MAAM,MAAM,GAAG,MAAM,SAAS,EAAE,CAAC;QAEjC,MAAM,cAAc,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,QAAQ,CAAC,CAAC;QACnE,MAAM,iBAAiB,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,QAAQ,CAAC,CAAC;QACtE,MAAM,aAAa,GAAG,cAAc,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACtE,MAAM,UAAU,GAAG,iBAAiB,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAEtE,MAAM,OAAO,GAAG,MAAM,MAAM,CAAC,aAAa,CAAC;YACzC,KAAK,EAAE,SAAS;YAChB,GAAG,CAAC,aAAa;gBACf,CAAC,CAAC,EAAE,aAAa,EAAE,EAAE,OAAO,EAAE,aAAa,EAAE,EAAE;gBAC/C,CAAC,CAAC,EAAE,CAAC;YACP,mBAAmB,EAAE,KAAK,IAAI,EAAE,CAAC,CAAC,EAAE,IAAI,EAAE,UAAU,EAAE,CAAC;SACxD,CAAC,CAAC;QAEH,IAAI,CAAC;YACH,MAAM,QAAQ,GAAG,MAAM,OAAO,CAAC,WAAW,CAAC,EAAE,MAAM,EAAE,UAAU,EAAE,CAAC,CAAC;YACnE,OAAO,CAAC,QAAQ,EAAE,IAAI,EAAE,OAAO,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC;QAChD,CAAC;gBAAS,CAAC;YACT,MAAM,OAAO,CAAC,UAAU,EAAE,CAAC;QAC7B,CAAC;IACH,CAAC;CACF"}
+{"version":3,"file":"copilot-sdk.js","sourceRoot":"","sources":["../../../../src/adapters/inference/copilot-sdk.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,SAAS,EAAE,MAAM,0BAA0B,CAAC;AAErD,MAAM,OAAO,mBAAmB;IACrB,IAAI,GAAG,aAAa,CAAC;IAE9B,KAAK,CAAC,IAAI,CAAC,QAAmB,EAAE,QAAsB;QACpD,MAAM,MAAM,GAAG,MAAM,SAAS,EAAE,CAAC;QAEjC,0DAA0D;QAC1D,MAAM,EAAE,UAAU,EAAE,GAAG,MAAM,MAAM,CAAC,qBAAqB,CAAC,CAAC;QAE3D,MAAM,cAAc,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,QAAQ,CAAC,CAAC;QACnE,MAAM,iBAAiB,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,QAAQ,CAAC,CAAC;QACtE,MAAM,aAAa,GAAG,cAAc,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACtE,MAAM,UAAU,GAAG,iBAAiB,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAEtE,MAAM,OAAO,GAAG,MAAM,MAAM,CAAC,aAAa,CAAC;YACzC,KAAK,EAAE,SAAS;YAChB,GAAG,CAAC,aAAa;gBACf,CAAC,CAAC,EAAE,aAAa,EAAE,EAAE,OAAO,EAAE,aAAa,EAAE,EAAE;gBAC/C,CAAC,CAAC,EAAE,CAAC;YACP,mBAAmB,EAAE,UAAU;YAC/B,gBAAgB,EAAE,EAAE,OAAO,EAAE,KAAK,EAAE;SACrC,CAAC,CAAC;QAEH,IAAI,CAAC;YACH,MAAM,QAAQ,GAAG,MAAM,OAAO,CAAC,WAAW,CAAC,EAAE,MAAM,EAAE,UAAU,EAAE,CAAC,CAAC;YACnE,OAAO,CAAC,QAAQ,EAAE,IAAI,EAAE,OAAO,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC;QAChD,CAAC;gBAAS,CAAC;YACT,MAAM,OAAO,CAAC,UAAU,EAAE,CAAC;QAC7B,CAAC;IACH,CAAC;CACF"}

package/dist/src/adapters/report/terminal.js

@@ -1,25 +1,105 @@
+import * as fs from 'node:fs';
+import * as path from 'node:path';
 import chalk from 'chalk';
+function loadPreviousIteration(iterationDir) {
+    const workspaceDir = path.dirname(iterationDir);
+    const currentName = path.basename(iterationDir);
+    const currentNum = parseInt(currentName.replace('iteration-', ''), 10);
+    if (isNaN(currentNum) || currentNum <= 1)
+        return null;
+    const prevDir = path.join(workspaceDir, `iteration-${currentNum - 1}`);
+    const prevBenchmarkPath = path.join(prevDir, 'benchmark.json');
+    if (!fs.existsSync(prevBenchmarkPath))
+        return null;
+    try {
+        const benchmark = JSON.parse(fs.readFileSync(prevBenchmarkPath, 'utf-8'));
+        const gradings = new Map();
+        const evalDirs = fs.readdirSync(prevDir).filter(d => d.startsWith('eval-'));
+        for (const evalDir of evalDirs) {
+            const wsPath = path.join(prevDir, evalDir, 'with_skill', 'grading.json');
+            const wosPath = path.join(prevDir, evalDir, 'without_skill', 'grading.json');
+            const ws = fs.existsSync(wsPath) ? JSON.parse(fs.readFileSync(wsPath, 'utf-8')) : undefined;
+            const wos = fs.existsSync(wosPath) ? JSON.parse(fs.readFileSync(wosPath, 'utf-8')) : undefined;
+            gradings.set(evalDir, { withSkill: ws, withoutSkill: wos });
+        }
+        return { benchmark, gradings };
+    }
+    catch {
+        return null;
+    }
+}
+function evalLabel(run) {
+    if (run.label)
+        return run.label;
+    if (run.slug && run.slug !== `${run.evalId}`)
+        return run.slug;
+    const firstLine = run.prompt.split('\n')[0].slice(0, 60);
+    return firstLine;
+}
 export class TerminalReporter {
     name = 'terminal';
     async report(results) {
         const { skillName, evalRuns, benchmark } = results;
         console.log(chalk.bold(`\nsnapeval — ${skillName}`));
-        console.log(chalk.dim('─'.repeat(50)));
+        console.log(chalk.dim(`Baseline = without SKILL.md (raw AI response)`));
+        console.log(chalk.dim('─'.repeat(60)));
+        const prev = loadPreviousIteration(results.iterationDir);
         for (const run of evalRuns) {
-            const wsRate = run.withSkill.grading?.summary.pass_rate;
+            const wsGrading = run.withSkill.grading;
+            const wsRate = wsGrading?.summary.pass_rate;
             const wosRate = run.withoutSkill.grading?.summary.pass_rate;
             const wsLabel = wsRate !== undefined ? `${(wsRate * 100).toFixed(0)}%` : 'n/a';
             const wosLabel = wosRate !== undefined ? `${(wosRate * 100).toFixed(0)}%` : 'n/a';
-            const tokens = run.withSkill.output.total_tokens;
-            const durationS = (run.withSkill.output.duration_ms / 1000).toFixed(2);
-            console.log(`  ${chalk.cyan(`#${run.evalId}`)} ${run.prompt.slice(0, 60)}`);
-            console.log(`    with_skill: ${wsLabel} | without_skill: ${wosLabel} | ${tokens} tokens, ${durationS}s`);
+            const wsColor = wsRate === 1 ? chalk.green : wsRate === 0 ? chalk.red : chalk.yellow;
+            const durationS = (run.withSkill.output.duration_ms / 1000).toFixed(1);
+            // Show per-eval delta from previous iteration
+            let perEvalDelta = '';
+            if (prev) {
+                const prevGrading = prev.gradings.get(`eval-${run.slug}`);
+                const prevRate = prevGrading?.withSkill?.summary.pass_rate;
+                if (prevRate !== undefined && wsRate !== undefined) {
+                    const change = wsRate - prevRate;
+                    if (change !== 0) {
+                        const arrow = change > 0 ? chalk.green('↑') : chalk.red('↓');
+                        perEvalDelta = ` ${arrow} was ${(prevRate * 100).toFixed(0)}%`;
+                    }
+                }
+            }
+            console.log(`  ${chalk.cyan(`#${run.evalId}`)} ${evalLabel(run)}`);
+            console.log(`    Skill: ${wsColor(wsLabel)}${perEvalDelta} | Baseline: ${wosLabel} | ${durationS}s`);
+            // Show failed assertions inline
+            if (wsGrading) {
+                const failed = wsGrading.assertion_results.filter((a) => !a.passed);
+                for (const f of failed) {
+                    console.log(chalk.red(`    FAIL: ${f.text}`));
+                    if (f.evidence) {
+                        console.log(chalk.dim(`          ${f.evidence.slice(0, 100)}`));
+                    }
+                }
+            }
         }
-        console.log(chalk.dim('─'.repeat(50)));
+        console.log(chalk.dim('─'.repeat(60)));
+        const ws = benchmark.run_summary.with_skill;
+        const wos = benchmark.run_summary.without_skill;
         const delta = benchmark.run_summary.delta;
         const deltaColor = delta.pass_rate > 0 ? chalk.green : delta.pass_rate < 0 ? chalk.red : chalk.dim;
-        console.log(`Delta: ${deltaColor(`${(delta.pass_rate * 100).toFixed(1)}% pass rate`)} | ${delta.time_seconds.toFixed(1)}s time | ${delta.tokens.toFixed(0)} tokens`);
-        console.log(chalk.dim(`with_skill avg: ${(benchmark.run_summary.with_skill.pass_rate.mean * 100).toFixed(1)}% | without_skill avg: ${(benchmark.run_summary.without_skill.pass_rate.mean * 100).toFixed(1)}%`));
+        console.log(chalk.bold('Summary:'));
+        console.log(`  Skill pass rate:    ${(ws.pass_rate.mean * 100).toFixed(1)}%`);
+        console.log(`  Baseline pass rate: ${(wos.pass_rate.mean * 100).toFixed(1)}%`);
+        console.log(`  Improvement:        ${deltaColor(`${delta.pass_rate > 0 ? '+' : ''}${(delta.pass_rate * 100).toFixed(1)}%`)}`);
+        if (prev) {
+            const prevRate = prev.benchmark.run_summary.with_skill.pass_rate.mean;
+            const currRate = ws.pass_rate.mean;
+            const change = currRate - prevRate;
+            const changeColor = change > 0 ? chalk.green : change < 0 ? chalk.red : chalk.dim;
+            console.log(`  vs previous:        ${changeColor(`${change > 0 ? '+' : ''}${(change * 100).toFixed(1)}%`)} (was ${(prevRate * 100).toFixed(1)}%)`);
+            // Note if eval set size changed
+            const prevEvalCount = prev.gradings.size;
+            const currEvalCount = evalRuns.length;
+            if (prevEvalCount !== currEvalCount) {
+                console.log(chalk.dim(`  Note: eval set changed (${prevEvalCount} → ${currEvalCount} evals)`));
+            }
+        }
     }
 }
 //# sourceMappingURL=terminal.js.map

package/dist/src/adapters/report/terminal.js.map

@@ -1 +1 @@
-{"version":3,"file":"terminal.js","sourceRoot":"","sources":["../../../../src/adapters/report/terminal.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,MAAM,OAAO,CAAC;AAG1B,MAAM,OAAO,gBAAgB;IAClB,IAAI,GAAG,UAAU,CAAC;IAE3B,KAAK,CAAC,MAAM,CAAC,OAAoB;QAC/B,MAAM,EAAE,SAAS,EAAE,QAAQ,EAAE,SAAS,EAAE,GAAG,OAAO,CAAC;QAEnD,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,gBAAgB,SAAS,EAAE,CAAC,CAAC,CAAC;QACrD,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;QAEvC,KAAK,MAAM,GAAG,IAAI,QAAQ,EAAE,CAAC;YAC3B,MAAM,MAAM,GAAG,GAAG,CAAC,SAAS,CAAC,OAAO,EAAE,OAAO,CAAC,SAAS,CAAC;YACxD,MAAM,OAAO,GAAG,GAAG,CAAC,YAAY,CAAC,OAAO,EAAE,OAAO,CAAC,SAAS,CAAC;YAC5D,MAAM,OAAO,GAAG,MAAM,KAAK,SAAS,CAAC,CAAC,CAAC,GAAG,CAAC,MAAM,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,KAAK,CAAC;YAC/E,MAAM,QAAQ,GAAG,OAAO,KAAK,SAAS,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,KAAK,CAAC;YAClF,MAAM,MAAM,GAAG,GAAG,CAAC,SAAS,CAAC,MAAM,CAAC,YAAY,CAAC;YACjD,MAAM,SAAS,GAAG,CAAC,GAAG,CAAC,SAAS,CAAC,MAAM,CAAC,WAAW,GAAG,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;YACvE,OAAO,CAAC,GAAG,CAAC,KAAK,KAAK,CAAC,IAAI,CAAC,IAAI,GAAG,CAAC,MAAM,EAAE,CAAC,IAAI,GAAG,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,EAAE,CAAC,CAAC;YAC5E,OAAO,CAAC,GAAG,CAAC,mBAAmB,OAAO,qBAAqB,QAAQ,MAAM,MAAM,YAAY,SAAS,GAAG,CAAC,CAAC;QAC3G,CAAC;QAED,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;QAEvC,MAAM,KAAK,GAAG,SAAS,CAAC,WAAW,CAAC,KAAK,CAAC;QAC1C,MAAM,UAAU,GAAG,KAAK,CAAC,SAAS,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,SAAS,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC;QACnG,OAAO,CAAC,GAAG,CAAC,UAAU,UAAU,CAAC,GAAG,CAAC,KAAK,CAAC,SAAS,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,aAAa,CAAC,MAAM,KAAK,CAAC,YAAY,CAAC,OAAO,CAAC,CAAC,CAAC,YAAY,KAAK,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC;QACrK,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,mBAAmB,CAAC,SAAS,CAAC,WAAW,CAAC,UAAU,CAAC,SAAS,CAAC,IAAI,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,0BAA0B,CAAC,SAAS,CAAC,WAAW,CAAC,aAAa,CAAC,SAAS,CAAC,IAAI,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;IAClN,CAAC;CACF"}
+{"version":3,"file":"terminal.js","sourceRoot":"","sources":["../../../../src/adapters/report/terminal.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,MAAM,SAAS,CAAC;AAC9B,OAAO,KAAK,IAAI,MAAM,WAAW,CAAC;AAClC,OAAO,KAAK,MAAM,OAAO,CAAC;AAQ1B,SAAS,qBAAqB,CAAC,YAAoB;IACjD,MAAM,YAAY,GAAG,IAAI,CAAC,OAAO,CAAC,YAAY,CAAC,CAAC;IAChD,MAAM,WAAW,GAAG,IAAI,CAAC,QAAQ,CAAC,YAAY,CAAC,CAAC;IAChD,MAAM,UAAU,GAAG,QAAQ,CAAC,WAAW,CAAC,OAAO,CAAC,YAAY,EAAE,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC;IACvE,IAAI,KAAK,CAAC,UAAU,CAAC,IAAI,UAAU,IAAI,CAAC;QAAE,OAAO,IAAI,CAAC;IACtD,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,CAAC,YAAY,EAAE,aAAa,UAAU,GAAG,CAAC,EAAE,CAAC,CAAC;IACvE,MAAM,iBAAiB,GAAG,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,gBAAgB,CAAC,CAAC;IAC/D,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,iBAAiB,CAAC;QAAE,OAAO,IAAI,CAAC;IACnD,IAAI,CAAC;QACH,MAAM,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,YAAY,CAAC,iBAAiB,EAAE,OAAO,CAAC,CAAC,CAAC;QAC1E,MAAM,QAAQ,GAAG,IAAI,GAAG,EAAuE,CAAC;QAChG,MAAM,QAAQ,GAAG,EAAE,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC,CAAC;QAC5E,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;YAC/B,MAAM,MAAM,GAAG,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,OAAO,EAAE,YAAY,EAAE,cAAc,CAAC,CAAC;YACzE,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,OAAO,EAAE,eAAe,EAAE,cAAc,CAAC,CAAC;YAC7E,MAAM,EAAE,GAAG,EAAE,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,YAAY,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;YAC5F,MAAM,GAAG,GAAG,EAAE,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,YAAY,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;YAC/F,QAAQ,CAAC,GAAG,CAAC,OAAO,EAAE,EAAE,SAAS,EAAE,EAAE,EAAE,YAAY,EAAE,GAAG,EAAE,CAAC,CAAC;QAC9D,CAAC;QACD,OAAO,EAAE,SAAS,EAAE,QAAQ,EAAE,CAAC;IACjC,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,IAAI,CAAC;IACd,CAAC;AACH,CAAC;AAED,SAAS,SAAS,CAAC,GAAqE;IACtF,IAAI,GAAG,CAAC,KAAK;QAAE,OAAO,GAAG,CAAC,KAAK,CAAC;IAChC,IAAI,GAAG,CAAC,IAAI,IAAI,GAAG,CAAC,IAAI,KAAK,GAAG,GAAG,CAAC,MAAM,EAAE;QAAE,OAAO,GAAG,CAAC,IAAI,CAAC;IAC9D,MAAM,SAAS,GAAG,GAAG,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;IACzD,OAAO,SAAS,CAAC;AACnB,CAAC;AAED,MAAM,OAAO,gBAAgB;IAClB,IAAI,GAAG,UAAU,CAAC;IAE3B,KAAK,CAAC,MAAM,CAAC,OAAoB;QAC/B,MAAM,EAAE,SAAS,EAAE,QAAQ,EAAE,SAAS,EAAE,GAAG,OAAO,CAAC;QAEnD,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,gBAAgB,SAAS,EAAE,CAAC,CAAC,CAAC;QACrD,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,+CAA+C,CAAC,CAAC,CAAC;QACxE,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;QAEvC,MAAM,IAAI,GAAG,qBAAqB,CAAC,OAAO,CAAC,YAAY,CAAC,CAAC;QAEzD,KAAK,MAAM,GAAG,IAAI,QAAQ,EAAE,CAAC;YAC3B,MAAM,SAAS,GAAG,GAAG,CAAC,SAAS,CAAC,OAAO,CAAC;YACxC,MAAM,MAAM,GAAG,SAAS,EAAE,OAAO,CAAC,SAAS,CAAC;YAC5C,MAAM,OAAO,GAAG,GAAG,CAAC,YAAY,CAAC,OAAO,EAAE,OAAO,CAAC,SAAS,CAAC;YAC5D,MAAM,OAAO,GAAG,MAAM,KAAK,SAAS,CAAC,CAAC,CAAC,GAAG,CAAC,MAAM,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,KAAK,CAAC;YAC/E,MAAM,QAAQ,GAAG,OAAO,KAAK,SAAS,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,KAAK,CAAC;YAClF,MAAM,OAAO,GAAG,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,KAAK,CAAC,MAAM,CAAC;YACrF,MAAM,SAAS,GAAG,CAAC,GAAG,CAAC,SAAS,CAAC,MAAM,CAAC,WAAW,GAAG,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;YAEvE,8CAA8C;YAC9C,IAAI,YAAY,GAAG,EAAE,CAAC;YACtB,IAAI,IAAI,EAAE,CAAC;gBACT,MAAM,WAAW,GAAG,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,QAAQ,GAAG,CAAC,IAAI,EAAE,CAAC,CAAC;gBAC1D,MAAM,QAAQ,GAAG,WAAW,EAAE,SAAS,EAAE,OAAO,CAAC,SAAS,CAAC;gBAC3D,IAAI,QAAQ,KAAK,SAAS,IAAI,MAAM,KAAK,SAAS,EAAE,CAAC;oBACnD,MAAM,MAAM,GAAG,MAAM,GAAG,QAAQ,CAAC;oBACjC,IAAI,MAAM,KAAK,CAAC,EAAE,CAAC;wBACjB,MAAM,KAAK,GAAG,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;wBAC7D,YAAY,GAAG,IAAI,KAAK,QAAQ,CAAC,QAAQ,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC;oBACjE,CAAC;gBACH,CAAC;YACH,CAAC;YAED,OAAO,CAAC,GAAG,CAAC,KAAK,KAAK,CAAC,IAAI,CAAC,IAAI,GAAG,CAAC,MAAM,EAAE,CAAC,IAAI,SAAS,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;YACnE,OAAO,CAAC,GAAG,CAAC,cAAc,OAAO,CAAC,OAAO,CAAC,GAAG,YAAY,gBAAgB,QAAQ,MAAM,SAAS,GAAG,CAAC,CAAC;YAErG,gCAAgC;YAChC,IAAI,SAAS,EAAE,CAAC;gBACd,MAAM,MAAM,GAAG,SAAS,CAAC,iBAAiB,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC;gBACpE,KAAK,MAAM,CAAC,IAAI,MAAM,EAAE,CAAC;oBACvB,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,aAAa,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC;oBAC9C,IAAI,CAAC,CAAC,QAAQ,EAAE,CAAC;wBACf,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,aAAa,CAAC,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC;oBAClE,CAAC;gBACH,CAAC;YACH,CAAC;QACH,CAAC;QAED,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;QAEvC,MAAM,EAAE,GAAG,SAAS,CAAC,WAAW,CAAC,UAAU,CAAC;QAC5C,MAAM,GAAG,GAAG,SAAS,CAAC,WAAW,CAAC,aAAa,CAAC;QAChD,MAAM,KAAK,GAAG,SAAS,CAAC,WAAW,CAAC,KAAK,CAAC;QAC1C,MAAM,UAAU,GAAG,KAAK,CAAC,SAAS,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,SAAS,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC;QAEnG,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC;QACpC,OAAO,CAAC,GAAG,CAAC,yBAAyB,CAAC,EAAE,CAAC,SAAS,CAAC,IAAI,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;QAC9E,OAAO,CAAC,GAAG,CAAC,yBAAyB,CAAC,GAAG,CAAC,SAAS,CAAC,IAAI,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;QAC/E,OAAO,CAAC,GAAG,CAAC,yBAAyB,UAAU,CAAC,GAAG,KAAK,CAAC,SAAS,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,GAAG,CAAC,KAAK,CAAC,SAAS,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QAE9H,IAAI,IAAI,EAAE,CAAC;YACT,MAAM,QAAQ,GAAG,IAAI,CAAC,SAAS,CAAC,WAAW,CAAC,UAAU,CAAC,SAAS,CAAC,IAAI,CAAC;YACtE,MAAM,QAAQ,GAAG,EAAE,CAAC,SAAS,CAAC,IAAI,CAAC;YACnC,MAAM,MAAM,GAAG,QAAQ,GAAG,QAAQ,CAAC;YACnC,MAAM,WAAW,GAAG,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC;YAClF,OAAO,CAAC,GAAG,CAAC,yBAAyB,WAAW,CAAC,GAAG,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,GAAG,CAAC,MAAM,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,SAAS,CAAC,QAAQ,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;YAEnJ,gCAAgC;YAChC,MAAM,aAAa,GAAG,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC;YACzC,MAAM,aAAa,GAAG,QAAQ,CAAC,MAAM,CAAC;YACtC,IAAI,aAAa,KAAK,aAAa,EAAE,CAAC;gBACpC,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,6BAA6B,aAAa,MAAM,aAAa,SAAS,CAAC,CAAC,CAAC;YACjG,CAAC;QACH,CAAC;IACH,CAAC;CACF"}

package/dist/src/commands/eval.d.ts

@@ -3,4 +3,7 @@ export declare function evalCommand(skillPath: string, harness: Harness, inferen
     workspace?: string;
     runs?: number;
     oldSkill?: string;
+    concurrency?: number;
+    only?: number[];
+    threshold?: number;
 }): Promise<EvalResults>;

package/dist/src/commands/eval.js

@@ -4,47 +4,176 @@ import { WorkspaceManager } from '../engine/workspace.js';
 import { runEval } from '../engine/runner.js';
 import { gradeAssertions } from '../engine/grader.js';
 import { computeBenchmark } from '../engine/aggregator.js';
-import { SnapevalError } from '../errors.js';
+import { SnapevalError, FileNotFoundError, ThresholdError } from '../errors.js';
+async function runWithConcurrency(tasks, limit) {
+    const results = new Array(tasks.length);
+    let index = 0;
+    async function worker() {
+        while (index < tasks.length) {
+            const i = index++;
+            results[i] = await tasks[i]();
+        }
+    }
+    await Promise.all(Array.from({ length: Math.min(limit, tasks.length) }, worker));
+    return results;
+}
+const MAX_CONCURRENCY = 10;
+/**
+ * Average pass rates across multiple grading runs.
+ * Uses the last run's assertion_results for display, but averages the
+ * pass_rate across all runs so --runs N provides statistical significance.
+ */
+function averageGradings(gradings) {
+    const valid = gradings.filter((g) => g !== null);
+    if (valid.length === 0)
+        return undefined;
+    if (valid.length === 1)
+        return valid[0];
+    const avgPassRate = valid.reduce((sum, g) => sum + g.summary.pass_rate, 0) / valid.length;
+    const avgPassed = valid.reduce((sum, g) => sum + g.summary.passed, 0) / valid.length;
+    const avgFailed = valid.reduce((sum, g) => sum + g.summary.failed, 0) / valid.length;
+    const last = valid[valid.length - 1];
+    return {
+        assertion_results: last.assertion_results,
+        summary: {
+            passed: Math.round(avgPassed),
+            failed: Math.round(avgFailed),
+            total: last.summary.total,
+            pass_rate: avgPassRate,
+        },
+    };
+}
+function validateEvalsFile(evalsFile, evalsPath) {
+    if (!evalsFile.skill_name || typeof evalsFile.skill_name !== 'string') {
+        throw new SnapevalError(`Invalid evals.json at ${evalsPath}: missing or invalid "skill_name" field.`);
+    }
+    if (!Array.isArray(evalsFile.evals)) {
+        throw new SnapevalError(`Invalid evals.json at ${evalsPath}: "evals" must be an array.`);
+    }
+    for (const [i, evalCase] of evalsFile.evals.entries()) {
+        const prefix = `Invalid evals.json at ${evalsPath}: evals[${i}]`;
+        if (typeof evalCase.id !== 'number') {
+            throw new SnapevalError(`${prefix} missing or invalid "id" (must be a number).`);
+        }
+        if (typeof evalCase.prompt !== 'string') {
+            throw new SnapevalError(`${prefix} (id:${evalCase.id}) missing "prompt" field.`);
+        }
+        if (typeof evalCase.expected_output !== 'string') {
+            throw new SnapevalError(`${prefix} (id:${evalCase.id}) missing "expected_output" field.`);
+        }
+        if (evalCase.assertions !== undefined && !Array.isArray(evalCase.assertions)) {
+            throw new SnapevalError(`${prefix} (id:${evalCase.id}) "assertions" must be an array of strings.`);
+        }
+    }
+}
 export async function evalCommand(skillPath, harness, inference, options) {
     const evalsPath = path.join(skillPath, 'evals', 'evals.json');
     if (!fs.existsSync(evalsPath)) {
-        throw new SnapevalError(`No evals.json found at ${evalsPath}. Create evals/evals.json with test scenarios first.`);
+        throw new FileNotFoundError(evalsPath, 'Create evals/evals.json with test scenarios first');
+    }
+    let evalsFile;
+    try {
+        evalsFile = JSON.parse(fs.readFileSync(evalsPath, 'utf-8'));
+    }
+    catch {
+        throw new SnapevalError(`Invalid JSON in ${evalsPath}. Check for syntax errors (missing commas, trailing commas, etc).`);
+    }
+    validateEvalsFile(evalsFile, evalsPath);
+    // Filter to specific eval IDs if --only is provided
+    if (options.only && options.only.length > 0) {
+        const ids = new Set(options.only);
+        const filtered = evalsFile.evals.filter((e) => ids.has(e.id));
+        if (filtered.length === 0) {
+            throw new SnapevalError(`No eval cases match --only ${options.only.join(',')}. Available IDs: ${evalsFile.evals.map((e) => e.id).join(', ')}`);
+        }
+        evalsFile = { ...evalsFile, evals: filtered };
+    }
+    if (options.threshold !== undefined && (options.threshold < 0 || options.threshold > 1)) {
+        throw new SnapevalError(`Threshold must be between 0 and 1 (e.g., 0.8 for 80%). Got: ${options.threshold}`);
     }
-    const evalsFile = JSON.parse(fs.readFileSync(evalsPath, 'utf-8'));
     const ws = new WorkspaceManager(skillPath, options.workspace);
     const iterationDir = ws.createIteration();
+    // Track which SKILL.md was used for this iteration
+    const skillMdPath = path.join(skillPath, 'SKILL.md');
+    if (fs.existsSync(skillMdPath)) {
+        fs.copyFileSync(skillMdPath, path.join(iterationDir, 'SKILL.md.snapshot'));
+    }
     const runs = options.runs ?? 1;
+    const concurrency = Math.min(Math.max(options.concurrency ?? 1, 1), MAX_CONCURRENCY);
     const baselineVariant = options.oldSkill ? 'old_skill' : 'without_skill';
     const scriptsDir = path.join(skillPath, 'evals', 'scripts');
-    const evalRuns = [];
-    for (const evalCase of evalsFile.evals) {
+    // Pre-create eval directories sequentially (filesystem setup)
+    const evalDirs = evalsFile.evals.map((evalCase) => {
         const slug = WorkspaceManager.getEvalSlug(evalCase).replace('eval-', '');
-        const evalDir = ws.createEvalDir(iterationDir, slug, baselineVariant);
+        return { evalCase, slug, evalDir: ws.createEvalDir(iterationDir, slug, baselineVariant) };
+    });
+    const tasks = evalDirs.map(({ evalCase, slug, evalDir }) => async () => {
+        const assertions = evalCase.assertions ?? [];
+        const allGradings = [];
         let lastRun = null;
         for (let i = 0; i < runs; i++) {
             lastRun = await runEval(evalCase, skillPath, evalDir, harness, options.oldSkill);
+            // Grade every run, not just the last
+            const [wsGrading, wosGrading] = await Promise.all([
+                gradeAssertions(assertions, lastRun.withSkill.output, path.join(evalDir, 'with_skill'), inference, fs.existsSync(scriptsDir) ? scriptsDir : undefined),
+                gradeAssertions(assertions, lastRun.withoutSkill.output, path.join(evalDir, baselineVariant), inference, fs.existsSync(scriptsDir) ? scriptsDir : undefined),
+            ]);
+            allGradings.push({ withSkill: wsGrading, withoutSkill: wosGrading });
         }
-        if (!lastRun)
-            continue;
-        const assertions = evalCase.assertions ?? [];
-        const withSkillGrading = await gradeAssertions(assertions, lastRun.withSkill.output, path.join(evalDir, 'with_skill'), inference, fs.existsSync(scriptsDir) ? scriptsDir : undefined);
-        const withoutSkillGrading = await gradeAssertions(assertions, lastRun.withoutSkill.output, path.join(evalDir, baselineVariant), inference, fs.existsSync(scriptsDir) ? scriptsDir : undefined);
-        evalRuns.push({
+        if (!lastRun) {
+            throw new SnapevalError(`No runs completed for eval ${evalCase.id}`);
+        }
+        // Average pass rates across all runs for statistical significance
+        const withSkillGrading = averageGradings(allGradings.map(g => g.withSkill));
+        const withoutSkillGrading = averageGradings(allGradings.map(g => g.withoutSkill));
+        // When runs > 1, overwrite grading.json with averaged results so
+        // artifacts match the benchmark (not just the last run's raw data)
+        if (runs > 1) {
+            if (withSkillGrading) {
+                fs.writeFileSync(path.join(evalDir, 'with_skill', 'grading.json'), JSON.stringify(withSkillGrading, null, 2));
+            }
+            if (withoutSkillGrading) {
+                fs.writeFileSync(path.join(evalDir, baselineVariant, 'grading.json'), JSON.stringify(withoutSkillGrading, null, 2));
+            }
+        }
+        return {
             evalId: evalCase.id,
             slug,
+            label: evalCase.label,
             prompt: evalCase.prompt,
             withSkill: {
                 output: lastRun.withSkill.output,
-                grading: withSkillGrading ?? undefined,
+                grading: withSkillGrading,
             },
             withoutSkill: {
                 output: lastRun.withoutSkill.output,
-                grading: withoutSkillGrading ?? undefined,
+                grading: withoutSkillGrading,
             },
-        });
-    }
+        };
+    });
+    const evalRuns = await runWithConcurrency(tasks, concurrency);
     const benchmark = computeBenchmark(evalRuns);
-    fs.writeFileSync(path.join(iterationDir, 'benchmark.json'), JSON.stringify(benchmark, null, 2));
+    // Add iteration metadata for cross-iteration comparison
+    const benchmarkWithMeta = {
+        ...benchmark,
+        metadata: {
+            eval_count: evalRuns.length,
+            eval_ids: evalRuns.map((r) => r.evalId),
+            skill_name: evalsFile.skill_name,
+            runs_per_eval: runs,
+            timestamp: new Date().toISOString(),
+        },
+    };
+    fs.writeFileSync(path.join(iterationDir, 'benchmark.json'), JSON.stringify(benchmarkWithMeta, (_key, value) => typeof value === 'number' ? Math.round(value * 10000) / 10000 : value, 2));
+    // Check threshold if set (for CI gating)
+    if (options.threshold !== undefined) {
+        const passRate = benchmark.run_summary.with_skill.pass_rate.mean;
+        if (passRate < options.threshold) {
+            // Still return results so the reporter can display them before the error
+            const results = { skillName: evalsFile.skill_name, evalRuns, benchmark, iterationDir };
+            throw Object.assign(new ThresholdError(passRate, options.threshold), { results });
+        }
+    }
     return {
         skillName: evalsFile.skill_name,
         evalRuns,

package/dist/src/commands/eval.js.map

@@ -1 +1 @@
-{"version":3,"file":"eval.js","sourceRoot":"","sources":["../../../src/commands/eval.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,MAAM,SAAS,CAAC;AAC9B,OAAO,KAAK,IAAI,MAAM,WAAW,CAAC;AAQlC,OAAO,EAAE,gBAAgB,EAAE,MAAM,wBAAwB,CAAC;AAC1D,OAAO,EAAE,OAAO,EAAE,MAAM,qBAAqB,CAAC;AAC9C,OAAO,EAAE,eAAe,EAAE,MAAM,qBAAqB,CAAC;AACtD,OAAO,EAAE,gBAAgB,EAAE,MAAM,yBAAyB,CAAC;AAC3D,OAAO,EAAE,aAAa,EAAE,MAAM,cAAc,CAAC;AAE7C,MAAM,CAAC,KAAK,UAAU,WAAW,CAC/B,SAAiB,EACjB,OAAgB,EAChB,SAA2B,EAC3B,OAAiE;IAEjE,MAAM,SAAS,GAAG,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,OAAO,EAAE,YAAY,CAAC,CAAC;IAC9D,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,SAAS,CAAC,EAAE,CAAC;QAC9B,MAAM,IAAI,aAAa,CAAC,0BAA0B,SAAS,sDAAsD,CAAC,CAAC;IACrH,CAAC;IAED,MAAM,SAAS,GAAc,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,YAAY,CAAC,SAAS,EAAE,OAAO,CAAC,CAAC,CAAC;IAC7E,MAAM,EAAE,GAAG,IAAI,gBAAgB,CAAC,SAAS,EAAE,OAAO,CAAC,SAAS,CAAC,CAAC;IAC9D,MAAM,YAAY,GAAG,EAAE,CAAC,eAAe,EAAE,CAAC;IAC1C,MAAM,IAAI,GAAG,OAAO,CAAC,IAAI,IAAI,CAAC,CAAC;IAC/B,MAAM,eAAe,GAAG,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,eAAe,CAAC;IACzE,MAAM,UAAU,GAAG,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,OAAO,EAAE,SAAS,CAAC,CAAC;IAE5D,MAAM,QAAQ,GAAoB,EAAE,CAAC;IAErC,KAAK,MAAM,QAAQ,IAAI,SAAS,CAAC,KAAK,EAAE,CAAC;QACvC,MAAM,IAAI,GAAG,gBAAgB,CAAC,WAAW,CAAC,QAAQ,CAAC,CAAC,OAAO,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC;QACzE,MAAM,OAAO,GAAG,EAAE,CAAC,aAAa,CAAC,YAAY,EAAE,IAAI,EAAE,eAAe,CAAC,CAAC;QAEtE,IAAI,OAAO,GAA+C,IAAI,CAAC;QAC/D,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,EAAE,CAAC,EAAE,EAAE,CAAC;YAC9B,OAAO,GAAG,MAAM,OAAO,CAAC,QAAQ,EAAE,SAAS,EAAE,OAAO,EAAE,OAAO,EAAE,OAAO,CAAC,QAAQ,CAAC,CAAC;QACnF,CAAC;QAED,IAAI,CAAC,OAAO;YAAE,SAAS;QAEvB,MAAM,UAAU,GAAG,QAAQ,CAAC,UAAU,IAAI,EAAE,CAAC;QAC7C,MAAM,gBAAgB,GAAG,MAAM,eAAe,CAC5C,UAAU,EACV,OAAO,CAAC,SAAS,CAAC,MAAM,EACxB,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,YAAY,CAAC,EAChC,SAAS,EACT,EAAE,CAAC,UAAU,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,SAAS,CACnD,CAAC;QACF,MAAM,mBAAmB,GAAG,MAAM,eAAe,CAC/C,UAAU,EACV,OAAO,CAAC,YAAY,CAAC,MAAM,EAC3B,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,eAAe,CAAC,EACnC,SAAS,EACT,EAAE,CAAC,UAAU,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,SAAS,CACnD,CAAC;QAEF,QAAQ,CAAC,IAAI,CAAC;YACZ,MAAM,EAAE,QAAQ,CAAC,EAAE;YACnB,IAAI;YACJ,MAAM,EAAE,QAAQ,CAAC,MAAM;YACvB,SAAS,EAAE;gBACT,MAAM,EAAE,OAAO,CAAC,SAAS,CAAC,MAAM;gBAChC,OAAO,EAAE,gBAAgB,IAAI,SAAS;aACvC;YACD,YAAY,EAAE;gBACZ,MAAM,EAAE,OAAO,CAAC,YAAY,CAAC,MAAM;gBACnC,OAAO,EAAE,mBAAmB,IAAI,SAAS;aAC1C;SACF,CAAC,CAAC;IACL,CAAC;IAED,MAAM,SAAS,GAAG,gBAAgB,CAAC,QAAQ,CAAC,CAAC;IAE7C,EAAE,CAAC,aAAa,CACd,IAAI,CAAC,IAAI,CAAC,YAAY,EAAE,gBAAgB,CAAC,EACzC,IAAI,CAAC,SAAS,CAAC,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC,CACnC,CAAC;IAEF,OAAO;QACL,SAAS,EAAE,SAAS,CAAC,UAAU;QAC/B,QAAQ;QACR,SAAS;QACT,YAAY;KACb,CAAC;AACJ,CAAC"}
+{"version":3,"file":"eval.js","sourceRoot":"","sources":["../../../src/commands/eval.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,MAAM,SAAS,CAAC;AAC9B,OAAO,KAAK,IAAI,MAAM,WAAW,CAAC;AASlC,OAAO,EAAE,gBAAgB,EAAE,MAAM,wBAAwB,CAAC;AAC1D,OAAO,EAAE,OAAO,EAAE,MAAM,qBAAqB,CAAC;AAC9C,OAAO,EAAE,eAAe,EAAE,MAAM,qBAAqB,CAAC;AACtD,OAAO,EAAE,gBAAgB,EAAE,MAAM,yBAAyB,CAAC;AAC3D,OAAO,EAAE,aAAa,EAAE,iBAAiB,EAAE,cAAc,EAAE,MAAM,cAAc,CAAC;AAEhF,KAAK,UAAU,kBAAkB,CAC/B,KAA2B,EAC3B,KAAa;IAEb,MAAM,OAAO,GAAQ,IAAI,KAAK,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;IAC7C,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,KAAK,UAAU,MAAM;QACnB,OAAO,KAAK,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC;YAC5B,MAAM,CAAC,GAAG,KAAK,EAAE,CAAC;YAClB,OAAO,CAAC,CAAC,CAAC,GAAG,MAAM,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC;QAChC,CAAC;IACH,CAAC;IACD,MAAM,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,GAAG,CAAC,KAAK,EAAE,KAAK,CAAC,MAAM,CAAC,EAAE,EAAE,MAAM,CAAC,CAAC,CAAC;IACjF,OAAO,OAAO,CAAC;AACjB,CAAC;AAED,MAAM,eAAe,GAAG,EAAE,CAAC;AAE3B;;;;GAIG;AACH,SAAS,eAAe,CAAC,QAAkC;IACzD,MAAM,KAAK,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,EAAsB,EAAE,CAAC,CAAC,KAAK,IAAI,CAAC,CAAC;IACrE,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,SAAS,CAAC;IACzC,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,KAAK,CAAC,CAAC,CAAC,CAAC;IAExC,MAAM,WAAW,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,SAAS,EAAE,CAAC,CAAC,GAAG,KAAK,CAAC,MAAM,CAAC;IAC1F,MAAM,SAAS,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC,GAAG,KAAK,CAAC,MAAM,CAAC;IACrF,MAAM,SAAS,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC,GAAG,KAAK,CAAC,MAAM,CAAC;IACrF,MAAM,IAAI,GAAG,KAAK,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;IAErC,OAAO;QACL,iBAAiB,EAAE,IAAI,CAAC,iBAAiB;QACzC,OAAO,EAAE;YACP,MAAM,EAAE,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC;YAC7B,MAAM,EAAE,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC;YAC7B,KAAK,EAAE,IAAI,CAAC,OAAO,CAAC,KAAK;YACzB,SAAS,EAAE,WAAW;SACvB;KACF,CAAC;AACJ,CAAC;AAED,SAAS,iBAAiB,CAAC,SAAoB,EAAE,SAAiB;IAChE,IAAI,CAAC,SAAS,CAAC,UAAU,IAAI,OAAO,SAAS,CAAC,UAAU,KAAK,QAAQ,EAAE,CAAC;QACtE,MAAM,IAAI,aAAa,CAAC,yBAAyB,SAAS,0CAA0C,CAAC,CAAC;IACxG,CAAC;IACD,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,SAAS,CAAC,KAAK,CAAC,EAAE,CAAC;QACpC,MAAM,IAAI,aAAa,CAAC,yBAAyB,SAAS,6BAA6B,CAAC,CAAC;IAC3F,CAAC;IACD,KAAK,MAAM,CAAC,CAAC,EAAE,QAAQ,CAAC,IAAI,SAAS,CAAC,KAAK,CAAC,OAAO,EAAE,EAAE,CAAC;QACtD,MAAM,MAAM,GAAG,yBAAyB,SAAS,WAAW,CAAC,GAAG,CAAC;QACjE,IAAI,OAAO,QAAQ,CAAC,EAAE,KAAK,QAAQ,EAAE,CAAC;YACpC,MAAM,IAAI,aAAa,CAAC,GAAG,MAAM,8CAA8C,CAAC,CAAC;QACnF,CAAC;QACD,IAAI,OAAO,QAAQ,CAAC,MAAM,KAAK,QAAQ,EAAE,CAAC;YACxC,MAAM,IAAI,aAAa,CAAC,GAAG,MAAM,QAAQ,QAAQ,CAAC,EAAE,2BAA2B,CAAC,CAAC;QACnF,CAAC;QACD,IAAI,OAAO,QAAQ,CAAC,eAAe,KAAK,QAAQ,EAAE,CAAC;YACjD,MAAM,IAAI,aAAa,CAAC,GAAG,MAAM,QAAQ,QAAQ,CAAC,EAAE,oCAAoC,CAAC,CAAC;QAC5F,CAAC;QACD,IAAI,QAAQ,CAAC,UAAU,KAAK,SAAS,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAC,EAAE,CAAC;YAC7E,MAAM,IAAI,aAAa,CAAC,GAAG,MAAM,QAAQ,QAAQ,CAAC,EAAE,6CAA6C,CAAC,CAAC;QACrG,CAAC;IACH,CAAC;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,WAAW,CAC/B,SAAiB,EACjB,OAAgB,EAChB,SAA2B,EAC3B,OAA4H;IAE5H,MAAM,SAAS,GAAG,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,OAAO,EAAE,YAAY,CAAC,CAAC;IAC9D,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,SAAS,CAAC,EAAE,CAAC;QAC9B,MAAM,IAAI,iBAAiB,CAAC,SAAS,EAAE,mDAAmD,CAAC,CAAC;IAC9F,CAAC;IAED,IAAI,SAAoB,CAAC;IACzB,IAAI,CAAC;QACH,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,YAAY,CAAC,SAAS,EAAE,OAAO,CAAC,CAAC,CAAC;IAC9D,CAAC;IAAC,MAAM,CAAC;QACP,MAAM,IAAI,aAAa,CAAC,mBAAmB,SAAS,mEAAmE,CAAC,CAAC;IAC3H,CAAC;IACD,iBAAiB,CAAC,SAAS,EAAE,SAAS,CAAC,CAAC;IAExC,oDAAoD;IACpD,IAAI,OAAO,CAAC,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC5C,MAAM,GAAG,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;QAClC,MAAM,QAAQ,GAAG,SAAS,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;QAC9D,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC1B,MAAM,IAAI,aAAa,CAAC,8BAA8B,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,oBAAoB,SAAS,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QACjJ,CAAC;QACD,SAAS,GAAG,EAAE,GAAG,SAAS,EAAE,KAAK,EAAE,QAAQ,EAAE,CAAC;IAChD,CAAC;IAED,IAAI,OAAO,CAAC,SAAS,KAAK,SAAS,IAAI,CAAC,OAAO,CAAC,SAAS,GAAG,CAAC,IAAI,OAAO,CAAC,SAAS,GAAG,CAAC,CAAC,EAAE,CAAC;QACxF,MAAM,IAAI,aAAa,CAAC,+DAA+D,OAAO,CAAC,SAAS,EAAE,CAAC,CAAC;IAC9G,CAAC;IAED,MAAM,EAAE,GAAG,IAAI,gBAAgB,CAAC,SAAS,EAAE,OAAO,CAAC,SAAS,CAAC,CAAC;IAC9D,MAAM,YAAY,GAAG,EAAE,CAAC,eAAe,EAAE,CAAC;IAE1C,mDAAmD;IACnD,MAAM,WAAW,GAAG,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,UAAU,CAAC,CAAC;IACrD,IAAI,EAAE,CAAC,UAAU,CAAC,WAAW,CAAC,EAAE,CAAC;QAC/B,EAAE,CAAC,YAAY,CAAC,WAAW,EAAE,IAAI,CAAC,IAAI,CAAC,YAAY,EAAE,mBAAmB,CAAC,CAAC,CAAC;IAC7E,CAAC;IACD,MAAM,IAAI,GAAG,OAAO,CAAC,IAAI,IAAI,CAAC,CAAC;IAC/B,MAAM,WAAW,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,WAAW,IAAI,CAAC,EAAE,CAAC,CAAC,EAAE,eAAe,CAAC,CAAC;IACrF,MAAM,eAAe,GAAG,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,eAAe,CAAC;IACzE,MAAM,UAAU,GAAG,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,OAAO,EAAE,SAAS,CAAC,CAAC;IAE5D,8DAA8D;IAC9D,MAAM,QAAQ,GAAG,SAAS,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE,EAAE;QAChD,MAAM,IAAI,GAAG,gBAAgB,CAAC,WAAW,CAAC,QAAQ,CAAC,CAAC,OAAO,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC;QACzE,OAAO,EAAE,QAAQ,EAAE,IAAI,EAAE,OAAO,EAAE,EAAE,CAAC,aAAa,CAAC,YAAY,EAAE,IAAI,EAAE,eAAe,CAAC,EAAE,CAAC;IAC5F,CAAC,CAAC,CAAC;IAEH,MAAM,KAAK,GAAG,QAAQ,CAAC,GAAG,CAAC,CAAC,EAAE,QAAQ,EAAE,IAAI,EAAE,OAAO,EAAE,EAAE,EAAE,CAAC,KAAK,IAA4B,EAAE;QAC7F,MAAM,UAAU,GAAG,QAAQ,CAAC,UAAU,IAAI,EAAE,CAAC;QAC7C,MAAM,WAAW,GAA8E,EAAE,CAAC;QAClG,IAAI,OAAO,GAA+C,IAAI,CAAC;QAE/D,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,EAAE,CAAC,EAAE,EAAE,CAAC;YAC9B,OAAO,GAAG,MAAM,OAAO,CAAC,QAAQ,EAAE,SAAS,EAAE,OAAO,EAAE,OAAO,EAAE,OAAO,CAAC,QAAQ,CAAC,CAAC;YAEjF,qCAAqC;YACrC,MAAM,CAAC,SAAS,EAAE,UAAU,CAAC,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC;gBAChD,eAAe,CACb,UAAU,EACV,OAAO,CAAC,SAAS,CAAC,MAAM,EACxB,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,YAAY,CAAC,EAChC,SAAS,EACT,EAAE,CAAC,UAAU,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,SAAS,CACnD;gBACD,eAAe,CACb,UAAU,EACV,OAAO,CAAC,YAAY,CAAC,MAAM,EAC3B,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,eAAe,CAAC,EACnC,SAAS,EACT,EAAE,CAAC,UAAU,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,SAAS,CACnD;aACF,CAAC,CAAC;YACH,WAAW,CAAC,IAAI,CAAC,EAAE,SAAS,EAAE,SAAS,EAAE,YAAY,EAAE,UAAU,EAAE,CAAC,CAAC;QACvE,CAAC;QAED,IAAI,CAAC,OAAO,EAAE,CAAC;YACb,MAAM,IAAI,aAAa,CAAC,8BAA8B,QAAQ,CAAC,EAAE,EAAE,CAAC,CAAC;QACvE,CAAC;QAED,kEAAkE;QAClE,MAAM,gBAAgB,GAAG,eAAe,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC;QAC5E,MAAM,mBAAmB,GAAG,eAAe,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC;QAElF,iEAAiE;QACjE,mEAAmE;QACnE,IAAI,IAAI,GAAG,CAAC,EAAE,CAAC;YACb,IAAI,gBAAgB,EAAE,CAAC;gBACrB,EAAE,CAAC,aAAa,CACd,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,YAAY,EAAE,cAAc,CAAC,EAChD,IAAI,CAAC,SAAS,CAAC,gBAAgB,EAAE,IAAI,EAAE,CAAC,CAAC,CAC1C,CAAC;YACJ,CAAC;YACD,IAAI,mBAAmB,EAAE,CAAC;gBACxB,EAAE,CAAC,aAAa,CACd,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,eAAe,EAAE,cAAc,CAAC,EACnD,IAAI,CAAC,SAAS,CAAC,mBAAmB,EAAE,IAAI,EAAE,CAAC,CAAC,CAC7C,CAAC;YACJ,CAAC;QACH,CAAC;QAED,OAAO;YACL,MAAM,EAAE,QAAQ,CAAC,EAAE;YACnB,IAAI;YACJ,KAAK,EAAE,QAAQ,CAAC,KAAK;YACrB,MAAM,EAAE,QAAQ,CAAC,MAAM;YACvB,SAAS,EAAE;gBACT,MAAM,EAAE,OAAO,CAAC,SAAS,CAAC,MAAM;gBAChC,OAAO,EAAE,gBAAgB;aAC1B;YACD,YAAY,EAAE;gBACZ,MAAM,EAAE,OAAO,CAAC,YAAY,CAAC,MAAM;gBACnC,OAAO,EAAE,mBAAmB;aAC7B;SACF,CAAC;IACJ,CAAC,CAAC,CAAC;IAEH,MAAM,QAAQ,GAAG,MAAM,kBAAkB,CAAC,KAAK,EAAE,WAAW,CAAC,CAAC;IAC9D,MAAM,SAAS,GAAG,gBAAgB,CAAC,QAAQ,CAAC,CAAC;IAE7C,wDAAwD;IACxD,MAAM,iBAAiB,GAAG;QACxB,GAAG,SAAS;QACZ,QAAQ,EAAE;YACR,UAAU,EAAE,QAAQ,CAAC,MAAM;YAC3B,QAAQ,EAAE,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC;YACvC,UAAU,EAAE,SAAS,CAAC,UAAU;YAChC,aAAa,EAAE,IAAI;YACnB,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;SACpC;KACF,CAAC;IAEF,EAAE,CAAC,aAAa,CACd,IAAI,CAAC,IAAI,CAAC,YAAY,EAAE,gBAAgB,CAAC,EACzC,IAAI,CAAC,SAAS,CAAC,iBAAiB,EAAE,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAChD,OAAO,KAAK,KAAK,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,GAAG,KAAK,CAAC,GAAG,KAAK,CAAC,CAAC,CAAC,KAAK,EAAE,CAAC,CAAC,CAC5E,CAAC;IAEF,yCAAyC;IACzC,IAAI,OAAO,CAAC,SAAS,KAAK,SAAS,EAAE,CAAC;QACpC,MAAM,QAAQ,GAAG,SAAS,CAAC,WAAW,CAAC,UAAU,CAAC,SAAS,CAAC,IAAI,CAAC;QACjE,IAAI,QAAQ,GAAG,OAAO,CAAC,SAAS,EAAE,CAAC;YACjC,yEAAyE;YACzE,MAAM,OAAO,GAAG,EAAE,SAAS,EAAE,SAAS,CAAC,UAAU,EAAE,QAAQ,EAAE,SAAS,EAAE,YAAY,EAAE,CAAC;YACvF,MAAM,MAAM,CAAC,MAAM,CAAC,IAAI,cAAc,CAAC,QAAQ,EAAE,OAAO,CAAC,SAAS,CAAC,EAAE,EAAE,OAAO,EAAE,CAAC,CAAC;QACpF,CAAC;IACH,CAAC;IAED,OAAO;QACL,SAAS,EAAE,SAAS,CAAC,UAAU;QAC/B,QAAQ;QACR,SAAS;QACT,YAAY;KACb,CAAC;AACJ,CAAC"}

package/dist/src/commands/review.d.ts

@@ -4,4 +4,5 @@ export declare function reviewCommand(skillPath: string, harness: Harness, infer
     runs?: number;
     oldSkill?: string;
     noOpen?: boolean;
+    concurrency?: number;
 }): Promise<void>;

package/dist/src/commands/review.js.map

@@ -1 +1 @@
-{"version":3,"file":"review.js","sourceRoot":"","sources":["../../../src/commands/review.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,oBAAoB,CAAC;AAC9C,OAAO,KAAK,EAAE,MAAM,SAAS,CAAC;AAC9B,OAAO,KAAK,IAAI,MAAM,WAAW,CAAC;AAClC,OAAO,KAAK,OAAO,MAAM,cAAc,CAAC;AAExC,OAAO,EAAE,WAAW,EAAE,MAAM,WAAW,CAAC;AACxC,OAAO,EAAE,gBAAgB,EAAE,MAAM,gCAAgC,CAAC;AAElE,MAAM,CAAC,KAAK,UAAU,aAAa,CACjC,SAAiB,EACjB,OAAgB,EAChB,SAA2B,EAC3B,OAAmF;IAEnF,MAAM,OAAO,GAAG,MAAM,WAAW,CAAC,SAAS,EAAE,OAAO,EAAE,SAAS,EAAE,OAAO,CAAC,CAAC;IAE1E,MAAM,QAAQ,GAAG,IAAI,gBAAgB,EAAE,CAAC;IACxC,MAAM,QAAQ,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;IAE/B,yBAAyB;IACzB,MAAM,QAAQ,GAAiB,EAAE,CAAC;IAClC,KAAK,MAAM,GAAG,IAAI,OAAO,CAAC,QAAQ,EAAE,CAAC;QACnC,QAAQ,CAAC,QAAQ,GAAG,CAAC,IAAI,EAAE,CAAC,GAAG,EAAE,CAAC;IACpC,CAAC;IACD,EAAE,CAAC,aAAa,CACd,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,YAAY,EAAE,eAAe,CAAC,EAChD,IAAI,CAAC,SAAS,CAAC,QAAQ,EAAE,IAAI,EAAE,CAAC,CAAC,CAClC,CAAC;IAEF,oEAAoE;IACpE,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC;QACpB,MAAM,UAAU,GAAG,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,YAAY,EAAE,gBAAgB,CAAC,CAAC;QACrE,aAAa,CAAC,UAAU,CAAC,CAAC;IAC5B,CAAC;AACH,CAAC;AAED,SAAS,aAAa,CAAC,QAAgB;IACrC,MAAM,GAAG,GACP,OAAO,CAAC,QAAQ,KAAK,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC;QACxC,OAAO,CAAC,QAAQ,KAAK,OAAO,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,UAAU,CAAC;IACpD,MAAM,IAAI,GACR,OAAO,CAAC,QAAQ,KAAK,OAAO,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,OAAO,EAAE,EAAE,EAAE,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC;IAC5E,QAAQ,CAAC,GAAG,EAAE,IAAI,EAAE,CAAC,GAAG,EAAE,EAAE;QAC1B,IAAI,GAAG;YAAE,OAAO,CAAC,IAAI,CAAC,2BAA2B,GAAG,CAAC,OAAO,EAAE,CAAC,CAAC;IAClE,CAAC,CAAC,CAAC;AACL,CAAC"}
+{"version":3,"file":"review.js","sourceRoot":"","sources":["../../../src/commands/review.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,oBAAoB,CAAC;AAC9C,OAAO,KAAK,EAAE,MAAM,SAAS,CAAC;AAC9B,OAAO,KAAK,IAAI,MAAM,WAAW,CAAC;AAClC,OAAO,KAAK,OAAO,MAAM,cAAc,CAAC;AAExC,OAAO,EAAE,WAAW,EAAE,MAAM,WAAW,CAAC;AACxC,OAAO,EAAE,gBAAgB,EAAE,MAAM,gCAAgC,CAAC;AAElE,MAAM,CAAC,KAAK,UAAU,aAAa,CACjC,SAAiB,EACjB,OAAgB,EAChB,SAA2B,EAC3B,OAAyG;IAEzG,MAAM,OAAO,GAAG,MAAM,WAAW,CAAC,SAAS,EAAE,OAAO,EAAE,SAAS,EAAE,OAAO,CAAC,CAAC;IAE1E,MAAM,QAAQ,GAAG,IAAI,gBAAgB,EAAE,CAAC;IACxC,MAAM,QAAQ,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;IAE/B,yBAAyB;IACzB,MAAM,QAAQ,GAAiB,EAAE,CAAC;IAClC,KAAK,MAAM,GAAG,IAAI,OAAO,CAAC,QAAQ,EAAE,CAAC;QACnC,QAAQ,CAAC,QAAQ,GAAG,CAAC,IAAI,EAAE,CAAC,GAAG,EAAE,CAAC;IACpC,CAAC;IACD,EAAE,CAAC,aAAa,CACd,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,YAAY,EAAE,eAAe,CAAC,EAChD,IAAI,CAAC,SAAS,CAAC,QAAQ,EAAE,IAAI,EAAE,CAAC,CAAC,CAClC,CAAC;IAEF,oEAAoE;IACpE,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC;QACpB,MAAM,UAAU,GAAG,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,YAAY,EAAE,gBAAgB,CAAC,CAAC;QACrE,aAAa,CAAC,UAAU,CAAC,CAAC;IAC5B,CAAC;AACH,CAAC;AAED,SAAS,aAAa,CAAC,QAAgB;IACrC,MAAM,GAAG,GACP,OAAO,CAAC,QAAQ,KAAK,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC;QACxC,OAAO,CAAC,QAAQ,KAAK,OAAO,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,UAAU,CAAC;IACpD,MAAM,IAAI,GACR,OAAO,CAAC,QAAQ,KAAK,OAAO,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,OAAO,EAAE,EAAE,EAAE,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC;IAC5E,QAAQ,CAAC,GAAG,EAAE,IAAI,EAAE,CAAC,GAAG,EAAE,EAAE;QAC1B,IAAI,GAAG;YAAE,OAAO,CAAC,IAAI,CAAC,2BAA2B,GAAG,CAAC,OAAO,EAAE,CAAC,CAAC;IAClE,CAAC,CAAC,CAAC;AACL,CAAC"}

package/dist/src/config.js

@@ -1,10 +1,11 @@
 import * as fs from 'node:fs';
 import * as path from 'node:path';
 export const DEFAULT_CONFIG = {
-    harness: 'copilot-cli',
+    harness: 'copilot-sdk',
     inference: 'auto',
     workspace: '../{skill_name}-workspace',
     runs: 1,
+    concurrency: 1,
 };
 function loadConfigFile(dirPath) {
     const configPath = path.join(dirPath, 'snapeval.config.json');

package/dist/src/config.js.map

@@ -1 +1 @@
-{"version":3,"file":"config.js","sourceRoot":"","sources":["../../src/config.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,MAAM,SAAS,CAAC;AAC9B,OAAO,KAAK,IAAI,MAAM,WAAW,CAAC;AAGlC,MAAM,CAAC,MAAM,cAAc,GAAmB;IAC5C,OAAO,EAAE,aAAa;IACtB,SAAS,EAAE,MAAM;IACjB,SAAS,EAAE,2BAA2B;IACtC,IAAI,EAAE,CAAC;CACR,CAAC;AAEF,SAAS,cAAc,CAAC,OAAe;IACrC,MAAM,UAAU,GAAG,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,sBAAsB,CAAC,CAAC;IAC9D,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,UAAU,CAAC;QAAE,OAAO,IAAI,CAAC;IAC5C,MAAM,GAAG,GAAG,EAAE,CAAC,YAAY,CAAC,UAAU,EAAE,OAAO,CAAC,CAAC;IACjD,OAAO,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;AACzB,CAAC;AAED,MAAM,UAAU,aAAa,CAC3B,QAAiC,EACjC,WAAmB,EACnB,QAAiB;IAEjB,MAAM,cAAc,GAAG,QAAQ,CAAC,CAAC,CAAC,cAAc,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;IAClE,MAAM,aAAa,GAAG,cAAc,CAAC,WAAW,CAAC,CAAC;IAClD,OAAO;QACL,GAAG,cAAc;QACjB,GAAG,CAAC,aAAa,IAAI,EAAE,CAAC;QACxB,GAAG,CAAC,cAAc,IAAI,EAAE,CAAC;QACzB,GAAG,cAAc,CAAC,QAAQ,CAAC;KAC5B,CAAC;AACJ,CAAC;AAED,SAAS,cAAc,CAAC,GAA4B;IAClD,OAAO,MAAM,CAAC,WAAW,CAAC,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,KAAK,SAAS,CAAC,CAAC,CAAC;AACpF,CAAC"}
+{"version":3,"file":"config.js","sourceRoot":"","sources":["../../src/config.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,MAAM,SAAS,CAAC;AAC9B,OAAO,KAAK,IAAI,MAAM,WAAW,CAAC;AAGlC,MAAM,CAAC,MAAM,cAAc,GAAmB;IAC5C,OAAO,EAAE,aAAa;IACtB,SAAS,EAAE,MAAM;IACjB,SAAS,EAAE,2BAA2B;IACtC,IAAI,EAAE,CAAC;IACP,WAAW,EAAE,CAAC;CACf,CAAC;AAEF,SAAS,cAAc,CAAC,OAAe;IACrC,MAAM,UAAU,GAAG,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,sBAAsB,CAAC,CAAC;IAC9D,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,UAAU,CAAC;QAAE,OAAO,IAAI,CAAC;IAC5C,MAAM,GAAG,GAAG,EAAE,CAAC,YAAY,CAAC,UAAU,EAAE,OAAO,CAAC,CAAC;IACjD,OAAO,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;AACzB,CAAC;AAED,MAAM,UAAU,aAAa,CAC3B,QAAiC,EACjC,WAAmB,EACnB,QAAiB;IAEjB,MAAM,cAAc,GAAG,QAAQ,CAAC,CAAC,CAAC,cAAc,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;IAClE,MAAM,aAAa,GAAG,cAAc,CAAC,WAAW,CAAC,CAAC;IAClD,OAAO;QACL,GAAG,cAAc;QACjB,GAAG,CAAC,aAAa,IAAI,EAAE,CAAC;QACxB,GAAG,CAAC,cAAc,IAAI,EAAE,CAAC;QACzB,GAAG,cAAc,CAAC,QAAQ,CAAC;KAC5B,CAAC;AACJ,CAAC;AAED,SAAS,cAAc,CAAC,GAA4B;IAClD,OAAO,MAAM,CAAC,WAAW,CAAC,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,KAAK,SAAS,CAAC,CAAC,CAAC;AACpF,CAAC"}