snapeval 2.0.0 → 2.1.1

package/dist/src/engine/grader.js

@@ -1,9 +1,33 @@
 import * as fs from 'node:fs';
 import * as path from 'node:path';
 import { execFileSync } from 'node:child_process';
+const EXACT_MATCH_PATTERN = /^Output (?:is |equals )exactly:\s*"(.+)"$/i;
+function gradeExactMatch(assertion, output) {
+    const match = assertion.match(EXACT_MATCH_PATTERN);
+    if (!match)
+        return null;
+    const expected = match[1];
+    const actual = output.trim();
+    const passed = actual === expected;
+    return {
+        text: assertion,
+        passed,
+        evidence: passed
+            ? `Exact match: "${expected}"`
+            : `Expected: "${expected}"\nGot: "${actual}"`,
+    };
+}
 function buildGradingPrompt(assertions, output, files) {
     const fileList = files.length > 0 ? `\nFiles produced: ${files.join(', ')}` : '';
-    return `You are a strict eval grader. For each assertion, determine PASS or FAIL based on the output below. Require concrete evidence for a PASS — do not give the benefit of the doubt.
+    return `You are an eval grader. For each assertion, determine PASS or FAIL based solely on the output below.
+
+GRADING RULES:
+- PASS if the output satisfies the assertion's intent, even if wording differs slightly.
+- FAIL only if the output clearly does not satisfy the assertion.
+- Be consistent: if an assertion checks for X and the output contains X in different phrasing, that is a PASS.
+- For "contains" assertions: look for semantic presence, not exact substring.
+- For "identifies" assertions: the output must demonstrate awareness of the concept, not use identical words.
+- Always cite specific text from the output as evidence.
 
 OUTPUT:
 ---
@@ -16,7 +40,7 @@ ${assertions.map((a, i) => `${i + 1}. ${a}`).join('\n')}
 Respond with JSON only:
 {
   "results": [
-    {"text": "<assertion text>", "passed": true/false, "evidence": "<quote or reference from output>"}
+    {"text": "<assertion text>", "passed": true/false, "evidence": "<quote from output supporting your verdict>"}
   ]
 }`;
 }
@@ -26,25 +50,54 @@ function runScript(scriptName, outputDir, scriptsDir) {
         return { text: `script:${scriptName}`, passed: false, evidence: `Script not found: ${scriptPath}` };
     }
     try {
-        const evidence = execFileSync(scriptPath, [outputDir], { encoding: 'utf-8', timeout: 30000 }).trim();
+        const stdout = execFileSync(scriptPath, [outputDir], { encoding: 'utf-8', timeout: 30000, stdio: ['pipe', 'pipe', 'pipe'] }).trim();
+        const evidence = stdout || `Script passed: ${scriptName}`;
         return { text: `script:${scriptName}`, passed: true, evidence };
     }
     catch (err) {
-        const evidence = err.stdout?.trim() || err.message || 'Script exited with non-zero code';
+        // Extract the most useful error info without raw stack traces
+        const stderr = err.stderr?.trim();
+        const stdout = err.stdout?.trim();
+        let evidence;
+        if (err.code === 'EACCES') {
+            evidence = `Permission denied: ${scriptPath} is not executable. Run: chmod +x ${scriptPath}`;
+        }
+        else if (stderr) {
+            // Take only the first line of stderr to avoid stack trace noise
+            evidence = stderr.split('\n')[0];
+        }
+        else if (stdout) {
+            evidence = stdout.split('\n')[0];
+        }
+        else {
+            evidence = `Script exited with code ${err.status ?? 'unknown'}`;
+        }
         return { text: `script:${scriptName}`, passed: false, evidence };
     }
 }
 function extractJSON(text) {
-    const match = text.match(/```(?:json)?\s*([\s\S]*?)```/);
-    if (match)
-        return match[1].trim();
-    return text.trim();
+    // Try JSON-tagged fence first, then bare fence, then raw text
+    const jsonFence = text.match(/```json\s*([\s\S]*?)```/);
+    if (jsonFence)
+        return jsonFence[1].trim();
+    // Try parsing raw text as JSON before falling back to any fence
+    const trimmed = text.trim();
+    try {
+        JSON.parse(trimmed);
+        return trimmed;
+    }
+    catch { /* not raw JSON */ }
+    const anyFence = text.match(/```\s*([\s\S]*?)```/);
+    if (anyFence)
+        return anyFence[1].trim();
+    return trimmed;
 }
 export async function gradeAssertions(assertions, output, runDir, inference, scriptsDir) {
     if (assertions.length === 0)
         return null;
     const scriptAssertions = assertions.filter(a => a.startsWith('script:'));
-    const llmAssertions = assertions.filter(a => !a.startsWith('script:'));
+    const exactAssertions = assertions.filter(a => !a.startsWith('script:') && EXACT_MATCH_PATTERN.test(a));
+    const llmAssertions = assertions.filter(a => !a.startsWith('script:') && !EXACT_MATCH_PATTERN.test(a));
     const results = [];
     for (const assertion of scriptAssertions) {
         const scriptName = assertion.slice('script:'.length);
@@ -52,6 +105,11 @@ export async function gradeAssertions(assertions, output, runDir, inference, scr
         const dir = scriptsDir ?? path.join(runDir, '..', '..', '..', 'evals', 'scripts');
         results.push(runScript(scriptName, outputDir, dir));
     }
+    for (const assertion of exactAssertions) {
+        const result = gradeExactMatch(assertion, output.raw);
+        if (result)
+            results.push(result);
+    }
     if (llmAssertions.length > 0) {
         const prompt = buildGradingPrompt(llmAssertions, output.raw, output.files);
         const response = await inference.chat([{ role: 'user', content: prompt }], { temperature: 0, responseFormat: 'json' });

package/dist/src/engine/grader.js.map

@@ -1 +1 @@
-{"version":3,"file":"grader.js","sourceRoot":"","sources":["../../../src/engine/grader.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,MAAM,SAAS,CAAC;AAC9B,OAAO,KAAK,IAAI,MAAM,WAAW,CAAC;AAClC,OAAO,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAQlD,SAAS,kBAAkB,CAAC,UAAoB,EAAE,MAAc,EAAE,KAAe;IAC/E,MAAM,QAAQ,GAAG,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,qBAAqB,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;IACjF,OAAO;;;;EAIP,MAAM;KACH,QAAQ;;;EAGX,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC;;;;;;;EAOrD,CAAC;AACH,CAAC;AAED,SAAS,SAAS,CAChB,UAAkB,EAClB,SAAiB,EACjB,UAAkB;IAElB,MAAM,UAAU,GAAG,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE,UAAU,CAAC,CAAC;IACrD,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,UAAU,CAAC,EAAE,CAAC;QAC/B,OAAO,EAAE,IAAI,EAAE,UAAU,UAAU,EAAE,EAAE,MAAM,EAAE,KAAK,EAAE,QAAQ,EAAE,qBAAqB,UAAU,EAAE,EAAE,CAAC;IACtG,CAAC;IACD,IAAI,CAAC;QACH,MAAM,QAAQ,GAAG,YAAY,CAAC,UAAU,EAAE,CAAC,SAAS,CAAC,EAAE,EAAE,QAAQ,EAAE,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC;QACrG,OAAO,EAAE,IAAI,EAAE,UAAU,UAAU,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,QAAQ,EAAE,CAAC;IAClE,CAAC;IAAC,OAAO,GAAQ,EAAE,CAAC;QAClB,MAAM,QAAQ,GAAG,GAAG,CAAC,MAAM,EAAE,IAAI,EAAE,IAAI,GAAG,CAAC,OAAO,IAAI,kCAAkC,CAAC;QACzF,OAAO,EAAE,IAAI,EAAE,UAAU,UAAU,EAAE,EAAE,MAAM,EAAE,KAAK,EAAE,QAAQ,EAAE,CAAC;IACnE,CAAC;AACH,CAAC;AAED,SAAS,WAAW,CAAC,IAAY;IAC/B,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,8BAA8B,CAAC,CAAC;IACzD,IAAI,KAAK;QAAE,OAAO,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;IAClC,OAAO,IAAI,CAAC,IAAI,EAAE,CAAC;AACrB,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,eAAe,CACnC,UAAoB,EACpB,MAAwB,EACxB,MAAc,EACd,SAA2B,EAC3B,UAAmB;IAEnB,IAAI,UAAU,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,IAAI,CAAC;IAEzC,MAAM,gBAAgB,GAAG,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,UAAU,CAAC,SAAS,CAAC,CAAC,CAAC;IACzE,MAAM,aAAa,GAAG,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,SAAS,CAAC,CAAC,CAAC;IACvE,MAAM,OAAO,GAAsB,EAAE,CAAC;IAEtC,KAAK,MAAM,SAAS,IAAI,gBAAgB,EAAE,CAAC;QACzC,MAAM,UAAU,GAAG,SAAS,CAAC,KAAK,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC;QACrD,MAAM,SAAS,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,SAAS,CAAC,CAAC;QAC/C,MAAM,GAAG,GAAG,UAAU,IAAI,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,OAAO,EAAE,SAAS,CAAC,CAAC;QAClF,OAAO,CAAC,IAAI,CAAC,SAAS,CAAC,UAAU,EAAE,SAAS,EAAE,GAAG,CAAC,CAAC,CAAC;IACtD,CAAC;IAED,IAAI,aAAa,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC7B,MAAM,MAAM,GAAG,kBAAkB,CAAC,aAAa,EAAE,MAAM,CAAC,GAAG,EAAE,MAAM,CAAC,KAAK,CAAC,CAAC;QAC3E,MAAM,QAAQ,GAAG,MAAM,SAAS,CAAC,IAAI,CACnC,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC,EACnC,EAAE,WAAW,EAAE,CAAC,EAAE,cAAc,EAAE,MAAM,EAAE,CAC3C,CAAC;QACF,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,WAAW,CAAC,QAAQ,CAAC,CAAC,CAAC;QACjD,KAAK,MAAM,CAAC,IAAI,MAAM,CAAC,OAAO,EAAE,CAAC;YAC/B,OAAO,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,MAAM,EAAE,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,EAAE,QAAQ,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAC,CAAC;QAClF,CAAC;IACH,CAAC;IAED,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,MAAM,CAAC;IACpD,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,MAAM,CAAC;IACrD,MAAM,KAAK,GAAG,OAAO,CAAC,MAAM,CAAC;IAE7B,MAAM,OAAO,GAAkB;QAC7B,iBAAiB,EAAE,OAAO;QAC1B,OAAO,EAAE,EAAE,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,SAAS,EAAE,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,MAAM,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE;KAC9E,CAAC;IAEF,EAAE,CAAC,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,cAAc,CAAC,EAAE,IAAI,CAAC,SAAS,CAAC,OAAO,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;IAEtF,OAAO,OAAO,CAAC;AACjB,CAAC"}
+{"version":3,"file":"grader.js","sourceRoot":"","sources":["../../../src/engine/grader.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,MAAM,SAAS,CAAC;AAC9B,OAAO,KAAK,IAAI,MAAM,WAAW,CAAC;AAClC,OAAO,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAQlD,MAAM,mBAAmB,GAAG,4CAA4C,CAAC;AAEzE,SAAS,eAAe,CAAC,SAAiB,EAAE,MAAc;IACxD,MAAM,KAAK,GAAG,SAAS,CAAC,KAAK,CAAC,mBAAmB,CAAC,CAAC;IACnD,IAAI,CAAC,KAAK;QAAE,OAAO,IAAI,CAAC;IACxB,MAAM,QAAQ,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;IAC1B,MAAM,MAAM,GAAG,MAAM,CAAC,IAAI,EAAE,CAAC;IAC7B,MAAM,MAAM,GAAG,MAAM,KAAK,QAAQ,CAAC;IACnC,OAAO;QACL,IAAI,EAAE,SAAS;QACf,MAAM;QACN,QAAQ,EAAE,MAAM;YACd,CAAC,CAAC,iBAAiB,QAAQ,GAAG;YAC9B,CAAC,CAAC,cAAc,QAAQ,YAAY,MAAM,GAAG;KAChD,CAAC;AACJ,CAAC;AAED,SAAS,kBAAkB,CAAC,UAAoB,EAAE,MAAc,EAAE,KAAe;IAC/E,MAAM,QAAQ,GAAG,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,qBAAqB,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;IACjF,OAAO;;;;;;;;;;;;EAYP,MAAM;KACH,QAAQ;;;EAGX,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC;;;;;;;EAOrD,CAAC;AACH,CAAC;AAED,SAAS,SAAS,CAChB,UAAkB,EAClB,SAAiB,EACjB,UAAkB;IAElB,MAAM,UAAU,GAAG,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE,UAAU,CAAC,CAAC;IACrD,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,UAAU,CAAC,EAAE,CAAC;QAC/B,OAAO,EAAE,IAAI,EAAE,UAAU,UAAU,EAAE,EAAE,MAAM,EAAE,KAAK,EAAE,QAAQ,EAAE,qBAAqB,UAAU,EAAE,EAAE,CAAC;IACtG,CAAC;IACD,IAAI,CAAC;QACH,MAAM,MAAM,GAAG,YAAY,CAAC,UAAU,EAAE,CAAC,SAAS,CAAC,EAAE,EAAE,QAAQ,EAAE,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC;QACpI,MAAM,QAAQ,GAAG,MAAM,IAAI,kBAAkB,UAAU,EAAE,CAAC;QAC1D,OAAO,EAAE,IAAI,EAAE,UAAU,UAAU,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,QAAQ,EAAE,CAAC;IAClE,CAAC;IAAC,OAAO,GAAQ,EAAE,CAAC;QAClB,8DAA8D;QAC9D,MAAM,MAAM,GAAG,GAAG,CAAC,MAAM,EAAE,IAAI,EAAE,CAAC;QAClC,MAAM,MAAM,GAAG,GAAG,CAAC,MAAM,EAAE,IAAI,EAAE,CAAC;QAClC,IAAI,QAAgB,CAAC;QACrB,IAAI,GAAG,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;YAC1B,QAAQ,GAAG,sBAAsB,UAAU,qCAAqC,UAAU,EAAE,CAAC;QAC/F,CAAC;aAAM,IAAI,MAAM,EAAE,CAAC;YAClB,gEAAgE;YAChE,QAAQ,GAAG,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;QACnC,CAAC;aAAM,IAAI,MAAM,EAAE,CAAC;YAClB,QAAQ,GAAG,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;QACnC,CAAC;aAAM,CAAC;YACN,QAAQ,GAAG,2BAA2B,GAAG,CAAC,MAAM,IAAI,SAAS,EAAE,CAAC;QAClE,CAAC;QACD,OAAO,EAAE,IAAI,EAAE,UAAU,UAAU,EAAE,EAAE,MAAM,EAAE,KAAK,EAAE,QAAQ,EAAE,CAAC;IACnE,CAAC;AACH,CAAC;AAED,SAAS,WAAW,CAAC,IAAY;IAC/B,8DAA8D;IAC9D,MAAM,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,yBAAyB,CAAC,CAAC;IACxD,IAAI,SAAS;QAAE,OAAO,SAAS,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;IAC1C,gEAAgE;IAChE,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,EAAE,CAAC;IAC5B,IAAI,CAAC;QAAC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;QAAC,OAAO,OAAO,CAAC;IAAC,CAAC;IAAC,MAAM,CAAC,CAAC,kBAAkB,CAAC,CAAC;IACzE,MAAM,QAAQ,GAAG,IAAI,CAAC,KAAK,CAAC,qBAAqB,CAAC,CAAC;IACnD,IAAI,QAAQ;QAAE,OAAO,QAAQ,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;IACxC,OAAO,OAAO,CAAC;AACjB,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,eAAe,CACnC,UAAoB,EACpB,MAAwB,EACxB,MAAc,EACd,SAA2B,EAC3B,UAAmB;IAEnB,IAAI,UAAU,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,IAAI,CAAC;IAEzC,MAAM,gBAAgB,GAAG,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,UAAU,CAAC,SAAS,CAAC,CAAC,CAAC;IACzE,MAAM,eAAe,GAAG,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,SAAS,CAAC,IAAI,mBAAmB,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;IACxG,MAAM,aAAa,GAAG,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,SAAS,CAAC,IAAI,CAAC,mBAAmB,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;IACvG,MAAM,OAAO,GAAsB,EAAE,CAAC;IAEtC,KAAK,MAAM,SAAS,IAAI,gBAAgB,EAAE,CAAC;QACzC,MAAM,UAAU,GAAG,SAAS,CAAC,KAAK,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC;QACrD,MAAM,SAAS,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,SAAS,CAAC,CAAC;QAC/C,MAAM,GAAG,GAAG,UAAU,IAAI,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,OAAO,EAAE,SAAS,CAAC,CAAC;QAClF,OAAO,CAAC,IAAI,CAAC,SAAS,CAAC,UAAU,EAAE,SAAS,EAAE,GAAG,CAAC,CAAC,CAAC;IACtD,CAAC;IAED,KAAK,MAAM,SAAS,IAAI,eAAe,EAAE,CAAC;QACxC,MAAM,MAAM,GAAG,eAAe,CAAC,SAAS,EAAE,MAAM,CAAC,GAAG,CAAC,CAAC;QACtD,IAAI,MAAM;YAAE,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;IACnC,CAAC;IAED,IAAI,aAAa,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC7B,MAAM,MAAM,GAAG,kBAAkB,CAAC,aAAa,EAAE,MAAM,CAAC,GAAG,EAAE,MAAM,CAAC,KAAK,CAAC,CAAC;QAC3E,MAAM,QAAQ,GAAG,MAAM,SAAS,CAAC,IAAI,CACnC,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC,EACnC,EAAE,WAAW,EAAE,CAAC,EAAE,cAAc,EAAE,MAAM,EAAE,CAC3C,CAAC;QACF,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,WAAW,CAAC,QAAQ,CAAC,CAAC,CAAC;QACjD,KAAK,MAAM,CAAC,IAAI,MAAM,CAAC,OAAO,EAAE,CAAC;YAC/B,OAAO,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,MAAM,EAAE,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,EAAE,QAAQ,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAC,CAAC;QAClF,CAAC;IACH,CAAC;IAED,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,MAAM,CAAC;IACpD,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,MAAM,CAAC;IACrD,MAAM,KAAK,GAAG,OAAO,CAAC,MAAM,CAAC;IAE7B,MAAM,OAAO,GAAkB;QAC7B,iBAAiB,EAAE,OAAO;QAC1B,OAAO,EAAE,EAAE,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,SAAS,EAAE,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,MAAM,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE;KAC9E,CAAC;IAEF,EAAE,CAAC,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,cAAc,CAAC,EAAE,IAAI,CAAC,SAAS,CAAC,OAAO,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;IAEtF,OAAO,OAAO,CAAC;AACjB,CAAC"}

package/dist/src/engine/runner.d.ts

@@ -2,6 +2,7 @@ import type { Harness, HarnessRunResult, EvalCase } from '../types.js';
 interface RunEvalResult {
     evalId: number;
     slug: string;
+    label?: string;
     prompt: string;
     withSkill: {
         output: HarnessRunResult;

package/dist/src/engine/runner.js

@@ -14,25 +14,28 @@ export async function runEval(evalCase, skillPath, evalDir, harness, oldSkillPat
     const withSkillDir = path.join(evalDir, 'with_skill');
     const baselineVariant = oldSkillPath ? 'old_skill' : 'without_skill';
     const baselineDir = path.join(evalDir, baselineVariant);
-    const withSkillResult = await harness.run({
-        skillPath,
-        prompt: evalCase.prompt,
-        files: evalCase.files,
-        outputDir: path.join(withSkillDir, 'outputs'),
-    });
+    const [withSkillResult, baselineResult] = await Promise.all([
+        harness.run({
+            skillPath,
+            prompt: evalCase.prompt,
+            files: evalCase.files,
+            outputDir: path.join(withSkillDir, 'outputs'),
+        }),
+        harness.run({
+            skillPath: oldSkillPath,
+            prompt: evalCase.prompt,
+            files: evalCase.files,
+            outputDir: path.join(baselineDir, 'outputs'),
+        }),
+    ]);
     writeTiming(withSkillDir, withSkillResult);
     writeOutput(withSkillDir, withSkillResult);
-    const baselineResult = await harness.run({
-        skillPath: oldSkillPath,
-        prompt: evalCase.prompt,
-        files: evalCase.files,
-        outputDir: path.join(baselineDir, 'outputs'),
-    });
     writeTiming(baselineDir, baselineResult);
     writeOutput(baselineDir, baselineResult);
     return {
         evalId: evalCase.id,
         slug: evalCase.slug ?? `${evalCase.id}`,
+        label: evalCase.label,
         prompt: evalCase.prompt,
         withSkill: { output: withSkillResult },
         withoutSkill: { output: baselineResult },

package/dist/src/engine/runner.js.map

@@ -1 +1 @@
-{"version":3,"file":"runner.js","sourceRoot":"","sources":["../../../src/engine/runner.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,MAAM,SAAS,CAAC;AAC9B,OAAO,KAAK,IAAI,MAAM,WAAW,CAAC;AAWlC,SAAS,WAAW,CAAC,GAAW,EAAE,MAAwB;IACxD,MAAM,MAAM,GAAe,EAAE,YAAY,EAAE,MAAM,CAAC,YAAY,EAAE,WAAW,EAAE,MAAM,CAAC,WAAW,EAAE,CAAC;IAClG,EAAE,CAAC,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,aAAa,CAAC,EAAE,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;AACnF,CAAC;AAED,SAAS,WAAW,CAAC,GAAW,EAAE,MAAwB;IACxD,EAAE,CAAC,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,SAAS,EAAE,YAAY,CAAC,EAAE,MAAM,CAAC,GAAG,CAAC,CAAC;IACtE,IAAI,MAAM,CAAC,UAAU,EAAE,CAAC;QACtB,EAAE,CAAC,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,gBAAgB,CAAC,EAAE,MAAM,CAAC,UAAU,CAAC,CAAC;IACxE,CAAC;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,OAAO,CAC3B,QAAkB,EAClB,SAAiB,EACjB,OAAe,EACf,OAAgB,EAChB,YAAqB;IAErB,MAAM,YAAY,GAAG,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,YAAY,CAAC,CAAC;IACtD,MAAM,eAAe,GAAG,YAAY,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,eAAe,CAAC;IACrE,MAAM,WAAW,GAAG,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,eAAe,CAAC,CAAC;IAExD,MAAM,eAAe,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC;QACxC,SAAS;QACT,MAAM,EAAE,QAAQ,CAAC,MAAM;QACvB,KAAK,EAAE,QAAQ,CAAC,KAAK;QACrB,SAAS,EAAE,IAAI,CAAC,IAAI,CAAC,YAAY,EAAE,SAAS,CAAC;KAC9C,CAAC,CAAC;IACH,WAAW,CAAC,YAAY,EAAE,eAAe,CAAC,CAAC;IAC3C,WAAW,CAAC,YAAY,EAAE,eAAe,CAAC,CAAC;IAE3C,MAAM,cAAc,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC;QACvC,SAAS,EAAE,YAAY;QACvB,MAAM,EAAE,QAAQ,CAAC,MAAM;QACvB,KAAK,EAAE,QAAQ,CAAC,KAAK;QACrB,SAAS,EAAE,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,SAAS,CAAC;KAC7C,CAAC,CAAC;IACH,WAAW,CAAC,WAAW,EAAE,cAAc,CAAC,CAAC;IACzC,WAAW,CAAC,WAAW,EAAE,cAAc,CAAC,CAAC;IAEzC,OAAO;QACL,MAAM,EAAE,QAAQ,CAAC,EAAE;QACnB,IAAI,EAAE,QAAQ,CAAC,IAAI,IAAI,GAAG,QAAQ,CAAC,EAAE,EAAE;QACvC,MAAM,EAAE,QAAQ,CAAC,MAAM;QACvB,SAAS,EAAE,EAAE,MAAM,EAAE,eAAe,EAAE;QACtC,YAAY,EAAE,EAAE,MAAM,EAAE,cAAc,EAAE;KACzC,CAAC;AACJ,CAAC"}
+{"version":3,"file":"runner.js","sourceRoot":"","sources":["../../../src/engine/runner.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,MAAM,SAAS,CAAC;AAC9B,OAAO,KAAK,IAAI,MAAM,WAAW,CAAC;AAYlC,SAAS,WAAW,CAAC,GAAW,EAAE,MAAwB;IACxD,MAAM,MAAM,GAAe,EAAE,YAAY,EAAE,MAAM,CAAC,YAAY,EAAE,WAAW,EAAE,MAAM,CAAC,WAAW,EAAE,CAAC;IAClG,EAAE,CAAC,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,aAAa,CAAC,EAAE,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;AACnF,CAAC;AAED,SAAS,WAAW,CAAC,GAAW,EAAE,MAAwB;IACxD,EAAE,CAAC,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,SAAS,EAAE,YAAY,CAAC,EAAE,MAAM,CAAC,GAAG,CAAC,CAAC;IACtE,IAAI,MAAM,CAAC,UAAU,EAAE,CAAC;QACtB,EAAE,CAAC,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,gBAAgB,CAAC,EAAE,MAAM,CAAC,UAAU,CAAC,CAAC;IACxE,CAAC;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,OAAO,CAC3B,QAAkB,EAClB,SAAiB,EACjB,OAAe,EACf,OAAgB,EAChB,YAAqB;IAErB,MAAM,YAAY,GAAG,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,YAAY,CAAC,CAAC;IACtD,MAAM,eAAe,GAAG,YAAY,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,eAAe,CAAC;IACrE,MAAM,WAAW,GAAG,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,eAAe,CAAC,CAAC;IAExD,MAAM,CAAC,eAAe,EAAE,cAAc,CAAC,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC;QAC1D,OAAO,CAAC,GAAG,CAAC;YACV,SAAS;YACT,MAAM,EAAE,QAAQ,CAAC,MAAM;YACvB,KAAK,EAAE,QAAQ,CAAC,KAAK;YACrB,SAAS,EAAE,IAAI,CAAC,IAAI,CAAC,YAAY,EAAE,SAAS,CAAC;SAC9C,CAAC;QACF,OAAO,CAAC,GAAG,CAAC;YACV,SAAS,EAAE,YAAY;YACvB,MAAM,EAAE,QAAQ,CAAC,MAAM;YACvB,KAAK,EAAE,QAAQ,CAAC,KAAK;YACrB,SAAS,EAAE,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,SAAS,CAAC;SAC7C,CAAC;KACH,CAAC,CAAC;IACH,WAAW,CAAC,YAAY,EAAE,eAAe,CAAC,CAAC;IAC3C,WAAW,CAAC,YAAY,EAAE,eAAe,CAAC,CAAC;IAC3C,WAAW,CAAC,WAAW,EAAE,cAAc,CAAC,CAAC;IACzC,WAAW,CAAC,WAAW,EAAE,cAAc,CAAC,CAAC;IAEzC,OAAO;QACL,MAAM,EAAE,QAAQ,CAAC,EAAE;QACnB,IAAI,EAAE,QAAQ,CAAC,IAAI,IAAI,GAAG,QAAQ,CAAC,EAAE,EAAE;QACvC,KAAK,EAAE,QAAQ,CAAC,KAAK;QACrB,MAAM,EAAE,QAAQ,CAAC,MAAM;QACvB,SAAS,EAAE,EAAE,MAAM,EAAE,eAAe,EAAE;QACtC,YAAY,EAAE,EAAE,MAAM,EAAE,cAAc,EAAE;KACzC,CAAC;AACJ,CAAC"}

package/dist/src/errors.d.ts

@@ -2,6 +2,12 @@ export declare class SnapevalError extends Error {
     exitCode: number;
     constructor(message: string, exitCode?: number);
 }
+export declare class FileNotFoundError extends SnapevalError {
+    constructor(filePath: string, hint?: string);
+}
+export declare class ThresholdError extends SnapevalError {
+    constructor(actual: number, threshold: number);
+}
 export declare class AdapterNotAvailableError extends SnapevalError {
     constructor(adapterName: string, installHint: string);
 }

package/dist/src/errors.js

@@ -1,3 +1,9 @@
+// Exit codes:
+// 0 = success
+// 1 = threshold not met (eval ran successfully but pass rate below threshold)
+// 2 = config/input error (bad JSON, missing fields, invalid flags)
+// 3 = file not found (missing skill dir, missing evals.json, missing script)
+// 4 = runtime error (harness failure, grading failure, timeout)
 export class SnapevalError extends Error {
     exitCode;
     constructor(message, exitCode = 2) {
@@ -6,9 +12,21 @@ export class SnapevalError extends Error {
         this.name = 'SnapevalError';
     }
 }
+export class FileNotFoundError extends SnapevalError {
+    constructor(filePath, hint) {
+        super(`File not found: ${filePath}${hint ? `. ${hint}` : ''}`, 3);
+        this.name = 'FileNotFoundError';
+    }
+}
+export class ThresholdError extends SnapevalError {
+    constructor(actual, threshold) {
+        super(`Skill pass rate ${(actual * 100).toFixed(1)}% is below threshold ${(threshold * 100).toFixed(1)}%`, 1);
+        this.name = 'ThresholdError';
+    }
+}
 export class AdapterNotAvailableError extends SnapevalError {
     constructor(adapterName, installHint) {
-        super(`${adapterName} is not available. ${installHint}`);
+        super(`${adapterName} is not available. ${installHint}`, 4);
         this.name = 'AdapterNotAvailableError';
     }
 }
@@ -20,13 +38,13 @@ export class RateLimitError extends SnapevalError {
 }
 export class TimeoutError extends SnapevalError {
     constructor(evalId, timeoutMs) {
-        super(`Eval ${evalId} timed out after ${timeoutMs}ms.`);
+        super(`Eval ${evalId} timed out after ${timeoutMs}ms.`, 4);
         this.name = 'TimeoutError';
     }
 }
 export class GradingError extends SnapevalError {
     constructor(evalId, detail) {
-        super(`Grading failed for eval ${evalId}: ${detail}`);
+        super(`Grading failed for eval ${evalId}: ${detail}`, 4);
         this.name = 'GradingError';
     }
 }

package/dist/src/errors.js.map

@@ -1 +1 @@
-{"version":3,"file":"errors.js","sourceRoot":"","sources":["../../src/errors.ts"],"names":[],"mappings":"AAAA,MAAM,OAAO,aAAc,SAAQ,KAAK;IACF;IAApC,YAAY,OAAe,EAAS,WAAmB,CAAC;QACtD,KAAK,CAAC,OAAO,CAAC,CAAC;QADmB,aAAQ,GAAR,QAAQ,CAAY;QAEtD,IAAI,CAAC,IAAI,GAAG,eAAe,CAAC;IAC9B,CAAC;CACF;AAED,MAAM,OAAO,wBAAyB,SAAQ,aAAa;IACzD,YAAY,WAAmB,EAAE,WAAmB;QAClD,KAAK,CAAC,GAAG,WAAW,sBAAsB,WAAW,EAAE,CAAC,CAAC;QACzD,IAAI,CAAC,IAAI,GAAG,0BAA0B,CAAC;IACzC,CAAC;CACF;AAED,MAAM,OAAO,cAAe,SAAQ,aAAa;IAC/C,YAAY,WAAmB;QAC7B,KAAK,CAAC,GAAG,WAAW,mEAAmE,CAAC,CAAC;QACzF,IAAI,CAAC,IAAI,GAAG,gBAAgB,CAAC;IAC/B,CAAC;CACF;AAED,MAAM,OAAO,YAAa,SAAQ,aAAa;IAC7C,YAAY,MAAc,EAAE,SAAiB;QAC3C,KAAK,CAAC,QAAQ,MAAM,oBAAoB,SAAS,KAAK,CAAC,CAAC;QACxD,IAAI,CAAC,IAAI,GAAG,cAAc,CAAC;IAC7B,CAAC;CACF;AAED,MAAM,OAAO,YAAa,SAAQ,aAAa;IAC7C,YAAY,MAAc,EAAE,MAAc;QACxC,KAAK,CAAC,2BAA2B,MAAM,KAAK,MAAM,EAAE,CAAC,CAAC;QACtD,IAAI,CAAC,IAAI,GAAG,cAAc,CAAC;IAC7B,CAAC;CACF"}
+{"version":3,"file":"errors.js","sourceRoot":"","sources":["../../src/errors.ts"],"names":[],"mappings":"AAAA,cAAc;AACd,cAAc;AACd,8EAA8E;AAC9E,mEAAmE;AACnE,6EAA6E;AAC7E,gEAAgE;AAEhE,MAAM,OAAO,aAAc,SAAQ,KAAK;IACF;IAApC,YAAY,OAAe,EAAS,WAAmB,CAAC;QACtD,KAAK,CAAC,OAAO,CAAC,CAAC;QADmB,aAAQ,GAAR,QAAQ,CAAY;QAEtD,IAAI,CAAC,IAAI,GAAG,eAAe,CAAC;IAC9B,CAAC;CACF;AAED,MAAM,OAAO,iBAAkB,SAAQ,aAAa;IAClD,YAAY,QAAgB,EAAE,IAAa;QACzC,KAAK,CAAC,mBAAmB,QAAQ,GAAG,IAAI,CAAC,CAAC,CAAC,KAAK,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC,CAAC;QAClE,IAAI,CAAC,IAAI,GAAG,mBAAmB,CAAC;IAClC,CAAC;CACF;AAED,MAAM,OAAO,cAAe,SAAQ,aAAa;IAC/C,YAAY,MAAc,EAAE,SAAiB;QAC3C,KAAK,CAAC,mBAAmB,CAAC,MAAM,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,wBAAwB,CAAC,SAAS,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC;QAC9G,IAAI,CAAC,IAAI,GAAG,gBAAgB,CAAC;IAC/B,CAAC;CACF;AAED,MAAM,OAAO,wBAAyB,SAAQ,aAAa;IACzD,YAAY,WAAmB,EAAE,WAAmB;QAClD,KAAK,CAAC,GAAG,WAAW,sBAAsB,WAAW,EAAE,EAAE,CAAC,CAAC,CAAC;QAC5D,IAAI,CAAC,IAAI,GAAG,0BAA0B,CAAC;IACzC,CAAC;CACF;AAED,MAAM,OAAO,cAAe,SAAQ,aAAa;IAC/C,YAAY,WAAmB;QAC7B,KAAK,CAAC,GAAG,WAAW,mEAAmE,CAAC,CAAC;QACzF,IAAI,CAAC,IAAI,GAAG,gBAAgB,CAAC;IAC/B,CAAC;CACF;AAED,MAAM,OAAO,YAAa,SAAQ,aAAa;IAC7C,YAAY,MAAc,EAAE,SAAiB;QAC3C,KAAK,CAAC,QAAQ,MAAM,oBAAoB,SAAS,KAAK,EAAE,CAAC,CAAC,CAAC;QAC3D,IAAI,CAAC,IAAI,GAAG,cAAc,CAAC;IAC7B,CAAC;CACF;AAED,MAAM,OAAO,YAAa,SAAQ,aAAa;IAC7C,YAAY,MAAc,EAAE,MAAc;QACxC,KAAK,CAAC,2BAA2B,MAAM,KAAK,MAAM,EAAE,EAAE,CAAC,CAAC,CAAC;QACzD,IAAI,CAAC,IAAI,GAAG,cAAc,CAAC;IAC7B,CAAC;CACF"}

package/dist/src/types.d.ts

@@ -32,6 +32,7 @@ export interface EvalCase {
     id: number;
     prompt: string;
     expected_output: string;
+    label?: string;
     slug?: string;
     files?: string[];
     assertions?: string[];
@@ -85,6 +86,7 @@ export interface FeedbackData {
 export interface EvalRunResult {
     evalId: number;
     slug: string;
+    label?: string;
     prompt: string;
     withSkill: {
         output: HarnessRunResult;
@@ -110,4 +112,5 @@ export interface SnapevalConfig {
     inference: string;
     workspace: string;
     runs: number;
+    concurrency: number;
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "snapeval",
-  "version": "2.0.0",
+  "version": "2.1.1",
   "description": "Harness-agnostic eval runner for agentskills.io skills",
   "type": "module",
   "bin": {
@@ -50,5 +50,8 @@
     "tsx": "^4.19.3",
     "typescript": "^5.8.2",
     "vitest": "^4.1.0"
+  },
+  "optionalDependencies": {
+    "@github/copilot-sdk": "^0.2.0"
   }
 }

package/plugin.json

@@ -1,6 +1,6 @@
 {
   "name": "snapeval",
-  "version": "2.0.0",
+  "version": "2.1.1",
   "description": "Semantic snapshot testing for AI skills. Zero assertions. AI-driven. Free inference.",
   "author": "Matan Tsach",
   "license": "MIT",

package/skills/snapeval/SKILL.md

@@ -1,9 +1,9 @@
 ---
 name: snapeval
-description: Evaluate AI skills using the agentskills.io eval spec. Generates test cases, runs with/without skill comparisons, grades assertions, and computes benchmarks. Use when the user wants to evaluate, test, or review any skill — including phrases like "test my skill", "run evals", "evaluate this", "set up evals", or "how good is my skill."
+description: Evaluate AI skills using the agentskills.io eval spec. Runs with/without skill comparisons, grades assertions, and computes benchmarks. Use when the user wants to evaluate, test, or review any skill — including phrases like "test my skill", "run evals", "evaluate this", "set up evals", or "how good is my skill."
 ---
 
-You are snapeval, a harness-agnostic eval runner for agentskills.io skills. You help developers evaluate AI skills by generating test scenarios, running with/without skill comparisons, grading assertions, and iterating on skill quality.
+You are snapeval, a harness-agnostic eval runner for agentskills.io skills. You help developers evaluate AI skills by understanding what matters to them, designing targeted test scenarios, running with/without skill comparisons, and iterating on skill quality.
 
 ## Mode Detection
 
@@ -12,61 +12,142 @@ Before acting, determine the current state by checking files in the skill direct
 | State | Condition | Mode |
 |-------|-----------|------|
 | **Fresh** | No `evals/evals.json` | First Evaluation |
-| **Has evals, no workspace** | `evals/evals.json` exists but no workspace directory | Run First Eval |
-| **Has results** | Workspace with `iteration-N/` exists | Review or Re-eval |
+| **Has evals, no workspace** | `evals/evals.json` exists but no workspace directory | Run Eval or Review (skip all interactive phases — go straight to running the command) |
+| **Has results** | Workspace with `iteration-N/` exists | Re-eval or Review (skip all interactive phases) |
+
+**Important:** The interactive phases (Discover, Analyze, Interview, Propose) only apply to the **First Evaluation** flow when no evals.json exists. When evals.json already exists, skip straight to running the `eval` or `review` command. If the user says "run", "just do it", or "without asking", always skip interactive phases.
 
 ## First Evaluation
 
-Triggered by: "evaluate", "test", "set up evals", "evaluate my skill"
+Triggered by: "evaluate", "test", "set up evals", "evaluate my skill", "how good is my skill"
 
 ### Phase 1 — Discover
 
-1. Ask the user which skill to evaluate (or accept the path they provide)
+1. Identify the skill to evaluate — accept the path the user provides, or infer it from context if they mention a skill name or directory
 2. Read the target skill's SKILL.md using the Read tool
 3. Summarize what the skill does in 1-2 sentences
 4. Confirm understanding: "This skill [summary]. Is that right?"
 
-### Phase 2 — Analyze & Propose
+**STOP. Do not proceed to Phase 2 until the user confirms your understanding is correct. Wait for the user to respond.**
 
-1. Decompose the skill into behaviors, input dimensions, and failure modes
-2. Present a brief skill profile: "Your skill has N core behaviors, handles N input variations, and I see N potential edge cases."
-3. Generate 5-8 test scenarios covering:
-   - Happy path scenarios (normal use cases)
-   - Edge cases (empty input, unusual input)
-   - At least one negative test
-4. Present scenarios as a numbered list. For each scenario show:
-   - The prompt (realistic — messy, with typos, abbreviations, personal context)
-   - What it tests
-   - Why it matters
-5. Ask: "Want to adjust any of these, or should I run them?"
+### Phase 2 — Deep Skill Analysis
 
-### Phase 3 — Handle Feedback
+Before asking the user anything, do your own homework. Study the skill thoroughly to map its surface area:
 
-- If the user wants changes, adjust conversationally
-- "Drop 3, add one about empty input" → adjust the list and re-present
-- Loop until confirmed
-- If the user says "just run it" → skip to Phase 4 immediately
+1. **Re-read the SKILL.md carefully** — not just the summary, but every instruction, rule, format spec, and example
+2. **Map the behavior space** — identify every distinct thing the skill does (e.g., "generates commit messages", "handles empty diffs", "detects breaking changes")
+3. **Map the input space** — what kinds of inputs does it accept? What dimensions vary? (language, length, complexity, format, edge cases)
+4. **Identify implicit assumptions** — what does the skill assume about context, user intent, or environment that could break?
+5. **Spot gaps and ambiguities** — where are the instructions vague, contradictory, or silent? These are often where failures hide
+
+Present this analysis to the user as a brief skill map:
+> "I've analyzed your skill in depth. Here's what I see:
+> - **N core behaviors**: [list them]
+> - **N input dimensions**: [list them]
+> - **N potential weak spots**: [list them — gaps, ambiguities, untested assumptions]"
+
+### Phase 3 — Interview
+
+Now ask targeted questions to fill gaps your analysis couldn't answer. You've done the work — your questions should be specific and informed, not generic.
+
+Ask 2-3 focused questions (one at a time) based on what you found in Phase 2. Examples:
 
-### Phase 4 — Init & First Eval
+- "Your skill says [X] but doesn't specify what happens when [Y]. What should it do?"
+- "I see the skill handles [A] and [B] but doesn't mention [C]. Is that a case you care about?"
+- "The output format section says [X]. In practice, do your users need exactly that, or is there flexibility?"
+- "I noticed the skill doesn't address [edge case]. Has that come up, or is it not a concern?"
 
-1. Run: `npx snapeval init <skill-path>` — generates evals.json (prompts + expected outputs, no assertions)
-2. Run: `npx snapeval eval <skill-path>` — runs each eval with and without the skill
-3. Report: "Ran N evals. With-skill vs without-skill outputs are in the workspace. Review the outputs and add assertions to evals.json for what 'good' looks like."
+Ask ONE question at a time. Wait for the answer before asking the next one. Two to three questions is usually enough — don't turn this into an interrogation. If the user seems impatient or says "just test it", respect that and move to Phase 4 (Propose Scenarios) with reasonable defaults.
 
-### Phase 5 — Add Assertions & Re-eval
+**STOP after each question. Wait for the user to respond before asking the next question or moving on.**
 
-After the user reviews outputs and adds assertions to evals.json:
+### Phase 4 — Propose Scenarios
 
-1. Run: `npx snapeval eval <skill-path>` — now grades assertions, produces grading.json + benchmark.json
-2. Interpret the benchmark:
-   > "With skill: X% pass rate. Without skill: Y% pass rate. Delta: +Z%. The skill adds value on [specific assertions]."
+Using your analysis and the user's answers, generate 5-8 test scenarios tailored to what actually matters.
+
+1. Present a brief skill profile: "Based on what you told me, I'll focus on [key concerns]. Your skill has N core behaviors and I see N areas worth testing."
+2. Present scenarios as a numbered list. For each scenario show:
+   - The prompt (realistic — messy, with typos, abbreviations, personal context)
+   - What it tests and why (connected back to the user's answers)
+   - Why it matters
+3. Ask: "Want to adjust any of these, or should I run them?"
+
+**STOP. Do not write evals.json or run any commands until the user approves the scenario list (or says "just run it", "looks good", "I trust you", etc). Wait for the user to respond.**
+
+### Phase 5 — Handle Feedback
+
+- If the user wants changes, adjust conversationally
+- "Drop 3, add one about empty input" → adjust the list and re-present
+- Loop until confirmed
+- If the user says "just run it", "looks good", "I trust you", or similar → skip to Phase 6 immediately
+
+### Phase 6 — Write evals.json & Run
+
+1. Write the approved scenarios to `<skill-path>/evals/evals.json`. Format:
+   ```json
+   {
+     "skill_name": "<skill-name>",
+     "evals": [
+       {
+         "id": 1,
+         "label": "short descriptive name",
+         "slug": "kebab-case-slug",
+         "prompt": "The realistic user prompt",
+         "expected_output": "Human description of expected behavior",
+         "assertions": ["Assertion 1", "Assertion 2"],
+         "files": []
+       }
+     ]
+   }
+   ```
+
+   **Writing good assertions:** Assertions are graded by an LLM that requires concrete evidence from the output to pass. Write specific, verifiable assertions — not vague ones.
+   - Good: `"Output contains a YAML block with an 'id' field for each issue"`
+   - Bad: `"Output is correct"`
+   - Good: `"Response declines to scout because the pipeline already has unclaimed issues"`
+   - Bad: `"Handles edge case properly"`
+
+   **Prefer semantic assertions for first evaluations.** Script assertions (`script:check.sh`) are powerful but add setup complexity (permissions, paths). Only suggest script assertions when the user specifically needs programmatic validation or has existing scripts.
+
+2. Run: `npx snapeval eval <skill-path>` — runs each eval with and without the skill, grades assertions, produces grading.json + benchmark.json
+
+3. Interpret the benchmark using these guidelines:
+
+   | Delta | Interpretation |
+   |-------|----------------|
+   | **+20% or more** | "Your skill adds significant value — it passes X% more assertions than raw AI." |
+   | **+1% to +19%** | "Your skill helps, but the improvement is modest. Here's where it adds value: [specific assertions]." |
+   | **0%** | "Your skill isn't measurably helping on these tests. The raw AI handles them equally well. Consider making the skill more specific or testing different scenarios." |
+   | **Negative** | "Your skill is actually hurting performance on these tests. The raw AI does better without it. Check [specific failing assertions] — the skill may be adding noise or wrong instructions." |
+
+## Adding or Modifying Evals
+
+When the user wants to add, edit, or remove specific eval cases (not regenerate from scratch):
+
+1. Read the existing `evals/evals.json`
+2. Make the requested change (add new eval, modify assertion, remove eval)
+3. Preserve all unchanged evals — never regenerate the full file. Never renumber existing eval IDs.
+4. For new evals, append with the next available ID (e.g., if max ID is 7, use 8)
+5. Run just the new/modified eval to verify it works: `npx snapeval eval <skill-path> --only <new-id>`
+
+## Re-eval After Skill Change
+
+When the user has modified their SKILL.md and wants to see if results improved:
+
+1. Detect that `evals/evals.json` already exists — do NOT regenerate scenarios
+2. Run: `npx snapeval eval <skill-path>` — this creates the next iteration automatically
+3. Compare the new iteration with the previous one:
+   - Read both `benchmark.json` files
+   - Show per-eval pass rate changes
+   - Highlight which evals improved, which regressed, and which stayed the same
+4. Give a verdict: "Your changes improved X evals, regressed Y evals, net delta: +Z%"
 
 ## Review & Iterate
 
 Triggered by: "review", "show results", "how did it do"
 
 1. Run: `npx snapeval review <skill-path>` — runs eval + creates feedback.json template
-2. Interpret results using the three signals from the spec:
+2. Interpret results using the three signals:
    - **Failed assertions** — specific gaps in the skill
    - **Human feedback** — broader quality issues (user fills in feedback.json)
    - **Benchmark delta** — where the skill adds value vs doesn't
@@ -76,7 +157,10 @@ Triggered by: "review", "show results", "how did it do"
    - **Always-fail assertions** — possibly broken, investigate
    - **Differentiating assertions** — pass with skill, fail without — this is where the skill shines
 
-4. Suggest iteration: "Want to feed these signals to an LLM to propose SKILL.md improvements?"
+4. Suggest concrete improvement strategies:
+   - Add few-shot examples to SKILL.md for failing scenarios
+   - Strengthen format constraints if output structure is inconsistent
+   - Remove redundant or conflicting instructions
 
 ## Comparing Skill Versions
 
@@ -91,14 +175,23 @@ Never show raw stack traces. Translate errors into plain language with a suggest
 
 | Error | Response |
 |-------|----------|
-| No SKILL.md found | "I can't find a SKILL.md in `<path>`. Is this the right directory?" |
-| No evals.json | "No test cases exist yet. Want me to generate them with `snapeval init`?" |
-| Inference unavailable | "I can't connect to the inference service. Check that Copilot CLI is authenticated (`copilot auth status`)." |
+| No evals.json | "No test cases exist yet. Want me to design scenarios and create evals.json?" |
+| Skill path doesn't exist | "I can't find a skill at that path. Check the directory exists and contains a SKILL.md." |
+| Harness unavailable | "The eval harness isn't available. Make sure `@github/copilot-sdk` is installed (`npm install @github/copilot-sdk`), or try `--harness copilot-cli`." |
+| Inference unavailable | "I can't connect to the inference service. Check that Copilot CLI is authenticated (`copilot auth status`) or set GITHUB_TOKEN." |
+| Eval command crashes | "The eval run failed: `<error>`. This might be a config issue — check the error message and try again." |
 | Skill invocation failure | "The skill failed to respond to eval N: `<error>`. This might be a bug in the skill — want to skip this eval and continue?" |
+| Invalid evals.json | "The evals.json file has a syntax error. Check for missing commas, trailing commas, or mismatched brackets." |
+
+If the same command fails twice, do not retry blindly. Explain the issue and ask the user how to proceed.
 
 ## Rules
 
 - Never ask the user to write evals.json or any config files manually
 - Always read the target skill's SKILL.md before generating scenarios
-- Only reference CLI commands that exist: `init`, `eval`, `review`
-- Only reference CLI flags that exist: `--harness`, `--inference`, `--workspace`, `--runs`, `--old-skill`, `--no-open`, `--verbose`
+- Only reference CLI commands that exist: `eval`, `review`
+- Only reference CLI flags that exist: `--harness`, `--inference`, `--workspace`, `--runs`, `--concurrency`, `--only`, `--threshold`, `--old-skill`, `--no-open`, `--verbose`
+- Use `--only <id>` to run specific eval IDs when the user wants to test a single eval (e.g., `--only 5` or `--only 1,3,7`)
+- Use `--concurrency 5` for parallel execution when running multiple evals
+- Use `--runs 3` when the user needs statistical confidence (averages pass rates across runs)
+- Use `--threshold 0.8` for CI gating (exits with code 1 if pass rate below threshold; value must be 0-1)

package/src/adapters/copilot-sdk-client.ts

@@ -33,7 +33,9 @@ export async function getClient(): Promise<any> {
     );
   }
 
-  clientInstance = new CopilotClient();
+  // Suppress ExperimentalWarning (e.g., SQLite) in the spawned CLI subprocess
+  const env = { ...process.env, NODE_OPTIONS: [process.env.NODE_OPTIONS, '--no-warnings'].filter(Boolean).join(' ') };
+  clientInstance = new CopilotClient({ logLevel: 'none', env });
   await clientInstance.start();
   clientStarted = true;
   return clientInstance;