snapeval 1.8.0 → 2.1.0

package/src/engine/runner.ts

@@ -33,21 +33,22 @@ export async function runEval(
   const baselineVariant = oldSkillPath ? 'old_skill' : 'without_skill';
   const baselineDir = path.join(evalDir, baselineVariant);
 
-  const withSkillResult = await harness.run({
-    skillPath,
-    prompt: evalCase.prompt,
-    files: evalCase.files,
-    outputDir: path.join(withSkillDir, 'outputs'),
-  });
+  const [withSkillResult, baselineResult] = await Promise.all([
+    harness.run({
+      skillPath,
+      prompt: evalCase.prompt,
+      files: evalCase.files,
+      outputDir: path.join(withSkillDir, 'outputs'),
+    }),
+    harness.run({
+      skillPath: oldSkillPath,
+      prompt: evalCase.prompt,
+      files: evalCase.files,
+      outputDir: path.join(baselineDir, 'outputs'),
+    }),
+  ]);
   writeTiming(withSkillDir, withSkillResult);
   writeOutput(withSkillDir, withSkillResult);
-
-  const baselineResult = await harness.run({
-    skillPath: oldSkillPath,
-    prompt: evalCase.prompt,
-    files: evalCase.files,
-    outputDir: path.join(baselineDir, 'outputs'),
-  });
   writeTiming(baselineDir, baselineResult);
   writeOutput(baselineDir, baselineResult);
 

package/src/errors.ts

@@ -1,3 +1,10 @@
+// Exit codes:
+// 0 = success
+// 1 = threshold not met (eval ran successfully but pass rate below threshold)
+// 2 = config/input error (bad JSON, missing fields, invalid flags)
+// 3 = file not found (missing skill dir, missing evals.json, missing script)
+// 4 = runtime error (harness failure, grading failure, timeout)
+
 export class SnapevalError extends Error {
   constructor(message: string, public exitCode: number = 2) {
     super(message);
@@ -5,9 +12,23 @@ export class SnapevalError extends Error {
   }
 }
 
+export class FileNotFoundError extends SnapevalError {
+  constructor(filePath: string, hint?: string) {
+    super(`File not found: ${filePath}${hint ? `. ${hint}` : ''}`, 3);
+    this.name = 'FileNotFoundError';
+  }
+}
+
+export class ThresholdError extends SnapevalError {
+  constructor(actual: number, threshold: number) {
+    super(`Skill pass rate ${(actual * 100).toFixed(1)}% is below threshold ${(threshold * 100).toFixed(1)}%`, 1);
+    this.name = 'ThresholdError';
+  }
+}
+
 export class AdapterNotAvailableError extends SnapevalError {
   constructor(adapterName: string, installHint: string) {
-    super(`${adapterName} is not available. ${installHint}`);
+    super(`${adapterName} is not available. ${installHint}`, 4);
     this.name = 'AdapterNotAvailableError';
   }
 }
@@ -21,14 +42,14 @@ export class RateLimitError extends SnapevalError {
 
 export class TimeoutError extends SnapevalError {
   constructor(evalId: number, timeoutMs: number) {
-    super(`Eval ${evalId} timed out after ${timeoutMs}ms.`);
+    super(`Eval ${evalId} timed out after ${timeoutMs}ms.`, 4);
     this.name = 'TimeoutError';
   }
 }
 
 export class GradingError extends SnapevalError {
   constructor(evalId: number, detail: string) {
-    super(`Grading failed for eval ${evalId}: ${detail}`);
+    super(`Grading failed for eval ${evalId}: ${detail}`, 4);
     this.name = 'GradingError';
   }
 }

package/src/types.ts

@@ -142,4 +142,5 @@ export interface SnapevalConfig {
   inference: string;
   workspace: string;
   runs: number;
+  concurrency: number;
 }

package/dist/src/commands/init.d.ts

@@ -1,2 +0,0 @@
-import type { InferenceAdapter } from '../types.js';
-export declare function initCommand(skillPath: string, inference: InferenceAdapter): Promise<void>;

package/dist/src/commands/init.js

@@ -1,27 +0,0 @@
-import * as fs from 'node:fs';
-import * as path from 'node:path';
-import { generateEvals } from '../engine/generator.js';
-import { SnapevalError } from '../errors.js';
-export async function initCommand(skillPath, inference) {
-    // Locate the skill definition file (SKILL.md or skill.md)
-    const candidates = ['SKILL.md', 'skill.md'];
-    let skillFilePath = null;
-    for (const name of candidates) {
-        const candidate = path.join(skillPath, name);
-        if (fs.existsSync(candidate)) {
-            skillFilePath = candidate;
-            break;
-        }
-    }
-    if (!skillFilePath) {
-        throw new SnapevalError(`No SKILL.md found at ${skillPath}. Create a SKILL.md file to describe your skill.`);
-    }
-    const skillContent = fs.readFileSync(skillFilePath, 'utf-8');
-    const skillName = path.basename(skillPath);
-    const evalsFile = await generateEvals(skillContent, skillName, inference);
-    const evalsDir = path.join(skillPath, 'evals');
-    fs.mkdirSync(evalsDir, { recursive: true });
-    const evalsPath = path.join(evalsDir, 'evals.json');
-    fs.writeFileSync(evalsPath, JSON.stringify(evalsFile, null, 2), 'utf-8');
-}
-//# sourceMappingURL=init.js.map

package/dist/src/commands/init.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"init.js","sourceRoot":"","sources":["../../../src/commands/init.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,MAAM,SAAS,CAAC;AAC9B,OAAO,KAAK,IAAI,MAAM,WAAW,CAAC;AAElC,OAAO,EAAE,aAAa,EAAE,MAAM,wBAAwB,CAAC;AACvD,OAAO,EAAE,aAAa,EAAE,MAAM,cAAc,CAAC;AAE7C,MAAM,CAAC,KAAK,UAAU,WAAW,CAC/B,SAAiB,EACjB,SAA2B;IAE3B,0DAA0D;IAC1D,MAAM,UAAU,GAAG,CAAC,UAAU,EAAE,UAAU,CAAC,CAAC;IAC5C,IAAI,aAAa,GAAkB,IAAI,CAAC;IACxC,KAAK,MAAM,IAAI,IAAI,UAAU,EAAE,CAAC;QAC9B,MAAM,SAAS,GAAG,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,IAAI,CAAC,CAAC;QAC7C,IAAI,EAAE,CAAC,UAAU,CAAC,SAAS,CAAC,EAAE,CAAC;YAC7B,aAAa,GAAG,SAAS,CAAC;YAC1B,MAAM;QACR,CAAC;IACH,CAAC;IAED,IAAI,CAAC,aAAa,EAAE,CAAC;QACnB,MAAM,IAAI,aAAa,CACrB,wBAAwB,SAAS,kDAAkD,CACpF,CAAC;IACJ,CAAC;IAED,MAAM,YAAY,GAAG,EAAE,CAAC,YAAY,CAAC,aAAa,EAAE,OAAO,CAAC,CAAC;IAC7D,MAAM,SAAS,GAAG,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC;IAE3C,MAAM,SAAS,GAAG,MAAM,aAAa,CAAC,YAAY,EAAE,SAAS,EAAE,SAAS,CAAC,CAAC;IAE1E,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,OAAO,CAAC,CAAC;IAC/C,EAAE,CAAC,SAAS,CAAC,QAAQ,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAE5C,MAAM,SAAS,GAAG,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,YAAY,CAAC,CAAC;IACpD,EAAE,CAAC,aAAa,CAAC,SAAS,EAAE,IAAI,CAAC,SAAS,CAAC,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC,EAAE,OAAO,CAAC,CAAC;AAC3E,CAAC"}

package/dist/src/engine/generator.d.ts

@@ -1,3 +0,0 @@
-import type { InferenceAdapter, EvalsFile } from '../types.js';
-export declare function buildGeneratorPrompt(skillContent: string): string;
-export declare function generateEvals(skillContent: string, skillName: string, inference: InferenceAdapter): Promise<EvalsFile>;

package/dist/src/engine/generator.js

@@ -1,51 +0,0 @@
-export function buildGeneratorPrompt(skillContent) {
-    return `You are a test case generator for AI skills. Read the following skill definition and generate 5-8 realistic test scenarios.
-
-SKILL DEFINITION:
----
-${skillContent}
----
-
-Generate test scenarios as JSON with this exact format:
-{
-  "skill_name": "<name from skill>",
-  "evals": [
-    {
-      "id": 1,
-      "slug": "<2-4-word-kebab-case-label>",
-      "prompt": "<realistic user prompt that would trigger this skill>",
-      "expected_output": "<human-readable description of expected behavior>"
-    }
-  ]
-}
-
-Requirements:
-- Include happy path scenarios (normal use cases)
-- Include edge cases (empty input, malformed input, boundary conditions)
-- Include at least one negative test (input the skill should handle gracefully)
-- Prompts should be realistic — the way a real user would type them
-- slug must be 2-4 words in kebab-case (e.g. "happy-path", "empty-input-edge-case")
-- Return ONLY the JSON, no markdown wrapping`;
-}
-function extractJSON(text) {
-    const match = text.match(/```(?:json)?\s*([\s\S]*?)```/);
-    if (match)
-        return match[1].trim();
-    return text.trim();
-}
-export async function generateEvals(skillContent, skillName, inference) {
-    const prompt = buildGeneratorPrompt(skillContent);
-    const response = await inference.chat([{ role: 'user', content: prompt }], { temperature: 0.7, responseFormat: 'json' });
-    const parsed = JSON.parse(extractJSON(response));
-    return {
-        skill_name: parsed.skill_name || skillName,
-        evals: parsed.evals.map((e, i) => ({
-            id: e.id || i + 1,
-            slug: e.slug,
-            prompt: e.prompt,
-            expected_output: e.expected_output || '',
-            files: e.files || [],
-        })),
-    };
-}
-//# sourceMappingURL=generator.js.map

package/dist/src/engine/generator.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"generator.js","sourceRoot":"","sources":["../../../src/engine/generator.ts"],"names":[],"mappings":"AAEA,MAAM,UAAU,oBAAoB,CAAC,YAAoB;IACvD,OAAO;;;;EAIP,YAAY;;;;;;;;;;;;;;;;;;;;;;6CAsB+B,CAAC;AAC9C,CAAC;AAED,SAAS,WAAW,CAAC,IAAY;IAC/B,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,8BAA8B,CAAC,CAAC;IACzD,IAAI,KAAK;QAAE,OAAO,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;IAClC,OAAO,IAAI,CAAC,IAAI,EAAE,CAAC;AACrB,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,aAAa,CACjC,YAAoB,EACpB,SAAiB,EACjB,SAA2B;IAE3B,MAAM,MAAM,GAAG,oBAAoB,CAAC,YAAY,CAAC,CAAC;IAClD,MAAM,QAAQ,GAAG,MAAM,SAAS,CAAC,IAAI,CACnC,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC,EACnC,EAAE,WAAW,EAAE,GAAG,EAAE,cAAc,EAAE,MAAM,EAAE,CAC7C,CAAC;IACF,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,WAAW,CAAC,QAAQ,CAAC,CAAC,CAAC;IACjD,OAAO;QACL,UAAU,EAAE,MAAM,CAAC,UAAU,IAAI,SAAS;QAC1C,KAAK,EAAE,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAM,EAAE,CAAS,EAAE,EAAE,CAAC,CAAC;YAC9C,EAAE,EAAE,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC;YACjB,IAAI,EAAE,CAAC,CAAC,IAAI;YACZ,MAAM,EAAE,CAAC,CAAC,MAAM;YAChB,eAAe,EAAE,CAAC,CAAC,eAAe,IAAI,EAAE;YACxC,KAAK,EAAE,CAAC,CAAC,KAAK,IAAI,EAAE;SACrB,CAAC,CAAC;KACJ,CAAC;AACJ,CAAC"}

package/src/commands/init.ts

@@ -1,38 +0,0 @@
-import * as fs from 'node:fs';
-import * as path from 'node:path';
-import type { InferenceAdapter } from '../types.js';
-import { generateEvals } from '../engine/generator.js';
-import { SnapevalError } from '../errors.js';
-
-export async function initCommand(
-  skillPath: string,
-  inference: InferenceAdapter
-): Promise<void> {
-  // Locate the skill definition file (SKILL.md or skill.md)
-  const candidates = ['SKILL.md', 'skill.md'];
-  let skillFilePath: string | null = null;
-  for (const name of candidates) {
-    const candidate = path.join(skillPath, name);
-    if (fs.existsSync(candidate)) {
-      skillFilePath = candidate;
-      break;
-    }
-  }
-
-  if (!skillFilePath) {
-    throw new SnapevalError(
-      `No SKILL.md found at ${skillPath}. Create a SKILL.md file to describe your skill.`
-    );
-  }
-
-  const skillContent = fs.readFileSync(skillFilePath, 'utf-8');
-  const skillName = path.basename(skillPath);
-
-  const evalsFile = await generateEvals(skillContent, skillName, inference);
-
-  const evalsDir = path.join(skillPath, 'evals');
-  fs.mkdirSync(evalsDir, { recursive: true });
-
-  const evalsPath = path.join(evalsDir, 'evals.json');
-  fs.writeFileSync(evalsPath, JSON.stringify(evalsFile, null, 2), 'utf-8');
-}

package/src/engine/generator.ts

@@ -1,60 +0,0 @@
-import type { InferenceAdapter, EvalsFile } from '../types.js';
-
-export function buildGeneratorPrompt(skillContent: string): string {
-  return `You are a test case generator for AI skills. Read the following skill definition and generate 5-8 realistic test scenarios.
-
-SKILL DEFINITION:
----
-${skillContent}
----
-
-Generate test scenarios as JSON with this exact format:
-{
-  "skill_name": "<name from skill>",
-  "evals": [
-    {
-      "id": 1,
-      "slug": "<2-4-word-kebab-case-label>",
-      "prompt": "<realistic user prompt that would trigger this skill>",
-      "expected_output": "<human-readable description of expected behavior>"
-    }
-  ]
-}
-
-Requirements:
-- Include happy path scenarios (normal use cases)
-- Include edge cases (empty input, malformed input, boundary conditions)
-- Include at least one negative test (input the skill should handle gracefully)
-- Prompts should be realistic — the way a real user would type them
-- slug must be 2-4 words in kebab-case (e.g. "happy-path", "empty-input-edge-case")
-- Return ONLY the JSON, no markdown wrapping`;
-}
-
-function extractJSON(text: string): string {
-  const match = text.match(/```(?:json)?\s*([\s\S]*?)```/);
-  if (match) return match[1].trim();
-  return text.trim();
-}
-
-export async function generateEvals(
-  skillContent: string,
-  skillName: string,
-  inference: InferenceAdapter
-): Promise<EvalsFile> {
-  const prompt = buildGeneratorPrompt(skillContent);
-  const response = await inference.chat(
-    [{ role: 'user', content: prompt }],
-    { temperature: 0.7, responseFormat: 'json' }
-  );
-  const parsed = JSON.parse(extractJSON(response));
-  return {
-    skill_name: parsed.skill_name || skillName,
-    evals: parsed.evals.map((e: any, i: number) => ({
-      id: e.id || i + 1,
-      slug: e.slug,
-      prompt: e.prompt,
-      expected_output: e.expected_output || '',
-      files: e.files || [],
-    })),
-  };
-}