snapeval 1.0.1

package/src/config.ts

@@ -0,0 +1,37 @@
+import * as fs from 'node:fs';
+import * as path from 'node:path';
+import type { SnapevalConfig } from './types.js';
+
+export const DEFAULT_CONFIG: SnapevalConfig = {
+  adapter: 'copilot-cli',
+  inference: 'auto',
+  threshold: 0.85,
+  runs: 3,
+  budget: 'unlimited',
+};
+
+function loadConfigFile(dirPath: string): Partial<SnapevalConfig> | null {
+  const configPath = path.join(dirPath, 'snapeval.config.json');
+  if (!fs.existsSync(configPath)) return null;
+  const raw = fs.readFileSync(configPath, 'utf-8');
+  return JSON.parse(raw);
+}
+
+export function resolveConfig(
+  cliFlags: Partial<SnapevalConfig>,
+  projectRoot: string,
+  skillDir?: string
+): SnapevalConfig {
+  const skillDirConfig = skillDir ? loadConfigFile(skillDir) : null;
+  const projectConfig = loadConfigFile(projectRoot);
+  return {
+    ...DEFAULT_CONFIG,
+    ...(projectConfig ?? {}),
+    ...(skillDirConfig ?? {}),
+    ...stripUndefined(cliFlags),
+  };
+}
+
+function stripUndefined(obj: Record<string, unknown>): Record<string, unknown> {
+  return Object.fromEntries(Object.entries(obj).filter(([, v]) => v !== undefined));
+}

package/src/engine/budget.ts

@@ -0,0 +1,27 @@
+export class BudgetEngine {
+  private spent = 0;
+  private cap: number | null;
+
+  constructor(budget: string) {
+    this.cap = budget === 'unlimited' ? null : parseFloat(budget);
+  }
+
+  get totalCost(): number { return this.spent; }
+
+  addCost(amount: number): void { this.spent += amount; }
+
+  isExceeded(): boolean {
+    if (this.cap === null) return false;
+    return this.spent > this.cap;
+  }
+
+  estimateScenarioCost(tokens: number, isFreeModel: boolean): number {
+    if (isFreeModel) return 0;
+    return (tokens / 1_000_000) * 0.15;
+  }
+
+  get remaining(): number | null {
+    if (this.cap === null) return null;
+    return Math.max(0, this.cap - this.spent);
+  }
+}

package/src/engine/comparison/embedding.ts

@@ -0,0 +1,26 @@
+import type { InferenceAdapter } from '../../types.js';
+
+export function cosineSimilarity(a: number[], b: number[]): number {
+  let dot = 0, magA = 0, magB = 0;
+  for (let i = 0; i < a.length; i++) {
+    dot += a[i] * b[i];
+    magA += a[i] * a[i];
+    magB += b[i] * b[i];
+  }
+  const denom = Math.sqrt(magA) * Math.sqrt(magB);
+  return denom === 0 ? 0 : dot / denom;
+}
+
+export async function embeddingCheck(
+  baseline: string,
+  current: string,
+  inference: InferenceAdapter,
+  threshold: number = 0.85
+): Promise<{ similarity: number; pass: boolean }> {
+  const [baselineEmb, currentEmb] = await Promise.all([
+    inference.embed(baseline),
+    inference.embed(current),
+  ]);
+  const similarity = cosineSimilarity(baselineEmb, currentEmb);
+  return { similarity, pass: similarity >= threshold };
+}

package/src/engine/comparison/judge.ts

@@ -0,0 +1,78 @@
+import type { InferenceAdapter, ComparisonVerdict } from '../../types.js';
+
+export function buildJudgePrompt(outputA: string, outputB: string): string {
+  return `You are an AI output comparison judge. Compare these two outputs and determine if they are semantically consistent (same meaning, same key information) or different (changed behavior, missing information, or contradictory content).
+
+OUTPUT A:
+---
+${outputA}
+---
+
+OUTPUT B:
+---
+${outputB}
+---
+
+Respond with JSON only: {"verdict": "consistent"} or {"verdict": "different"}`;
+}
+
+interface JudgeResult {
+  verdict: ComparisonVerdict;
+  details: string;
+}
+
+function parseJudgeResponse(response: string): 'consistent' | 'different' | null {
+  try {
+    const parsed = JSON.parse(response);
+    if (parsed.verdict === 'consistent' || parsed.verdict === 'different') return parsed.verdict;
+    return null;
+  } catch {
+    return null;
+  }
+}
+
+async function runJudgePair(
+  baseline: string,
+  current: string,
+  inference: InferenceAdapter
+): Promise<{ forward: string | null; reverse: string | null }> {
+  const [forwardResp, reverseResp] = await Promise.all([
+    inference.chat([{ role: 'user', content: buildJudgePrompt(baseline, current) }], {
+      temperature: 0,
+      responseFormat: 'json',
+    }),
+    inference.chat([{ role: 'user', content: buildJudgePrompt(current, baseline) }], {
+      temperature: 0,
+      responseFormat: 'json',
+    }),
+  ]);
+  return { forward: parseJudgeResponse(forwardResp), reverse: parseJudgeResponse(reverseResp) };
+}
+
+export async function llmJudge(
+  baseline: string,
+  current: string,
+  inference: InferenceAdapter
+): Promise<JudgeResult> {
+  for (let attempt = 0; attempt < 2; attempt++) {
+    const { forward, reverse } = await runJudgePair(baseline, current, inference);
+    if (forward === null || reverse === null) {
+      if (attempt === 0) continue;
+      return {
+        verdict: 'inconclusive',
+        details: 'LLM judge returned unparseable response after retry',
+      };
+    }
+    if (forward === reverse) {
+      return {
+        verdict: forward === 'consistent' ? 'pass' : 'regressed',
+        details: `LLM Judge: both orderings agree — ${forward}`,
+      };
+    }
+    return {
+      verdict: 'inconclusive',
+      details: `LLM Judge: orderings disagree (forward=${forward}, reverse=${reverse})`,
+    };
+  }
+  return { verdict: 'inconclusive', details: 'LLM judge exhausted retries' };
+}

package/src/engine/comparison/pipeline.ts

@@ -0,0 +1,43 @@
+import type { InferenceAdapter, ComparisonResult } from '../../types.js';
+import { schemaCheck } from './schema.js';
+import { embeddingCheck } from './embedding.js';
+import { llmJudge } from './judge.js';
+
+export interface PipelineOptions {
+  threshold: number;
+  skipEmbedding?: boolean;
+}
+
+export async function comparePipeline(
+  baseline: string,
+  current: string,
+  inference: InferenceAdapter,
+  options: PipelineOptions
+): Promise<ComparisonResult> {
+  // Tier 1: Schema check (FREE)
+  if (schemaCheck(baseline, current)) {
+    return { scenarioId: 0, verdict: 'pass', tier: 1, details: 'Schema match' };
+  }
+
+  // Tier 2: Embedding similarity (CHEAP) — skip if unavailable
+  if (!options.skipEmbedding) {
+    try {
+      const embResult = await embeddingCheck(baseline, current, inference, options.threshold);
+      if (embResult.pass) {
+        return {
+          scenarioId: 0,
+          verdict: 'pass',
+          tier: 2,
+          similarity: embResult.similarity,
+          details: `Embedding similarity: ${embResult.similarity.toFixed(4)}`,
+        };
+      }
+    } catch {
+      // Embedding not available — fall through to Tier 3
+    }
+  }
+
+  // Tier 3: LLM Judge (EXPENSIVE)
+  const judgeResult = await llmJudge(baseline, current, inference);
+  return { scenarioId: 0, verdict: judgeResult.verdict, tier: 3, details: judgeResult.details };
+}

package/src/engine/comparison/schema.ts

@@ -0,0 +1,22 @@
+export function extractSchema(text: string): string {
+  if (!text) return '';
+  return text
+    .split('\n')
+    .map((line) => {
+      const trimmed = line.trim();
+      if (/^#{1,6}\s/.test(trimmed)) return trimmed.replace(/^(#{1,6}\s).*/, '$1[heading]');
+      if (/^[-*+]\s/.test(trimmed)) return '- [item]';
+      if (/^\d+\.\s/.test(trimmed)) return '1. [item]';
+      if (/^```/.test(trimmed)) return '```';
+      if (trimmed === '') return '';
+      return '[content]';
+    })
+    .join('\n')
+    .replace(/(\[content\]\n)+\[content\]/g, '[content]')
+    .replace(/(\[content\]\n)+/g, '[content]\n')
+    .trim();
+}
+
+export function schemaCheck(baseline: string, current: string): boolean {
+  return extractSchema(baseline) === extractSchema(current);
+}

package/src/engine/comparison/variance.ts

@@ -0,0 +1,31 @@
+import type { VarianceEnvelope, VarianceEnvelopeRun } from '../../types.js';
+import { cosineSimilarity } from './embedding.js';
+
+export function computeEnvelope(scenarioId: number, runs: VarianceEnvelopeRun[]): VarianceEnvelope {
+  const dims = runs[0].embedding.length;
+  const centroid = new Array(dims).fill(0);
+  for (const run of runs) {
+    for (let i = 0; i < dims; i++) {
+      centroid[i] += run.embedding[i];
+    }
+  }
+  for (let i = 0; i < dims; i++) {
+    centroid[i] /= runs.length;
+  }
+  let maxDistance = 0;
+  for (const run of runs) {
+    const sim = cosineSimilarity(run.embedding, centroid);
+    const distance = 1 - sim;
+    if (distance > maxDistance) maxDistance = distance;
+  }
+  return { scenario_id: scenarioId, runs, centroid, radius: maxDistance };
+}
+
+export function isWithinEnvelope(
+  embedding: number[],
+  envelope: VarianceEnvelope,
+  threshold: number
+): boolean {
+  const sim = cosineSimilarity(embedding, envelope.centroid);
+  return sim >= threshold - envelope.radius;
+}

package/src/engine/generator.ts

@@ -0,0 +1,61 @@
+import type { InferenceAdapter, EvalsFile } from '../types.js';
+
+export function buildGeneratorPrompt(skillContent: string): string {
+  return `You are a test case generator for AI skills. Read the following skill definition and generate 5-8 realistic test scenarios.
+
+SKILL DEFINITION:
+---
+${skillContent}
+---
+
+Generate test scenarios as JSON with this exact format:
+{
+  "skill_name": "<name from skill>",
+  "evals": [
+    {
+      "id": 1,
+      "prompt": "<realistic user prompt that would trigger this skill>",
+      "expected_output": "<human-readable description of expected behavior>",
+      "assertions": ["<verifiable statement about the output>"]
+    }
+  ]
+}
+
+Requirements:
+- Include happy path scenarios (normal use cases)
+- Include edge cases (empty input, malformed input, boundary conditions)
+- Include at least one negative test (input the skill should handle gracefully)
+- Prompts should be realistic — the way a real user would type them
+- Each assertion should be specific and verifiable
+- Return ONLY the JSON, no markdown wrapping`;
+}
+
+function extractJSON(text: string): string {
+  const match = text.match(/```(?:json)?\s*([\s\S]*?)```/);
+  if (match) return match[1].trim();
+  return text.trim();
+}
+
+export async function generateEvals(
+  skillContent: string,
+  skillName: string,
+  inference: InferenceAdapter
+): Promise<EvalsFile> {
+  const prompt = buildGeneratorPrompt(skillContent);
+  const response = await inference.chat(
+    [{ role: 'user', content: prompt }],
+    { temperature: 0.7, responseFormat: 'json' }
+  );
+  const parsed = JSON.parse(extractJSON(response));
+  return {
+    skill_name: parsed.skill_name || skillName,
+    generated_by: 'snapeval v1.0.0',
+    evals: parsed.evals.map((e: any, i: number) => ({
+      id: e.id || i + 1,
+      prompt: e.prompt,
+      expected_output: e.expected_output || '',
+      files: e.files || [],
+      assertions: e.assertions || [],
+    })),
+  };
+}

package/src/engine/snapshot.ts

@@ -0,0 +1,48 @@
+import * as fs from 'node:fs';
+import * as path from 'node:path';
+import * as crypto from 'node:crypto';
+import type { SkillOutput, Snapshot } from '../types.js';
+
+export class SnapshotManager {
+  private snapshotsDir: string;
+  constructor(private evalsDir: string) {
+    this.snapshotsDir = path.join(evalsDir, 'snapshots');
+  }
+
+  private snapshotPath(scenarioId: number): string {
+    return path.join(this.snapshotsDir, `scenario-${scenarioId}.snap.json`);
+  }
+
+  saveSnapshot(scenarioId: number, prompt: string, output: SkillOutput, runs: number = 1): void {
+    const snapshot: Snapshot = {
+      scenario_id: scenarioId, prompt, output,
+      captured_at: new Date().toISOString(), runs, approved_by: null,
+    };
+    fs.mkdirSync(this.snapshotsDir, { recursive: true });
+    fs.writeFileSync(this.snapshotPath(scenarioId), JSON.stringify(snapshot, null, 2));
+  }
+
+  loadSnapshot(scenarioId: number): Snapshot | null {
+    const p = this.snapshotPath(scenarioId);
+    if (!fs.existsSync(p)) return null;
+    return JSON.parse(fs.readFileSync(p, 'utf-8'));
+  }
+
+  approve(scenarioId: number, prompt: string, newOutput: SkillOutput): void {
+    const old = this.loadSnapshot(scenarioId);
+    const previousHash = old ? crypto.createHash('sha256').update(old.output.raw).digest('hex').slice(0, 8) : 'none';
+    const newHash = crypto.createHash('sha256').update(newOutput.raw).digest('hex').slice(0, 8);
+    this.saveSnapshot(scenarioId, prompt, newOutput);
+    const auditEntry = { scenario_id: scenarioId, approved_at: new Date().toISOString(), previous_hash: previousHash, new_hash: newHash };
+    const auditPath = path.join(this.snapshotsDir, '.audit-log.jsonl');
+    fs.appendFileSync(auditPath, JSON.stringify(auditEntry) + '\n');
+  }
+
+  listSnapshotIds(): number[] {
+    if (!fs.existsSync(this.snapshotsDir)) return [];
+    return fs.readdirSync(this.snapshotsDir)
+      .filter((f) => f.match(/^scenario-\d+\.snap\.json$/))
+      .map((f) => parseInt(f.match(/scenario-(\d+)/)![1]))
+      .sort((a, b) => a - b);
+  }
+}

package/src/errors.ts

@@ -0,0 +1,34 @@
+export class SnapevalError extends Error {
+  constructor(message: string, public exitCode: number = 2) {
+    super(message);
+    this.name = 'SnapevalError';
+  }
+}
+
+export class AdapterNotAvailableError extends SnapevalError {
+  constructor(adapterName: string, installHint: string) {
+    super(`${adapterName} is not available. ${installHint}`);
+    this.name = 'AdapterNotAvailableError';
+  }
+}
+
+export class RateLimitError extends SnapevalError {
+  constructor(adapterName: string) {
+    super(`${adapterName} rate limit exceeded. Try again later or use a different adapter.`);
+    this.name = 'RateLimitError';
+  }
+}
+
+export class TimeoutError extends SnapevalError {
+  constructor(scenarioId: number, timeoutMs: number) {
+    super(`Scenario ${scenarioId} timed out after ${timeoutMs}ms.`);
+    this.name = 'TimeoutError';
+  }
+}
+
+export class NoBaselineError extends SnapevalError {
+  constructor(skillPath: string) {
+    super(`No baselines found at ${skillPath}/evals/snapshots/. Run \`snapeval capture\` first.`, 2);
+    this.name = 'NoBaselineError';
+  }
+}

package/src/types.ts

@@ -0,0 +1,153 @@
+// === Adapter Interfaces ===
+export interface SkillOutput {
+  raw: string;
+  metadata: {
+    tokens: number;
+    durationMs: number;
+    model: string;
+    adapter: string;
+  };
+}
+
+export interface SkillAdapter {
+  name: string;
+  invoke(skillPath: string, prompt: string, files?: string[]): Promise<SkillOutput>;
+  isAvailable(): Promise<boolean>;
+}
+
+export interface Message {
+  role: 'system' | 'user' | 'assistant';
+  content: string;
+}
+
+export interface ChatOptions {
+  temperature?: number;
+  maxTokens?: number;
+  responseFormat?: 'text' | 'json';
+}
+
+export interface InferenceAdapter {
+  name: string;
+  chat(messages: Message[], options?: ChatOptions): Promise<string>;
+  embed(text: string): Promise<number[]>;
+  estimateCost(tokens: number): number;
+}
+
+export interface EvalResults {
+  skillName: string;
+  scenarios: ScenarioResult[];
+  summary: BenchmarkSummary;
+  timing: TimingData;
+}
+
+export interface ReportAdapter {
+  name: string;
+  report(results: EvalResults): Promise<void>;
+}
+
+// === Eval Format (agentskills.io) ===
+export interface EvalCase {
+  id: number;
+  prompt: string;
+  expected_output: string;
+  files?: string[];
+  assertions?: string[];
+}
+
+export interface EvalsFile {
+  skill_name: string;
+  generated_by: string;
+  evals: EvalCase[];
+}
+
+// === Snapshot Format ===
+export interface Snapshot {
+  scenario_id: number;
+  prompt: string;
+  output: SkillOutput;
+  captured_at: string;
+  runs: number;
+  approved_by: string | null;
+}
+
+export interface VarianceEnvelopeRun {
+  raw: string;
+  embedding: number[];
+}
+
+export interface VarianceEnvelope {
+  scenario_id: number;
+  runs: VarianceEnvelopeRun[];
+  centroid: number[];
+  radius: number;
+}
+
+// === Comparison Results ===
+export type ComparisonVerdict = 'pass' | 'regressed' | 'inconclusive' | 'error';
+
+export interface ComparisonResult {
+  scenarioId: number;
+  verdict: ComparisonVerdict;
+  tier: 1 | 2 | 3;
+  similarity?: number;
+  details: string;
+}
+
+// === Grading ===
+export interface AssertionResult {
+  text: string;
+  passed: boolean;
+  evidence: string;
+}
+
+export interface GradingSummary {
+  passed: number;
+  failed: number;
+  total: number;
+  pass_rate: number;
+}
+
+export interface GradingFile {
+  assertion_results: AssertionResult[];
+  summary: GradingSummary;
+}
+
+// === Timing & Benchmark ===
+export interface TimingData {
+  total_tokens: number;
+  duration_ms: number;
+}
+
+export interface BenchmarkSummary {
+  total_scenarios: number;
+  passed: number;
+  regressed: number;
+  pass_rate: number;
+  total_tokens: number;
+  total_cost_usd: number;
+  total_duration_ms: number;
+  tier_breakdown: {
+    tier1_schema: number;
+    tier2_embedding: number;
+    tier3_llm_judge: number;
+  };
+}
+
+// === Scenario Result ===
+export interface ScenarioResult {
+  scenarioId: number;
+  prompt: string;
+  comparison: ComparisonResult;
+  grading?: GradingFile;
+  timing: TimingData;
+  newOutput: SkillOutput;
+}
+
+// === Config ===
+export interface SnapevalConfig {
+  adapter: string;
+  inference: string;
+  threshold: number;
+  runs: number;
+  budget: string;
+}