snapeval 2.2.0 → 4.0.0

package/src/adapters/harness/copilot-sdk.ts

@@ -1,13 +1,18 @@
 import * as fs from 'node:fs';
 import * as path from 'node:path';
 import type { Harness, HarnessRunResult } from '../../types.js';
-import { getClient, isSDKInstalled } from '../copilot-sdk-client.js';
+import { getClient } from '../copilot-sdk-client.js';
+import {
+  createSkillBlockingPermissionHandler,
+  createSkillBlockingHook,
+} from './skill-blocker.js';
 
 export class CopilotSDKHarness implements Harness {
   readonly name = 'copilot-sdk';
 
   async run(options: {
     skillPath?: string;
+    blockedSkillPath?: string;
     prompt: string;
     files?: string[];
     outputDir: string;
@@ -18,7 +23,6 @@ export class CopilotSDKHarness implements Harness {
     fs.mkdirSync(options.outputDir, { recursive: true });
 
     // Dynamically import SDK for approveAll
-    // @ts-ignore — module may not be installed (optional dep)
     const { approveAll } = await import('@github/copilot-sdk');
 
     // Build session config
@@ -31,7 +35,16 @@ export class CopilotSDKHarness implements Harness {
 
     // Native skill loading: point skillDirectories at the skill's parent
     if (options.skillPath) {
-      sessionConfig.skillDirectories = [options.skillPath];
+      sessionConfig.skillDirectories = [path.dirname(options.skillPath)];
+    }
+
+    // Skill blocking: replace approveAll with blocking handler + hook
+    if (options.blockedSkillPath) {
+      const blockedDir = path.dirname(options.blockedSkillPath);
+      sessionConfig.onPermissionRequest = createSkillBlockingPermissionHandler(blockedDir);
+      sessionConfig.hooks = {
+        onPreToolUse: createSkillBlockingHook(blockedDir),
+      };
     }
 
     const session = await client.createSession(sessionConfig);
@@ -62,8 +75,8 @@ export class CopilotSDKHarness implements Harness {
       const events = await session.getMessages();
       const transcript = buildTranscript(events);
 
-      // Extract token count from events if available
-      const totalTokens = extractTokenCount(events);
+      // SDK assistant.usage events are ephemeral and not available via getMessages()
+      const totalTokens = 0;
 
       const durationMs = Date.now() - startMs;
 
@@ -80,7 +93,7 @@ export class CopilotSDKHarness implements Harness {
   }
 
   async isAvailable(): Promise<boolean> {
-    return isSDKInstalled();
+    return true;
   }
 }
 
@@ -98,7 +111,7 @@ function buildTranscript(events: any[]): string {
         lines.push(`[tool:start] ${event.data?.toolName ?? 'unknown'}(${JSON.stringify(event.data?.arguments ?? {})})`);
         break;
       case 'tool.execution_complete':
-        lines.push(`[tool:done] ${event.data?.toolName ?? 'unknown'} → ${truncate(event.data?.result ?? '', 200)}`);
+        lines.push(`[tool:done] ${event.data?.toolName ?? 'unknown'} → ${truncate(event.data?.result?.content ?? '', 200)}`);
         break;
       case 'skill.invoked':
         lines.push(`[skill] ${event.data?.name ?? 'unknown'} (${event.data?.path ?? ''})`);
@@ -111,16 +124,6 @@ function buildTranscript(events: any[]): string {
   return lines.join('\n');
 }
 
-function extractTokenCount(events: any[]): number {
-  let total = 0;
-  for (const event of events) {
-    if (event.type === 'assistant.usage') {
-      total += (event.data?.inputTokens ?? 0) + (event.data?.outputTokens ?? 0);
-    }
-  }
-  return total;
-}
-
 function truncate(str: string, max: number): string {
   return str.length > max ? str.slice(0, max) + '...' : str;
 }

package/src/adapters/harness/resolve.ts

@@ -1,17 +1,10 @@
 import type { Harness } from '../../types.js';
 import { CopilotCLIHarness } from './copilot-cli.js';
 import { CopilotSDKHarness } from './copilot-sdk.js';
-import { AdapterNotAvailableError, SnapevalError } from '../../errors.js';
-import { isSDKInstalled } from '../copilot-sdk-client.js';
+import { SnapevalError } from '../../errors.js';
 
 export function resolveHarness(name: string): Harness {
   if (name === 'copilot-sdk') {
-    if (!isSDKInstalled()) {
-      throw new AdapterNotAvailableError(
-        'copilot-sdk',
-        '@github/copilot-sdk is not installed. Install with: npm install @github/copilot-sdk'
-      );
-    }
     return new CopilotSDKHarness();
   }
   if (name === 'copilot-cli') {

package/src/adapters/harness/skill-blocker.ts

@@ -0,0 +1,61 @@
+import * as fs from 'node:fs';
+import * as path from 'node:path';
+import type { PermissionHandler } from '@github/copilot-sdk';
+
+// PreToolUseHandler is not re-exported from @github/copilot-sdk's public API,
+// so we define the minimal type inline matching the SDK's internal definition.
+type PreToolUseHookInput = { toolName: string; toolArgs: unknown; timestamp: number; cwd: string };
+type PreToolUseHookOutput = { permissionDecision?: 'allow' | 'deny' | 'ask'; permissionDecisionReason?: string };
+type PreToolUseHandler = (input: PreToolUseHookInput, invocation: { sessionId: string }) => PreToolUseHookOutput | void;
+
+function resolveDir(dir: string): string {
+  try {
+    return fs.realpathSync(dir) + path.sep;
+  } catch {
+    return path.resolve(dir) + path.sep;
+  }
+}
+
+export function createSkillBlockingHook(
+  blockedDir: string,
+): PreToolUseHandler {
+  const resolved = resolveDir(blockedDir);
+  const raw = path.resolve(blockedDir) + path.sep;
+
+  return (input) => {
+    const argsStr = typeof input.toolArgs === 'string'
+      ? input.toolArgs
+      : JSON.stringify(input.toolArgs ?? '');
+
+    if (argsStr.includes(resolved) || argsStr.includes(raw)) {
+      return {
+        permissionDecision: 'deny' as const,
+        permissionDecisionReason: `Blocked: tool "${input.toolName}" references blocked skill directory`,
+      };
+    }
+    return {};
+  };
+}
+
+export function createSkillBlockingPermissionHandler(
+  blockedDir: string,
+): PermissionHandler {
+  const resolved = resolveDir(blockedDir);
+
+  return (request) => {
+    if (request.kind === 'read') {
+      const readPath = (request as { path?: unknown }).path;
+      if (typeof readPath === 'string') {
+        try {
+          const resolvedPath = fs.realpathSync(readPath);
+          if (resolvedPath === resolved.slice(0, -1) || resolvedPath.startsWith(resolved)) {
+            return { kind: 'denied-by-rules', rules: [{ reason: 'skill-access-blocked' }] };
+          }
+        } catch {
+          // Path doesn't exist — can't be the skill dir
+        }
+      }
+    }
+    return { kind: 'approved' };
+  };
+}

package/src/adapters/inference/copilot-sdk.ts

@@ -4,10 +4,9 @@ import { getClient } from '../copilot-sdk-client.js';
 export class CopilotSDKInference implements InferenceAdapter {
   readonly name = 'copilot-sdk';
 
-  async chat(messages: Message[], _options?: ChatOptions): Promise<string> {
+  async chat(messages: Message[], options?: ChatOptions): Promise<string> {
     const client = await getClient();
 
-    // @ts-ignore — module may not be installed (optional dep)
     const { approveAll } = await import('@github/copilot-sdk');
 
     const systemMessages = messages.filter((m) => m.role === 'system');
@@ -15,6 +14,9 @@ export class CopilotSDKInference implements InferenceAdapter {
     const systemContent = systemMessages.map((m) => m.content).join('\n');
     const userPrompt = nonSystemMessages.map((m) => m.content).join('\n');
 
+    // Note: ChatOptions (temperature, responseFormat) are not supported by the
+    // SDK's session config. The SDK controls these at the server level.
+    void options;
     const session = await client.createSession({
       model: 'gpt-4.1',
       ...(systemContent

package/src/adapters/inference/github-models.ts

@@ -40,6 +40,9 @@ export class GitHubModelsInference implements InferenceAdapter {
     const data = (await response.json()) as {
       choices: Array<{ message: { content: string } }>;
     };
+    if (!data.choices?.length) {
+      throw new Error('GitHub Models API returned no choices');
+    }
     return data.choices[0].message.content;
   }
 }

package/src/adapters/inference/resolve.ts

@@ -1,19 +1,7 @@
-import { execFileSync } from 'node:child_process';
 import type { InferenceAdapter } from '../../types.js';
 import { AdapterNotAvailableError } from '../../errors.js';
 import { GitHubModelsInference } from './github-models.js';
-import { CopilotInference } from './copilot.js';
 import { CopilotSDKInference } from './copilot-sdk.js';
-import { isSDKInstalled } from '../copilot-sdk-client.js';
-
-function isCopilotAvailable(): boolean {
-  try {
-    execFileSync('copilot', ['--version'], { encoding: 'utf-8', stdio: 'pipe' });
-    return true;
-  } catch {
-    return false;
-  }
-}
 
 function isGitHubTokenAvailable(): boolean {
   return Boolean(process.env.GITHUB_TOKEN);
@@ -21,31 +9,18 @@ function isGitHubTokenAvailable(): boolean {
 
 export function resolveInference(preference: string): InferenceAdapter {
   if (preference === 'auto') {
-    const copilotAvailable = isCopilotAvailable();
-    const tokenAvailable = isGitHubTokenAvailable();
-
-    if (copilotAvailable) {
-      return new CopilotInference();
-    }
-
-    if (tokenAvailable) {
-      return new GitHubModelsInference();
-    }
+    return new CopilotSDKInference();
+  }
 
+  if (preference === 'copilot') {
     throw new AdapterNotAvailableError(
-      'inference',
-      'No inference adapter available. Install GitHub Copilot CLI (`npm install -g @github/copilot`) or set GITHUB_TOKEN.'
+      'copilot',
+      'The copilot CLI inference adapter has been removed. Use --inference copilot-sdk instead.'
     );
   }
 
-  if (preference === 'copilot') {
-    if (!isCopilotAvailable()) {
-      throw new AdapterNotAvailableError(
-        'copilot',
-        'GitHub Copilot CLI is not available. Install with: npm install -g @github/copilot'
-      );
-    }
-    return new CopilotInference();
+  if (preference === 'copilot-sdk') {
+    return new CopilotSDKInference();
   }
 
   if (preference === 'github-models') {
@@ -58,18 +33,8 @@ export function resolveInference(preference: string): InferenceAdapter {
     return new GitHubModelsInference();
   }
 
-  if (preference === 'copilot-sdk') {
-    if (!isSDKInstalled()) {
-      throw new AdapterNotAvailableError(
-        'copilot-sdk',
-        '@github/copilot-sdk is not installed. Install with: npm install @github/copilot-sdk'
-      );
-    }
-    return new CopilotSDKInference();
-  }
-
   throw new AdapterNotAvailableError(
     preference,
-    `Unknown inference adapter "${preference}". Valid options: auto, copilot, copilot-sdk, github-models.`
+    `Unknown inference adapter "${preference}". Valid options: auto, copilot-sdk, github-models.`
   );
 }

package/src/commands/eval.ts

@@ -7,6 +7,7 @@ import type {
   EvalResults,
   EvalRunResult,
   GradingResult,
+  FeedbackData,
 } from '../types.js';
 import { WorkspaceManager } from '../engine/workspace.js';
 import { runEval } from '../engine/runner.js';
@@ -86,7 +87,7 @@ export async function evalCommand(
   skillPath: string,
   harness: Harness,
   inference: InferenceAdapter,
-  options: { workspace?: string; runs?: number; oldSkill?: string; concurrency?: number; only?: number[]; threshold?: number }
+  options: { workspace?: string; runs?: number; oldSkill?: string; concurrency?: number; only?: number[]; threshold?: number; feedback?: boolean }
 ): Promise<EvalResults> {
   const evalsPath = path.join(skillPath, 'evals', 'evals.json');
   if (!fs.existsSync(evalsPath)) {
@@ -224,6 +225,18 @@ export async function evalCommand(
       typeof value === 'number' ? Math.round(value * 10000) / 10000 : value, 2)
   );
 
+  // Write feedback template if requested
+  if (options.feedback) {
+    const feedback: FeedbackData = {};
+    for (const run of evalRuns) {
+      feedback[`eval-${run.slug}`] = '';
+    }
+    fs.writeFileSync(
+      path.join(iterationDir, 'feedback.json'),
+      JSON.stringify(feedback, null, 2)
+    );
+  }
+
   // Check threshold if set (for CI gating)
   if (options.threshold !== undefined) {
     const passRate = benchmark.run_summary.with_skill.pass_rate.mean;

package/src/engine/runner.ts

@@ -43,6 +43,7 @@ export async function runEval(
     }),
     harness.run({
       skillPath: oldSkillPath,
+      blockedSkillPath: skillPath,
       prompt: evalCase.prompt,
       files: evalCase.files,
       outputDir: path.join(baselineDir, 'outputs'),

package/src/errors.ts

@@ -35,21 +35,7 @@ export class AdapterNotAvailableError extends SnapevalError {
 
 export class RateLimitError extends SnapevalError {
   constructor(adapterName: string) {
-    super(`${adapterName} rate limit exceeded. Try again later or use a different adapter.`);
+    super(`${adapterName} rate limit exceeded. Try again later or use a different adapter.`, 4);
     this.name = 'RateLimitError';
   }
 }
-
-export class TimeoutError extends SnapevalError {
-  constructor(evalId: number, timeoutMs: number) {
-    super(`Eval ${evalId} timed out after ${timeoutMs}ms.`, 4);
-    this.name = 'TimeoutError';
-  }
-}
-
-export class GradingError extends SnapevalError {
-  constructor(evalId: number, detail: string) {
-    super(`Grading failed for eval ${evalId}: ${detail}`, 4);
-    this.name = 'GradingError';
-  }
-}

package/src/types.ts

@@ -15,7 +15,7 @@ export interface Harness {
     prompt: string;
     files?: string[];
     outputDir: string;
-  }): Promise<HarnessRunResult>;
+  } & Record<string, unknown>): Promise<HarnessRunResult>;
   isAvailable(): Promise<boolean>;
 }