snapeval 3.0.0 → 4.0.0

package/dist/src/adapters/harness/copilot-sdk.d.ts

@@ -3,6 +3,7 @@ export declare class CopilotSDKHarness implements Harness {
     readonly name = "copilot-sdk";
     run(options: {
         skillPath?: string;
+        blockedSkillPath?: string;
         prompt: string;
         files?: string[];
         outputDir: string;

package/dist/src/adapters/harness/copilot-sdk.js

@@ -1,6 +1,7 @@
 import * as fs from 'node:fs';
 import * as path from 'node:path';
 import { getClient } from '../copilot-sdk-client.js';
+import { createSkillBlockingPermissionHandler, createSkillBlockingHook, } from './skill-blocker.js';
 export class CopilotSDKHarness {
     name = 'copilot-sdk';
     async run(options) {
@@ -20,6 +21,14 @@ export class CopilotSDKHarness {
         if (options.skillPath) {
             sessionConfig.skillDirectories = [path.dirname(options.skillPath)];
         }
+        // Skill blocking: replace approveAll with blocking handler + hook
+        if (options.blockedSkillPath) {
+            const blockedDir = path.dirname(options.blockedSkillPath);
+            sessionConfig.onPermissionRequest = createSkillBlockingPermissionHandler(blockedDir);
+            sessionConfig.hooks = {
+                onPreToolUse: createSkillBlockingHook(blockedDir),
+            };
+        }
         const session = await client.createSession(sessionConfig);
         try {
             // Attach input files if provided

package/dist/src/adapters/harness/copilot-sdk.js.map

@@ -1 +1 @@
-{"version":3,"file":"copilot-sdk.js","sourceRoot":"","sources":["../../../../src/adapters/harness/copilot-sdk.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,MAAM,SAAS,CAAC;AAC9B,OAAO,KAAK,IAAI,MAAM,WAAW,CAAC;AAElC,OAAO,EAAE,SAAS,EAAE,MAAM,0BAA0B,CAAC;AAErD,MAAM,OAAO,iBAAiB;IACnB,IAAI,GAAG,aAAa,CAAC;IAE9B,KAAK,CAAC,GAAG,CAAC,OAKT;QACC,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAC3B,MAAM,MAAM,GAAG,MAAM,SAAS,EAAE,CAAC;QAEjC,EAAE,CAAC,SAAS,CAAC,OAAO,CAAC,SAAS,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QAErD,wCAAwC;QACxC,MAAM,EAAE,UAAU,EAAE,GAAG,MAAM,MAAM,CAAC,qBAAqB,CAAC,CAAC;QAE3D,uBAAuB;QACvB,MAAM,aAAa,GAA4B;YAC7C,KAAK,EAAE,SAAS;YAChB,mBAAmB,EAAE,UAAU;YAC/B,gBAAgB,EAAE,OAAO,CAAC,SAAS;YACnC,gBAAgB,EAAE,EAAE,OAAO,EAAE,KAAK,EAAE;SACrC,CAAC;QAEF,qEAAqE;QACrE,IAAI,OAAO,CAAC,SAAS,EAAE,CAAC;YACtB,aAAa,CAAC,gBAAgB,GAAG,CAAC,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC,CAAC;QACrE,CAAC;QAED,MAAM,OAAO,GAAG,MAAM,MAAM,CAAC,aAAa,CAAC,aAAa,CAAC,CAAC;QAE1D,IAAI,CAAC;YACH,iCAAiC;YACjC,MAAM,WAAW,GAAgE,EAAE,CAAC;YACpF,IAAI,OAAO,CAAC,KAAK,EAAE,CAAC;gBAClB,KAAK,MAAM,IAAI,IAAI,OAAO,CAAC,KAAK,EAAE,CAAC;oBACjC,oEAAoE;oBACpE,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,SAAS,EAAE,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC;oBAC/D,EAAE,CAAC,YAAY,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;oBAC5B,WAAW,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,WAAW,EAAE,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;gBACnF,CAAC;YACH,CAAC;YAED,MAAM,QAAQ,GAAG,MAAM,OAAO,CAAC,WAAW,CACxC;gBACE,MAAM,EAAE,OAAO,CAAC,MAAM;gBACtB,GAAG,CAAC,WAAW,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,WAAW,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;aACnD,EACD,OAAO,CACR,CAAC;YAEF,MAAM,GAAG,GAAG,QAAQ,EAAE,IAAI,EAAE,OAAO,IAAI,EAAE,CAAC;YAE1C,8CAA8C;YAC9C,MAAM,MAAM,GAAG,MAAM,OAAO,CAAC,WAAW,EAAE,CAAC;YAC3C,MAAM,UAAU,GAAG,eAAe,CAAC,MAAM,CAAC,CAAC;YAE3C,+EAA+E;YAC/E,MAAM,WAAW,GAAG,CAAC,CAAC;YAEtB,MAAM,UAAU,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,OAAO,CAAC;YAExC,OAAO;gBACL,GAAG,EAAE,GAAG,CAAC,IAAI,EAAE;gBACf,UAAU;gBACV,KAAK,EAAE,EAAE;gBACT,YAAY,EAAE,WAAW;gBACzB,WAAW,EAAE,UAAU;aACxB,CAAC;QACJ,CAAC;gBAAS,CAAC;YACT,MAAM,OAAO,CAAC,UAAU,EAAE,CAAC;QAC7B,CAAC;IACH,CAAC;IAED,KAAK,CAAC,WAAW;QACf,OAAO,IAAI,CAAC;IACd,CAAC;CACF;AAED,SAAS,eAAe,CAAC,MAAa;IACpC,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;QAC3B,QAAQ,KAAK,CAAC,IAAI,EAAE,CAAC;YACnB,KAAK,cAAc;gBACjB,KAAK,CAAC,IAAI,CAAC,UAAU,KAAK,CAAC,IAAI,EAAE,OAAO,IAAI,EAAE,EAAE,CAAC,CAAC;gBAClD,MAAM;YACR,KAAK,mBAAmB;gBACtB,KAAK,CAAC,IAAI,CAAC,eAAe,KAAK,CAAC,IAAI,EAAE,OAAO,IAAI,EAAE,EAAE,CAAC,CAAC;gBACvD,MAAM;YACR,KAAK,sBAAsB;gBACzB,KAAK,CAAC,IAAI,CAAC,gBAAgB,KAAK,CAAC,IAAI,EAAE,QAAQ,IAAI,SAAS,IAAI,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,IAAI,EAAE,SAAS,IAAI,EAAE,CAAC,GAAG,CAAC,CAAC;gBAChH,MAAM;YACR,KAAK,yBAAyB;gBAC5B,KAAK,CAAC,IAAI,CAAC,eAAe,KAAK,CAAC,IAAI,EAAE,QAAQ,IAAI,SAAS,MAAM,QAAQ,CAAC,KAAK,CAAC,IAAI,EAAE,MAAM,EAAE,OAAO,IAAI,EAAE,EAAE,GAAG,CAAC,EAAE,CAAC,CAAC;gBACrH,MAAM;YACR,KAAK,eAAe;gBAClB,KAAK,CAAC,IAAI,CAAC,WAAW,KAAK,CAAC,IAAI,EAAE,IAAI,IAAI,SAAS,KAAK,KAAK,CAAC,IAAI,EAAE,IAAI,IAAI,EAAE,GAAG,CAAC,CAAC;gBACnF,MAAM;YACR,KAAK,eAAe;gBAClB,KAAK,CAAC,IAAI,CAAC,WAAW,KAAK,CAAC,IAAI,EAAE,OAAO,IAAI,EAAE,EAAE,CAAC,CAAC;gBACnD,MAAM;QACV,CAAC;IACH,CAAC;IACD,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAC1B,CAAC;AAED,SAAS,QAAQ,CAAC,GAAW,EAAE,GAAW;IACxC,OAAO,GAAG,CAAC,MAAM,GAAG,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,GAAG,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC;AAC5D,CAAC"}
+{"version":3,"file":"copilot-sdk.js","sourceRoot":"","sources":["../../../../src/adapters/harness/copilot-sdk.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,MAAM,SAAS,CAAC;AAC9B,OAAO,KAAK,IAAI,MAAM,WAAW,CAAC;AAElC,OAAO,EAAE,SAAS,EAAE,MAAM,0BAA0B,CAAC;AACrD,OAAO,EACL,oCAAoC,EACpC,uBAAuB,GACxB,MAAM,oBAAoB,CAAC;AAE5B,MAAM,OAAO,iBAAiB;IACnB,IAAI,GAAG,aAAa,CAAC;IAE9B,KAAK,CAAC,GAAG,CAAC,OAMT;QACC,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAC3B,MAAM,MAAM,GAAG,MAAM,SAAS,EAAE,CAAC;QAEjC,EAAE,CAAC,SAAS,CAAC,OAAO,CAAC,SAAS,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QAErD,wCAAwC;QACxC,MAAM,EAAE,UAAU,EAAE,GAAG,MAAM,MAAM,CAAC,qBAAqB,CAAC,CAAC;QAE3D,uBAAuB;QACvB,MAAM,aAAa,GAA4B;YAC7C,KAAK,EAAE,SAAS;YAChB,mBAAmB,EAAE,UAAU;YAC/B,gBAAgB,EAAE,OAAO,CAAC,SAAS;YACnC,gBAAgB,EAAE,EAAE,OAAO,EAAE,KAAK,EAAE;SACrC,CAAC;QAEF,qEAAqE;QACrE,IAAI,OAAO,CAAC,SAAS,EAAE,CAAC;YACtB,aAAa,CAAC,gBAAgB,GAAG,CAAC,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC,CAAC;QACrE,CAAC;QAED,kEAAkE;QAClE,IAAI,OAAO,CAAC,gBAAgB,EAAE,CAAC;YAC7B,MAAM,UAAU,GAAG,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,gBAAgB,CAAC,CAAC;YAC1D,aAAa,CAAC,mBAAmB,GAAG,oCAAoC,CAAC,UAAU,CAAC,CAAC;YACrF,aAAa,CAAC,KAAK,GAAG;gBACpB,YAAY,EAAE,uBAAuB,CAAC,UAAU,CAAC;aAClD,CAAC;QACJ,CAAC;QAED,MAAM,OAAO,GAAG,MAAM,MAAM,CAAC,aAAa,CAAC,aAAa,CAAC,CAAC;QAE1D,IAAI,CAAC;YACH,iCAAiC;YACjC,MAAM,WAAW,GAAgE,EAAE,CAAC;YACpF,IAAI,OAAO,CAAC,KAAK,EAAE,CAAC;gBAClB,KAAK,MAAM,IAAI,IAAI,OAAO,CAAC,KAAK,EAAE,CAAC;oBACjC,oEAAoE;oBACpE,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,SAAS,EAAE,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC;oBAC/D,EAAE,CAAC,YAAY,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;oBAC5B,WAAW,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,WAAW,EAAE,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;gBACnF,CAAC;YACH,CAAC;YAED,MAAM,QAAQ,GAAG,MAAM,OAAO,CAAC,WAAW,CACxC;gBACE,MAAM,EAAE,OAAO,CAAC,MAAM;gBACtB,GAAG,CAAC,WAAW,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,WAAW,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;aACnD,EACD,OAAO,CACR,CAAC;YAEF,MAAM,GAAG,GAAG,QAAQ,EAAE,IAAI,EAAE,OAAO,IAAI,EAAE,CAAC;YAE1C,8CAA8C;YAC9C,MAAM,MAAM,GAAG,MAAM,OAAO,CAAC,WAAW,EAAE,CAAC;YAC3C,MAAM,UAAU,GAAG,eAAe,CAAC,MAAM,CAAC,CAAC;YAE3C,+EAA+E;YAC/E,MAAM,WAAW,GAAG,CAAC,CAAC;YAEtB,MAAM,UAAU,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,OAAO,CAAC;YAExC,OAAO;gBACL,GAAG,EAAE,GAAG,CAAC,IAAI,EAAE;gBACf,UAAU;gBACV,KAAK,EAAE,EAAE;gBACT,YAAY,EAAE,WAAW;gBACzB,WAAW,EAAE,UAAU;aACxB,CAAC;QACJ,CAAC;gBAAS,CAAC;YACT,MAAM,OAAO,CAAC,UAAU,EAAE,CAAC;QAC7B,CAAC;IACH,CAAC;IAED,KAAK,CAAC,WAAW;QACf,OAAO,IAAI,CAAC;IACd,CAAC;CACF;AAED,SAAS,eAAe,CAAC,MAAa;IACpC,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;QAC3B,QAAQ,KAAK,CAAC,IAAI,EAAE,CAAC;YACnB,KAAK,cAAc;gBACjB,KAAK,CAAC,IAAI,CAAC,UAAU,KAAK,CAAC,IAAI,EAAE,OAAO,IAAI,EAAE,EAAE,CAAC,CAAC;gBAClD,MAAM;YACR,KAAK,mBAAmB;gBACtB,KAAK,CAAC,IAAI,CAAC,eAAe,KAAK,CAAC,IAAI,EAAE,OAAO,IAAI,EAAE,EAAE,CAAC,CAAC;gBACvD,MAAM;YACR,KAAK,sBAAsB;gBACzB,KAAK,CAAC,IAAI,CAAC,gBAAgB,KAAK,CAAC,IAAI,EAAE,QAAQ,IAAI,SAAS,IAAI,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,IAAI,EAAE,SAAS,IAAI,EAAE,CAAC,GAAG,CAAC,CAAC;gBAChH,MAAM;YACR,KAAK,yBAAyB;gBAC5B,KAAK,CAAC,IAAI,CAAC,eAAe,KAAK,CAAC,IAAI,EAAE,QAAQ,IAAI,SAAS,MAAM,QAAQ,CAAC,KAAK,CAAC,IAAI,EAAE,MAAM,EAAE,OAAO,IAAI,EAAE,EAAE,GAAG,CAAC,EAAE,CAAC,CAAC;gBACrH,MAAM;YACR,KAAK,eAAe;gBAClB,KAAK,CAAC,IAAI,CAAC,WAAW,KAAK,CAAC,IAAI,EAAE,IAAI,IAAI,SAAS,KAAK,KAAK,CAAC,IAAI,EAAE,IAAI,IAAI,EAAE,GAAG,CAAC,CAAC;gBACnF,MAAM;YACR,KAAK,eAAe;gBAClB,KAAK,CAAC,IAAI,CAAC,WAAW,KAAK,CAAC,IAAI,EAAE,OAAO,IAAI,EAAE,EAAE,CAAC,CAAC;gBACnD,MAAM;QACV,CAAC;IACH,CAAC;IACD,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAC1B,CAAC;AAED,SAAS,QAAQ,CAAC,GAAW,EAAE,GAAW;IACxC,OAAO,GAAG,CAAC,MAAM,GAAG,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,GAAG,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC;AAC5D,CAAC"}

package/dist/src/adapters/harness/skill-blocker.d.ts

@@ -0,0 +1,17 @@
+import type { PermissionHandler } from '@github/copilot-sdk';
+type PreToolUseHookInput = {
+    toolName: string;
+    toolArgs: unknown;
+    timestamp: number;
+    cwd: string;
+};
+type PreToolUseHookOutput = {
+    permissionDecision?: 'allow' | 'deny' | 'ask';
+    permissionDecisionReason?: string;
+};
+type PreToolUseHandler = (input: PreToolUseHookInput, invocation: {
+    sessionId: string;
+}) => PreToolUseHookOutput | void;
+export declare function createSkillBlockingHook(blockedDir: string): PreToolUseHandler;
+export declare function createSkillBlockingPermissionHandler(blockedDir: string): PermissionHandler;
+export {};

package/dist/src/adapters/harness/skill-blocker.js

@@ -0,0 +1,47 @@
+import * as fs from 'node:fs';
+import * as path from 'node:path';
+function resolveDir(dir) {
+    try {
+        return fs.realpathSync(dir) + path.sep;
+    }
+    catch {
+        return path.resolve(dir) + path.sep;
+    }
+}
+export function createSkillBlockingHook(blockedDir) {
+    const resolved = resolveDir(blockedDir);
+    const raw = path.resolve(blockedDir) + path.sep;
+    return (input) => {
+        const argsStr = typeof input.toolArgs === 'string'
+            ? input.toolArgs
+            : JSON.stringify(input.toolArgs ?? '');
+        if (argsStr.includes(resolved) || argsStr.includes(raw)) {
+            return {
+                permissionDecision: 'deny',
+                permissionDecisionReason: `Blocked: tool "${input.toolName}" references blocked skill directory`,
+            };
+        }
+        return {};
+    };
+}
+export function createSkillBlockingPermissionHandler(blockedDir) {
+    const resolved = resolveDir(blockedDir);
+    return (request) => {
+        if (request.kind === 'read') {
+            const readPath = request.path;
+            if (typeof readPath === 'string') {
+                try {
+                    const resolvedPath = fs.realpathSync(readPath);
+                    if (resolvedPath === resolved.slice(0, -1) || resolvedPath.startsWith(resolved)) {
+                        return { kind: 'denied-by-rules', rules: [{ reason: 'skill-access-blocked' }] };
+                    }
+                }
+                catch {
+                    // Path doesn't exist — can't be the skill dir
+                }
+            }
+        }
+        return { kind: 'approved' };
+    };
+}
+//# sourceMappingURL=skill-blocker.js.map

package/dist/src/adapters/harness/skill-blocker.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"skill-blocker.js","sourceRoot":"","sources":["../../../../src/adapters/harness/skill-blocker.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,MAAM,SAAS,CAAC;AAC9B,OAAO,KAAK,IAAI,MAAM,WAAW,CAAC;AASlC,SAAS,UAAU,CAAC,GAAW;IAC7B,IAAI,CAAC;QACH,OAAO,EAAE,CAAC,YAAY,CAAC,GAAG,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC;IACzC,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC;IACtC,CAAC;AACH,CAAC;AAED,MAAM,UAAU,uBAAuB,CACrC,UAAkB;IAElB,MAAM,QAAQ,GAAG,UAAU,CAAC,UAAU,CAAC,CAAC;IACxC,MAAM,GAAG,GAAG,IAAI,CAAC,OAAO,CAAC,UAAU,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC;IAEhD,OAAO,CAAC,KAAK,EAAE,EAAE;QACf,MAAM,OAAO,GAAG,OAAO,KAAK,CAAC,QAAQ,KAAK,QAAQ;YAChD,CAAC,CAAC,KAAK,CAAC,QAAQ;YAChB,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,QAAQ,IAAI,EAAE,CAAC,CAAC;QAEzC,IAAI,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAC,IAAI,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;YACxD,OAAO;gBACL,kBAAkB,EAAE,MAAe;gBACnC,wBAAwB,EAAE,kBAAkB,KAAK,CAAC,QAAQ,sCAAsC;aACjG,CAAC;QACJ,CAAC;QACD,OAAO,EAAE,CAAC;IACZ,CAAC,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,oCAAoC,CAClD,UAAkB;IAElB,MAAM,QAAQ,GAAG,UAAU,CAAC,UAAU,CAAC,CAAC;IAExC,OAAO,CAAC,OAAO,EAAE,EAAE;QACjB,IAAI,OAAO,CAAC,IAAI,KAAK,MAAM,EAAE,CAAC;YAC5B,MAAM,QAAQ,GAAI,OAA8B,CAAC,IAAI,CAAC;YACtD,IAAI,OAAO,QAAQ,KAAK,QAAQ,EAAE,CAAC;gBACjC,IAAI,CAAC;oBACH,MAAM,YAAY,GAAG,EAAE,CAAC,YAAY,CAAC,QAAQ,CAAC,CAAC;oBAC/C,IAAI,YAAY,KAAK,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,YAAY,CAAC,UAAU,CAAC,QAAQ,CAAC,EAAE,CAAC;wBAChF,OAAO,EAAE,IAAI,EAAE,iBAAiB,EAAE,KAAK,EAAE,CAAC,EAAE,MAAM,EAAE,sBAAsB,EAAE,CAAC,EAAE,CAAC;oBAClF,CAAC;gBACH,CAAC;gBAAC,MAAM,CAAC;oBACP,8CAA8C;gBAChD,CAAC;YACH,CAAC;QACH,CAAC;QACD,OAAO,EAAE,IAAI,EAAE,UAAU,EAAE,CAAC;IAC9B,CAAC,CAAC;AACJ,CAAC"}

package/dist/src/engine/runner.js

@@ -23,6 +23,7 @@ export async function runEval(evalCase, skillPath, evalDir, harness, oldSkillPat
         }),
         harness.run({
             skillPath: oldSkillPath,
+            blockedSkillPath: skillPath,
             prompt: evalCase.prompt,
             files: evalCase.files,
             outputDir: path.join(baselineDir, 'outputs'),

package/dist/src/engine/runner.js.map

@@ -1 +1 @@
-{"version":3,"file":"runner.js","sourceRoot":"","sources":["../../../src/engine/runner.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,MAAM,SAAS,CAAC;AAC9B,OAAO,KAAK,IAAI,MAAM,WAAW,CAAC;AAYlC,SAAS,WAAW,CAAC,GAAW,EAAE,MAAwB;IACxD,MAAM,MAAM,GAAe,EAAE,YAAY,EAAE,MAAM,CAAC,YAAY,EAAE,WAAW,EAAE,MAAM,CAAC,WAAW,EAAE,CAAC;IAClG,EAAE,CAAC,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,aAAa,CAAC,EAAE,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;AACnF,CAAC;AAED,SAAS,WAAW,CAAC,GAAW,EAAE,MAAwB;IACxD,EAAE,CAAC,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,SAAS,EAAE,YAAY,CAAC,EAAE,MAAM,CAAC,GAAG,CAAC,CAAC;IACtE,IAAI,MAAM,CAAC,UAAU,EAAE,CAAC;QACtB,EAAE,CAAC,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,gBAAgB,CAAC,EAAE,MAAM,CAAC,UAAU,CAAC,CAAC;IACxE,CAAC;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,OAAO,CAC3B,QAAkB,EAClB,SAAiB,EACjB,OAAe,EACf,OAAgB,EAChB,YAAqB;IAErB,MAAM,YAAY,GAAG,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,YAAY,CAAC,CAAC;IACtD,MAAM,eAAe,GAAG,YAAY,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,eAAe,CAAC;IACrE,MAAM,WAAW,GAAG,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,eAAe,CAAC,CAAC;IAExD,MAAM,CAAC,eAAe,EAAE,cAAc,CAAC,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC;QAC1D,OAAO,CAAC,GAAG,CAAC;YACV,SAAS;YACT,MAAM,EAAE,QAAQ,CAAC,MAAM;YACvB,KAAK,EAAE,QAAQ,CAAC,KAAK;YACrB,SAAS,EAAE,IAAI,CAAC,IAAI,CAAC,YAAY,EAAE,SAAS,CAAC;SAC9C,CAAC;QACF,OAAO,CAAC,GAAG,CAAC;YACV,SAAS,EAAE,YAAY;YACvB,MAAM,EAAE,QAAQ,CAAC,MAAM;YACvB,KAAK,EAAE,QAAQ,CAAC,KAAK;YACrB,SAAS,EAAE,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,SAAS,CAAC;SAC7C,CAAC;KACH,CAAC,CAAC;IACH,WAAW,CAAC,YAAY,EAAE,eAAe,CAAC,CAAC;IAC3C,WAAW,CAAC,YAAY,EAAE,eAAe,CAAC,CAAC;IAC3C,WAAW,CAAC,WAAW,EAAE,cAAc,CAAC,CAAC;IACzC,WAAW,CAAC,WAAW,EAAE,cAAc,CAAC,CAAC;IAEzC,OAAO;QACL,MAAM,EAAE,QAAQ,CAAC,EAAE;QACnB,IAAI,EAAE,QAAQ,CAAC,IAAI,IAAI,GAAG,QAAQ,CAAC,EAAE,EAAE;QACvC,KAAK,EAAE,QAAQ,CAAC,KAAK;QACrB,MAAM,EAAE,QAAQ,CAAC,MAAM;QACvB,SAAS,EAAE,EAAE,MAAM,EAAE,eAAe,EAAE;QACtC,YAAY,EAAE,EAAE,MAAM,EAAE,cAAc,EAAE;KACzC,CAAC;AACJ,CAAC"}
+{"version":3,"file":"runner.js","sourceRoot":"","sources":["../../../src/engine/runner.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,MAAM,SAAS,CAAC;AAC9B,OAAO,KAAK,IAAI,MAAM,WAAW,CAAC;AAYlC,SAAS,WAAW,CAAC,GAAW,EAAE,MAAwB;IACxD,MAAM,MAAM,GAAe,EAAE,YAAY,EAAE,MAAM,CAAC,YAAY,EAAE,WAAW,EAAE,MAAM,CAAC,WAAW,EAAE,CAAC;IAClG,EAAE,CAAC,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,aAAa,CAAC,EAAE,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;AACnF,CAAC;AAED,SAAS,WAAW,CAAC,GAAW,EAAE,MAAwB;IACxD,EAAE,CAAC,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,SAAS,EAAE,YAAY,CAAC,EAAE,MAAM,CAAC,GAAG,CAAC,CAAC;IACtE,IAAI,MAAM,CAAC,UAAU,EAAE,CAAC;QACtB,EAAE,CAAC,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,gBAAgB,CAAC,EAAE,MAAM,CAAC,UAAU,CAAC,CAAC;IACxE,CAAC;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,OAAO,CAC3B,QAAkB,EAClB,SAAiB,EACjB,OAAe,EACf,OAAgB,EAChB,YAAqB;IAErB,MAAM,YAAY,GAAG,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,YAAY,CAAC,CAAC;IACtD,MAAM,eAAe,GAAG,YAAY,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,eAAe,CAAC;IACrE,MAAM,WAAW,GAAG,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,eAAe,CAAC,CAAC;IAExD,MAAM,CAAC,eAAe,EAAE,cAAc,CAAC,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC;QAC1D,OAAO,CAAC,GAAG,CAAC;YACV,SAAS;YACT,MAAM,EAAE,QAAQ,CAAC,MAAM;YACvB,KAAK,EAAE,QAAQ,CAAC,KAAK;YACrB,SAAS,EAAE,IAAI,CAAC,IAAI,CAAC,YAAY,EAAE,SAAS,CAAC;SAC9C,CAAC;QACF,OAAO,CAAC,GAAG,CAAC;YACV,SAAS,EAAE,YAAY;YACvB,gBAAgB,EAAE,SAAS;YAC3B,MAAM,EAAE,QAAQ,CAAC,MAAM;YACvB,KAAK,EAAE,QAAQ,CAAC,KAAK;YACrB,SAAS,EAAE,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,SAAS,CAAC;SAC7C,CAAC;KACH,CAAC,CAAC;IACH,WAAW,CAAC,YAAY,EAAE,eAAe,CAAC,CAAC;IAC3C,WAAW,CAAC,YAAY,EAAE,eAAe,CAAC,CAAC;IAC3C,WAAW,CAAC,WAAW,EAAE,cAAc,CAAC,CAAC;IACzC,WAAW,CAAC,WAAW,EAAE,cAAc,CAAC,CAAC;IAEzC,OAAO;QACL,MAAM,EAAE,QAAQ,CAAC,EAAE;QACnB,IAAI,EAAE,QAAQ,CAAC,IAAI,IAAI,GAAG,QAAQ,CAAC,EAAE,EAAE;QACvC,KAAK,EAAE,QAAQ,CAAC,KAAK;QACrB,MAAM,EAAE,QAAQ,CAAC,MAAM;QACvB,SAAS,EAAE,EAAE,MAAM,EAAE,eAAe,EAAE;QACtC,YAAY,EAAE,EAAE,MAAM,EAAE,cAAc,EAAE;KACzC,CAAC;AACJ,CAAC"}

package/dist/src/types.d.ts

@@ -12,7 +12,7 @@ export interface Harness {
         prompt: string;
         files?: string[];
         outputDir: string;
-    }): Promise<HarnessRunResult>;
+    } & Record<string, unknown>): Promise<HarnessRunResult>;
     isAvailable(): Promise<boolean>;
 }
 export interface Message {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "snapeval",
-  "version": "3.0.0",
+  "version": "4.0.0",
   "description": "Harness-agnostic eval runner for agentskills.io skills",
   "type": "module",
   "bin": {

package/plugin.json

@@ -1,6 +1,6 @@
 {
   "name": "snapeval",
-  "version": "3.0.0",
+  "version": "4.0.0",
   "description": "Eval runner for AI skills. Design test scenarios, run with/without skill comparisons, grade assertions, iterate on quality.",
   "author": {
     "name": "Matan Tsach"

package/src/adapters/harness/copilot-sdk.ts

@@ -2,12 +2,17 @@ import * as fs from 'node:fs';
 import * as path from 'node:path';
 import type { Harness, HarnessRunResult } from '../../types.js';
 import { getClient } from '../copilot-sdk-client.js';
+import {
+  createSkillBlockingPermissionHandler,
+  createSkillBlockingHook,
+} from './skill-blocker.js';
 
 export class CopilotSDKHarness implements Harness {
   readonly name = 'copilot-sdk';
 
   async run(options: {
     skillPath?: string;
+    blockedSkillPath?: string;
     prompt: string;
     files?: string[];
     outputDir: string;
@@ -33,6 +38,15 @@ export class CopilotSDKHarness implements Harness {
       sessionConfig.skillDirectories = [path.dirname(options.skillPath)];
     }
 
+    // Skill blocking: replace approveAll with blocking handler + hook
+    if (options.blockedSkillPath) {
+      const blockedDir = path.dirname(options.blockedSkillPath);
+      sessionConfig.onPermissionRequest = createSkillBlockingPermissionHandler(blockedDir);
+      sessionConfig.hooks = {
+        onPreToolUse: createSkillBlockingHook(blockedDir),
+      };
+    }
+
     const session = await client.createSession(sessionConfig);
 
     try {

package/src/adapters/harness/skill-blocker.ts

@@ -0,0 +1,61 @@
+import * as fs from 'node:fs';
+import * as path from 'node:path';
+import type { PermissionHandler } from '@github/copilot-sdk';
+
+// PreToolUseHandler is not re-exported from @github/copilot-sdk's public API,
+// so we define the minimal type inline matching the SDK's internal definition.
+type PreToolUseHookInput = { toolName: string; toolArgs: unknown; timestamp: number; cwd: string };
+type PreToolUseHookOutput = { permissionDecision?: 'allow' | 'deny' | 'ask'; permissionDecisionReason?: string };
+type PreToolUseHandler = (input: PreToolUseHookInput, invocation: { sessionId: string }) => PreToolUseHookOutput | void;
+
+function resolveDir(dir: string): string {
+  try {
+    return fs.realpathSync(dir) + path.sep;
+  } catch {
+    return path.resolve(dir) + path.sep;
+  }
+}
+
+export function createSkillBlockingHook(
+  blockedDir: string,
+): PreToolUseHandler {
+  const resolved = resolveDir(blockedDir);
+  const raw = path.resolve(blockedDir) + path.sep;
+
+  return (input) => {
+    const argsStr = typeof input.toolArgs === 'string'
+      ? input.toolArgs
+      : JSON.stringify(input.toolArgs ?? '');
+
+    if (argsStr.includes(resolved) || argsStr.includes(raw)) {
+      return {
+        permissionDecision: 'deny' as const,
+        permissionDecisionReason: `Blocked: tool "${input.toolName}" references blocked skill directory`,
+      };
+    }
+    return {};
+  };
+}
+
+export function createSkillBlockingPermissionHandler(
+  blockedDir: string,
+): PermissionHandler {
+  const resolved = resolveDir(blockedDir);
+
+  return (request) => {
+    if (request.kind === 'read') {
+      const readPath = (request as { path?: unknown }).path;
+      if (typeof readPath === 'string') {
+        try {
+          const resolvedPath = fs.realpathSync(readPath);
+          if (resolvedPath === resolved.slice(0, -1) || resolvedPath.startsWith(resolved)) {
+            return { kind: 'denied-by-rules', rules: [{ reason: 'skill-access-blocked' }] };
+          }
+        } catch {
+          // Path doesn't exist — can't be the skill dir
+        }
+      }
+    }
+    return { kind: 'approved' };
+  };
+}

package/src/engine/runner.ts

@@ -43,6 +43,7 @@ export async function runEval(
     }),
     harness.run({
       skillPath: oldSkillPath,
+      blockedSkillPath: skillPath,
       prompt: evalCase.prompt,
       files: evalCase.files,
       outputDir: path.join(baselineDir, 'outputs'),

package/src/types.ts

@@ -15,7 +15,7 @@ export interface Harness {
     prompt: string;
     files?: string[];
     outputDir: string;
-  }): Promise<HarnessRunResult>;
+  } & Record<string, unknown>): Promise<HarnessRunResult>;
   isAvailable(): Promise<boolean>;
 }