snapeval 1.3.1 → 1.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bin/snapeval.ts +43 -1
- package/dist/bin/snapeval.js +37 -1
- package/dist/bin/snapeval.js.map +1 -1
- package/dist/src/adapters/copilot-sdk-client.d.ts +13 -0
- package/dist/src/adapters/copilot-sdk-client.js +59 -0
- package/dist/src/adapters/copilot-sdk-client.js.map +1 -0
- package/dist/src/adapters/inference/copilot-sdk.d.ts +9 -0
- package/dist/src/adapters/inference/copilot-sdk.js +41 -0
- package/dist/src/adapters/inference/copilot-sdk.js.map +1 -0
- package/dist/src/adapters/inference/resolve.js +10 -1
- package/dist/src/adapters/inference/resolve.js.map +1 -1
- package/dist/src/adapters/skill/copilot-sdk.d.ts +6 -0
- package/dist/src/adapters/skill/copilot-sdk.js +68 -0
- package/dist/src/adapters/skill/copilot-sdk.js.map +1 -0
- package/dist/src/commands/review.d.ts +7 -0
- package/dist/src/commands/review.js +34 -0
- package/dist/src/commands/review.js.map +1 -0
- package/package.json +10 -1
- package/plugin.json +1 -1
- package/skills/snapeval/SKILL.md +94 -18
- package/src/adapters/copilot-sdk-client.ts +66 -0
- package/src/adapters/inference/copilot-sdk.ts +48 -0
- package/src/adapters/inference/resolve.ts +14 -1
- package/src/adapters/skill/copilot-sdk.ts +72 -0
- package/src/commands/review.ts +48 -0
package/bin/snapeval.ts
CHANGED
|
@@ -3,6 +3,7 @@ import { Command } from 'commander';
|
|
|
3
3
|
import { resolveConfig } from '../src/config.js';
|
|
4
4
|
import { resolveInference } from '../src/adapters/inference/resolve.js';
|
|
5
5
|
import { CopilotCLIAdapter } from '../src/adapters/skill/copilot-cli.js';
|
|
6
|
+
import { CopilotSDKAdapter } from '../src/adapters/skill/copilot-sdk.js';
|
|
6
7
|
import { TerminalReporter } from '../src/adapters/report/terminal.js';
|
|
7
8
|
import { initCommand } from '../src/commands/init.js';
|
|
8
9
|
import { captureCommand } from '../src/commands/capture.js';
|
|
@@ -10,6 +11,7 @@ import { checkCommand } from '../src/commands/check.js';
|
|
|
10
11
|
import { approveCommand, approveFromResults } from '../src/commands/approve.js';
|
|
11
12
|
import { reportCommand } from '../src/commands/report.js';
|
|
12
13
|
import { ideateCommand } from '../src/commands/ideate.js';
|
|
14
|
+
import { reviewCommand } from '../src/commands/review.js';
|
|
13
15
|
import { SnapevalError } from '../src/errors.js';
|
|
14
16
|
import * as path from 'node:path';
|
|
15
17
|
|
|
@@ -194,6 +196,43 @@ program
|
|
|
194
196
|
}
|
|
195
197
|
});
|
|
196
198
|
|
|
199
|
+
// --- review ---
|
|
200
|
+
program
|
|
201
|
+
.command('review')
|
|
202
|
+
.description('Run checks, generate HTML report, and open in browser')
|
|
203
|
+
.option('--adapter <adapter>', 'Skill adapter to use', 'copilot-cli')
|
|
204
|
+
.option('--inference <inference>', 'Inference adapter to use', 'auto')
|
|
205
|
+
.option('--budget <amount>', 'Spend cap in USD (or "unlimited")', 'unlimited')
|
|
206
|
+
.option('--verbose', 'Verbose output')
|
|
207
|
+
.argument('[skill-dir]', 'Path to skill directory', process.cwd())
|
|
208
|
+
.action(async (skillDir: string, opts: Record<string, string | boolean>) => {
|
|
209
|
+
try {
|
|
210
|
+
const skillPath = path.resolve(skillDir);
|
|
211
|
+
const config = resolveConfig(
|
|
212
|
+
{
|
|
213
|
+
adapter: opts.adapter as string,
|
|
214
|
+
inference: opts.inference as string,
|
|
215
|
+
budget: opts.budget as string,
|
|
216
|
+
},
|
|
217
|
+
process.cwd(),
|
|
218
|
+
skillPath
|
|
219
|
+
);
|
|
220
|
+
const skillAdapter = resolveSkillAdapter(config.adapter);
|
|
221
|
+
const inference = resolveInference(config.inference);
|
|
222
|
+
|
|
223
|
+
const { hasRegressions } = await reviewCommand(skillPath, skillAdapter, inference, {
|
|
224
|
+
budget: config.budget,
|
|
225
|
+
});
|
|
226
|
+
|
|
227
|
+
if (hasRegressions) {
|
|
228
|
+
process.exit(1);
|
|
229
|
+
}
|
|
230
|
+
process.exit(0);
|
|
231
|
+
} catch (err) {
|
|
232
|
+
handleError(err);
|
|
233
|
+
}
|
|
234
|
+
});
|
|
235
|
+
|
|
197
236
|
// --- ideate ---
|
|
198
237
|
program
|
|
199
238
|
.command('ideate')
|
|
@@ -216,8 +255,11 @@ function resolveSkillAdapter(adapterName: string) {
|
|
|
216
255
|
if (adapterName === 'copilot-cli') {
|
|
217
256
|
return new CopilotCLIAdapter();
|
|
218
257
|
}
|
|
258
|
+
if (adapterName === 'copilot-sdk') {
|
|
259
|
+
return new CopilotSDKAdapter();
|
|
260
|
+
}
|
|
219
261
|
throw new SnapevalError(
|
|
220
|
-
`Unknown skill adapter "${adapterName}". Valid options: copilot-cli.`
|
|
262
|
+
`Unknown skill adapter "${adapterName}". Valid options: copilot-cli, copilot-sdk.`
|
|
221
263
|
);
|
|
222
264
|
}
|
|
223
265
|
|
package/dist/bin/snapeval.js
CHANGED
|
@@ -3,6 +3,7 @@ import { Command } from 'commander';
|
|
|
3
3
|
import { resolveConfig } from '../src/config.js';
|
|
4
4
|
import { resolveInference } from '../src/adapters/inference/resolve.js';
|
|
5
5
|
import { CopilotCLIAdapter } from '../src/adapters/skill/copilot-cli.js';
|
|
6
|
+
import { CopilotSDKAdapter } from '../src/adapters/skill/copilot-sdk.js';
|
|
6
7
|
import { TerminalReporter } from '../src/adapters/report/terminal.js';
|
|
7
8
|
import { initCommand } from '../src/commands/init.js';
|
|
8
9
|
import { captureCommand } from '../src/commands/capture.js';
|
|
@@ -10,6 +11,7 @@ import { checkCommand } from '../src/commands/check.js';
|
|
|
10
11
|
import { approveCommand } from '../src/commands/approve.js';
|
|
11
12
|
import { reportCommand } from '../src/commands/report.js';
|
|
12
13
|
import { ideateCommand } from '../src/commands/ideate.js';
|
|
14
|
+
import { reviewCommand } from '../src/commands/review.js';
|
|
13
15
|
import { SnapevalError } from '../src/errors.js';
|
|
14
16
|
import * as path from 'node:path';
|
|
15
17
|
const program = new Command();
|
|
@@ -163,6 +165,37 @@ program
|
|
|
163
165
|
handleError(err);
|
|
164
166
|
}
|
|
165
167
|
});
|
|
168
|
+
// --- review ---
|
|
169
|
+
program
|
|
170
|
+
.command('review')
|
|
171
|
+
.description('Run checks, generate HTML report, and open in browser')
|
|
172
|
+
.option('--adapter <adapter>', 'Skill adapter to use', 'copilot-cli')
|
|
173
|
+
.option('--inference <inference>', 'Inference adapter to use', 'auto')
|
|
174
|
+
.option('--budget <amount>', 'Spend cap in USD (or "unlimited")', 'unlimited')
|
|
175
|
+
.option('--verbose', 'Verbose output')
|
|
176
|
+
.argument('[skill-dir]', 'Path to skill directory', process.cwd())
|
|
177
|
+
.action(async (skillDir, opts) => {
|
|
178
|
+
try {
|
|
179
|
+
const skillPath = path.resolve(skillDir);
|
|
180
|
+
const config = resolveConfig({
|
|
181
|
+
adapter: opts.adapter,
|
|
182
|
+
inference: opts.inference,
|
|
183
|
+
budget: opts.budget,
|
|
184
|
+
}, process.cwd(), skillPath);
|
|
185
|
+
const skillAdapter = resolveSkillAdapter(config.adapter);
|
|
186
|
+
const inference = resolveInference(config.inference);
|
|
187
|
+
const { hasRegressions } = await reviewCommand(skillPath, skillAdapter, inference, {
|
|
188
|
+
budget: config.budget,
|
|
189
|
+
});
|
|
190
|
+
if (hasRegressions) {
|
|
191
|
+
process.exit(1);
|
|
192
|
+
}
|
|
193
|
+
process.exit(0);
|
|
194
|
+
}
|
|
195
|
+
catch (err) {
|
|
196
|
+
handleError(err);
|
|
197
|
+
}
|
|
198
|
+
});
|
|
166
199
|
// --- ideate ---
|
|
167
200
|
program
|
|
168
201
|
.command('ideate')
|
|
@@ -184,7 +217,10 @@ function resolveSkillAdapter(adapterName) {
|
|
|
184
217
|
if (adapterName === 'copilot-cli') {
|
|
185
218
|
return new CopilotCLIAdapter();
|
|
186
219
|
}
|
|
187
|
-
|
|
220
|
+
if (adapterName === 'copilot-sdk') {
|
|
221
|
+
return new CopilotSDKAdapter();
|
|
222
|
+
}
|
|
223
|
+
throw new SnapevalError(`Unknown skill adapter "${adapterName}". Valid options: copilot-cli, copilot-sdk.`);
|
|
188
224
|
}
|
|
189
225
|
function handleError(err) {
|
|
190
226
|
if (err instanceof SnapevalError) {
|
package/dist/bin/snapeval.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"snapeval.js","sourceRoot":"","sources":["../../bin/snapeval.ts"],"names":[],"mappings":";AACA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AACpC,OAAO,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAC;AACjD,OAAO,EAAE,gBAAgB,EAAE,MAAM,sCAAsC,CAAC;AACxE,OAAO,EAAE,iBAAiB,EAAE,MAAM,sCAAsC,CAAC;AACzE,OAAO,EAAE,gBAAgB,EAAE,MAAM,oCAAoC,CAAC;AACtE,OAAO,EAAE,WAAW,EAAE,MAAM,yBAAyB,CAAC;AACtD,OAAO,EAAE,cAAc,EAAE,MAAM,4BAA4B,CAAC;AAC5D,OAAO,EAAE,YAAY,EAAE,MAAM,0BAA0B,CAAC;AACxD,OAAO,EAAE,cAAc,EAAsB,MAAM,4BAA4B,CAAC;AAChF,OAAO,EAAE,aAAa,EAAE,MAAM,2BAA2B,CAAC;AAC1D,OAAO,EAAE,aAAa,EAAE,MAAM,2BAA2B,CAAC;AAC1D,OAAO,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAC;AACjD,OAAO,KAAK,IAAI,MAAM,WAAW,CAAC;AAElC,MAAM,OAAO,GAAG,IAAI,OAAO,EAAE,CAAC;AAE9B,OAAO;KACJ,IAAI,CAAC,UAAU,CAAC;KAChB,WAAW,CAAC,yCAAyC,CAAC;KACtD,OAAO,CAAC,OAAO,CAAC,CAAC;AAEpB,eAAe;AACf,OAAO;KACJ,OAAO,CAAC,MAAM,CAAC;KACf,WAAW,CAAC,4CAA4C,CAAC;KACzD,MAAM,CAAC,qBAAqB,EAAE,sBAAsB,EAAE,aAAa,CAAC;KACpE,MAAM,CAAC,yBAAyB,EAAE,0BAA0B,EAAE,MAAM,CAAC;KACrE,MAAM,CAAC,WAAW,EAAE,gBAAgB,CAAC;KACrC,QAAQ,CAAC,aAAa,EAAE,yBAAyB,EAAE,OAAO,CAAC,GAAG,EAAE,CAAC;KACjE,MAAM,CAAC,KAAK,EAAE,QAAgB,EAAE,IAAsC,EAAE,EAAE;IACzE,IAAI,CAAC;QACH,MAAM,SAAS,GAAG,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;QACzC,MAAM,MAAM,GAAG,aAAa,CAC1B,EAAE,OAAO,EAAE,IAAI,CAAC,OAAiB,EAAE,SAAS,EAAE,IAAI,CAAC,SAAmB,EAAE,EACxE,OAAO,CAAC,GAAG,EAAE,EACb,SAAS,CACV,CAAC;QACF,MAAM,SAAS,GAAG,gBAAgB,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;QACrD,MAAM,WAAW,CAAC,SAAS,EAAE,SAAS,CAAC,CAAC;QACxC,OAAO,CAAC,GAAG,CAAC,sBAAsB,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,OAAO,EAAE,YAAY,CAAC,EAAE,CAAC,CAAC;QACjF,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,WAAW,CAAC,GAAG,CAAC,CAAC;IACnB,CAAC;AACH,CAAC,CAAC,CAAC;AAEL,kBAAkB;AAClB,OAAO;KACJ,OAAO,CAAC,SAAS,CAAC;KAClB,WAAW,CAAC,6DAA6D,CAAC;KAC1E,MAAM,CAAC,qBAAqB,EAAE,sBAAsB,EAAE,aAAa,CAAC;KACpE,MAAM,CAAC,yBAAyB,EAAE,0BAA0B,EAAE,MAAM,CAAC;KACrE,MAAM,CAAC,YAAY,EAAE,6BAA6B,EAAE,GAAG,CAAC;KACxD,MAAM,CAAC,WAAW,EAAE,gBAAgB,CAAC;KACrC,QAAQ,CAAC,aAAa,EAAE,yBAAyB,EAAE,OAAO,CAAC,GAAG,EAAE,CAAC;KACjE,MAAM,CAAC,KAAK,EAAE,QAAgB,EAAE,IAAsC,EAAE,EAAE;IACzE,IAAI,CAAC;QACH,MAAM,SAAS,GAAG,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;QACzC,MAAM,MAAM,GAAG,aAAa,CAC1B;YACE,OAAO,EAAE,IAAI,CAAC,OAAiB;YAC/B,SAAS,EAAE,IAAI,CAAC,SAAmB;YACnC,IAAI,EAAE,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAc,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,SAAS;SAChE,EACD,OAAO,CAAC,GAAG,EAAE,EACb,SAAS,CACV,CAAC;QACF,MAAM,YAAY,GAAG,mBAAmB,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;QACzD,MAAM,cAAc,CAAC,SAAS,EAAE,YAAY,EAAE,EAAE,IAAI,EAAE,MAAM,CAAC,IAAI,EAAE,CAAC,CAAC;QACrE,OAAO,CAAC,GAAG,CAAC,yBAAyB,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,OAAO,EAAE,WAAW,CAAC,EAAE,CAAC,CAAC;QACnF,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,WAAW,CAAC,GAAG,CAAC,CAAC;IACnB,CAAC;AACH,CAAC,CAAC,CAAC;AAEL,gBAAgB;AAChB,OAAO;KACJ,OAAO,CAAC,OAAO,CAAC;KAChB,WAAW,CAAC,gDAAgD,CAAC;KAC7D,MAAM,CAAC,qBAAqB,EAAE,sBAAsB,EAAE,aAAa,CAAC;KACpE,MAAM,CAAC,yBAAyB,EAAE,0BAA0B,EAAE,MAAM,CAAC;KACrE,MAAM,CAAC,mBAAmB,EAAE,mCAAmC,EAAE,WAAW,CAAC;KAC7E,MAAM,CAAC,MAAM,EAAE,wDAAwD,CAAC;KACxE,MAAM,CAAC,WAAW,EAAE,gBAAgB,CAAC;KACrC,MAAM,CAAC,kBAAkB,EAAE,uCAAuC,CAAC;KACnE,QAAQ,CAAC,aAAa,EAAE,yBAAyB,EAAE,OAAO,CAAC,GAAG,EAAE,CAAC;KACjE,MAAM,CAAC,KAAK,EAAE,QAAgB,EAAE,IAAsC,EAAE,EAAE;IACzE,IAAI,CAAC;QACH,MAAM,SAAS,GAAG,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;QACzC,MAAM,MAAM,GAAG,aAAa,CAC1B;YACE,OAAO,EAAE,IAAI,CAAC,OAAiB;YAC/B,SAAS,EAAE,IAAI,CAAC,SAAmB;YACnC,MAAM,EAAE,IAAI,CAAC,MAAgB;SAC9B,EACD,OAAO,CAAC,GAAG,EAAE,EACb,SAAS,CACV,CAAC;QACF,MAAM,YAAY,GAAG,mBAAmB,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;QACzD,MAAM,SAAS,GAAG,gBAAgB,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;QAErD,MAAM,OAAO,GAAG,MAAM,YAAY,CAAC,SAAS,EAAE,YAAY,EAAE,SAAS,EAAE;YACrE,MAAM,EAAE,MAAM,CAAC,MAAM;SACtB,CAAC,CAAC;QAEH,+BAA+B;QAC/B,MAAM,QAAQ,GAAG,IAAI,gBAAgB,EAAE,CAAC;QACxC,MAAM,QAAQ,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;QAE/B,MAAM,cAAc,GAAG,OAAO,CAAC,OAAO,CAAC,SAAS,GAAG,CAAC,CAAC;QACrD,IAAI,cAAc,EAAE,CAAC;YACnB,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAClB,CAAC;QACD,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,WAAW,CAAC,GAAG,CAAC,CAAC;IACnB,CAAC;AACH,CAAC,CAAC,CAAC;AAEL,kBAAkB;AAClB,OAAO;KACJ,OAAO,CAAC,SAAS,CAAC;KAClB,WAAW,CAAC,8CAA8C,CAAC;KAC3D,MAAM,CAAC,qBAAqB,EAAE,sBAAsB,EAAE,aAAa,CAAC;KACpE,MAAM,CAAC,yBAAyB,EAAE,0BAA0B,EAAE,MAAM,CAAC;KACrE,MAAM,CAAC,kBAAkB,EAAE,wDAAwD,CAAC;KACpF,MAAM,CAAC,WAAW,EAAE,gBAAgB,CAAC;KACrC,QAAQ,CAAC,aAAa,EAAE,yBAAyB,EAAE,OAAO,CAAC,GAAG,EAAE,CAAC;KACjE,MAAM,CAAC,KAAK,EAAE,QAAgB,EAAE,IAAsC,EAAE,EAAE;IACzE,IAAI,CAAC;QACH,MAAM,SAAS,GAAG,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;QACzC,MAAM,MAAM,GAAG,aAAa,CAC1B,EAAE,OAAO,EAAE,IAAI,CAAC,OAAiB,EAAE,SAAS,EAAE,IAAI,CAAC,SAAmB,EAAE,EACxE,OAAO,CAAC,GAAG,EAAE,EACb,SAAS,CACV,CAAC;QACF,MAAM,YAAY,GAAG,mBAAmB,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;QAEzD,MAAM,WAAW,GAAG,IAAI,CAAC,QAAQ;YAC/B,CAAC,CAAE,IAAI,CAAC,QAAmB,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,EAAE,EAAE,EAAE,CAAC,CAAC;YACzE,CAAC,CAAC,SAAS,CAAC;QAEd,MAAM,cAAc,CAAC,SAAS,EAAE,YAAY,EAAE,EAAE,WAAW,EAAE,CAAC,CAAC;QAC/D,OAAO,CAAC,GAAG,CAAC,6BAA6B,CAAC,CAAC;QAC3C,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,WAAW,CAAC,GAAG,CAAC,CAAC;IACnB,CAAC;AACH,CAAC,CAAC,CAAC;AAEL,iBAAiB;AACjB,OAAO;KACJ,OAAO,CAAC,QAAQ,CAAC;KACjB,WAAW,CAAC,0DAA0D,CAAC;KACvE,MAAM,CAAC,qBAAqB,EAAE,sBAAsB,EAAE,aAAa,CAAC;KACpE,MAAM,CAAC,yBAAyB,EAAE,0BAA0B,EAAE,MAAM,CAAC;KACrE,MAAM,CAAC,mBAAmB,EAAE,mCAAmC,EAAE,WAAW,CAAC;KAC7E,MAAM,CAAC,WAAW,EAAE,gBAAgB,CAAC;KACrC,MAAM,CAAC,QAAQ,EAAE,6BAA6B,CAAC;KAC/C,QAAQ,CAAC,aAAa,EAAE,yBAAyB,EAAE,OAAO,CAAC,GAAG,EAAE,CAAC;KACjE,MAAM,CAAC,KAAK,EAAE,QAAgB,EAAE,IAAsC,EAAE,EAAE;IACzE,IAAI,CAAC;QACH,MAAM,SAAS,GAAG,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;QACzC,MAAM,MAAM,GAAG,aAAa,CAC1B;YACE,OAAO,EAAE,IAAI,CAAC,OAAiB;YAC/B,SAAS,EAAE,IAAI,CAAC,SAAmB;YACnC,MAAM,EAAE,IAAI,CAAC,MAAgB;SAC9B,EACD,OAAO,CAAC,GAAG,EAAE,EACb,SAAS,CACV,CAAC;QACF,MAAM,YAAY,GAAG,mBAAmB,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;QACzD,MAAM,SAAS,GAAG,gBAAgB,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;QAErD,MAAM,OAAO,GAAG,MAAM,YAAY,CAAC,SAAS,EAAE,YAAY,EAAE,SAAS,EAAE;YACrE,MAAM,EAAE,MAAM,CAAC,MAAM;SACtB,CAAC,CAAC;QAEH,MAAM,aAAa,CAAC,SAAS,EAAE,OAAO,EAAE;YACtC,OAAO,EAAE,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC;YAC9B,IAAI,EAAE,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC;SACzB,CAAC,CAAC;QAEH,MAAM,cAAc,GAAG,OAAO,CAAC,OAAO,CAAC,SAAS,GAAG,CAAC,CAAC;QACrD,IAAI,cAAc,EAAE,CAAC;YACnB,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAClB,CAAC;QACD,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,WAAW,CAAC,GAAG,CAAC,CAAC;IACnB,CAAC;AACH,CAAC,CAAC,CAAC;AAEL,iBAAiB;AACjB,OAAO;KACJ,OAAO,CAAC,QAAQ,CAAC;KACjB,WAAW,CAAC,+CAA+C,CAAC;KAC5D,QAAQ,CAAC,aAAa,EAAE,yBAAyB,EAAE,OAAO,CAAC,GAAG,EAAE,CAAC;KACjE,MAAM,CAAC,KAAK,EAAE,QAAgB,EAAE,EAAE;IACjC,IAAI,CAAC;QACH,MAAM,SAAS,GAAG,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;QACzC,MAAM,UAAU,GAAG,MAAM,aAAa,CAAC,SAAS,CAAC,CAAC;QAClD,OAAO,CAAC,GAAG,CAAC,2BAA2B,UAAU,EAAE,CAAC,CAAC;QACrD,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,WAAW,CAAC,GAAG,CAAC,CAAC;IACnB,CAAC;AACH,CAAC,CAAC,CAAC;AAEL,kBAAkB;AAElB,SAAS,mBAAmB,CAAC,WAAmB;IAC9C,IAAI,WAAW,KAAK,aAAa,EAAE,CAAC;QAClC,OAAO,IAAI,iBAAiB,EAAE,CAAC;IACjC,CAAC;IACD,MAAM,IAAI,aAAa,CACrB,0BAA0B,WAAW,
|
|
1
|
+
{"version":3,"file":"snapeval.js","sourceRoot":"","sources":["../../bin/snapeval.ts"],"names":[],"mappings":";AACA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AACpC,OAAO,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAC;AACjD,OAAO,EAAE,gBAAgB,EAAE,MAAM,sCAAsC,CAAC;AACxE,OAAO,EAAE,iBAAiB,EAAE,MAAM,sCAAsC,CAAC;AACzE,OAAO,EAAE,iBAAiB,EAAE,MAAM,sCAAsC,CAAC;AACzE,OAAO,EAAE,gBAAgB,EAAE,MAAM,oCAAoC,CAAC;AACtE,OAAO,EAAE,WAAW,EAAE,MAAM,yBAAyB,CAAC;AACtD,OAAO,EAAE,cAAc,EAAE,MAAM,4BAA4B,CAAC;AAC5D,OAAO,EAAE,YAAY,EAAE,MAAM,0BAA0B,CAAC;AACxD,OAAO,EAAE,cAAc,EAAsB,MAAM,4BAA4B,CAAC;AAChF,OAAO,EAAE,aAAa,EAAE,MAAM,2BAA2B,CAAC;AAC1D,OAAO,EAAE,aAAa,EAAE,MAAM,2BAA2B,CAAC;AAC1D,OAAO,EAAE,aAAa,EAAE,MAAM,2BAA2B,CAAC;AAC1D,OAAO,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAC;AACjD,OAAO,KAAK,IAAI,MAAM,WAAW,CAAC;AAElC,MAAM,OAAO,GAAG,IAAI,OAAO,EAAE,CAAC;AAE9B,OAAO;KACJ,IAAI,CAAC,UAAU,CAAC;KAChB,WAAW,CAAC,yCAAyC,CAAC;KACtD,OAAO,CAAC,OAAO,CAAC,CAAC;AAEpB,eAAe;AACf,OAAO;KACJ,OAAO,CAAC,MAAM,CAAC;KACf,WAAW,CAAC,4CAA4C,CAAC;KACzD,MAAM,CAAC,qBAAqB,EAAE,sBAAsB,EAAE,aAAa,CAAC;KACpE,MAAM,CAAC,yBAAyB,EAAE,0BAA0B,EAAE,MAAM,CAAC;KACrE,MAAM,CAAC,WAAW,EAAE,gBAAgB,CAAC;KACrC,QAAQ,CAAC,aAAa,EAAE,yBAAyB,EAAE,OAAO,CAAC,GAAG,EAAE,CAAC;KACjE,MAAM,CAAC,KAAK,EAAE,QAAgB,EAAE,IAAsC,EAAE,EAAE;IACzE,IAAI,CAAC;QACH,MAAM,SAAS,GAAG,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;QACzC,MAAM,MAAM,GAAG,aAAa,CAC1B,EAAE,OAAO,EAAE,IAAI,CAAC,OAAiB,EAAE,SAAS,EAAE,IAAI,CAAC,SAAmB,EAAE,EACxE,OAAO,CAAC,GAAG,EAAE,EACb,SAAS,CACV,CAAC;QACF,MAAM,SAAS,GAAG,gBAAgB,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;QACrD,MAAM,WAAW,CAAC,SAAS,EAAE,SAAS,CAAC,CAAC;QACxC,OAAO,CAAC,GAAG,CAAC,sBAAsB,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,OAAO,EAAE,YAAY,CAAC,EAAE,CAAC,CAAC;QACjF,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,WAAW,CAAC,GAAG,CAAC,CAAC;IACnB,CAAC;AACH,CAAC,CAAC,CAAC;AAEL,kBAAkB;AAClB,OAAO;KACJ,OAAO,CAAC,SAAS,CAAC;KAClB,WAAW,CAAC,6DAA6D,CAAC;KAC1E,MAAM,CAAC,qBAAqB,EAAE,sBAAsB,EAAE,aAAa,CAAC;KACpE,MAAM,CAAC,yBAAyB,EAAE,0BAA0B,EAAE,MAAM,CAAC;KACrE,MAAM,CAAC,YAAY,EAAE,6BAA6B,EAAE,GAAG,CAAC;KACxD,MAAM,CAAC,WAAW,EAAE,gBAAgB,CAAC;KACrC,QAAQ,CAAC,aAAa,EAAE,yBAAyB,EAAE,OAAO,CAAC,GAAG,EAAE,CAAC;KACjE,MAAM,CAAC,KAAK,EAAE,QAAgB,EAAE,IAAsC,EAAE,EAAE;IACzE,IAAI,CAAC;QACH,MAAM,SAAS,GAAG,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;QACzC,MAAM,MAAM,GAAG,aAAa,CAC1B;YACE,OAAO,EAAE,IAAI,CAAC,OAAiB;YAC/B,SAAS,EAAE,IAAI,CAAC,SAAmB;YACnC,IAAI,EAAE,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAc,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,SAAS;SAChE,EACD,OAAO,CAAC,GAAG,EAAE,EACb,SAAS,CACV,CAAC;QACF,MAAM,YAAY,GAAG,mBAAmB,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;QACzD,MAAM,cAAc,CAAC,SAAS,EAAE,YAAY,EAAE,EAAE,IAAI,EAAE,MAAM,CAAC,IAAI,EAAE,CAAC,CAAC;QACrE,OAAO,CAAC,GAAG,CAAC,yBAAyB,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,OAAO,EAAE,WAAW,CAAC,EAAE,CAAC,CAAC;QACnF,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,WAAW,CAAC,GAAG,CAAC,CAAC;IACnB,CAAC;AACH,CAAC,CAAC,CAAC;AAEL,gBAAgB;AAChB,OAAO;KACJ,OAAO,CAAC,OAAO,CAAC;KAChB,WAAW,CAAC,gDAAgD,CAAC;KAC7D,MAAM,CAAC,qBAAqB,EAAE,sBAAsB,EAAE,aAAa,CAAC;KACpE,MAAM,CAAC,yBAAyB,EAAE,0BAA0B,EAAE,MAAM,CAAC;KACrE,MAAM,CAAC,mBAAmB,EAAE,mCAAmC,EAAE,WAAW,CAAC;KAC7E,MAAM,CAAC,MAAM,EAAE,wDAAwD,CAAC;KACxE,MAAM,CAAC,WAAW,EAAE,gBAAgB,CAAC;KACrC,MAAM,CAAC,kBAAkB,EAAE,uCAAuC,CAAC;KACnE,QAAQ,CAAC,aAAa,EAAE,yBAAyB,EAAE,OAAO,CAAC,GAAG,EAAE,CAAC;KACjE,MAAM,CAAC,KAAK,EAAE,QAAgB,EAAE,IAAsC,EAAE,EAAE;IACzE,IAAI,CAAC;QACH,MAAM,SAAS,GAAG,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;QACzC,MAAM,MAAM,GAAG,aAAa,CAC1B;YACE,OAAO,EAAE,IAAI,CAAC,OAAiB;YAC/B,SAAS,EAAE,IAAI,CAAC,SAAmB;YACnC,MAAM,EAAE,IAAI,CAAC,MAAgB;SAC9B,EACD,OAAO,CAAC,GAAG,EAAE,EACb,SAAS,CACV,CAAC;QACF,MAAM,YAAY,GAAG,mBAAmB,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;QACzD,MAAM,SAAS,GAAG,gBAAgB,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;QAErD,MAAM,OAAO,GAAG,MAAM,YAAY,CAAC,SAAS,EAAE,YAAY,EAAE,SAAS,EAAE;YACrE,MAAM,EAAE,MAAM,CAAC,MAAM;SACtB,CAAC,CAAC;QAEH,+BAA+B;QAC/B,MAAM,QAAQ,GAAG,IAAI,gBAAgB,EAAE,CAAC;QACxC,MAAM,QAAQ,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;QAE/B,MAAM,cAAc,GAAG,OAAO,CAAC,OAAO,CAAC,SAAS,GAAG,CAAC,CAAC;QACrD,IAAI,cAAc,EAAE,CAAC;YACnB,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAClB,CAAC;QACD,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,WAAW,CAAC,GAAG,CAAC,CAAC;IACnB,CAAC;AACH,CAAC,CAAC,CAAC;AAEL,kBAAkB;AAClB,OAAO;KACJ,OAAO,CAAC,SAAS,CAAC;KAClB,WAAW,CAAC,8CAA8C,CAAC;KAC3D,MAAM,CAAC,qBAAqB,EAAE,sBAAsB,EAAE,aAAa,CAAC;KACpE,MAAM,CAAC,yBAAyB,EAAE,0BAA0B,EAAE,MAAM,CAAC;KACrE,MAAM,CAAC,kBAAkB,EAAE,wDAAwD,CAAC;KACpF,MAAM,CAAC,WAAW,EAAE,gBAAgB,CAAC;KACrC,QAAQ,CAAC,aAAa,EAAE,yBAAyB,EAAE,OAAO,CAAC,GAAG,EAAE,CAAC;KACjE,MAAM,CAAC,KAAK,EAAE,QAAgB,EAAE,IAAsC,EAAE,EAAE;IACzE,IAAI,CAAC;QACH,MAAM,SAAS,GAAG,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;QACzC,MAAM,MAAM,GAAG,aAAa,CAC1B,EAAE,OAAO,EAAE,IAAI,CAAC,OAAiB,EAAE,SAAS,EAAE,IAAI,CAAC,SAAmB,EAAE,EACxE,OAAO,CAAC,GAAG,EAAE,EACb,SAAS,CACV,CAAC;QACF,MAAM,YAAY,GAAG,mBAAmB,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;QAEzD,MAAM,WAAW,GAAG,IAAI,CAAC,QAAQ;YAC/B,CAAC,CAAE,IAAI,CAAC,QAAmB,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,EAAE,EAAE,EAAE,CAAC,CAAC;YACzE,CAAC,CAAC,SAAS,CAAC;QAEd,MAAM,cAAc,CAAC,SAAS,EAAE,YAAY,EAAE,EAAE,WAAW,EAAE,CAAC,CAAC;QAC/D,OAAO,CAAC,GAAG,CAAC,6BAA6B,CAAC,CAAC;QAC3C,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,WAAW,CAAC,GAAG,CAAC,CAAC;IACnB,CAAC;AACH,CAAC,CAAC,CAAC;AAEL,iBAAiB;AACjB,OAAO;KACJ,OAAO,CAAC,QAAQ,CAAC;KACjB,WAAW,CAAC,0DAA0D,CAAC;KACvE,MAAM,CAAC,qBAAqB,EAAE,sBAAsB,EAAE,aAAa,CAAC;KACpE,MAAM,CAAC,yBAAyB,EAAE,0BAA0B,EAAE,MAAM,CAAC;KACrE,MAAM,CAAC,mBAAmB,EAAE,mCAAmC,EAAE,WAAW,CAAC;KAC7E,MAAM,CAAC,WAAW,EAAE,gBAAgB,CAAC;KACrC,MAAM,CAAC,QAAQ,EAAE,6BAA6B,CAAC;KAC/C,QAAQ,CAAC,aAAa,EAAE,yBAAyB,EAAE,OAAO,CAAC,GAAG,EAAE,CAAC;KACjE,MAAM,CAAC,KAAK,EAAE,QAAgB,EAAE,IAAsC,EAAE,EAAE;IACzE,IAAI,CAAC;QACH,MAAM,SAAS,GAAG,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;QACzC,MAAM,MAAM,GAAG,aAAa,CAC1B;YACE,OAAO,EAAE,IAAI,CAAC,OAAiB;YAC/B,SAAS,EAAE,IAAI,CAAC,SAAmB;YACnC,MAAM,EAAE,IAAI,CAAC,MAAgB;SAC9B,EACD,OAAO,CAAC,GAAG,EAAE,EACb,SAAS,CACV,CAAC;QACF,MAAM,YAAY,GAAG,mBAAmB,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;QACzD,MAAM,SAAS,GAAG,gBAAgB,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;QAErD,MAAM,OAAO,GAAG,MAAM,YAAY,CAAC,SAAS,EAAE,YAAY,EAAE,SAAS,EAAE;YACrE,MAAM,EAAE,MAAM,CAAC,MAAM;SACtB,CAAC,CAAC;QAEH,MAAM,aAAa,CAAC,SAAS,EAAE,OAAO,EAAE;YACtC,OAAO,EAAE,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC;YAC9B,IAAI,EAAE,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC;SACzB,CAAC,CAAC;QAEH,MAAM,cAAc,GAAG,OAAO,CAAC,OAAO,CAAC,SAAS,GAAG,CAAC,CAAC;QACrD,IAAI,cAAc,EAAE,CAAC;YACnB,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAClB,CAAC;QACD,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,WAAW,CAAC,GAAG,CAAC,CAAC;IACnB,CAAC;AACH,CAAC,CAAC,CAAC;AAEL,iBAAiB;AACjB,OAAO;KACJ,OAAO,CAAC,QAAQ,CAAC;KACjB,WAAW,CAAC,uDAAuD,CAAC;KACpE,MAAM,CAAC,qBAAqB,EAAE,sBAAsB,EAAE,aAAa,CAAC;KACpE,MAAM,CAAC,yBAAyB,EAAE,0BAA0B,EAAE,MAAM,CAAC;KACrE,MAAM,CAAC,mBAAmB,EAAE,mCAAmC,EAAE,WAAW,CAAC;KAC7E,MAAM,CAAC,WAAW,EAAE,gBAAgB,CAAC;KACrC,QAAQ,CAAC,aAAa,EAAE,yBAAyB,EAAE,OAAO,CAAC,GAAG,EAAE,CAAC;KACjE,MAAM,CAAC,KAAK,EAAE,QAAgB,EAAE,IAAsC,EAAE,EAAE;IACzE,IAAI,CAAC;QACH,MAAM,SAAS,GAAG,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;QACzC,MAAM,MAAM,GAAG,aAAa,CAC1B;YACE,OAAO,EAAE,IAAI,CAAC,OAAiB;YAC/B,SAAS,EAAE,IAAI,CAAC,SAAmB;YACnC,MAAM,EAAE,IAAI,CAAC,MAAgB;SAC9B,EACD,OAAO,CAAC,GAAG,EAAE,EACb,SAAS,CACV,CAAC;QACF,MAAM,YAAY,GAAG,mBAAmB,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;QACzD,MAAM,SAAS,GAAG,gBAAgB,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;QAErD,MAAM,EAAE,cAAc,EAAE,GAAG,MAAM,aAAa,CAAC,SAAS,EAAE,YAAY,EAAE,SAAS,EAAE;YACjF,MAAM,EAAE,MAAM,CAAC,MAAM;SACtB,CAAC,CAAC;QAEH,IAAI,cAAc,EAAE,CAAC;YACnB,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAClB,CAAC;QACD,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,WAAW,CAAC,GAAG,CAAC,CAAC;IACnB,CAAC;AACH,CAAC,CAAC,CAAC;AAEL,iBAAiB;AACjB,OAAO;KACJ,OAAO,CAAC,QAAQ,CAAC;KACjB,WAAW,CAAC,+CAA+C,CAAC;KAC5D,QAAQ,CAAC,aAAa,EAAE,yBAAyB,EAAE,OAAO,CAAC,GAAG,EAAE,CAAC;KACjE,MAAM,CAAC,KAAK,EAAE,QAAgB,EAAE,EAAE;IACjC,IAAI,CAAC;QACH,MAAM,SAAS,GAAG,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;QACzC,MAAM,UAAU,GAAG,MAAM,aAAa,CAAC,SAAS,CAAC,CAAC;QAClD,OAAO,CAAC,GAAG,CAAC,2BAA2B,UAAU,EAAE,CAAC,CAAC;QACrD,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,WAAW,CAAC,GAAG,CAAC,CAAC;IACnB,CAAC;AACH,CAAC,CAAC,CAAC;AAEL,kBAAkB;AAElB,SAAS,mBAAmB,CAAC,WAAmB;IAC9C,IAAI,WAAW,KAAK,aAAa,EAAE,CAAC;QAClC,OAAO,IAAI,iBAAiB,EAAE,CAAC;IACjC,CAAC;IACD,IAAI,WAAW,KAAK,aAAa,EAAE,CAAC;QAClC,OAAO,IAAI,iBAAiB,EAAE,CAAC;IACjC,CAAC;IACD,MAAM,IAAI,aAAa,CACrB,0BAA0B,WAAW,6CAA6C,CACnF,CAAC;AACJ,CAAC;AAED,SAAS,WAAW,CAAC,GAAY;IAC/B,IAAI,GAAG,YAAY,aAAa,EAAE,CAAC;QACjC,OAAO,CAAC,KAAK,CAAC,UAAU,GAAG,CAAC,OAAO,EAAE,CAAC,CAAC;QACvC,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,QAAQ,IAAI,CAAC,CAAC,CAAC;IAClC,CAAC;IACD,IAAI,GAAG,YAAY,KAAK,EAAE,CAAC;QACzB,OAAO,CAAC,KAAK,CAAC,UAAU,GAAG,CAAC,OAAO,EAAE,CAAC,CAAC;QACvC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;IACD,OAAO,CAAC,KAAK,CAAC,4BAA4B,CAAC,CAAC;IAC5C,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;AAClB,CAAC;AAED,OAAO,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC"}
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Shared lazy CopilotClient singleton.
|
|
3
|
+
*
|
|
4
|
+
* Both CopilotSDKAdapter (SkillAdapter) and CopilotSDKInference
|
|
5
|
+
* (InferenceAdapter) share a single client to avoid spawning
|
|
6
|
+
* multiple CLI server processes.
|
|
7
|
+
*
|
|
8
|
+
* The SDK is dynamically imported so that users who don't install
|
|
9
|
+
* @github/copilot-sdk pay no cost.
|
|
10
|
+
*/
|
|
11
|
+
export declare function getClient(): Promise<any>;
|
|
12
|
+
export declare function stopClient(): Promise<void>;
|
|
13
|
+
export declare function isSDKInstalled(): boolean;
|
|
@@ -0,0 +1,59 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Shared lazy CopilotClient singleton.
|
|
3
|
+
*
|
|
4
|
+
* Both CopilotSDKAdapter (SkillAdapter) and CopilotSDKInference
|
|
5
|
+
* (InferenceAdapter) share a single client to avoid spawning
|
|
6
|
+
* multiple CLI server processes.
|
|
7
|
+
*
|
|
8
|
+
* The SDK is dynamically imported so that users who don't install
|
|
9
|
+
* @github/copilot-sdk pay no cost.
|
|
10
|
+
*/
|
|
11
|
+
import * as fs from 'node:fs';
|
|
12
|
+
import * as path from 'node:path';
|
|
13
|
+
// We store the client as `any` to avoid a hard import dependency
|
|
14
|
+
// on @github/copilot-sdk. The module may not be installed.
|
|
15
|
+
let clientInstance = null;
|
|
16
|
+
let clientStarted = false;
|
|
17
|
+
export async function getClient() {
|
|
18
|
+
if (clientInstance && clientStarted)
|
|
19
|
+
return clientInstance;
|
|
20
|
+
let sdk;
|
|
21
|
+
try {
|
|
22
|
+
// @ts-ignore — module may not be installed (optional peer dep)
|
|
23
|
+
sdk = await import('@github/copilot-sdk');
|
|
24
|
+
}
|
|
25
|
+
catch {
|
|
26
|
+
throw new Error('CopilotSDK adapter requires @github/copilot-sdk. Install it with: npm install @github/copilot-sdk');
|
|
27
|
+
}
|
|
28
|
+
const CopilotClient = sdk.CopilotClient ?? sdk.default?.CopilotClient;
|
|
29
|
+
if (!CopilotClient) {
|
|
30
|
+
throw new Error('Could not find CopilotClient export in @github/copilot-sdk. The package may have changed its API.');
|
|
31
|
+
}
|
|
32
|
+
clientInstance = new CopilotClient();
|
|
33
|
+
await clientInstance.start();
|
|
34
|
+
clientStarted = true;
|
|
35
|
+
return clientInstance;
|
|
36
|
+
}
|
|
37
|
+
export async function stopClient() {
|
|
38
|
+
if (clientInstance && clientStarted) {
|
|
39
|
+
await clientInstance.stop();
|
|
40
|
+
clientStarted = false;
|
|
41
|
+
clientInstance = null;
|
|
42
|
+
}
|
|
43
|
+
}
|
|
44
|
+
export function isSDKInstalled() {
|
|
45
|
+
// Walk up from cwd looking for node_modules/@github/copilot-sdk.
|
|
46
|
+
// This avoids createRequire/import issues across ESM/CJS contexts.
|
|
47
|
+
let dir = process.cwd();
|
|
48
|
+
while (true) {
|
|
49
|
+
const candidate = path.join(dir, 'node_modules', '@github', 'copilot-sdk', 'package.json');
|
|
50
|
+
if (fs.existsSync(candidate))
|
|
51
|
+
return true;
|
|
52
|
+
const parent = path.dirname(dir);
|
|
53
|
+
if (parent === dir)
|
|
54
|
+
break;
|
|
55
|
+
dir = parent;
|
|
56
|
+
}
|
|
57
|
+
return false;
|
|
58
|
+
}
|
|
59
|
+
//# sourceMappingURL=copilot-sdk-client.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"copilot-sdk-client.js","sourceRoot":"","sources":["../../../src/adapters/copilot-sdk-client.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AAEH,OAAO,KAAK,EAAE,MAAM,SAAS,CAAC;AAC9B,OAAO,KAAK,IAAI,MAAM,WAAW,CAAC;AAElC,iEAAiE;AACjE,4DAA4D;AAC5D,IAAI,cAAc,GAAQ,IAAI,CAAC;AAC/B,IAAI,aAAa,GAAG,KAAK,CAAC;AAE1B,MAAM,CAAC,KAAK,UAAU,SAAS;IAC7B,IAAI,cAAc,IAAI,aAAa;QAAE,OAAO,cAAc,CAAC;IAE3D,IAAI,GAAQ,CAAC;IACb,IAAI,CAAC;QACH,+DAA+D;QAC/D,GAAG,GAAG,MAAM,MAAM,CAAC,qBAAqB,CAAC,CAAC;IAC5C,CAAC;IAAC,MAAM,CAAC;QACP,MAAM,IAAI,KAAK,CACb,mGAAmG,CACpG,CAAC;IACJ,CAAC;IAED,MAAM,aAAa,GAAG,GAAG,CAAC,aAAa,IAAI,GAAG,CAAC,OAAO,EAAE,aAAa,CAAC;IACtE,IAAI,CAAC,aAAa,EAAE,CAAC;QACnB,MAAM,IAAI,KAAK,CACb,mGAAmG,CACpG,CAAC;IACJ,CAAC;IAED,cAAc,GAAG,IAAI,aAAa,EAAE,CAAC;IACrC,MAAM,cAAc,CAAC,KAAK,EAAE,CAAC;IAC7B,aAAa,GAAG,IAAI,CAAC;IACrB,OAAO,cAAc,CAAC;AACxB,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,UAAU;IAC9B,IAAI,cAAc,IAAI,aAAa,EAAE,CAAC;QACpC,MAAM,cAAc,CAAC,IAAI,EAAE,CAAC;QAC5B,aAAa,GAAG,KAAK,CAAC;QACtB,cAAc,GAAG,IAAI,CAAC;IACxB,CAAC;AACH,CAAC;AAED,MAAM,UAAU,cAAc;IAC5B,iEAAiE;IACjE,mEAAmE;IACnE,IAAI,GAAG,GAAG,OAAO,CAAC,GAAG,EAAE,CAAC;IACxB,OAAO,IAAI,EAAE,CAAC;QACZ,MAAM,SAAS,GAAG,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,cAAc,EAAE,SAAS,EAAE,aAAa,EAAE,cAAc,CAAC,CAAC;QAC3F,IAAI,EAAE,CAAC,UAAU,CAAC,SAAS,CAAC;YAAE,OAAO,IAAI,CAAC;QAC1C,MAAM,MAAM,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;QACjC,IAAI,MAAM,KAAK,GAAG;YAAE,MAAM;QAC1B,GAAG,GAAG,MAAM,CAAC;IACf,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC"}
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
import type { InferenceAdapter, Message, ChatOptions } from '../../types.js';
|
|
2
|
+
export declare class CopilotSDKInference implements InferenceAdapter {
|
|
3
|
+
private readonly fallback?;
|
|
4
|
+
readonly name = "copilot-sdk";
|
|
5
|
+
constructor(fallback?: InferenceAdapter | undefined);
|
|
6
|
+
chat(messages: Message[], _options?: ChatOptions): Promise<string>;
|
|
7
|
+
embed(text: string): Promise<number[]>;
|
|
8
|
+
estimateCost(_tokens: number): number;
|
|
9
|
+
}
|
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
import { AdapterNotAvailableError } from '../../errors.js';
|
|
2
|
+
import { getClient } from '../copilot-sdk-client.js';
|
|
3
|
+
export class CopilotSDKInference {
|
|
4
|
+
fallback;
|
|
5
|
+
name = 'copilot-sdk';
|
|
6
|
+
constructor(fallback) {
|
|
7
|
+
this.fallback = fallback;
|
|
8
|
+
}
|
|
9
|
+
async chat(messages, _options) {
|
|
10
|
+
const client = await getClient();
|
|
11
|
+
// Extract system message if present
|
|
12
|
+
const systemMessages = messages.filter((m) => m.role === 'system');
|
|
13
|
+
const nonSystemMessages = messages.filter((m) => m.role !== 'system');
|
|
14
|
+
const systemContent = systemMessages.map((m) => m.content).join('\n');
|
|
15
|
+
const userPrompt = nonSystemMessages.map((m) => m.content).join('\n');
|
|
16
|
+
const session = await client.createSession({
|
|
17
|
+
model: 'gpt-4.1',
|
|
18
|
+
...(systemContent
|
|
19
|
+
? { systemMessage: { content: systemContent } }
|
|
20
|
+
: {}),
|
|
21
|
+
onPermissionRequest: async () => ({ kind: 'approved' }),
|
|
22
|
+
});
|
|
23
|
+
try {
|
|
24
|
+
const response = await session.sendAndWait({ prompt: userPrompt });
|
|
25
|
+
return (response?.data?.content ?? '').trim();
|
|
26
|
+
}
|
|
27
|
+
finally {
|
|
28
|
+
await session.disconnect();
|
|
29
|
+
}
|
|
30
|
+
}
|
|
31
|
+
async embed(text) {
|
|
32
|
+
if (this.fallback) {
|
|
33
|
+
return this.fallback.embed(text);
|
|
34
|
+
}
|
|
35
|
+
throw new AdapterNotAvailableError('copilot-sdk-embed', 'Copilot SDK does not support embeddings. Provide a fallback InferenceAdapter (e.g. GitHubModelsInference).');
|
|
36
|
+
}
|
|
37
|
+
estimateCost(_tokens) {
|
|
38
|
+
return 0;
|
|
39
|
+
}
|
|
40
|
+
}
|
|
41
|
+
//# sourceMappingURL=copilot-sdk.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"copilot-sdk.js","sourceRoot":"","sources":["../../../../src/adapters/inference/copilot-sdk.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,wBAAwB,EAAE,MAAM,iBAAiB,CAAC;AAC3D,OAAO,EAAE,SAAS,EAAE,MAAM,0BAA0B,CAAC;AAErD,MAAM,OAAO,mBAAmB;IAGD;IAFpB,IAAI,GAAG,aAAa,CAAC;IAE9B,YAA6B,QAA2B;QAA3B,aAAQ,GAAR,QAAQ,CAAmB;IAAG,CAAC;IAE5D,KAAK,CAAC,IAAI,CAAC,QAAmB,EAAE,QAAsB;QACpD,MAAM,MAAM,GAAG,MAAM,SAAS,EAAE,CAAC;QAEjC,oCAAoC;QACpC,MAAM,cAAc,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,QAAQ,CAAC,CAAC;QACnE,MAAM,iBAAiB,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,QAAQ,CAAC,CAAC;QACtE,MAAM,aAAa,GAAG,cAAc,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACtE,MAAM,UAAU,GAAG,iBAAiB,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAEtE,MAAM,OAAO,GAAG,MAAM,MAAM,CAAC,aAAa,CAAC;YACzC,KAAK,EAAE,SAAS;YAChB,GAAG,CAAC,aAAa;gBACf,CAAC,CAAC,EAAE,aAAa,EAAE,EAAE,OAAO,EAAE,aAAa,EAAE,EAAE;gBAC/C,CAAC,CAAC,EAAE,CAAC;YACP,mBAAmB,EAAE,KAAK,IAAI,EAAE,CAAC,CAAC,EAAE,IAAI,EAAE,UAAU,EAAE,CAAC;SACxD,CAAC,CAAC;QAEH,IAAI,CAAC;YACH,MAAM,QAAQ,GAAG,MAAM,OAAO,CAAC,WAAW,CAAC,EAAE,MAAM,EAAE,UAAU,EAAE,CAAC,CAAC;YACnE,OAAO,CAAC,QAAQ,EAAE,IAAI,EAAE,OAAO,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC;QAChD,CAAC;gBAAS,CAAC;YACT,MAAM,OAAO,CAAC,UAAU,EAAE,CAAC;QAC7B,CAAC;IACH,CAAC;IAED,KAAK,CAAC,KAAK,CAAC,IAAY;QACtB,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC;YAClB,OAAO,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QACnC,CAAC;QACD,MAAM,IAAI,wBAAwB,CAChC,mBAAmB,EACnB,4GAA4G,CAC7G,CAAC;IACJ,CAAC;IAED,YAAY,CAAC,OAAe;QAC1B,OAAO,CAAC,CAAC;IACX,CAAC;CACF"}
|
|
@@ -2,6 +2,8 @@ import { execFileSync } from 'node:child_process';
|
|
|
2
2
|
import { AdapterNotAvailableError } from '../../errors.js';
|
|
3
3
|
import { GitHubModelsInference } from './github-models.js';
|
|
4
4
|
import { CopilotInference } from './copilot.js';
|
|
5
|
+
import { CopilotSDKInference } from './copilot-sdk.js';
|
|
6
|
+
import { isSDKInstalled } from '../copilot-sdk-client.js';
|
|
5
7
|
function isCopilotAvailable() {
|
|
6
8
|
try {
|
|
7
9
|
execFileSync('copilot', ['--version'], { encoding: 'utf-8', stdio: 'pipe' });
|
|
@@ -44,6 +46,13 @@ export function resolveInference(preference) {
|
|
|
44
46
|
}
|
|
45
47
|
return new GitHubModelsInference();
|
|
46
48
|
}
|
|
47
|
-
|
|
49
|
+
if (preference === 'copilot-sdk') {
|
|
50
|
+
if (!isSDKInstalled()) {
|
|
51
|
+
throw new AdapterNotAvailableError('copilot-sdk', '@github/copilot-sdk is not installed. Install with: npm install @github/copilot-sdk');
|
|
52
|
+
}
|
|
53
|
+
const fallback = isGitHubTokenAvailable() ? new GitHubModelsInference() : undefined;
|
|
54
|
+
return new CopilotSDKInference(fallback);
|
|
55
|
+
}
|
|
56
|
+
throw new AdapterNotAvailableError(preference, `Unknown inference adapter "${preference}". Valid options: auto, copilot, copilot-sdk, github-models.`);
|
|
48
57
|
}
|
|
49
58
|
//# sourceMappingURL=resolve.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"resolve.js","sourceRoot":"","sources":["../../../../src/adapters/inference/resolve.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAElD,OAAO,EAAE,wBAAwB,EAAE,MAAM,iBAAiB,CAAC;AAC3D,OAAO,EAAE,qBAAqB,EAAE,MAAM,oBAAoB,CAAC;AAC3D,OAAO,EAAE,gBAAgB,EAAE,MAAM,cAAc,CAAC;
|
|
1
|
+
{"version":3,"file":"resolve.js","sourceRoot":"","sources":["../../../../src/adapters/inference/resolve.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAElD,OAAO,EAAE,wBAAwB,EAAE,MAAM,iBAAiB,CAAC;AAC3D,OAAO,EAAE,qBAAqB,EAAE,MAAM,oBAAoB,CAAC;AAC3D,OAAO,EAAE,gBAAgB,EAAE,MAAM,cAAc,CAAC;AAChD,OAAO,EAAE,mBAAmB,EAAE,MAAM,kBAAkB,CAAC;AACvD,OAAO,EAAE,cAAc,EAAE,MAAM,0BAA0B,CAAC;AAE1D,SAAS,kBAAkB;IACzB,IAAI,CAAC;QACH,YAAY,CAAC,SAAS,EAAE,CAAC,WAAW,CAAC,EAAE,EAAE,QAAQ,EAAE,OAAO,EAAE,KAAK,EAAE,MAAM,EAAE,CAAC,CAAC;QAC7E,OAAO,IAAI,CAAC;IACd,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,KAAK,CAAC;IACf,CAAC;AACH,CAAC;AAED,SAAS,sBAAsB;IAC7B,OAAO,OAAO,CAAC,OAAO,CAAC,GAAG,CAAC,YAAY,CAAC,CAAC;AAC3C,CAAC;AAED,MAAM,UAAU,gBAAgB,CAAC,UAAkB;IACjD,IAAI,UAAU,KAAK,MAAM,EAAE,CAAC;QAC1B,MAAM,gBAAgB,GAAG,kBAAkB,EAAE,CAAC;QAC9C,MAAM,cAAc,GAAG,sBAAsB,EAAE,CAAC;QAEhD,IAAI,gBAAgB,IAAI,cAAc,EAAE,CAAC;YACvC,uDAAuD;YACvD,MAAM,YAAY,GAAG,IAAI,qBAAqB,EAAE,CAAC;YACjD,OAAO,IAAI,gBAAgB,CAAC,YAAY,CAAC,CAAC;QAC5C,CAAC;QAED,IAAI,gBAAgB,EAAE,CAAC;YACrB,OAAO,IAAI,gBAAgB,EAAE,CAAC;QAChC,CAAC;QAED,IAAI,cAAc,EAAE,CAAC;YACnB,OAAO,IAAI,qBAAqB,EAAE,CAAC;QACrC,CAAC;QAED,MAAM,IAAI,wBAAwB,CAChC,WAAW,EACX,oHAAoH,CACrH,CAAC;IACJ,CAAC;IAED,IAAI,UAAU,KAAK,SAAS,EAAE,CAAC;QAC7B,IAAI,CAAC,kBAAkB,EAAE,EAAE,CAAC;YAC1B,MAAM,IAAI,wBAAwB,CAChC,SAAS,EACT,mFAAmF,CACpF,CAAC;QACJ,CAAC;QACD,MAAM,QAAQ,GAAG,sBAAsB,EAAE,CAAC,CAAC,CAAC,IAAI,qBAAqB,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC;QACpF,OAAO,IAAI,gBAAgB,CAAC,QAAQ,CAAC,CAAC;IACxC,CAAC;IAED,IAAI,UAAU,KAAK,eAAe,EAAE,CAAC;QACnC,IAAI,CAAC,sBAAsB,EAAE,EAAE,CAAC;YAC9B,MAAM,IAAI,wBAAwB,CAChC,eAAe,EACf,+CAA+C,CAChD,CAAC;QACJ,CAAC;QACD,OAAO,IAAI,qBAAqB,EAAE,CAAC;IACrC,CAAC;IAED,IAAI,UAAU,KAAK,aAAa,EAAE,CAAC;QACjC,IAAI,CAAC,cAAc,EAAE,EAAE,CAAC;YACtB,MAAM,IAAI,wBAAwB,CAChC,aAAa,EACb,qFAAqF,CACtF,CAAC;QACJ,CAAC;QACD,MAAM,QAAQ,GAAG,sBAAsB,EAAE,CAAC,CAAC,CAAC,IAAI,qBAAqB,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC;QACpF,OAAO,IAAI,mBAAmB,CAAC,QAAQ,CAAC,CAAC;IAC3C,CAAC;IAED,MAAM,IAAI,wBAAwB,CAChC,UAAU,EACV,8BAA8B,UAAU,8DAA8D,CACvG,CAAC;AACJ,CAAC"}
|
|
@@ -0,0 +1,6 @@
|
|
|
1
|
+
import type { SkillAdapter, SkillOutput } from '../../types.js';
|
|
2
|
+
export declare class CopilotSDKAdapter implements SkillAdapter {
|
|
3
|
+
readonly name = "copilot-sdk";
|
|
4
|
+
invoke(skillPath: string, prompt: string, _files?: string[]): Promise<SkillOutput>;
|
|
5
|
+
isAvailable(): Promise<boolean>;
|
|
6
|
+
}
|
|
@@ -0,0 +1,68 @@
|
|
|
1
|
+
import { readFile } from 'node:fs/promises';
|
|
2
|
+
import * as path from 'node:path';
|
|
3
|
+
import { getClient } from '../copilot-sdk-client.js';
|
|
4
|
+
export class CopilotSDKAdapter {
|
|
5
|
+
name = 'copilot-sdk';
|
|
6
|
+
async invoke(skillPath, prompt, _files) {
|
|
7
|
+
const startMs = Date.now();
|
|
8
|
+
// Read SKILL.md for system message context
|
|
9
|
+
let skillMd = '';
|
|
10
|
+
try {
|
|
11
|
+
const skillFile = path.join(skillPath, 'SKILL.md');
|
|
12
|
+
skillMd = await readFile(skillFile, { encoding: 'utf-8' });
|
|
13
|
+
}
|
|
14
|
+
catch {
|
|
15
|
+
// ignore missing SKILL.md
|
|
16
|
+
}
|
|
17
|
+
const client = await getClient();
|
|
18
|
+
// Track token usage from events
|
|
19
|
+
let inputTokens = 0;
|
|
20
|
+
let outputTokens = 0;
|
|
21
|
+
let model = 'copilot-sdk';
|
|
22
|
+
const session = await client.createSession({
|
|
23
|
+
model: 'gpt-4.1',
|
|
24
|
+
...(skillMd
|
|
25
|
+
? { systemMessage: { content: skillMd } }
|
|
26
|
+
: {}),
|
|
27
|
+
onPermissionRequest: async () => ({ kind: 'approved' }),
|
|
28
|
+
});
|
|
29
|
+
const unsubscribe = session.on((event) => {
|
|
30
|
+
if (event.type === 'assistant.usage') {
|
|
31
|
+
if (event.data.inputTokens != null)
|
|
32
|
+
inputTokens += event.data.inputTokens;
|
|
33
|
+
if (event.data.outputTokens != null)
|
|
34
|
+
outputTokens += event.data.outputTokens;
|
|
35
|
+
if (event.data.model)
|
|
36
|
+
model = event.data.model;
|
|
37
|
+
}
|
|
38
|
+
});
|
|
39
|
+
try {
|
|
40
|
+
const response = await session.sendAndWait({ prompt });
|
|
41
|
+
const raw = response?.data?.content ?? '';
|
|
42
|
+
const durationMs = Date.now() - startMs;
|
|
43
|
+
return {
|
|
44
|
+
raw: raw.trim(),
|
|
45
|
+
metadata: {
|
|
46
|
+
tokens: inputTokens + outputTokens,
|
|
47
|
+
durationMs,
|
|
48
|
+
model,
|
|
49
|
+
adapter: this.name,
|
|
50
|
+
},
|
|
51
|
+
};
|
|
52
|
+
}
|
|
53
|
+
finally {
|
|
54
|
+
unsubscribe();
|
|
55
|
+
await session.disconnect();
|
|
56
|
+
}
|
|
57
|
+
}
|
|
58
|
+
async isAvailable() {
|
|
59
|
+
try {
|
|
60
|
+
await getClient();
|
|
61
|
+
return true;
|
|
62
|
+
}
|
|
63
|
+
catch {
|
|
64
|
+
return false;
|
|
65
|
+
}
|
|
66
|
+
}
|
|
67
|
+
}
|
|
68
|
+
//# sourceMappingURL=copilot-sdk.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"copilot-sdk.js","sourceRoot":"","sources":["../../../../src/adapters/skill/copilot-sdk.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAC;AAC5C,OAAO,KAAK,IAAI,MAAM,WAAW,CAAC;AAElC,OAAO,EAAE,SAAS,EAAc,MAAM,0BAA0B,CAAC;AAEjE,MAAM,OAAO,iBAAiB;IACnB,IAAI,GAAG,aAAa,CAAC;IAE9B,KAAK,CAAC,MAAM,CAAC,SAAiB,EAAE,MAAc,EAAE,MAAiB;QAC/D,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAE3B,2CAA2C;QAC3C,IAAI,OAAO,GAAG,EAAE,CAAC;QACjB,IAAI,CAAC;YACH,MAAM,SAAS,GAAG,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,UAAU,CAAC,CAAC;YACnD,OAAO,GAAG,MAAM,QAAQ,CAAC,SAAS,EAAE,EAAE,QAAQ,EAAE,OAAO,EAAE,CAAC,CAAC;QAC7D,CAAC;QAAC,MAAM,CAAC;YACP,0BAA0B;QAC5B,CAAC;QAED,MAAM,MAAM,GAAG,MAAM,SAAS,EAAE,CAAC;QAEjC,gCAAgC;QAChC,IAAI,WAAW,GAAG,CAAC,CAAC;QACpB,IAAI,YAAY,GAAG,CAAC,CAAC;QACrB,IAAI,KAAK,GAAG,aAAa,CAAC;QAE1B,MAAM,OAAO,GAAG,MAAM,MAAM,CAAC,aAAa,CAAC;YACzC,KAAK,EAAE,SAAS;YAChB,GAAG,CAAC,OAAO;gBACT,CAAC,CAAC,EAAE,aAAa,EAAE,EAAE,OAAO,EAAE,OAAO,EAAE,EAAE;gBACzC,CAAC,CAAC,EAAE,CAAC;YACP,mBAAmB,EAAE,KAAK,IAAI,EAAE,CAAC,CAAC,EAAE,IAAI,EAAE,UAAU,EAAE,CAAC;SACxD,CAAC,CAAC;QAEH,MAAM,WAAW,GAAG,OAAO,CAAC,EAAE,CAAC,CAAC,KAAU,EAAE,EAAE;YAC5C,IAAI,KAAK,CAAC,IAAI,KAAK,iBAAiB,EAAE,CAAC;gBACrC,IAAI,KAAK,CAAC,IAAI,CAAC,WAAW,IAAI,IAAI;oBAAE,WAAW,IAAI,KAAK,CAAC,IAAI,CAAC,WAAW,CAAC;gBAC1E,IAAI,KAAK,CAAC,IAAI,CAAC,YAAY,IAAI,IAAI;oBAAE,YAAY,IAAI,KAAK,CAAC,IAAI,CAAC,YAAY,CAAC;gBAC7E,IAAI,KAAK,CAAC,IAAI,CAAC,KAAK;oBAAE,KAAK,GAAG,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC;YACjD,CAAC;QACH,CAAC,CAAC,CAAC;QAEH,IAAI,CAAC;YACH,MAAM,QAAQ,GAAG,MAAM,OAAO,CAAC,WAAW,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC;YACvD,MAAM,GAAG,GAAG,QAAQ,EAAE,IAAI,EAAE,OAAO,IAAI,EAAE,CAAC;YAC1C,MAAM,UAAU,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,OAAO,CAAC;YAExC,OAAO;gBACL,GAAG,EAAE,GAAG,CAAC,IAAI,EAAE;gBACf,QAAQ,EAAE;oBACR,MAAM,EAAE,WAAW,GAAG,YAAY;oBAClC,UAAU;oBACV,KAAK;oBACL,OAAO,EAAE,IAAI,CAAC,IAAI;iBACnB;aACF,CAAC;QACJ,CAAC;gBAAS,CAAC;YACT,WAAW,EAAE,CAAC;YACd,MAAM,OAAO,CAAC,UAAU,EAAE,CAAC;QAC7B,CAAC;IACH,CAAC;IAED,KAAK,CAAC,WAAW;QACf,IAAI,CAAC;YACH,MAAM,SAAS,EAAE,CAAC;YAClB,OAAO,IAAI,CAAC;QACd,CAAC;QAAC,MAAM,CAAC;YACP,OAAO,KAAK,CAAC;QACf,CAAC;IACH,CAAC;CACF"}
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
import type { SkillAdapter, InferenceAdapter } from '../types.js';
|
|
2
|
+
export declare function reviewCommand(skillPath: string, skillAdapter: SkillAdapter, inference: InferenceAdapter, options: {
|
|
3
|
+
budget: string;
|
|
4
|
+
}): Promise<{
|
|
5
|
+
iterationDir: string;
|
|
6
|
+
hasRegressions: boolean;
|
|
7
|
+
}>;
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
import { execFile } from 'node:child_process';
|
|
2
|
+
import * as path from 'node:path';
|
|
3
|
+
import * as process from 'node:process';
|
|
4
|
+
import { checkCommand } from './check.js';
|
|
5
|
+
import { reportCommand } from './report.js';
|
|
6
|
+
export async function reviewCommand(skillPath, skillAdapter, inference, options) {
|
|
7
|
+
const results = await checkCommand(skillPath, skillAdapter, inference, options);
|
|
8
|
+
const iterationDir = await reportCommand(skillPath, results, {
|
|
9
|
+
verbose: true,
|
|
10
|
+
html: true,
|
|
11
|
+
});
|
|
12
|
+
const reportPath = path.join(iterationDir, 'report.html');
|
|
13
|
+
openInBrowser(reportPath);
|
|
14
|
+
return {
|
|
15
|
+
iterationDir,
|
|
16
|
+
hasRegressions: results.summary.regressed > 0,
|
|
17
|
+
};
|
|
18
|
+
}
|
|
19
|
+
function openInBrowser(filePath) {
|
|
20
|
+
const cmd = process.platform === 'darwin'
|
|
21
|
+
? 'open'
|
|
22
|
+
: process.platform === 'win32'
|
|
23
|
+
? 'cmd'
|
|
24
|
+
: 'xdg-open';
|
|
25
|
+
const args = process.platform === 'win32'
|
|
26
|
+
? ['/c', 'start', '', filePath]
|
|
27
|
+
: [filePath];
|
|
28
|
+
execFile(cmd, args, (err) => {
|
|
29
|
+
if (err) {
|
|
30
|
+
console.warn(`Could not open browser: ${err.message}`);
|
|
31
|
+
}
|
|
32
|
+
});
|
|
33
|
+
}
|
|
34
|
+
//# sourceMappingURL=review.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"review.js","sourceRoot":"","sources":["../../../src/commands/review.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,oBAAoB,CAAC;AAC9C,OAAO,KAAK,IAAI,MAAM,WAAW,CAAC;AAClC,OAAO,KAAK,OAAO,MAAM,cAAc,CAAC;AAExC,OAAO,EAAE,YAAY,EAAE,MAAM,YAAY,CAAC;AAC1C,OAAO,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AAE5C,MAAM,CAAC,KAAK,UAAU,aAAa,CACjC,SAAiB,EACjB,YAA0B,EAC1B,SAA2B,EAC3B,OAA2B;IAE3B,MAAM,OAAO,GAAG,MAAM,YAAY,CAAC,SAAS,EAAE,YAAY,EAAE,SAAS,EAAE,OAAO,CAAC,CAAC;IAEhF,MAAM,YAAY,GAAG,MAAM,aAAa,CAAC,SAAS,EAAE,OAAO,EAAE;QAC3D,OAAO,EAAE,IAAI;QACb,IAAI,EAAE,IAAI;KACX,CAAC,CAAC;IAEH,MAAM,UAAU,GAAG,IAAI,CAAC,IAAI,CAAC,YAAY,EAAE,aAAa,CAAC,CAAC;IAC1D,aAAa,CAAC,UAAU,CAAC,CAAC;IAE1B,OAAO;QACL,YAAY;QACZ,cAAc,EAAE,OAAO,CAAC,OAAO,CAAC,SAAS,GAAG,CAAC;KAC9C,CAAC;AACJ,CAAC;AAED,SAAS,aAAa,CAAC,QAAgB;IACrC,MAAM,GAAG,GACP,OAAO,CAAC,QAAQ,KAAK,QAAQ;QAC3B,CAAC,CAAC,MAAM;QACR,CAAC,CAAC,OAAO,CAAC,QAAQ,KAAK,OAAO;YAC5B,CAAC,CAAC,KAAK;YACP,CAAC,CAAC,UAAU,CAAC;IAEnB,MAAM,IAAI,GACR,OAAO,CAAC,QAAQ,KAAK,OAAO;QAC1B,CAAC,CAAC,CAAC,IAAI,EAAE,OAAO,EAAE,EAAE,EAAE,QAAQ,CAAC;QAC/B,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC;IAEjB,QAAQ,CAAC,GAAG,EAAE,IAAI,EAAE,CAAC,GAAG,EAAE,EAAE;QAC1B,IAAI,GAAG,EAAE,CAAC;YACR,OAAO,CAAC,IAAI,CAAC,2BAA2B,GAAG,CAAC,OAAO,EAAE,CAAC,CAAC;QACzD,CAAC;IACH,CAAC,CAAC,CAAC;AACL,CAAC"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "snapeval",
|
|
3
|
-
"version": "1.
|
|
3
|
+
"version": "1.5.0",
|
|
4
4
|
"description": "Semantic snapshot testing for AI skills. Zero assertions. AI-driven. Free inference.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"bin": {
|
|
@@ -18,6 +18,7 @@
|
|
|
18
18
|
"ai-skills",
|
|
19
19
|
"snapshot-testing",
|
|
20
20
|
"copilot-cli",
|
|
21
|
+
"copilot-sdk",
|
|
21
22
|
"agentskills",
|
|
22
23
|
"evaluation",
|
|
23
24
|
"semantic-testing"
|
|
@@ -46,6 +47,14 @@
|
|
|
46
47
|
"chalk": "^5.4.1",
|
|
47
48
|
"commander": "^14.0.3"
|
|
48
49
|
},
|
|
50
|
+
"peerDependencies": {
|
|
51
|
+
"@github/copilot-sdk": "~0.1.0"
|
|
52
|
+
},
|
|
53
|
+
"peerDependenciesMeta": {
|
|
54
|
+
"@github/copilot-sdk": {
|
|
55
|
+
"optional": true
|
|
56
|
+
}
|
|
57
|
+
},
|
|
49
58
|
"devDependencies": {
|
|
50
59
|
"@types/node": "^25.5.0",
|
|
51
60
|
"tsx": "^4.19.3",
|
package/plugin.json
CHANGED
package/skills/snapeval/SKILL.md
CHANGED
|
@@ -1,23 +1,87 @@
|
|
|
1
1
|
---
|
|
2
2
|
name: snapeval
|
|
3
|
-
description: Evaluate AI skills through interactive scenario ideation. Analyzes skill behaviors, dimensions, and failure modes, then collaborates with the user to design a test strategy. Use when the user wants to evaluate, test, check, or review any skill — including phrases like "did I break anything", "test my skill", "run evals", or "
|
|
3
|
+
description: Evaluate AI skills through interactive scenario ideation and detect regressions. Analyzes skill behaviors, dimensions, and failure modes, then collaborates with the user to design a test strategy. Use when the user wants to evaluate, test, check, or review any skill — including phrases like "did I break anything", "test my skill", "run evals", "evaluate this", "set up evals", "check for regressions", or "I have a new skill."
|
|
4
4
|
---
|
|
5
5
|
|
|
6
6
|
You are snapeval, a skill evaluation assistant. You help users design thorough test strategies for AI skills and detect regressions.
|
|
7
7
|
|
|
8
|
-
##
|
|
8
|
+
## Phase 0 — Validate & Route
|
|
9
9
|
|
|
10
|
-
|
|
10
|
+
Every interaction starts here. Determine what the user needs and route to the right flow.
|
|
11
11
|
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
#### Phase 0 — Validate
|
|
12
|
+
### Step 1: Identify the skill
|
|
15
13
|
|
|
16
14
|
1. Identify the skill to evaluate — ask for the path if not provided
|
|
17
15
|
2. Verify the skill directory exists and contains a SKILL.md (or skill.md)
|
|
18
16
|
3. If not found, tell the user: "No SKILL.md found at `<path>`. This tool evaluates skills that follow the agentskills.io standard."
|
|
19
17
|
|
|
20
|
-
|
|
18
|
+
### Step 2: Detect state and intent
|
|
19
|
+
|
|
20
|
+
Check whether `<skill-path>/evals/snapshots/` exists and contains `.snap.json` files.
|
|
21
|
+
|
|
22
|
+
**If NO baselines exist:**
|
|
23
|
+
- Route to **Quick Onboard** (below). The user needs baselines before anything else.
|
|
24
|
+
- Exception: if the user explicitly asks for the full evaluation flow ("evaluate my skill", "full analysis"), route to **Full Ideation** instead.
|
|
25
|
+
|
|
26
|
+
**If baselines exist, detect intent:**
|
|
27
|
+
|
|
28
|
+
| User says | Route to |
|
|
29
|
+
|-----------|----------|
|
|
30
|
+
| "did I break anything", "quick check", "run my tests", "check for regressions", "check" | **Quick Check** |
|
|
31
|
+
| "review", "show me the report", "visual review" | **Review** |
|
|
32
|
+
| "approve", "accept the changes" | **Approve** |
|
|
33
|
+
| "evaluate", "test my skill", "full analysis", "re-evaluate", "expand coverage" | **Full Ideation** |
|
|
34
|
+
| Ambiguous | Ask: "You have baselines already. Want me to check for regressions, or do a full re-evaluation?" |
|
|
35
|
+
|
|
36
|
+
---
|
|
37
|
+
|
|
38
|
+
## Quick Onboard (no baselines)
|
|
39
|
+
|
|
40
|
+
A fast path to "you have baselines now." No browser viewer, no analysis.json — just scenarios, confirmation, and capture.
|
|
41
|
+
|
|
42
|
+
1. Read the target skill's SKILL.md completely. If it references files in `scripts/`, `references/`, or `assets/`, read those too.
|
|
43
|
+
|
|
44
|
+
2. Generate 3-5 test scenarios covering the skill's core behaviors. For each scenario:
|
|
45
|
+
- Write a realistic, messy user prompt (see Prompt Realism below)
|
|
46
|
+
- Briefly explain what it tests
|
|
47
|
+
|
|
48
|
+
Focus on covering distinct behaviors rather than exhaustive dimensional coverage. Fewer scenarios, same quality.
|
|
49
|
+
|
|
50
|
+
3. Present scenarios inline:
|
|
51
|
+
> "I've read your skill and generated N scenarios to get you started. Here they are:"
|
|
52
|
+
>
|
|
53
|
+
> **1.** `"hey can you greet my colleague eleanor? make it formal"` — tests formal greeting with a name
|
|
54
|
+
> **2.** `"greet me in pirate style plz"` — tests style selection
|
|
55
|
+
> ...
|
|
56
|
+
>
|
|
57
|
+
> "Look good? I'll capture baselines so you can detect regressions going forward."
|
|
58
|
+
|
|
59
|
+
4. On confirmation, write scenarios to `evals/evals.json`:
|
|
60
|
+
```json
|
|
61
|
+
{
|
|
62
|
+
"skill_name": "<name>",
|
|
63
|
+
"generated_by": "snapeval quick-onboard",
|
|
64
|
+
"evals": [{ "id": 1, "prompt": "...", "expected_output": "...", "files": [], "assertions": [] }]
|
|
65
|
+
}
|
|
66
|
+
```
|
|
67
|
+
|
|
68
|
+
5. Run capture:
|
|
69
|
+
```bash
|
|
70
|
+
npx snapeval capture <skill-path>
|
|
71
|
+
```
|
|
72
|
+
|
|
73
|
+
6. Report results:
|
|
74
|
+
> "Baselines captured (N scenarios, $0.00). You now have regression detection — just say 'did I break anything?' anytime to check."
|
|
75
|
+
>
|
|
76
|
+
> "Want more thorough coverage? Say 'evaluate my skill' for the full analysis with the interactive viewer."
|
|
77
|
+
|
|
78
|
+
---
|
|
79
|
+
|
|
80
|
+
## Full Ideation (evaluate / test)
|
|
81
|
+
|
|
82
|
+
When the user asks for a full evaluation, or explicitly requests the deep analysis flow. Do NOT skip phases or collapse them into a single step.
|
|
83
|
+
|
|
84
|
+
### Phase 1 — Analyze the Skill
|
|
21
85
|
|
|
22
86
|
Read the target skill's SKILL.md completely. If it references files in `scripts/`, `references/`, or `assets/`, read those too.
|
|
23
87
|
|
|
@@ -62,7 +126,7 @@ Write the analysis as JSON to `<skill-path>/evals/analysis.json`:
|
|
|
62
126
|
|
|
63
127
|
Give a brief terminal summary: "I've analyzed your skill — found N behaviors, N dimensions, and N potential gaps. Opening the analysis viewer."
|
|
64
128
|
|
|
65
|
-
|
|
129
|
+
### Phase 2 — Visual Presentation
|
|
66
130
|
|
|
67
131
|
Open the interactive ideation viewer:
|
|
68
132
|
|
|
@@ -75,7 +139,7 @@ Tell the user:
|
|
|
75
139
|
|
|
76
140
|
Wait for the user to return.
|
|
77
141
|
|
|
78
|
-
|
|
142
|
+
### Phase 3 — Ingest Feedback
|
|
79
143
|
|
|
80
144
|
When the user says they're done, find the exported plan:
|
|
81
145
|
1. Check `~/Downloads/scenario_plan.json`
|
|
@@ -90,7 +154,7 @@ Read the plan and acknowledge changes:
|
|
|
90
154
|
|
|
91
155
|
If the user marked ambiguities as in-scope, generate additional scenarios covering them and ask for quick confirmation.
|
|
92
156
|
|
|
93
|
-
|
|
157
|
+
### Phase 4 — Write & Run
|
|
94
158
|
|
|
95
159
|
Write the finalized scenarios to `evals/evals.json`. Map fields:
|
|
96
160
|
- `confirmed_scenarios[].prompt` → `evals[].prompt`
|
|
@@ -105,7 +169,9 @@ npx snapeval capture <skill-path>
|
|
|
105
169
|
|
|
106
170
|
Report results: how many scenarios captured, total cost, location of snapshots.
|
|
107
171
|
|
|
108
|
-
|
|
172
|
+
---
|
|
173
|
+
|
|
174
|
+
## Quick Check (regression detection)
|
|
109
175
|
|
|
110
176
|
1. Run: `npx snapeval check <skill-path>`
|
|
111
177
|
2. Parse the terminal output
|
|
@@ -117,20 +183,29 @@ Report results: how many scenarios captured, total cost, location of snapshots.
|
|
|
117
183
|
- Fix the skill and re-check
|
|
118
184
|
- Run `@snapeval approve` to accept new behavior
|
|
119
185
|
|
|
120
|
-
|
|
186
|
+
---
|
|
187
|
+
|
|
188
|
+
## Review (visual review)
|
|
121
189
|
|
|
122
|
-
After running check, generate a visual report:
|
|
123
|
-
1. Run: `npx snapeval
|
|
124
|
-
2.
|
|
125
|
-
3.
|
|
126
|
-
4. If the user provides feedback
|
|
190
|
+
After running check, generate a visual report and open it:
|
|
191
|
+
1. Run: `npx snapeval review <skill-path>`
|
|
192
|
+
2. This runs check, generates an HTML report, and opens it in the browser automatically
|
|
193
|
+
3. Tell the user: "Opening the report in your browser — it shows baseline vs current output with diffs, comparison analysis, and benchmark stats"
|
|
194
|
+
4. If the user provides feedback, use it to guide skill improvements
|
|
195
|
+
5. If regressions found, present options:
|
|
196
|
+
- Fix the skill and re-review
|
|
197
|
+
- Run `@snapeval approve` to accept new behavior
|
|
127
198
|
|
|
128
|
-
|
|
199
|
+
---
|
|
200
|
+
|
|
201
|
+
## Approve
|
|
129
202
|
|
|
130
203
|
1. Run: `npx snapeval approve --scenario <N>` (or without --scenario for all)
|
|
131
204
|
2. Confirm what was approved
|
|
132
205
|
3. Remind user to commit the updated snapshots
|
|
133
206
|
|
|
207
|
+
---
|
|
208
|
+
|
|
134
209
|
## Prompt Realism
|
|
135
210
|
|
|
136
211
|
When generating scenario prompts, make them realistic — the way a real user would actually type them. Not abstract test cases, but the kind of messy, specific, contextual prompts real people write.
|
|
@@ -153,3 +228,4 @@ Vary style across scenarios: some terse, some with backstory, some with typos or
|
|
|
153
228
|
- Report costs prominently (should be $0.00 for Copilot gpt-5-mini)
|
|
154
229
|
- When reporting regressions, explain what changed in plain language
|
|
155
230
|
- The ideation viewer and eval viewer are separate tools for separate stages — don't confuse them
|
|
231
|
+
- Quick Onboard is for getting started fast — if users want thorough coverage, guide them to Full Ideation
|
|
@@ -0,0 +1,66 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Shared lazy CopilotClient singleton.
|
|
3
|
+
*
|
|
4
|
+
* Both CopilotSDKAdapter (SkillAdapter) and CopilotSDKInference
|
|
5
|
+
* (InferenceAdapter) share a single client to avoid spawning
|
|
6
|
+
* multiple CLI server processes.
|
|
7
|
+
*
|
|
8
|
+
* The SDK is dynamically imported so that users who don't install
|
|
9
|
+
* @github/copilot-sdk pay no cost.
|
|
10
|
+
*/
|
|
11
|
+
|
|
12
|
+
import * as fs from 'node:fs';
|
|
13
|
+
import * as path from 'node:path';
|
|
14
|
+
|
|
15
|
+
// We store the client as `any` to avoid a hard import dependency
|
|
16
|
+
// on @github/copilot-sdk. The module may not be installed.
|
|
17
|
+
let clientInstance: any = null;
|
|
18
|
+
let clientStarted = false;
|
|
19
|
+
|
|
20
|
+
export async function getClient(): Promise<any> {
|
|
21
|
+
if (clientInstance && clientStarted) return clientInstance;
|
|
22
|
+
|
|
23
|
+
let sdk: any;
|
|
24
|
+
try {
|
|
25
|
+
// @ts-ignore — module may not be installed (optional peer dep)
|
|
26
|
+
sdk = await import('@github/copilot-sdk');
|
|
27
|
+
} catch {
|
|
28
|
+
throw new Error(
|
|
29
|
+
'CopilotSDK adapter requires @github/copilot-sdk. Install it with: npm install @github/copilot-sdk'
|
|
30
|
+
);
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
const CopilotClient = sdk.CopilotClient ?? sdk.default?.CopilotClient;
|
|
34
|
+
if (!CopilotClient) {
|
|
35
|
+
throw new Error(
|
|
36
|
+
'Could not find CopilotClient export in @github/copilot-sdk. The package may have changed its API.'
|
|
37
|
+
);
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
clientInstance = new CopilotClient();
|
|
41
|
+
await clientInstance.start();
|
|
42
|
+
clientStarted = true;
|
|
43
|
+
return clientInstance;
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
export async function stopClient(): Promise<void> {
|
|
47
|
+
if (clientInstance && clientStarted) {
|
|
48
|
+
await clientInstance.stop();
|
|
49
|
+
clientStarted = false;
|
|
50
|
+
clientInstance = null;
|
|
51
|
+
}
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
export function isSDKInstalled(): boolean {
|
|
55
|
+
// Walk up from cwd looking for node_modules/@github/copilot-sdk.
|
|
56
|
+
// This avoids createRequire/import issues across ESM/CJS contexts.
|
|
57
|
+
let dir = process.cwd();
|
|
58
|
+
while (true) {
|
|
59
|
+
const candidate = path.join(dir, 'node_modules', '@github', 'copilot-sdk', 'package.json');
|
|
60
|
+
if (fs.existsSync(candidate)) return true;
|
|
61
|
+
const parent = path.dirname(dir);
|
|
62
|
+
if (parent === dir) break;
|
|
63
|
+
dir = parent;
|
|
64
|
+
}
|
|
65
|
+
return false;
|
|
66
|
+
}
|
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
import type { InferenceAdapter, Message, ChatOptions } from '../../types.js';
|
|
2
|
+
import { AdapterNotAvailableError } from '../../errors.js';
|
|
3
|
+
import { getClient } from '../copilot-sdk-client.js';
|
|
4
|
+
|
|
5
|
+
export class CopilotSDKInference implements InferenceAdapter {
|
|
6
|
+
readonly name = 'copilot-sdk';
|
|
7
|
+
|
|
8
|
+
constructor(private readonly fallback?: InferenceAdapter) {}
|
|
9
|
+
|
|
10
|
+
async chat(messages: Message[], _options?: ChatOptions): Promise<string> {
|
|
11
|
+
const client = await getClient();
|
|
12
|
+
|
|
13
|
+
// Extract system message if present
|
|
14
|
+
const systemMessages = messages.filter((m) => m.role === 'system');
|
|
15
|
+
const nonSystemMessages = messages.filter((m) => m.role !== 'system');
|
|
16
|
+
const systemContent = systemMessages.map((m) => m.content).join('\n');
|
|
17
|
+
const userPrompt = nonSystemMessages.map((m) => m.content).join('\n');
|
|
18
|
+
|
|
19
|
+
const session = await client.createSession({
|
|
20
|
+
model: 'gpt-4.1',
|
|
21
|
+
...(systemContent
|
|
22
|
+
? { systemMessage: { content: systemContent } }
|
|
23
|
+
: {}),
|
|
24
|
+
onPermissionRequest: async () => ({ kind: 'approved' }),
|
|
25
|
+
});
|
|
26
|
+
|
|
27
|
+
try {
|
|
28
|
+
const response = await session.sendAndWait({ prompt: userPrompt });
|
|
29
|
+
return (response?.data?.content ?? '').trim();
|
|
30
|
+
} finally {
|
|
31
|
+
await session.disconnect();
|
|
32
|
+
}
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
async embed(text: string): Promise<number[]> {
|
|
36
|
+
if (this.fallback) {
|
|
37
|
+
return this.fallback.embed(text);
|
|
38
|
+
}
|
|
39
|
+
throw new AdapterNotAvailableError(
|
|
40
|
+
'copilot-sdk-embed',
|
|
41
|
+
'Copilot SDK does not support embeddings. Provide a fallback InferenceAdapter (e.g. GitHubModelsInference).'
|
|
42
|
+
);
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
estimateCost(_tokens: number): number {
|
|
46
|
+
return 0;
|
|
47
|
+
}
|
|
48
|
+
}
|
|
@@ -3,6 +3,8 @@ import type { InferenceAdapter } from '../../types.js';
|
|
|
3
3
|
import { AdapterNotAvailableError } from '../../errors.js';
|
|
4
4
|
import { GitHubModelsInference } from './github-models.js';
|
|
5
5
|
import { CopilotInference } from './copilot.js';
|
|
6
|
+
import { CopilotSDKInference } from './copilot-sdk.js';
|
|
7
|
+
import { isSDKInstalled } from '../copilot-sdk-client.js';
|
|
6
8
|
|
|
7
9
|
function isCopilotAvailable(): boolean {
|
|
8
10
|
try {
|
|
@@ -63,8 +65,19 @@ export function resolveInference(preference: string): InferenceAdapter {
|
|
|
63
65
|
return new GitHubModelsInference();
|
|
64
66
|
}
|
|
65
67
|
|
|
68
|
+
if (preference === 'copilot-sdk') {
|
|
69
|
+
if (!isSDKInstalled()) {
|
|
70
|
+
throw new AdapterNotAvailableError(
|
|
71
|
+
'copilot-sdk',
|
|
72
|
+
'@github/copilot-sdk is not installed. Install with: npm install @github/copilot-sdk'
|
|
73
|
+
);
|
|
74
|
+
}
|
|
75
|
+
const fallback = isGitHubTokenAvailable() ? new GitHubModelsInference() : undefined;
|
|
76
|
+
return new CopilotSDKInference(fallback);
|
|
77
|
+
}
|
|
78
|
+
|
|
66
79
|
throw new AdapterNotAvailableError(
|
|
67
80
|
preference,
|
|
68
|
-
`Unknown inference adapter "${preference}". Valid options: auto, copilot, github-models.`
|
|
81
|
+
`Unknown inference adapter "${preference}". Valid options: auto, copilot, copilot-sdk, github-models.`
|
|
69
82
|
);
|
|
70
83
|
}
|
|
@@ -0,0 +1,72 @@
|
|
|
1
|
+
import { readFile } from 'node:fs/promises';
|
|
2
|
+
import * as path from 'node:path';
|
|
3
|
+
import type { SkillAdapter, SkillOutput } from '../../types.js';
|
|
4
|
+
import { getClient, stopClient } from '../copilot-sdk-client.js';
|
|
5
|
+
|
|
6
|
+
export class CopilotSDKAdapter implements SkillAdapter {
|
|
7
|
+
readonly name = 'copilot-sdk';
|
|
8
|
+
|
|
9
|
+
async invoke(skillPath: string, prompt: string, _files?: string[]): Promise<SkillOutput> {
|
|
10
|
+
const startMs = Date.now();
|
|
11
|
+
|
|
12
|
+
// Read SKILL.md for system message context
|
|
13
|
+
let skillMd = '';
|
|
14
|
+
try {
|
|
15
|
+
const skillFile = path.join(skillPath, 'SKILL.md');
|
|
16
|
+
skillMd = await readFile(skillFile, { encoding: 'utf-8' });
|
|
17
|
+
} catch {
|
|
18
|
+
// ignore missing SKILL.md
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
const client = await getClient();
|
|
22
|
+
|
|
23
|
+
// Track token usage from events
|
|
24
|
+
let inputTokens = 0;
|
|
25
|
+
let outputTokens = 0;
|
|
26
|
+
let model = 'copilot-sdk';
|
|
27
|
+
|
|
28
|
+
const session = await client.createSession({
|
|
29
|
+
model: 'gpt-4.1',
|
|
30
|
+
...(skillMd
|
|
31
|
+
? { systemMessage: { content: skillMd } }
|
|
32
|
+
: {}),
|
|
33
|
+
onPermissionRequest: async () => ({ kind: 'approved' }),
|
|
34
|
+
});
|
|
35
|
+
|
|
36
|
+
const unsubscribe = session.on((event: any) => {
|
|
37
|
+
if (event.type === 'assistant.usage') {
|
|
38
|
+
if (event.data.inputTokens != null) inputTokens += event.data.inputTokens;
|
|
39
|
+
if (event.data.outputTokens != null) outputTokens += event.data.outputTokens;
|
|
40
|
+
if (event.data.model) model = event.data.model;
|
|
41
|
+
}
|
|
42
|
+
});
|
|
43
|
+
|
|
44
|
+
try {
|
|
45
|
+
const response = await session.sendAndWait({ prompt });
|
|
46
|
+
const raw = response?.data?.content ?? '';
|
|
47
|
+
const durationMs = Date.now() - startMs;
|
|
48
|
+
|
|
49
|
+
return {
|
|
50
|
+
raw: raw.trim(),
|
|
51
|
+
metadata: {
|
|
52
|
+
tokens: inputTokens + outputTokens,
|
|
53
|
+
durationMs,
|
|
54
|
+
model,
|
|
55
|
+
adapter: this.name,
|
|
56
|
+
},
|
|
57
|
+
};
|
|
58
|
+
} finally {
|
|
59
|
+
unsubscribe();
|
|
60
|
+
await session.disconnect();
|
|
61
|
+
}
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
async isAvailable(): Promise<boolean> {
|
|
65
|
+
try {
|
|
66
|
+
await getClient();
|
|
67
|
+
return true;
|
|
68
|
+
} catch {
|
|
69
|
+
return false;
|
|
70
|
+
}
|
|
71
|
+
}
|
|
72
|
+
}
|
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
import { execFile } from 'node:child_process';
|
|
2
|
+
import * as path from 'node:path';
|
|
3
|
+
import * as process from 'node:process';
|
|
4
|
+
import type { SkillAdapter, InferenceAdapter } from '../types.js';
|
|
5
|
+
import { checkCommand } from './check.js';
|
|
6
|
+
import { reportCommand } from './report.js';
|
|
7
|
+
|
|
8
|
+
export async function reviewCommand(
|
|
9
|
+
skillPath: string,
|
|
10
|
+
skillAdapter: SkillAdapter,
|
|
11
|
+
inference: InferenceAdapter,
|
|
12
|
+
options: { budget: string }
|
|
13
|
+
): Promise<{ iterationDir: string; hasRegressions: boolean }> {
|
|
14
|
+
const results = await checkCommand(skillPath, skillAdapter, inference, options);
|
|
15
|
+
|
|
16
|
+
const iterationDir = await reportCommand(skillPath, results, {
|
|
17
|
+
verbose: true,
|
|
18
|
+
html: true,
|
|
19
|
+
});
|
|
20
|
+
|
|
21
|
+
const reportPath = path.join(iterationDir, 'report.html');
|
|
22
|
+
openInBrowser(reportPath);
|
|
23
|
+
|
|
24
|
+
return {
|
|
25
|
+
iterationDir,
|
|
26
|
+
hasRegressions: results.summary.regressed > 0,
|
|
27
|
+
};
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
function openInBrowser(filePath: string): void {
|
|
31
|
+
const cmd =
|
|
32
|
+
process.platform === 'darwin'
|
|
33
|
+
? 'open'
|
|
34
|
+
: process.platform === 'win32'
|
|
35
|
+
? 'cmd'
|
|
36
|
+
: 'xdg-open';
|
|
37
|
+
|
|
38
|
+
const args =
|
|
39
|
+
process.platform === 'win32'
|
|
40
|
+
? ['/c', 'start', '', filePath]
|
|
41
|
+
: [filePath];
|
|
42
|
+
|
|
43
|
+
execFile(cmd, args, (err) => {
|
|
44
|
+
if (err) {
|
|
45
|
+
console.warn(`Could not open browser: ${err.message}`);
|
|
46
|
+
}
|
|
47
|
+
});
|
|
48
|
+
}
|