snapeval 1.5.0 → 1.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bin/snapeval.ts +49 -197
- package/dist/bin/snapeval.js +43 -159
- package/dist/bin/snapeval.js.map +1 -1
- package/dist/src/adapters/copilot-sdk-client.d.ts +0 -4
- package/dist/src/adapters/copilot-sdk-client.js +0 -4
- package/dist/src/adapters/copilot-sdk-client.js.map +1 -1
- package/dist/src/adapters/harness/copilot-cli.d.ts +12 -0
- package/dist/src/adapters/harness/copilot-cli.js +54 -0
- package/dist/src/adapters/harness/copilot-cli.js.map +1 -0
- package/dist/src/adapters/harness/resolve.d.ts +2 -0
- package/dist/src/adapters/harness/resolve.js +9 -0
- package/dist/src/adapters/harness/resolve.js.map +1 -0
- package/dist/src/adapters/inference/copilot-sdk.d.ts +0 -4
- package/dist/src/adapters/inference/copilot-sdk.js +0 -15
- package/dist/src/adapters/inference/copilot-sdk.js.map +1 -1
- package/dist/src/adapters/inference/copilot.d.ts +0 -4
- package/dist/src/adapters/inference/copilot.js +0 -14
- package/dist/src/adapters/inference/copilot.js.map +1 -1
- package/dist/src/adapters/inference/github-models.d.ts +0 -2
- package/dist/src/adapters/inference/github-models.js +0 -22
- package/dist/src/adapters/inference/github-models.js.map +1 -1
- package/dist/src/adapters/inference/resolve.js +2 -9
- package/dist/src/adapters/inference/resolve.js.map +1 -1
- package/dist/src/adapters/report/terminal.js +15 -32
- package/dist/src/adapters/report/terminal.js.map +1 -1
- package/dist/src/commands/eval.d.ts +6 -0
- package/dist/src/commands/eval.js +55 -0
- package/dist/src/commands/eval.js.map +1 -0
- package/dist/src/commands/review.d.ts +7 -7
- package/dist/src/commands/review.js +22 -24
- package/dist/src/commands/review.js.map +1 -1
- package/dist/src/config.js +3 -3
- package/dist/src/config.js.map +1 -1
- package/dist/src/engine/aggregator.d.ts +2 -0
- package/dist/src/engine/aggregator.js +53 -0
- package/dist/src/engine/aggregator.js.map +1 -0
- package/dist/src/engine/generator.d.ts +0 -5
- package/dist/src/engine/generator.js +4 -18
- package/dist/src/engine/generator.js.map +1 -1
- package/dist/src/engine/grader.d.ts +2 -0
- package/dist/src/engine/grader.js +73 -0
- package/dist/src/engine/grader.js.map +1 -0
- package/dist/src/engine/runner.d.ts +14 -0
- package/dist/src/engine/runner.js +41 -0
- package/dist/src/engine/runner.js.map +1 -0
- package/dist/src/engine/workspace.d.ts +8 -0
- package/dist/src/engine/workspace.js +37 -0
- package/dist/src/engine/workspace.js.map +1 -0
- package/dist/src/errors.d.ts +3 -3
- package/dist/src/errors.js +6 -6
- package/dist/src/errors.js.map +1 -1
- package/dist/src/types.d.ts +58 -98
- package/dist/src/types.js +1 -0
- package/dist/src/types.js.map +1 -1
- package/package.json +4 -14
- package/plugin.json +1 -1
- package/skills/snapeval/SKILL.md +71 -198
- package/src/adapters/copilot-sdk-client.ts +0 -4
- package/src/adapters/harness/copilot-cli.ts +64 -0
- package/src/adapters/harness/resolve.ts +10 -0
- package/src/adapters/inference/copilot-sdk.ts +0 -18
- package/src/adapters/inference/copilot.ts +0 -17
- package/src/adapters/inference/github-models.ts +0 -29
- package/src/adapters/inference/resolve.ts +2 -10
- package/src/adapters/report/terminal.ts +16 -44
- package/src/commands/eval.ts +91 -0
- package/src/commands/review.ts +28 -30
- package/src/config.ts +3 -3
- package/src/engine/aggregator.ts +59 -0
- package/src/engine/generator.ts +4 -26
- package/src/engine/grader.ts +99 -0
- package/src/engine/runner.ts +61 -0
- package/src/engine/workspace.ts +40 -0
- package/src/errors.ts +6 -6
- package/src/types.ts +70 -103
- package/dist/src/adapters/report/html.d.ts +0 -8
- package/dist/src/adapters/report/html.js +0 -283
- package/dist/src/adapters/report/html.js.map +0 -1
- package/dist/src/adapters/report/json.d.ts +0 -7
- package/dist/src/adapters/report/json.js +0 -39
- package/dist/src/adapters/report/json.js.map +0 -1
- package/dist/src/adapters/skill/copilot-cli.d.ts +0 -7
- package/dist/src/adapters/skill/copilot-cli.js +0 -62
- package/dist/src/adapters/skill/copilot-cli.js.map +0 -1
- package/dist/src/adapters/skill/copilot-sdk.d.ts +0 -6
- package/dist/src/adapters/skill/copilot-sdk.js +0 -68
- package/dist/src/adapters/skill/copilot-sdk.js.map +0 -1
- package/dist/src/commands/approve.d.ts +0 -5
- package/dist/src/commands/approve.js +0 -40
- package/dist/src/commands/approve.js.map +0 -1
- package/dist/src/commands/capture.d.ts +0 -4
- package/dist/src/commands/capture.js +0 -18
- package/dist/src/commands/capture.js.map +0 -1
- package/dist/src/commands/check.d.ts +0 -4
- package/dist/src/commands/check.js +0 -67
- package/dist/src/commands/check.js.map +0 -1
- package/dist/src/commands/ideate.d.ts +0 -1
- package/dist/src/commands/ideate.js +0 -69
- package/dist/src/commands/ideate.js.map +0 -1
- package/dist/src/commands/report.d.ts +0 -5
- package/dist/src/commands/report.js +0 -35
- package/dist/src/commands/report.js.map +0 -1
- package/dist/src/engine/budget.d.ts +0 -10
- package/dist/src/engine/budget.js +0 -25
- package/dist/src/engine/budget.js.map +0 -1
- package/dist/src/engine/comparison/embedding.d.ts +0 -6
- package/dist/src/engine/comparison/embedding.js +0 -19
- package/dist/src/engine/comparison/embedding.js.map +0 -1
- package/dist/src/engine/comparison/judge.d.ts +0 -12
- package/dist/src/engine/comparison/judge.js +0 -73
- package/dist/src/engine/comparison/judge.js.map +0 -1
- package/dist/src/engine/comparison/pipeline.d.ts +0 -2
- package/dist/src/engine/comparison/pipeline.js +0 -18
- package/dist/src/engine/comparison/pipeline.js.map +0 -1
- package/dist/src/engine/comparison/schema.d.ts +0 -2
- package/dist/src/engine/comparison/schema.js +0 -28
- package/dist/src/engine/comparison/schema.js.map +0 -1
- package/dist/src/engine/comparison/variance.d.ts +0 -3
- package/dist/src/engine/comparison/variance.js +0 -26
- package/dist/src/engine/comparison/variance.js.map +0 -1
- package/dist/src/engine/snapshot.d.ts +0 -11
- package/dist/src/engine/snapshot.js +0 -46
- package/dist/src/engine/snapshot.js.map +0 -1
- package/src/adapters/report/html.ts +0 -304
- package/src/adapters/report/json.ts +0 -64
- package/src/adapters/skill/copilot-cli.ts +0 -74
- package/src/adapters/skill/copilot-sdk.ts +0 -72
- package/src/commands/approve.ts +0 -58
- package/src/commands/capture.ts +0 -25
- package/src/commands/check.ts +0 -85
- package/src/commands/ideate.ts +0 -101
- package/src/commands/report.ts +0 -47
- package/src/engine/budget.ts +0 -27
- package/src/engine/comparison/embedding.ts +0 -26
- package/src/engine/comparison/judge.ts +0 -88
- package/src/engine/comparison/pipeline.ts +0 -24
- package/src/engine/comparison/schema.ts +0 -22
- package/src/engine/comparison/variance.ts +0 -31
- package/src/engine/snapshot.ts +0 -48
package/bin/snapeval.ts
CHANGED
|
@@ -2,16 +2,11 @@
|
|
|
2
2
|
import { Command } from 'commander';
|
|
3
3
|
import { resolveConfig } from '../src/config.js';
|
|
4
4
|
import { resolveInference } from '../src/adapters/inference/resolve.js';
|
|
5
|
-
import {
|
|
6
|
-
import { CopilotSDKAdapter } from '../src/adapters/skill/copilot-sdk.js';
|
|
7
|
-
import { TerminalReporter } from '../src/adapters/report/terminal.js';
|
|
5
|
+
import { resolveHarness } from '../src/adapters/harness/resolve.js';
|
|
8
6
|
import { initCommand } from '../src/commands/init.js';
|
|
9
|
-
import {
|
|
10
|
-
import { checkCommand } from '../src/commands/check.js';
|
|
11
|
-
import { approveCommand, approveFromResults } from '../src/commands/approve.js';
|
|
12
|
-
import { reportCommand } from '../src/commands/report.js';
|
|
13
|
-
import { ideateCommand } from '../src/commands/ideate.js';
|
|
7
|
+
import { evalCommand } from '../src/commands/eval.js';
|
|
14
8
|
import { reviewCommand } from '../src/commands/review.js';
|
|
9
|
+
import { TerminalReporter } from '../src/adapters/report/terminal.js';
|
|
15
10
|
import { SnapevalError } from '../src/errors.js';
|
|
16
11
|
import * as path from 'node:path';
|
|
17
12
|
|
|
@@ -19,41 +14,40 @@ const program = new Command();
|
|
|
19
14
|
|
|
20
15
|
program
|
|
21
16
|
.name('snapeval')
|
|
22
|
-
.description('
|
|
23
|
-
.version('
|
|
17
|
+
.description('Harness-agnostic eval runner for agentskills.io skills')
|
|
18
|
+
.version('2.0.0');
|
|
24
19
|
|
|
25
20
|
// --- init ---
|
|
26
21
|
program
|
|
27
22
|
.command('init')
|
|
28
|
-
.description('Generate
|
|
29
|
-
.option('--
|
|
30
|
-
.option('--inference <inference>', 'Inference adapter to use'
|
|
23
|
+
.description('Generate evals.json from SKILL.md (prompts + expected outputs, no assertions)')
|
|
24
|
+
.option('--harness <harness>', 'Harness to use')
|
|
25
|
+
.option('--inference <inference>', 'Inference adapter to use')
|
|
31
26
|
.option('--verbose', 'Verbose output')
|
|
32
27
|
.argument('[skill-dir]', 'Path to skill directory', process.cwd())
|
|
33
28
|
.action(async (skillDir: string, opts: Record<string, string | boolean>) => {
|
|
34
29
|
try {
|
|
35
30
|
const skillPath = path.resolve(skillDir);
|
|
36
31
|
const config = resolveConfig(
|
|
37
|
-
{
|
|
38
|
-
process.cwd(),
|
|
39
|
-
skillPath
|
|
32
|
+
{ harness: opts.harness as string, inference: opts.inference as string },
|
|
33
|
+
process.cwd(), skillPath
|
|
40
34
|
);
|
|
41
35
|
const inference = resolveInference(config.inference);
|
|
42
36
|
await initCommand(skillPath, inference);
|
|
43
37
|
console.log(`Generated evals at ${path.join(skillPath, 'evals', 'evals.json')}`);
|
|
44
38
|
process.exit(0);
|
|
45
|
-
} catch (err) {
|
|
46
|
-
handleError(err);
|
|
47
|
-
}
|
|
39
|
+
} catch (err) { handleError(err); }
|
|
48
40
|
});
|
|
49
41
|
|
|
50
|
-
// ---
|
|
42
|
+
// --- eval ---
|
|
51
43
|
program
|
|
52
|
-
.command('
|
|
53
|
-
.description('Run
|
|
54
|
-
.option('--
|
|
55
|
-
.option('--inference <inference>', 'Inference adapter to use'
|
|
56
|
-
.option('--
|
|
44
|
+
.command('eval')
|
|
45
|
+
.description('Run evals (with/without skill), grade assertions, compute benchmark')
|
|
46
|
+
.option('--harness <harness>', 'Harness to use')
|
|
47
|
+
.option('--inference <inference>', 'Inference adapter to use')
|
|
48
|
+
.option('--workspace <path>', 'Workspace directory')
|
|
49
|
+
.option('--runs <n>', 'Runs per eval for statistical significance', '1')
|
|
50
|
+
.option('--old-skill <path>', 'Compare against old skill version instead of no-skill')
|
|
57
51
|
.option('--verbose', 'Verbose output')
|
|
58
52
|
.argument('[skill-dir]', 'Path to skill directory', process.cwd())
|
|
59
53
|
.action(async (skillDir: string, opts: Record<string, string | boolean>) => {
|
|
@@ -61,148 +55,39 @@ program
|
|
|
61
55
|
const skillPath = path.resolve(skillDir);
|
|
62
56
|
const config = resolveConfig(
|
|
63
57
|
{
|
|
64
|
-
|
|
58
|
+
harness: opts.harness as string,
|
|
65
59
|
inference: opts.inference as string,
|
|
60
|
+
workspace: opts.workspace as string,
|
|
66
61
|
runs: opts.runs ? parseInt(opts.runs as string, 10) : undefined,
|
|
67
62
|
},
|
|
68
|
-
process.cwd(),
|
|
69
|
-
skillPath
|
|
70
|
-
);
|
|
71
|
-
const skillAdapter = resolveSkillAdapter(config.adapter);
|
|
72
|
-
await captureCommand(skillPath, skillAdapter, { runs: config.runs });
|
|
73
|
-
console.log(`Captured baselines at ${path.join(skillPath, 'evals', 'snapshots')}`);
|
|
74
|
-
process.exit(0);
|
|
75
|
-
} catch (err) {
|
|
76
|
-
handleError(err);
|
|
77
|
-
}
|
|
78
|
-
});
|
|
79
|
-
|
|
80
|
-
// --- check ---
|
|
81
|
-
program
|
|
82
|
-
.command('check')
|
|
83
|
-
.description('Compare current skill output against baselines')
|
|
84
|
-
.option('--adapter <adapter>', 'Skill adapter to use', 'copilot-cli')
|
|
85
|
-
.option('--inference <inference>', 'Inference adapter to use', 'auto')
|
|
86
|
-
.option('--budget <amount>', 'Spend cap in USD (or "unlimited")', 'unlimited')
|
|
87
|
-
.option('--ci', 'CI mode: exit 1 on regressions, no interactive prompts')
|
|
88
|
-
.option('--verbose', 'Verbose output')
|
|
89
|
-
.option('--scenario <ids>', 'Comma-separated scenario IDs to check')
|
|
90
|
-
.argument('[skill-dir]', 'Path to skill directory', process.cwd())
|
|
91
|
-
.action(async (skillDir: string, opts: Record<string, string | boolean>) => {
|
|
92
|
-
try {
|
|
93
|
-
const skillPath = path.resolve(skillDir);
|
|
94
|
-
const config = resolveConfig(
|
|
95
|
-
{
|
|
96
|
-
adapter: opts.adapter as string,
|
|
97
|
-
inference: opts.inference as string,
|
|
98
|
-
budget: opts.budget as string,
|
|
99
|
-
},
|
|
100
|
-
process.cwd(),
|
|
101
|
-
skillPath
|
|
102
|
-
);
|
|
103
|
-
const skillAdapter = resolveSkillAdapter(config.adapter);
|
|
104
|
-
const inference = resolveInference(config.inference);
|
|
105
|
-
|
|
106
|
-
const results = await checkCommand(skillPath, skillAdapter, inference, {
|
|
107
|
-
budget: config.budget,
|
|
108
|
-
});
|
|
109
|
-
|
|
110
|
-
// Always print terminal report
|
|
111
|
-
const reporter = new TerminalReporter();
|
|
112
|
-
await reporter.report(results);
|
|
113
|
-
|
|
114
|
-
const hasRegressions = results.summary.regressed > 0;
|
|
115
|
-
if (hasRegressions) {
|
|
116
|
-
process.exit(1);
|
|
117
|
-
}
|
|
118
|
-
process.exit(0);
|
|
119
|
-
} catch (err) {
|
|
120
|
-
handleError(err);
|
|
121
|
-
}
|
|
122
|
-
});
|
|
123
|
-
|
|
124
|
-
// --- approve ---
|
|
125
|
-
program
|
|
126
|
-
.command('approve')
|
|
127
|
-
.description('Approve regressed scenarios as new baselines')
|
|
128
|
-
.option('--adapter <adapter>', 'Skill adapter to use', 'copilot-cli')
|
|
129
|
-
.option('--inference <inference>', 'Inference adapter to use', 'auto')
|
|
130
|
-
.option('--scenario <ids>', 'Comma-separated scenario IDs to approve (default: all)')
|
|
131
|
-
.option('--verbose', 'Verbose output')
|
|
132
|
-
.argument('[skill-dir]', 'Path to skill directory', process.cwd())
|
|
133
|
-
.action(async (skillDir: string, opts: Record<string, string | boolean>) => {
|
|
134
|
-
try {
|
|
135
|
-
const skillPath = path.resolve(skillDir);
|
|
136
|
-
const config = resolveConfig(
|
|
137
|
-
{ adapter: opts.adapter as string, inference: opts.inference as string },
|
|
138
|
-
process.cwd(),
|
|
139
|
-
skillPath
|
|
140
|
-
);
|
|
141
|
-
const skillAdapter = resolveSkillAdapter(config.adapter);
|
|
142
|
-
|
|
143
|
-
const scenarioIds = opts.scenario
|
|
144
|
-
? (opts.scenario as string).split(',').map((s) => parseInt(s.trim(), 10))
|
|
145
|
-
: undefined;
|
|
146
|
-
|
|
147
|
-
await approveCommand(skillPath, skillAdapter, { scenarioIds });
|
|
148
|
-
console.log('Approved snapshots updated.');
|
|
149
|
-
process.exit(0);
|
|
150
|
-
} catch (err) {
|
|
151
|
-
handleError(err);
|
|
152
|
-
}
|
|
153
|
-
});
|
|
154
|
-
|
|
155
|
-
// --- report ---
|
|
156
|
-
program
|
|
157
|
-
.command('report')
|
|
158
|
-
.description('Write latest check results to evals/results/iteration-N/')
|
|
159
|
-
.option('--adapter <adapter>', 'Skill adapter to use', 'copilot-cli')
|
|
160
|
-
.option('--inference <inference>', 'Inference adapter to use', 'auto')
|
|
161
|
-
.option('--budget <amount>', 'Spend cap in USD (or "unlimited")', 'unlimited')
|
|
162
|
-
.option('--verbose', 'Verbose output')
|
|
163
|
-
.option('--html', 'Generate HTML report viewer')
|
|
164
|
-
.argument('[skill-dir]', 'Path to skill directory', process.cwd())
|
|
165
|
-
.action(async (skillDir: string, opts: Record<string, string | boolean>) => {
|
|
166
|
-
try {
|
|
167
|
-
const skillPath = path.resolve(skillDir);
|
|
168
|
-
const config = resolveConfig(
|
|
169
|
-
{
|
|
170
|
-
adapter: opts.adapter as string,
|
|
171
|
-
inference: opts.inference as string,
|
|
172
|
-
budget: opts.budget as string,
|
|
173
|
-
},
|
|
174
|
-
process.cwd(),
|
|
175
|
-
skillPath
|
|
63
|
+
process.cwd(), skillPath
|
|
176
64
|
);
|
|
177
|
-
const
|
|
65
|
+
const harness = resolveHarness(config.harness);
|
|
178
66
|
const inference = resolveInference(config.inference);
|
|
179
67
|
|
|
180
|
-
const results = await
|
|
181
|
-
|
|
68
|
+
const results = await evalCommand(skillPath, harness, inference, {
|
|
69
|
+
workspace: config.workspace,
|
|
70
|
+
runs: config.runs,
|
|
71
|
+
oldSkill: opts['old-skill'] as string | undefined,
|
|
182
72
|
});
|
|
183
73
|
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
});
|
|
188
|
-
|
|
189
|
-
const hasRegressions = results.summary.regressed > 0;
|
|
190
|
-
if (hasRegressions) {
|
|
191
|
-
process.exit(1);
|
|
192
|
-
}
|
|
74
|
+
const terminal = new TerminalReporter();
|
|
75
|
+
await terminal.report(results);
|
|
76
|
+
console.log(`Results at ${results.iterationDir}`);
|
|
193
77
|
process.exit(0);
|
|
194
|
-
} catch (err) {
|
|
195
|
-
handleError(err);
|
|
196
|
-
}
|
|
78
|
+
} catch (err) { handleError(err); }
|
|
197
79
|
});
|
|
198
80
|
|
|
199
81
|
// --- review ---
|
|
200
82
|
program
|
|
201
83
|
.command('review')
|
|
202
|
-
.description('Run
|
|
203
|
-
.option('--
|
|
204
|
-
.option('--inference <inference>', 'Inference adapter to use'
|
|
205
|
-
.option('--
|
|
84
|
+
.description('Run eval + generate HTML report + open in browser')
|
|
85
|
+
.option('--harness <harness>', 'Harness to use')
|
|
86
|
+
.option('--inference <inference>', 'Inference adapter to use')
|
|
87
|
+
.option('--workspace <path>', 'Workspace directory')
|
|
88
|
+
.option('--runs <n>', 'Runs per eval for statistical significance', '1')
|
|
89
|
+
.option('--old-skill <path>', 'Compare against old skill version instead of no-skill')
|
|
90
|
+
.option('--no-open', 'Do not open browser')
|
|
206
91
|
.option('--verbose', 'Verbose output')
|
|
207
92
|
.argument('[skill-dir]', 'Path to skill directory', process.cwd())
|
|
208
93
|
.action(async (skillDir: string, opts: Record<string, string | boolean>) => {
|
|
@@ -210,59 +95,26 @@ program
|
|
|
210
95
|
const skillPath = path.resolve(skillDir);
|
|
211
96
|
const config = resolveConfig(
|
|
212
97
|
{
|
|
213
|
-
|
|
98
|
+
harness: opts.harness as string,
|
|
214
99
|
inference: opts.inference as string,
|
|
215
|
-
|
|
100
|
+
workspace: opts.workspace as string,
|
|
101
|
+
runs: opts.runs ? parseInt(opts.runs as string, 10) : undefined,
|
|
216
102
|
},
|
|
217
|
-
process.cwd(),
|
|
218
|
-
skillPath
|
|
103
|
+
process.cwd(), skillPath
|
|
219
104
|
);
|
|
220
|
-
const
|
|
105
|
+
const harness = resolveHarness(config.harness);
|
|
221
106
|
const inference = resolveInference(config.inference);
|
|
222
107
|
|
|
223
|
-
|
|
224
|
-
|
|
108
|
+
await reviewCommand(skillPath, harness, inference, {
|
|
109
|
+
workspace: config.workspace,
|
|
110
|
+
runs: config.runs,
|
|
111
|
+
oldSkill: opts['old-skill'] as string | undefined,
|
|
112
|
+
noOpen: opts.open === false,
|
|
225
113
|
});
|
|
226
|
-
|
|
227
|
-
if (hasRegressions) {
|
|
228
|
-
process.exit(1);
|
|
229
|
-
}
|
|
230
|
-
process.exit(0);
|
|
231
|
-
} catch (err) {
|
|
232
|
-
handleError(err);
|
|
233
|
-
}
|
|
234
|
-
});
|
|
235
|
-
|
|
236
|
-
// --- ideate ---
|
|
237
|
-
program
|
|
238
|
-
.command('ideate')
|
|
239
|
-
.description('Open the interactive scenario ideation viewer')
|
|
240
|
-
.argument('[skill-dir]', 'Path to skill directory', process.cwd())
|
|
241
|
-
.action(async (skillDir: string) => {
|
|
242
|
-
try {
|
|
243
|
-
const skillPath = path.resolve(skillDir);
|
|
244
|
-
const outputPath = await ideateCommand(skillPath);
|
|
245
|
-
console.log(`Ideation viewer opened: ${outputPath}`);
|
|
246
114
|
process.exit(0);
|
|
247
|
-
} catch (err) {
|
|
248
|
-
handleError(err);
|
|
249
|
-
}
|
|
115
|
+
} catch (err) { handleError(err); }
|
|
250
116
|
});
|
|
251
117
|
|
|
252
|
-
// --- helpers ---
|
|
253
|
-
|
|
254
|
-
function resolveSkillAdapter(adapterName: string) {
|
|
255
|
-
if (adapterName === 'copilot-cli') {
|
|
256
|
-
return new CopilotCLIAdapter();
|
|
257
|
-
}
|
|
258
|
-
if (adapterName === 'copilot-sdk') {
|
|
259
|
-
return new CopilotSDKAdapter();
|
|
260
|
-
}
|
|
261
|
-
throw new SnapevalError(
|
|
262
|
-
`Unknown skill adapter "${adapterName}". Valid options: copilot-cli, copilot-sdk.`
|
|
263
|
-
);
|
|
264
|
-
}
|
|
265
|
-
|
|
266
118
|
function handleError(err: unknown): never {
|
|
267
119
|
if (err instanceof SnapevalError) {
|
|
268
120
|
console.error(`Error: ${err.message}`);
|
package/dist/bin/snapeval.js
CHANGED
|
@@ -2,35 +2,30 @@
|
|
|
2
2
|
import { Command } from 'commander';
|
|
3
3
|
import { resolveConfig } from '../src/config.js';
|
|
4
4
|
import { resolveInference } from '../src/adapters/inference/resolve.js';
|
|
5
|
-
import {
|
|
6
|
-
import { CopilotSDKAdapter } from '../src/adapters/skill/copilot-sdk.js';
|
|
7
|
-
import { TerminalReporter } from '../src/adapters/report/terminal.js';
|
|
5
|
+
import { resolveHarness } from '../src/adapters/harness/resolve.js';
|
|
8
6
|
import { initCommand } from '../src/commands/init.js';
|
|
9
|
-
import {
|
|
10
|
-
import { checkCommand } from '../src/commands/check.js';
|
|
11
|
-
import { approveCommand } from '../src/commands/approve.js';
|
|
12
|
-
import { reportCommand } from '../src/commands/report.js';
|
|
13
|
-
import { ideateCommand } from '../src/commands/ideate.js';
|
|
7
|
+
import { evalCommand } from '../src/commands/eval.js';
|
|
14
8
|
import { reviewCommand } from '../src/commands/review.js';
|
|
9
|
+
import { TerminalReporter } from '../src/adapters/report/terminal.js';
|
|
15
10
|
import { SnapevalError } from '../src/errors.js';
|
|
16
11
|
import * as path from 'node:path';
|
|
17
12
|
const program = new Command();
|
|
18
13
|
program
|
|
19
14
|
.name('snapeval')
|
|
20
|
-
.description('
|
|
21
|
-
.version('
|
|
15
|
+
.description('Harness-agnostic eval runner for agentskills.io skills')
|
|
16
|
+
.version('2.0.0');
|
|
22
17
|
// --- init ---
|
|
23
18
|
program
|
|
24
19
|
.command('init')
|
|
25
|
-
.description('Generate
|
|
26
|
-
.option('--
|
|
27
|
-
.option('--inference <inference>', 'Inference adapter to use'
|
|
20
|
+
.description('Generate evals.json from SKILL.md (prompts + expected outputs, no assertions)')
|
|
21
|
+
.option('--harness <harness>', 'Harness to use')
|
|
22
|
+
.option('--inference <inference>', 'Inference adapter to use')
|
|
28
23
|
.option('--verbose', 'Verbose output')
|
|
29
24
|
.argument('[skill-dir]', 'Path to skill directory', process.cwd())
|
|
30
25
|
.action(async (skillDir, opts) => {
|
|
31
26
|
try {
|
|
32
27
|
const skillPath = path.resolve(skillDir);
|
|
33
|
-
const config = resolveConfig({
|
|
28
|
+
const config = resolveConfig({ harness: opts.harness, inference: opts.inference }, process.cwd(), skillPath);
|
|
34
29
|
const inference = resolveInference(config.inference);
|
|
35
30
|
await initCommand(skillPath, inference);
|
|
36
31
|
console.log(`Generated evals at ${path.join(skillPath, 'evals', 'evals.json')}`);
|
|
@@ -40,125 +35,36 @@ program
|
|
|
40
35
|
handleError(err);
|
|
41
36
|
}
|
|
42
37
|
});
|
|
43
|
-
// ---
|
|
38
|
+
// --- eval ---
|
|
44
39
|
program
|
|
45
|
-
.command('
|
|
46
|
-
.description('Run
|
|
47
|
-
.option('--
|
|
48
|
-
.option('--inference <inference>', 'Inference adapter to use'
|
|
49
|
-
.option('--
|
|
40
|
+
.command('eval')
|
|
41
|
+
.description('Run evals (with/without skill), grade assertions, compute benchmark')
|
|
42
|
+
.option('--harness <harness>', 'Harness to use')
|
|
43
|
+
.option('--inference <inference>', 'Inference adapter to use')
|
|
44
|
+
.option('--workspace <path>', 'Workspace directory')
|
|
45
|
+
.option('--runs <n>', 'Runs per eval for statistical significance', '1')
|
|
46
|
+
.option('--old-skill <path>', 'Compare against old skill version instead of no-skill')
|
|
50
47
|
.option('--verbose', 'Verbose output')
|
|
51
48
|
.argument('[skill-dir]', 'Path to skill directory', process.cwd())
|
|
52
49
|
.action(async (skillDir, opts) => {
|
|
53
50
|
try {
|
|
54
51
|
const skillPath = path.resolve(skillDir);
|
|
55
52
|
const config = resolveConfig({
|
|
56
|
-
|
|
53
|
+
harness: opts.harness,
|
|
57
54
|
inference: opts.inference,
|
|
55
|
+
workspace: opts.workspace,
|
|
58
56
|
runs: opts.runs ? parseInt(opts.runs, 10) : undefined,
|
|
59
57
|
}, process.cwd(), skillPath);
|
|
60
|
-
const
|
|
61
|
-
await captureCommand(skillPath, skillAdapter, { runs: config.runs });
|
|
62
|
-
console.log(`Captured baselines at ${path.join(skillPath, 'evals', 'snapshots')}`);
|
|
63
|
-
process.exit(0);
|
|
64
|
-
}
|
|
65
|
-
catch (err) {
|
|
66
|
-
handleError(err);
|
|
67
|
-
}
|
|
68
|
-
});
|
|
69
|
-
// --- check ---
|
|
70
|
-
program
|
|
71
|
-
.command('check')
|
|
72
|
-
.description('Compare current skill output against baselines')
|
|
73
|
-
.option('--adapter <adapter>', 'Skill adapter to use', 'copilot-cli')
|
|
74
|
-
.option('--inference <inference>', 'Inference adapter to use', 'auto')
|
|
75
|
-
.option('--budget <amount>', 'Spend cap in USD (or "unlimited")', 'unlimited')
|
|
76
|
-
.option('--ci', 'CI mode: exit 1 on regressions, no interactive prompts')
|
|
77
|
-
.option('--verbose', 'Verbose output')
|
|
78
|
-
.option('--scenario <ids>', 'Comma-separated scenario IDs to check')
|
|
79
|
-
.argument('[skill-dir]', 'Path to skill directory', process.cwd())
|
|
80
|
-
.action(async (skillDir, opts) => {
|
|
81
|
-
try {
|
|
82
|
-
const skillPath = path.resolve(skillDir);
|
|
83
|
-
const config = resolveConfig({
|
|
84
|
-
adapter: opts.adapter,
|
|
85
|
-
inference: opts.inference,
|
|
86
|
-
budget: opts.budget,
|
|
87
|
-
}, process.cwd(), skillPath);
|
|
88
|
-
const skillAdapter = resolveSkillAdapter(config.adapter);
|
|
58
|
+
const harness = resolveHarness(config.harness);
|
|
89
59
|
const inference = resolveInference(config.inference);
|
|
90
|
-
const results = await
|
|
91
|
-
|
|
60
|
+
const results = await evalCommand(skillPath, harness, inference, {
|
|
61
|
+
workspace: config.workspace,
|
|
62
|
+
runs: config.runs,
|
|
63
|
+
oldSkill: opts['old-skill'],
|
|
92
64
|
});
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
const hasRegressions = results.summary.regressed > 0;
|
|
97
|
-
if (hasRegressions) {
|
|
98
|
-
process.exit(1);
|
|
99
|
-
}
|
|
100
|
-
process.exit(0);
|
|
101
|
-
}
|
|
102
|
-
catch (err) {
|
|
103
|
-
handleError(err);
|
|
104
|
-
}
|
|
105
|
-
});
|
|
106
|
-
// --- approve ---
|
|
107
|
-
program
|
|
108
|
-
.command('approve')
|
|
109
|
-
.description('Approve regressed scenarios as new baselines')
|
|
110
|
-
.option('--adapter <adapter>', 'Skill adapter to use', 'copilot-cli')
|
|
111
|
-
.option('--inference <inference>', 'Inference adapter to use', 'auto')
|
|
112
|
-
.option('--scenario <ids>', 'Comma-separated scenario IDs to approve (default: all)')
|
|
113
|
-
.option('--verbose', 'Verbose output')
|
|
114
|
-
.argument('[skill-dir]', 'Path to skill directory', process.cwd())
|
|
115
|
-
.action(async (skillDir, opts) => {
|
|
116
|
-
try {
|
|
117
|
-
const skillPath = path.resolve(skillDir);
|
|
118
|
-
const config = resolveConfig({ adapter: opts.adapter, inference: opts.inference }, process.cwd(), skillPath);
|
|
119
|
-
const skillAdapter = resolveSkillAdapter(config.adapter);
|
|
120
|
-
const scenarioIds = opts.scenario
|
|
121
|
-
? opts.scenario.split(',').map((s) => parseInt(s.trim(), 10))
|
|
122
|
-
: undefined;
|
|
123
|
-
await approveCommand(skillPath, skillAdapter, { scenarioIds });
|
|
124
|
-
console.log('Approved snapshots updated.');
|
|
125
|
-
process.exit(0);
|
|
126
|
-
}
|
|
127
|
-
catch (err) {
|
|
128
|
-
handleError(err);
|
|
129
|
-
}
|
|
130
|
-
});
|
|
131
|
-
// --- report ---
|
|
132
|
-
program
|
|
133
|
-
.command('report')
|
|
134
|
-
.description('Write latest check results to evals/results/iteration-N/')
|
|
135
|
-
.option('--adapter <adapter>', 'Skill adapter to use', 'copilot-cli')
|
|
136
|
-
.option('--inference <inference>', 'Inference adapter to use', 'auto')
|
|
137
|
-
.option('--budget <amount>', 'Spend cap in USD (or "unlimited")', 'unlimited')
|
|
138
|
-
.option('--verbose', 'Verbose output')
|
|
139
|
-
.option('--html', 'Generate HTML report viewer')
|
|
140
|
-
.argument('[skill-dir]', 'Path to skill directory', process.cwd())
|
|
141
|
-
.action(async (skillDir, opts) => {
|
|
142
|
-
try {
|
|
143
|
-
const skillPath = path.resolve(skillDir);
|
|
144
|
-
const config = resolveConfig({
|
|
145
|
-
adapter: opts.adapter,
|
|
146
|
-
inference: opts.inference,
|
|
147
|
-
budget: opts.budget,
|
|
148
|
-
}, process.cwd(), skillPath);
|
|
149
|
-
const skillAdapter = resolveSkillAdapter(config.adapter);
|
|
150
|
-
const inference = resolveInference(config.inference);
|
|
151
|
-
const results = await checkCommand(skillPath, skillAdapter, inference, {
|
|
152
|
-
budget: config.budget,
|
|
153
|
-
});
|
|
154
|
-
await reportCommand(skillPath, results, {
|
|
155
|
-
verbose: Boolean(opts.verbose),
|
|
156
|
-
html: Boolean(opts.html),
|
|
157
|
-
});
|
|
158
|
-
const hasRegressions = results.summary.regressed > 0;
|
|
159
|
-
if (hasRegressions) {
|
|
160
|
-
process.exit(1);
|
|
161
|
-
}
|
|
65
|
+
const terminal = new TerminalReporter();
|
|
66
|
+
await terminal.report(results);
|
|
67
|
+
console.log(`Results at ${results.iterationDir}`);
|
|
162
68
|
process.exit(0);
|
|
163
69
|
}
|
|
164
70
|
catch (err) {
|
|
@@ -168,60 +74,38 @@ program
|
|
|
168
74
|
// --- review ---
|
|
169
75
|
program
|
|
170
76
|
.command('review')
|
|
171
|
-
.description('Run
|
|
172
|
-
.option('--
|
|
173
|
-
.option('--inference <inference>', 'Inference adapter to use'
|
|
174
|
-
.option('--
|
|
77
|
+
.description('Run eval + generate HTML report + open in browser')
|
|
78
|
+
.option('--harness <harness>', 'Harness to use')
|
|
79
|
+
.option('--inference <inference>', 'Inference adapter to use')
|
|
80
|
+
.option('--workspace <path>', 'Workspace directory')
|
|
81
|
+
.option('--runs <n>', 'Runs per eval for statistical significance', '1')
|
|
82
|
+
.option('--old-skill <path>', 'Compare against old skill version instead of no-skill')
|
|
83
|
+
.option('--no-open', 'Do not open browser')
|
|
175
84
|
.option('--verbose', 'Verbose output')
|
|
176
85
|
.argument('[skill-dir]', 'Path to skill directory', process.cwd())
|
|
177
86
|
.action(async (skillDir, opts) => {
|
|
178
87
|
try {
|
|
179
88
|
const skillPath = path.resolve(skillDir);
|
|
180
89
|
const config = resolveConfig({
|
|
181
|
-
|
|
90
|
+
harness: opts.harness,
|
|
182
91
|
inference: opts.inference,
|
|
183
|
-
|
|
92
|
+
workspace: opts.workspace,
|
|
93
|
+
runs: opts.runs ? parseInt(opts.runs, 10) : undefined,
|
|
184
94
|
}, process.cwd(), skillPath);
|
|
185
|
-
const
|
|
95
|
+
const harness = resolveHarness(config.harness);
|
|
186
96
|
const inference = resolveInference(config.inference);
|
|
187
|
-
|
|
188
|
-
|
|
97
|
+
await reviewCommand(skillPath, harness, inference, {
|
|
98
|
+
workspace: config.workspace,
|
|
99
|
+
runs: config.runs,
|
|
100
|
+
oldSkill: opts['old-skill'],
|
|
101
|
+
noOpen: opts.open === false,
|
|
189
102
|
});
|
|
190
|
-
if (hasRegressions) {
|
|
191
|
-
process.exit(1);
|
|
192
|
-
}
|
|
193
103
|
process.exit(0);
|
|
194
104
|
}
|
|
195
105
|
catch (err) {
|
|
196
106
|
handleError(err);
|
|
197
107
|
}
|
|
198
108
|
});
|
|
199
|
-
// --- ideate ---
|
|
200
|
-
program
|
|
201
|
-
.command('ideate')
|
|
202
|
-
.description('Open the interactive scenario ideation viewer')
|
|
203
|
-
.argument('[skill-dir]', 'Path to skill directory', process.cwd())
|
|
204
|
-
.action(async (skillDir) => {
|
|
205
|
-
try {
|
|
206
|
-
const skillPath = path.resolve(skillDir);
|
|
207
|
-
const outputPath = await ideateCommand(skillPath);
|
|
208
|
-
console.log(`Ideation viewer opened: ${outputPath}`);
|
|
209
|
-
process.exit(0);
|
|
210
|
-
}
|
|
211
|
-
catch (err) {
|
|
212
|
-
handleError(err);
|
|
213
|
-
}
|
|
214
|
-
});
|
|
215
|
-
// --- helpers ---
|
|
216
|
-
function resolveSkillAdapter(adapterName) {
|
|
217
|
-
if (adapterName === 'copilot-cli') {
|
|
218
|
-
return new CopilotCLIAdapter();
|
|
219
|
-
}
|
|
220
|
-
if (adapterName === 'copilot-sdk') {
|
|
221
|
-
return new CopilotSDKAdapter();
|
|
222
|
-
}
|
|
223
|
-
throw new SnapevalError(`Unknown skill adapter "${adapterName}". Valid options: copilot-cli, copilot-sdk.`);
|
|
224
|
-
}
|
|
225
109
|
function handleError(err) {
|
|
226
110
|
if (err instanceof SnapevalError) {
|
|
227
111
|
console.error(`Error: ${err.message}`);
|
package/dist/bin/snapeval.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"snapeval.js","sourceRoot":"","sources":["../../bin/snapeval.ts"],"names":[],"mappings":";AACA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AACpC,OAAO,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAC;AACjD,OAAO,EAAE,gBAAgB,EAAE,MAAM,sCAAsC,CAAC;AACxE,OAAO,EAAE,
|
|
1
|
+
{"version":3,"file":"snapeval.js","sourceRoot":"","sources":["../../bin/snapeval.ts"],"names":[],"mappings":";AACA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AACpC,OAAO,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAC;AACjD,OAAO,EAAE,gBAAgB,EAAE,MAAM,sCAAsC,CAAC;AACxE,OAAO,EAAE,cAAc,EAAE,MAAM,oCAAoC,CAAC;AACpE,OAAO,EAAE,WAAW,EAAE,MAAM,yBAAyB,CAAC;AACtD,OAAO,EAAE,WAAW,EAAE,MAAM,yBAAyB,CAAC;AACtD,OAAO,EAAE,aAAa,EAAE,MAAM,2BAA2B,CAAC;AAC1D,OAAO,EAAE,gBAAgB,EAAE,MAAM,oCAAoC,CAAC;AACtE,OAAO,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAC;AACjD,OAAO,KAAK,IAAI,MAAM,WAAW,CAAC;AAElC,MAAM,OAAO,GAAG,IAAI,OAAO,EAAE,CAAC;AAE9B,OAAO;KACJ,IAAI,CAAC,UAAU,CAAC;KAChB,WAAW,CAAC,wDAAwD,CAAC;KACrE,OAAO,CAAC,OAAO,CAAC,CAAC;AAEpB,eAAe;AACf,OAAO;KACJ,OAAO,CAAC,MAAM,CAAC;KACf,WAAW,CAAC,+EAA+E,CAAC;KAC5F,MAAM,CAAC,qBAAqB,EAAE,gBAAgB,CAAC;KAC/C,MAAM,CAAC,yBAAyB,EAAE,0BAA0B,CAAC;KAC7D,MAAM,CAAC,WAAW,EAAE,gBAAgB,CAAC;KACrC,QAAQ,CAAC,aAAa,EAAE,yBAAyB,EAAE,OAAO,CAAC,GAAG,EAAE,CAAC;KACjE,MAAM,CAAC,KAAK,EAAE,QAAgB,EAAE,IAAsC,EAAE,EAAE;IACzE,IAAI,CAAC;QACH,MAAM,SAAS,GAAG,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;QACzC,MAAM,MAAM,GAAG,aAAa,CAC1B,EAAE,OAAO,EAAE,IAAI,CAAC,OAAiB,EAAE,SAAS,EAAE,IAAI,CAAC,SAAmB,EAAE,EACxE,OAAO,CAAC,GAAG,EAAE,EAAE,SAAS,CACzB,CAAC;QACF,MAAM,SAAS,GAAG,gBAAgB,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;QACrD,MAAM,WAAW,CAAC,SAAS,EAAE,SAAS,CAAC,CAAC;QACxC,OAAO,CAAC,GAAG,CAAC,sBAAsB,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,OAAO,EAAE,YAAY,CAAC,EAAE,CAAC,CAAC;QACjF,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QAAC,WAAW,CAAC,GAAG,CAAC,CAAC;IAAC,CAAC;AACrC,CAAC,CAAC,CAAC;AAEL,eAAe;AACf,OAAO;KACJ,OAAO,CAAC,MAAM,CAAC;KACf,WAAW,CAAC,qEAAqE,CAAC;KAClF,MAAM,CAAC,qBAAqB,EAAE,gBAAgB,CAAC;KAC/C,MAAM,CAAC,yBAAyB,EAAE,0BAA0B,CAAC;KAC7D,MAAM,CAAC,oBAAoB,EAAE,qBAAqB,CAAC;KACnD,MAAM,CAAC,YAAY,EAAE,4CAA4C,EAAE,GAAG,CAAC;KACvE,MAAM,CAAC,oBAAoB,EAAE,uDAAuD,CAAC;KACrF,MAAM,CAAC,WAAW,EAAE,gBAAgB,CAAC;KACrC,QAAQ,CAAC,aAAa,EAAE,yBAAyB,EAAE,OAAO,CAAC,GAAG,EAAE,CAAC;KACjE,MAAM,CAAC,KAAK,EAAE,QAAgB,EAAE,IAAsC,EAAE,EAAE;IACzE,IAAI,CAAC;QACH,MAAM,SAAS,GAAG,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;QACzC,MAAM,MAAM,GAAG,aAAa,CAC1B;YACE,OAAO,EAAE,IAAI,CAAC,OAAiB;YAC/B,SAAS,EAAE,IAAI,CAAC,SAAmB;YACnC,SAAS,EAAE,IAAI,CAAC,SAAmB;YACnC,IAAI,EAAE,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAc,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,SAAS;SAChE,EACD,OAAO,CAAC,GAAG,EAAE,EAAE,SAAS,CACzB,CAAC;QACF,MAAM,OAAO,GAAG,cAAc,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;QAC/C,MAAM,SAAS,GAAG,gBAAgB,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;QAErD,MAAM,OAAO,GAAG,MAAM,WAAW,CAAC,SAAS,EAAE,OAAO,EAAE,SAAS,EAAE;YAC/D,SAAS,EAAE,MAAM,CAAC,SAAS;YAC3B,IAAI,EAAE,MAAM,CAAC,IAAI;YACjB,QAAQ,EAAE,IAAI,CAAC,WAAW,CAAuB;SAClD,CAAC,CAAC;QAEH,MAAM,QAAQ,GAAG,IAAI,gBAAgB,EAAE,CAAC;QACxC,MAAM,QAAQ,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;QAC/B,OAAO,CAAC,GAAG,CAAC,cAAc,OAAO,CAAC,YAAY,EAAE,CAAC,CAAC;QAClD,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QAAC,WAAW,CAAC,GAAG,CAAC,CAAC;IAAC,CAAC;AACrC,CAAC,CAAC,CAAC;AAEL,iBAAiB;AACjB,OAAO;KACJ,OAAO,CAAC,QAAQ,CAAC;KACjB,WAAW,CAAC,mDAAmD,CAAC;KAChE,MAAM,CAAC,qBAAqB,EAAE,gBAAgB,CAAC;KAC/C,MAAM,CAAC,yBAAyB,EAAE,0BAA0B,CAAC;KAC7D,MAAM,CAAC,oBAAoB,EAAE,qBAAqB,CAAC;KACnD,MAAM,CAAC,YAAY,EAAE,4CAA4C,EAAE,GAAG,CAAC;KACvE,MAAM,CAAC,oBAAoB,EAAE,uDAAuD,CAAC;KACrF,MAAM,CAAC,WAAW,EAAE,qBAAqB,CAAC;KAC1C,MAAM,CAAC,WAAW,EAAE,gBAAgB,CAAC;KACrC,QAAQ,CAAC,aAAa,EAAE,yBAAyB,EAAE,OAAO,CAAC,GAAG,EAAE,CAAC;KACjE,MAAM,CAAC,KAAK,EAAE,QAAgB,EAAE,IAAsC,EAAE,EAAE;IACzE,IAAI,CAAC;QACH,MAAM,SAAS,GAAG,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;QACzC,MAAM,MAAM,GAAG,aAAa,CAC1B;YACE,OAAO,EAAE,IAAI,CAAC,OAAiB;YAC/B,SAAS,EAAE,IAAI,CAAC,SAAmB;YACnC,SAAS,EAAE,IAAI,CAAC,SAAmB;YACnC,IAAI,EAAE,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAc,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,SAAS;SAChE,EACD,OAAO,CAAC,GAAG,EAAE,EAAE,SAAS,CACzB,CAAC;QACF,MAAM,OAAO,GAAG,cAAc,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;QAC/C,MAAM,SAAS,GAAG,gBAAgB,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;QAErD,MAAM,aAAa,CAAC,SAAS,EAAE,OAAO,EAAE,SAAS,EAAE;YACjD,SAAS,EAAE,MAAM,CAAC,SAAS;YAC3B,IAAI,EAAE,MAAM,CAAC,IAAI;YACjB,QAAQ,EAAE,IAAI,CAAC,WAAW,CAAuB;YACjD,MAAM,EAAE,IAAI,CAAC,IAAI,KAAK,KAAK;SAC5B,CAAC,CAAC;QACH,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QAAC,WAAW,CAAC,GAAG,CAAC,CAAC;IAAC,CAAC;AACrC,CAAC,CAAC,CAAC;AAEL,SAAS,WAAW,CAAC,GAAY;IAC/B,IAAI,GAAG,YAAY,aAAa,EAAE,CAAC;QACjC,OAAO,CAAC,KAAK,CAAC,UAAU,GAAG,CAAC,OAAO,EAAE,CAAC,CAAC;QACvC,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,QAAQ,IAAI,CAAC,CAAC,CAAC;IAClC,CAAC;IACD,IAAI,GAAG,YAAY,KAAK,EAAE,CAAC;QACzB,OAAO,CAAC,KAAK,CAAC,UAAU,GAAG,CAAC,OAAO,EAAE,CAAC,CAAC;QACvC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;IACD,OAAO,CAAC,KAAK,CAAC,4BAA4B,CAAC,CAAC;IAC5C,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;AAClB,CAAC;AAED,OAAO,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC"}
|
|
@@ -1,10 +1,6 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* Shared lazy CopilotClient singleton.
|
|
3
3
|
*
|
|
4
|
-
* Both CopilotSDKAdapter (SkillAdapter) and CopilotSDKInference
|
|
5
|
-
* (InferenceAdapter) share a single client to avoid spawning
|
|
6
|
-
* multiple CLI server processes.
|
|
7
|
-
*
|
|
8
4
|
* The SDK is dynamically imported so that users who don't install
|
|
9
5
|
* @github/copilot-sdk pay no cost.
|
|
10
6
|
*/
|
|
@@ -1,10 +1,6 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* Shared lazy CopilotClient singleton.
|
|
3
3
|
*
|
|
4
|
-
* Both CopilotSDKAdapter (SkillAdapter) and CopilotSDKInference
|
|
5
|
-
* (InferenceAdapter) share a single client to avoid spawning
|
|
6
|
-
* multiple CLI server processes.
|
|
7
|
-
*
|
|
8
4
|
* The SDK is dynamically imported so that users who don't install
|
|
9
5
|
* @github/copilot-sdk pay no cost.
|
|
10
6
|
*/
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"copilot-sdk-client.js","sourceRoot":"","sources":["../../../src/adapters/copilot-sdk-client.ts"],"names":[],"mappings":"AAAA
|
|
1
|
+
{"version":3,"file":"copilot-sdk-client.js","sourceRoot":"","sources":["../../../src/adapters/copilot-sdk-client.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,KAAK,EAAE,MAAM,SAAS,CAAC;AAC9B,OAAO,KAAK,IAAI,MAAM,WAAW,CAAC;AAElC,iEAAiE;AACjE,4DAA4D;AAC5D,IAAI,cAAc,GAAQ,IAAI,CAAC;AAC/B,IAAI,aAAa,GAAG,KAAK,CAAC;AAE1B,MAAM,CAAC,KAAK,UAAU,SAAS;IAC7B,IAAI,cAAc,IAAI,aAAa;QAAE,OAAO,cAAc,CAAC;IAE3D,IAAI,GAAQ,CAAC;IACb,IAAI,CAAC;QACH,+DAA+D;QAC/D,GAAG,GAAG,MAAM,MAAM,CAAC,qBAAqB,CAAC,CAAC;IAC5C,CAAC;IAAC,MAAM,CAAC;QACP,MAAM,IAAI,KAAK,CACb,mGAAmG,CACpG,CAAC;IACJ,CAAC;IAED,MAAM,aAAa,GAAG,GAAG,CAAC,aAAa,IAAI,GAAG,CAAC,OAAO,EAAE,aAAa,CAAC;IACtE,IAAI,CAAC,aAAa,EAAE,CAAC;QACnB,MAAM,IAAI,KAAK,CACb,mGAAmG,CACpG,CAAC;IACJ,CAAC;IAED,cAAc,GAAG,IAAI,aAAa,EAAE,CAAC;IACrC,MAAM,cAAc,CAAC,KAAK,EAAE,CAAC;IAC7B,aAAa,GAAG,IAAI,CAAC;IACrB,OAAO,cAAc,CAAC;AACxB,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,UAAU;IAC9B,IAAI,cAAc,IAAI,aAAa,EAAE,CAAC;QACpC,MAAM,cAAc,CAAC,IAAI,EAAE,CAAC;QAC5B,aAAa,GAAG,KAAK,CAAC;QACtB,cAAc,GAAG,IAAI,CAAC;IACxB,CAAC;AACH,CAAC;AAED,MAAM,UAAU,cAAc;IAC5B,iEAAiE;IACjE,mEAAmE;IACnE,IAAI,GAAG,GAAG,OAAO,CAAC,GAAG,EAAE,CAAC;IACxB,OAAO,IAAI,EAAE,CAAC;QACZ,MAAM,SAAS,GAAG,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,cAAc,EAAE,SAAS,EAAE,aAAa,EAAE,cAAc,CAAC,CAAC;QAC3F,IAAI,EAAE,CAAC,UAAU,CAAC,SAAS,CAAC;YAAE,OAAO,IAAI,CAAC;QAC1C,MAAM,MAAM,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;QACjC,IAAI,MAAM,KAAK,GAAG;YAAE,MAAM;QAC1B,GAAG,GAAG,MAAM,CAAC;IACf,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC"}
|