snapeval 2.1.2 → 3.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -13
- package/bin/snapeval.ts +7 -44
- package/dist/bin/snapeval.d.ts +1 -1
- package/dist/bin/snapeval.js +6 -42
- package/dist/bin/snapeval.js.map +1 -1
- package/dist/src/adapters/copilot-sdk-client.d.ts +0 -4
- package/dist/src/adapters/copilot-sdk-client.js +2 -23
- package/dist/src/adapters/copilot-sdk-client.js.map +1 -1
- package/dist/src/adapters/harness/copilot-cli.js +1 -0
- package/dist/src/adapters/harness/copilot-cli.js.map +1 -1
- package/dist/src/adapters/harness/copilot-sdk.js +6 -16
- package/dist/src/adapters/harness/copilot-sdk.js.map +1 -1
- package/dist/src/adapters/harness/resolve.js +1 -5
- package/dist/src/adapters/harness/resolve.js.map +1 -1
- package/dist/src/adapters/inference/copilot-sdk.d.ts +1 -1
- package/dist/src/adapters/inference/copilot-sdk.js +4 -2
- package/dist/src/adapters/inference/copilot-sdk.js.map +1 -1
- package/dist/src/adapters/inference/github-models.js +3 -0
- package/dist/src/adapters/inference/github-models.js.map +1 -1
- package/dist/src/adapters/inference/resolve.js +6 -32
- package/dist/src/adapters/inference/resolve.js.map +1 -1
- package/dist/src/commands/eval.d.ts +1 -0
- package/dist/src/commands/eval.js +8 -0
- package/dist/src/commands/eval.js.map +1 -1
- package/dist/src/errors.d.ts +0 -6
- package/dist/src/errors.js +1 -13
- package/dist/src/errors.js.map +1 -1
- package/package.json +8 -11
- package/plugin.json +7 -4
- package/skills/create-evals/SKILL.md +152 -0
- package/skills/run-evals/SKILL.md +132 -0
- package/src/adapters/copilot-sdk-client.ts +2 -22
- package/src/adapters/harness/copilot-cli.ts +1 -0
- package/src/adapters/harness/copilot-sdk.ts +6 -17
- package/src/adapters/harness/resolve.ts +1 -8
- package/src/adapters/inference/copilot-sdk.ts +4 -2
- package/src/adapters/inference/github-models.ts +3 -0
- package/src/adapters/inference/resolve.ts +8 -43
- package/src/commands/eval.ts +14 -1
- package/src/errors.ts +1 -15
- package/assets/ideation-viewer.html +0 -469
- package/dist/src/adapters/inference/copilot.d.ts +0 -5
- package/dist/src/adapters/inference/copilot.js +0 -10
- package/dist/src/adapters/inference/copilot.js.map +0 -1
- package/dist/src/commands/review.d.ts +0 -8
- package/dist/src/commands/review.js +0 -32
- package/dist/src/commands/review.js.map +0 -1
- package/skills/snapeval/SKILL.md +0 -197
- package/src/adapters/inference/copilot.ts +0 -12
- package/src/commands/review.ts +0 -46
package/README.md
CHANGED
|
@@ -127,17 +127,7 @@ npx snapeval eval [skill-dir] [options]
|
|
|
127
127
|
| `--only <ids>` | Run specific eval IDs (e.g. `--only 1,3,5`) | all |
|
|
128
128
|
| `--threshold <rate>` | Minimum pass rate 0-1 for exit code 0 | none |
|
|
129
129
|
| `--old-skill <path>` | Compare against old skill version | none |
|
|
130
|
-
| `--
|
|
131
|
-
|
|
132
|
-
### `review`
|
|
133
|
-
|
|
134
|
-
Run eval + generate HTML report + open in browser.
|
|
135
|
-
|
|
136
|
-
```bash
|
|
137
|
-
npx snapeval review [skill-dir] [options]
|
|
138
|
-
```
|
|
139
|
-
|
|
140
|
-
Same flags as `eval`, plus `--no-open` to skip opening the browser.
|
|
130
|
+
| `--feedback` | Write feedback.json template for human review | off |
|
|
141
131
|
|
|
142
132
|
### Exit codes
|
|
143
133
|
|
|
@@ -221,8 +211,7 @@ The SDK harness loads skills natively via `skillDirectories`, captures full tran
|
|
|
221
211
|
|
|
222
212
|
| Adapter | Description |
|
|
223
213
|
|---------|-------------|
|
|
224
|
-
| `auto` |
|
|
225
|
-
| `copilot` | Copilot CLI (`copilot` binary) |
|
|
214
|
+
| `auto` | Uses `@github/copilot-sdk` by default, falls back to GitHub Models API |
|
|
226
215
|
| `copilot-sdk` | `@github/copilot-sdk` programmatic |
|
|
227
216
|
| `github-models` | GitHub Models API (requires `GITHUB_TOKEN`) |
|
|
228
217
|
|
package/bin/snapeval.ts
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
#!/usr/bin/env
|
|
1
|
+
#!/usr/bin/env node
|
|
2
2
|
|
|
3
3
|
// Suppress Node.js ExperimentalWarning (e.g., SQLite) from polluting output
|
|
4
4
|
const _origEmit = process.emit;
|
|
@@ -8,12 +8,14 @@ process.emit = function (event: string, ...args: any[]) {
|
|
|
8
8
|
return _origEmit.apply(process, [event, ...args] as any);
|
|
9
9
|
};
|
|
10
10
|
|
|
11
|
+
import { readFileSync } from 'node:fs';
|
|
12
|
+
const pkg = JSON.parse(readFileSync(new URL('../package.json', import.meta.url), 'utf-8'));
|
|
13
|
+
|
|
11
14
|
import { Command } from 'commander';
|
|
12
15
|
import { resolveConfig } from '../src/config.js';
|
|
13
16
|
import { resolveInference } from '../src/adapters/inference/resolve.js';
|
|
14
17
|
import { resolveHarness } from '../src/adapters/harness/resolve.js';
|
|
15
18
|
import { evalCommand } from '../src/commands/eval.js';
|
|
16
|
-
import { reviewCommand } from '../src/commands/review.js';
|
|
17
19
|
import { TerminalReporter } from '../src/adapters/report/terminal.js';
|
|
18
20
|
import { SnapevalError } from '../src/errors.js';
|
|
19
21
|
import { stopClient } from '../src/adapters/copilot-sdk-client.js';
|
|
@@ -24,7 +26,7 @@ const program = new Command();
|
|
|
24
26
|
program
|
|
25
27
|
.name('snapeval')
|
|
26
28
|
.description('Harness-agnostic eval runner for agentskills.io skills')
|
|
27
|
-
.version(
|
|
29
|
+
.version(pkg.version);
|
|
28
30
|
|
|
29
31
|
// --- eval ---
|
|
30
32
|
program
|
|
@@ -38,7 +40,7 @@ program
|
|
|
38
40
|
.option('--only <ids>', 'Run only specific eval IDs (comma-separated, e.g. --only 1,3,5)')
|
|
39
41
|
.option('--threshold <rate>', 'Minimum pass rate (0-1) for exit code 0. Below threshold exits with code 1.')
|
|
40
42
|
.option('--old-skill <path>', 'Compare against old skill version instead of no-skill')
|
|
41
|
-
.option('--
|
|
43
|
+
.option('--feedback', 'Write feedback.json template for human review')
|
|
42
44
|
.argument('[skill-dir]', 'Path to skill directory', process.cwd())
|
|
43
45
|
.action(async (skillDir: string, opts: Record<string, string | boolean>) => {
|
|
44
46
|
try {
|
|
@@ -70,6 +72,7 @@ program
|
|
|
70
72
|
only,
|
|
71
73
|
threshold,
|
|
72
74
|
oldSkill: opts.oldSkill as string | undefined,
|
|
75
|
+
feedback: opts.feedback as boolean | undefined,
|
|
73
76
|
});
|
|
74
77
|
|
|
75
78
|
const terminal = new TerminalReporter();
|
|
@@ -87,46 +90,6 @@ program
|
|
|
87
90
|
}
|
|
88
91
|
});
|
|
89
92
|
|
|
90
|
-
// --- review ---
|
|
91
|
-
program
|
|
92
|
-
.command('review')
|
|
93
|
-
.description('Run eval + generate HTML report + open in browser')
|
|
94
|
-
.option('--harness <harness>', 'Harness to use')
|
|
95
|
-
.option('--inference <inference>', 'Inference adapter to use')
|
|
96
|
-
.option('--workspace <path>', 'Workspace directory')
|
|
97
|
-
.option('--runs <n>', 'Runs per eval for statistical significance', '1')
|
|
98
|
-
.option('--concurrency <n>', 'Number of eval cases to run in parallel (1-10)', '1')
|
|
99
|
-
.option('--old-skill <path>', 'Compare against old skill version instead of no-skill')
|
|
100
|
-
.option('--no-open', 'Do not open browser')
|
|
101
|
-
.option('--verbose', 'Verbose output')
|
|
102
|
-
.argument('[skill-dir]', 'Path to skill directory', process.cwd())
|
|
103
|
-
.action(async (skillDir: string, opts: Record<string, string | boolean>) => {
|
|
104
|
-
try {
|
|
105
|
-
const skillPath = path.resolve(skillDir);
|
|
106
|
-
const config = resolveConfig(
|
|
107
|
-
{
|
|
108
|
-
harness: opts.harness as string,
|
|
109
|
-
inference: opts.inference as string,
|
|
110
|
-
workspace: opts.workspace as string,
|
|
111
|
-
runs: opts.runs ? parseInt(opts.runs as string, 10) : undefined,
|
|
112
|
-
concurrency: opts.concurrency ? parseInt(opts.concurrency as string, 10) : undefined,
|
|
113
|
-
},
|
|
114
|
-
process.cwd(), skillPath
|
|
115
|
-
);
|
|
116
|
-
const harness = resolveHarness(config.harness);
|
|
117
|
-
const inference = resolveInference(config.inference);
|
|
118
|
-
|
|
119
|
-
await reviewCommand(skillPath, harness, inference, {
|
|
120
|
-
workspace: config.workspace,
|
|
121
|
-
runs: config.runs,
|
|
122
|
-
concurrency: config.concurrency,
|
|
123
|
-
oldSkill: opts.oldSkill as string | undefined,
|
|
124
|
-
noOpen: opts.open === false,
|
|
125
|
-
});
|
|
126
|
-
process.exit(0);
|
|
127
|
-
} catch (err) { handleError(err); }
|
|
128
|
-
});
|
|
129
|
-
|
|
130
93
|
// Clean up SDK client on exit (no-op if never started)
|
|
131
94
|
process.on('exit', () => { stopClient().catch(() => {}); });
|
|
132
95
|
|
package/dist/bin/snapeval.d.ts
CHANGED
|
@@ -1,2 +1,2 @@
|
|
|
1
|
-
#!/usr/bin/env
|
|
1
|
+
#!/usr/bin/env node
|
|
2
2
|
export {};
|
package/dist/bin/snapeval.js
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
#!/usr/bin/env
|
|
1
|
+
#!/usr/bin/env node
|
|
2
2
|
// Suppress Node.js ExperimentalWarning (e.g., SQLite) from polluting output
|
|
3
3
|
const _origEmit = process.emit;
|
|
4
4
|
// @ts-ignore — override to filter warnings
|
|
@@ -7,12 +7,13 @@ process.emit = function (event, ...args) {
|
|
|
7
7
|
return false;
|
|
8
8
|
return _origEmit.apply(process, [event, ...args]);
|
|
9
9
|
};
|
|
10
|
+
import { readFileSync } from 'node:fs';
|
|
11
|
+
const pkg = JSON.parse(readFileSync(new URL('../package.json', import.meta.url), 'utf-8'));
|
|
10
12
|
import { Command } from 'commander';
|
|
11
13
|
import { resolveConfig } from '../src/config.js';
|
|
12
14
|
import { resolveInference } from '../src/adapters/inference/resolve.js';
|
|
13
15
|
import { resolveHarness } from '../src/adapters/harness/resolve.js';
|
|
14
16
|
import { evalCommand } from '../src/commands/eval.js';
|
|
15
|
-
import { reviewCommand } from '../src/commands/review.js';
|
|
16
17
|
import { TerminalReporter } from '../src/adapters/report/terminal.js';
|
|
17
18
|
import { SnapevalError } from '../src/errors.js';
|
|
18
19
|
import { stopClient } from '../src/adapters/copilot-sdk-client.js';
|
|
@@ -21,7 +22,7 @@ const program = new Command();
|
|
|
21
22
|
program
|
|
22
23
|
.name('snapeval')
|
|
23
24
|
.description('Harness-agnostic eval runner for agentskills.io skills')
|
|
24
|
-
.version(
|
|
25
|
+
.version(pkg.version);
|
|
25
26
|
// --- eval ---
|
|
26
27
|
program
|
|
27
28
|
.command('eval')
|
|
@@ -34,7 +35,7 @@ program
|
|
|
34
35
|
.option('--only <ids>', 'Run only specific eval IDs (comma-separated, e.g. --only 1,3,5)')
|
|
35
36
|
.option('--threshold <rate>', 'Minimum pass rate (0-1) for exit code 0. Below threshold exits with code 1.')
|
|
36
37
|
.option('--old-skill <path>', 'Compare against old skill version instead of no-skill')
|
|
37
|
-
.option('--
|
|
38
|
+
.option('--feedback', 'Write feedback.json template for human review')
|
|
38
39
|
.argument('[skill-dir]', 'Path to skill directory', process.cwd())
|
|
39
40
|
.action(async (skillDir, opts) => {
|
|
40
41
|
try {
|
|
@@ -61,6 +62,7 @@ program
|
|
|
61
62
|
only,
|
|
62
63
|
threshold,
|
|
63
64
|
oldSkill: opts.oldSkill,
|
|
65
|
+
feedback: opts.feedback,
|
|
64
66
|
});
|
|
65
67
|
const terminal = new TerminalReporter();
|
|
66
68
|
await terminal.report(results);
|
|
@@ -77,44 +79,6 @@ program
|
|
|
77
79
|
handleError(err);
|
|
78
80
|
}
|
|
79
81
|
});
|
|
80
|
-
// --- review ---
|
|
81
|
-
program
|
|
82
|
-
.command('review')
|
|
83
|
-
.description('Run eval + generate HTML report + open in browser')
|
|
84
|
-
.option('--harness <harness>', 'Harness to use')
|
|
85
|
-
.option('--inference <inference>', 'Inference adapter to use')
|
|
86
|
-
.option('--workspace <path>', 'Workspace directory')
|
|
87
|
-
.option('--runs <n>', 'Runs per eval for statistical significance', '1')
|
|
88
|
-
.option('--concurrency <n>', 'Number of eval cases to run in parallel (1-10)', '1')
|
|
89
|
-
.option('--old-skill <path>', 'Compare against old skill version instead of no-skill')
|
|
90
|
-
.option('--no-open', 'Do not open browser')
|
|
91
|
-
.option('--verbose', 'Verbose output')
|
|
92
|
-
.argument('[skill-dir]', 'Path to skill directory', process.cwd())
|
|
93
|
-
.action(async (skillDir, opts) => {
|
|
94
|
-
try {
|
|
95
|
-
const skillPath = path.resolve(skillDir);
|
|
96
|
-
const config = resolveConfig({
|
|
97
|
-
harness: opts.harness,
|
|
98
|
-
inference: opts.inference,
|
|
99
|
-
workspace: opts.workspace,
|
|
100
|
-
runs: opts.runs ? parseInt(opts.runs, 10) : undefined,
|
|
101
|
-
concurrency: opts.concurrency ? parseInt(opts.concurrency, 10) : undefined,
|
|
102
|
-
}, process.cwd(), skillPath);
|
|
103
|
-
const harness = resolveHarness(config.harness);
|
|
104
|
-
const inference = resolveInference(config.inference);
|
|
105
|
-
await reviewCommand(skillPath, harness, inference, {
|
|
106
|
-
workspace: config.workspace,
|
|
107
|
-
runs: config.runs,
|
|
108
|
-
concurrency: config.concurrency,
|
|
109
|
-
oldSkill: opts.oldSkill,
|
|
110
|
-
noOpen: opts.open === false,
|
|
111
|
-
});
|
|
112
|
-
process.exit(0);
|
|
113
|
-
}
|
|
114
|
-
catch (err) {
|
|
115
|
-
handleError(err);
|
|
116
|
-
}
|
|
117
|
-
});
|
|
118
82
|
// Clean up SDK client on exit (no-op if never started)
|
|
119
83
|
process.on('exit', () => { stopClient().catch(() => { }); });
|
|
120
84
|
function handleError(err) {
|
package/dist/bin/snapeval.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"snapeval.js","sourceRoot":"","sources":["../../bin/snapeval.ts"],"names":[],"mappings":";AAEA,4EAA4E;AAC5E,MAAM,SAAS,GAAG,OAAO,CAAC,IAAI,CAAC;AAC/B,2CAA2C;AAC3C,OAAO,CAAC,IAAI,GAAG,UAAU,KAAa,EAAE,GAAG,IAAW;IACpD,IAAI,KAAK,KAAK,SAAS,IAAI,IAAI,CAAC,CAAC,CAAC,EAAE,IAAI,KAAK,qBAAqB;QAAE,OAAO,KAAK,CAAC;IACjF,OAAO,SAAS,CAAC,KAAK,CAAC,OAAO,EAAE,CAAC,KAAK,EAAE,GAAG,IAAI,CAAQ,CAAC,CAAC;AAC3D,CAAC,CAAC;AAEF,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AACpC,OAAO,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAC;AACjD,OAAO,EAAE,gBAAgB,EAAE,MAAM,sCAAsC,CAAC;AACxE,OAAO,EAAE,cAAc,EAAE,MAAM,oCAAoC,CAAC;AACpE,OAAO,EAAE,WAAW,EAAE,MAAM,yBAAyB,CAAC;AACtD,OAAO,EAAE,
|
|
1
|
+
{"version":3,"file":"snapeval.js","sourceRoot":"","sources":["../../bin/snapeval.ts"],"names":[],"mappings":";AAEA,4EAA4E;AAC5E,MAAM,SAAS,GAAG,OAAO,CAAC,IAAI,CAAC;AAC/B,2CAA2C;AAC3C,OAAO,CAAC,IAAI,GAAG,UAAU,KAAa,EAAE,GAAG,IAAW;IACpD,IAAI,KAAK,KAAK,SAAS,IAAI,IAAI,CAAC,CAAC,CAAC,EAAE,IAAI,KAAK,qBAAqB;QAAE,OAAO,KAAK,CAAC;IACjF,OAAO,SAAS,CAAC,KAAK,CAAC,OAAO,EAAE,CAAC,KAAK,EAAE,GAAG,IAAI,CAAQ,CAAC,CAAC;AAC3D,CAAC,CAAC;AAEF,OAAO,EAAE,YAAY,EAAE,MAAM,SAAS,CAAC;AACvC,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,IAAI,GAAG,CAAC,iBAAiB,EAAE,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,OAAO,CAAC,CAAC,CAAC;AAE3F,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AACpC,OAAO,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAC;AACjD,OAAO,EAAE,gBAAgB,EAAE,MAAM,sCAAsC,CAAC;AACxE,OAAO,EAAE,cAAc,EAAE,MAAM,oCAAoC,CAAC;AACpE,OAAO,EAAE,WAAW,EAAE,MAAM,yBAAyB,CAAC;AACtD,OAAO,EAAE,gBAAgB,EAAE,MAAM,oCAAoC,CAAC;AACtE,OAAO,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAC;AACjD,OAAO,EAAE,UAAU,EAAE,MAAM,uCAAuC,CAAC;AACnE,OAAO,KAAK,IAAI,MAAM,WAAW,CAAC;AAElC,MAAM,OAAO,GAAG,IAAI,OAAO,EAAE,CAAC;AAE9B,OAAO;KACJ,IAAI,CAAC,UAAU,CAAC;KAChB,WAAW,CAAC,wDAAwD,CAAC;KACrE,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;AAExB,eAAe;AACf,OAAO;KACJ,OAAO,CAAC,MAAM,CAAC;KACf,WAAW,CAAC,qEAAqE,CAAC;KAClF,MAAM,CAAC,qBAAqB,EAAE,gBAAgB,CAAC;KAC/C,MAAM,CAAC,yBAAyB,EAAE,0BAA0B,CAAC;KAC7D,MAAM,CAAC,oBAAoB,EAAE,qBAAqB,CAAC;KACnD,MAAM,CAAC,YAAY,EAAE,4CAA4C,EAAE,GAAG,CAAC;KACvE,MAAM,CAAC,mBAAmB,EAAE,gDAAgD,EAAE,GAAG,CAAC;KAClF,MAAM,CAAC,cAAc,EAAE,iEAAiE,CAAC;KACzF,MAAM,CAAC,oBAAoB,EAAE,6EAA6E,CAAC;KAC3G,MAAM,CAAC,oBAAoB,EAAE,uDAAuD,CAAC;KACrF,MAAM,CAAC,YAAY,EAAE,+CAA+C,CAAC;KACrE,QAAQ,CAAC,aAAa,EAAE,yBAAyB,EAAE,OAAO,CAAC,GAAG,EAAE,CAAC;KACjE,MAAM,CAAC,KAAK,EAAE,QAAgB,EAAE,IAAsC,EAAE,EAAE;IACzE,IAAI,CAAC;QACH,MAAM,SAAS,GAAG,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;QACzC,MAAM,MAAM,GAAG,aAAa,CAC1B;YACE,OAAO,EAAE,IAAI,CAAC,OAAiB;YAC/B,SAAS,EAAE,IAAI,CAAC,SAAmB;YACnC,SAAS,EAAE,IAAI,CAAC,SAAmB;YACnC,IAAI,EAAE,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAc,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,SAAS;YAC/D,WAAW,EAAE,IAAI,CAAC,WAAW,CAAC,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,WAAqB,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,SAAS;SACrF,EACD,OAAO,CAAC,GAAG,EAAE,EAAE,SAAS,CACzB,CAAC;QACF,MAAM,OAAO,GAAG,cAAc,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;QAC/C,MAAM,SAAS,GAAG,gBAAgB,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;QAErD,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI;YACpB,CAAC,CAAE,IAAI,CAAC,IAAe,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,EAAE,EAAE,EAAE,CAAC,CAAC;YACrE,CAAC,CAAC,SAAS,CAAC;QACd,MAAM,SAAS,GAAG,IAAI,CAAC,SAAS;YAC9B,CAAC,CAAC,UAAU,CAAC,IAAI,CAAC,SAAmB,CAAC;YACtC,CAAC,CAAC,SAAS,CAAC;QAEd,MAAM,OAAO,GAAG,MAAM,WAAW,CAAC,SAAS,EAAE,OAAO,EAAE,SAAS,EAAE;YAC/D,SAAS,EAAE,MAAM,CAAC,SAAS;YAC3B,IAAI,EAAE,MAAM,CAAC,IAAI;YACjB,WAAW,EAAE,MAAM,CAAC,WAAW;YAC/B,IAAI;YACJ,SAAS;YACT,QAAQ,EAAE,IAAI,CAAC,QAA8B;YAC7C,QAAQ,EAAE,IAAI,CAAC,QAA+B;SAC/C,CAAC,CAAC;QAEH,MAAM,QAAQ,GAAG,IAAI,gBAAgB,EAAE,CAAC;QACxC,MAAM,QAAQ,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;QAC/B,OAAO,CAAC,GAAG,CAAC,cAAc,OAAO,CAAC,YAAY,EAAE,CAAC,CAAC;QAClD,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;IAAC,OAAO,GAAQ,EAAE,CAAC;QAClB,iEAAiE;QACjE,IAAI,GAAG,CAAC,OAAO,EAAE,CAAC;YAChB,MAAM,QAAQ,GAAG,IAAI,gBAAgB,EAAE,CAAC;YACxC,MAAM,QAAQ,CAAC,MAAM,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;YACnC,OAAO,CAAC,GAAG,CAAC,cAAc,GAAG,CAAC,OAAO,CAAC,YAAY,EAAE,CAAC,CAAC;QACxD,CAAC;QACD,WAAW,CAAC,GAAG,CAAC,CAAC;IACnB,CAAC;AACH,CAAC,CAAC,CAAC;AAEL,uDAAuD;AACvD,OAAO,CAAC,EAAE,CAAC,MAAM,EAAE,GAAG,EAAE,GAAG,UAAU,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;AAE5D,SAAS,WAAW,CAAC,GAAY;IAC/B,IAAI,GAAG,YAAY,aAAa,EAAE,CAAC;QACjC,OAAO,CAAC,KAAK,CAAC,UAAU,GAAG,CAAC,OAAO,EAAE,CAAC,CAAC;QACvC,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,QAAQ,IAAI,CAAC,CAAC,CAAC;IAClC,CAAC;IACD,IAAI,GAAG,YAAY,KAAK,EAAE,CAAC;QACzB,OAAO,CAAC,KAAK,CAAC,UAAU,GAAG,CAAC,OAAO,EAAE,CAAC,CAAC;QACvC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;IACD,OAAO,CAAC,KAAK,CAAC,4BAA4B,CAAC,CAAC;IAC5C,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;AAClB,CAAC;AAED,OAAO,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC"}
|
|
@@ -1,9 +1,5 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* Shared lazy CopilotClient singleton.
|
|
3
|
-
*
|
|
4
|
-
* The SDK is dynamically imported so that users who don't install
|
|
5
|
-
* @github/copilot-sdk pay no cost.
|
|
6
3
|
*/
|
|
7
4
|
export declare function getClient(): Promise<any>;
|
|
8
5
|
export declare function stopClient(): Promise<void>;
|
|
9
|
-
export declare function isSDKInstalled(): boolean;
|
|
@@ -1,13 +1,8 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* Shared lazy CopilotClient singleton.
|
|
3
|
-
*
|
|
4
|
-
* The SDK is dynamically imported so that users who don't install
|
|
5
|
-
* @github/copilot-sdk pay no cost.
|
|
6
3
|
*/
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
// We store the client as `any` to avoid a hard import dependency
|
|
10
|
-
// on @github/copilot-sdk. The module may not be installed.
|
|
4
|
+
// We store the client as `any` to avoid a static import until the module is
|
|
5
|
+
// confirmed to export the expected shape at runtime.
|
|
11
6
|
let clientInstance = null;
|
|
12
7
|
let clientStarted = false;
|
|
13
8
|
export async function getClient() {
|
|
@@ -15,7 +10,6 @@ export async function getClient() {
|
|
|
15
10
|
return clientInstance;
|
|
16
11
|
let sdk;
|
|
17
12
|
try {
|
|
18
|
-
// @ts-ignore — module may not be installed (optional peer dep)
|
|
19
13
|
sdk = await import('@github/copilot-sdk');
|
|
20
14
|
}
|
|
21
15
|
catch {
|
|
@@ -39,19 +33,4 @@ export async function stopClient() {
|
|
|
39
33
|
clientInstance = null;
|
|
40
34
|
}
|
|
41
35
|
}
|
|
42
|
-
export function isSDKInstalled() {
|
|
43
|
-
// Walk up from cwd looking for node_modules/@github/copilot-sdk.
|
|
44
|
-
// This avoids createRequire/import issues across ESM/CJS contexts.
|
|
45
|
-
let dir = process.cwd();
|
|
46
|
-
while (true) {
|
|
47
|
-
const candidate = path.join(dir, 'node_modules', '@github', 'copilot-sdk', 'package.json');
|
|
48
|
-
if (fs.existsSync(candidate))
|
|
49
|
-
return true;
|
|
50
|
-
const parent = path.dirname(dir);
|
|
51
|
-
if (parent === dir)
|
|
52
|
-
break;
|
|
53
|
-
dir = parent;
|
|
54
|
-
}
|
|
55
|
-
return false;
|
|
56
|
-
}
|
|
57
36
|
//# sourceMappingURL=copilot-sdk-client.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"copilot-sdk-client.js","sourceRoot":"","sources":["../../../src/adapters/copilot-sdk-client.ts"],"names":[],"mappings":"AAAA
|
|
1
|
+
{"version":3,"file":"copilot-sdk-client.js","sourceRoot":"","sources":["../../../src/adapters/copilot-sdk-client.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,4EAA4E;AAC5E,qDAAqD;AACrD,IAAI,cAAc,GAAQ,IAAI,CAAC;AAC/B,IAAI,aAAa,GAAG,KAAK,CAAC;AAE1B,MAAM,CAAC,KAAK,UAAU,SAAS;IAC7B,IAAI,cAAc,IAAI,aAAa;QAAE,OAAO,cAAc,CAAC;IAE3D,IAAI,GAAQ,CAAC;IACb,IAAI,CAAC;QACH,GAAG,GAAG,MAAM,MAAM,CAAC,qBAAqB,CAAC,CAAC;IAC5C,CAAC;IAAC,MAAM,CAAC;QACP,MAAM,IAAI,KAAK,CACb,mGAAmG,CACpG,CAAC;IACJ,CAAC;IAED,MAAM,aAAa,GAAG,GAAG,CAAC,aAAa,IAAI,GAAG,CAAC,OAAO,EAAE,aAAa,CAAC;IACtE,IAAI,CAAC,aAAa,EAAE,CAAC;QACnB,MAAM,IAAI,KAAK,CACb,mGAAmG,CACpG,CAAC;IACJ,CAAC;IAED,4EAA4E;IAC5E,MAAM,GAAG,GAAG,EAAE,GAAG,OAAO,CAAC,GAAG,EAAE,YAAY,EAAE,CAAC,OAAO,CAAC,GAAG,CAAC,YAAY,EAAE,eAAe,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC;IACpH,cAAc,GAAG,IAAI,aAAa,CAAC,EAAE,QAAQ,EAAE,MAAM,EAAE,GAAG,EAAE,CAAC,CAAC;IAC9D,MAAM,cAAc,CAAC,KAAK,EAAE,CAAC;IAC7B,aAAa,GAAG,IAAI,CAAC;IACrB,OAAO,cAAc,CAAC;AACxB,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,UAAU;IAC9B,IAAI,cAAc,IAAI,aAAa,EAAE,CAAC;QACpC,MAAM,cAAc,CAAC,IAAI,EAAE,CAAC;QAC5B,aAAa,GAAG,KAAK,CAAC;QACtB,cAAc,GAAG,IAAI,CAAC;IACxB,CAAC;AACH,CAAC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"copilot-cli.js","sourceRoot":"","sources":["../../../../src/adapters/harness/copilot-cli.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAC5D,OAAO,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAC;AAC5C,OAAO,KAAK,EAAE,MAAM,SAAS,CAAC;AAC9B,OAAO,KAAK,IAAI,MAAM,WAAW,CAAC;AAGlC,MAAM,OAAO,iBAAiB;IACnB,IAAI,GAAG,aAAa,CAAC;IAE9B,KAAK,CAAC,GAAG,CAAC,OAKT;QACC,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAC3B,MAAM,IAAI,CAAC,eAAe,EAAE,CAAC;QAE7B,EAAE,CAAC,SAAS,CAAC,OAAO,CAAC,SAAS,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QAErD,IAAI,OAAO,CAAC,KAAK,EAAE,CAAC;YAClB,KAAK,MAAM,IAAI,IAAI,OAAO,CAAC,KAAK,EAAE,CAAC;gBACjC,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,SAAS,EAAE,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC;gBAC/D,EAAE,CAAC,YAAY,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;YAC9B,CAAC;QACH,CAAC;QAED,IAAI,WAAW,GAAG,OAAO,CAAC,MAAM,CAAC;QACjC,IAAI,OAAO,CAAC,SAAS,EAAE,CAAC;YACtB,IAAI,CAAC;gBACH,MAAM,SAAS,GAAG,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,SAAS,EAAE,UAAU,CAAC,CAAC;gBAC3D,MAAM,OAAO,GAAG,MAAM,QAAQ,CAAC,SAAS,EAAE,EAAE,QAAQ,EAAE,OAAO,EAAE,CAAC,CAAC;gBACjE,WAAW,GAAG,GAAG,OAAO,OAAO,OAAO,CAAC,MAAM,EAAE,CAAC;YAClD,CAAC;YAAC,MAAM,CAAC;gBACP,oBAAoB;YACtB,CAAC;QACH,CAAC;QAED,OAAO,IAAI,OAAO,CAAmB,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;YACvD,QAAQ,CACN,SAAS,EACT,CAAC,IAAI,EAAE,eAAe,EAAE,mBAAmB,EAAE,SAAS,EAAE,SAAS,EAAE,IAAI,EAAE,WAAW,CAAC,EACrF,EAAE,QAAQ,EAAE,OAAO,EAAE,SAAS,EAAE,EAAE,GAAG,IAAI,GAAG,IAAI,EAAE,EAClD,CAAC,KAAK,EAAE,MAAM,EAAE,OAAO,EAAE,EAAE;gBACzB,IAAI,KAAK,EAAE,CAAC;oBAAC,MAAM,CAAC,KAAK,CAAC,CAAC;oBAAC,OAAO;gBAAC,CAAC;gBACrC,MAAM,UAAU,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,OAAO,CAAC;gBACxC,OAAO,CAAC,EAAE,GAAG,EAAE,MAAM,CAAC,IAAI,EAAE,EAAE,KAAK,EAAE,EAAE,EAAE,YAAY,EAAE,CAAC,EAAE,WAAW,EAAE,UAAU,EAAE,CAAC,CAAC;YACvF,CAAC,CACF,CAAC;QACJ,CAAC,CAAC,CAAC;IACL,CAAC;IAED,KAAK,CAAC,WAAW;QACf,IAAI,CAAC;YACH,YAAY,CAAC,SAAS,EAAE,CAAC,WAAW,CAAC,EAAE,EAAE,QAAQ,EAAE,OAAO,EAAE,KAAK,EAAE,MAAM,EAAE,CAAC,CAAC;YAC7E,OAAO,IAAI,CAAC;QACd,CAAC;QAAC,MAAM,CAAC;YAAC,OAAO,KAAK,CAAC;QAAC,CAAC;IAC3B,CAAC;IAEO,KAAK,CAAC,eAAe;QAC3B,IAAI,MAAM,IAAI,CAAC,WAAW,EAAE;YAAE,OAAO;QACrC,MAAM,IAAI,KAAK,CAAC,mFAAmF,CAAC,CAAC;IACvG,CAAC;CACF"}
|
|
1
|
+
{"version":3,"file":"copilot-cli.js","sourceRoot":"","sources":["../../../../src/adapters/harness/copilot-cli.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAC5D,OAAO,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAC;AAC5C,OAAO,KAAK,EAAE,MAAM,SAAS,CAAC;AAC9B,OAAO,KAAK,IAAI,MAAM,WAAW,CAAC;AAGlC,MAAM,OAAO,iBAAiB;IACnB,IAAI,GAAG,aAAa,CAAC;IAE9B,KAAK,CAAC,GAAG,CAAC,OAKT;QACC,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAC3B,MAAM,IAAI,CAAC,eAAe,EAAE,CAAC;QAE7B,EAAE,CAAC,SAAS,CAAC,OAAO,CAAC,SAAS,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QAErD,IAAI,OAAO,CAAC,KAAK,EAAE,CAAC;YAClB,KAAK,MAAM,IAAI,IAAI,OAAO,CAAC,KAAK,EAAE,CAAC;gBACjC,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,SAAS,EAAE,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC;gBAC/D,EAAE,CAAC,YAAY,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;YAC9B,CAAC;QACH,CAAC;QAED,IAAI,WAAW,GAAG,OAAO,CAAC,MAAM,CAAC;QACjC,IAAI,OAAO,CAAC,SAAS,EAAE,CAAC;YACtB,IAAI,CAAC;gBACH,MAAM,SAAS,GAAG,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,SAAS,EAAE,UAAU,CAAC,CAAC;gBAC3D,MAAM,OAAO,GAAG,MAAM,QAAQ,CAAC,SAAS,EAAE,EAAE,QAAQ,EAAE,OAAO,EAAE,CAAC,CAAC;gBACjE,WAAW,GAAG,GAAG,OAAO,OAAO,OAAO,CAAC,MAAM,EAAE,CAAC;YAClD,CAAC;YAAC,MAAM,CAAC;gBACP,oBAAoB;YACtB,CAAC;QACH,CAAC;QAED,OAAO,IAAI,OAAO,CAAmB,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;YACvD,QAAQ,CACN,SAAS,EACT,CAAC,IAAI,EAAE,eAAe,EAAE,mBAAmB,EAAE,SAAS,EAAE,SAAS,EAAE,IAAI,EAAE,WAAW,CAAC,EACrF,EAAE,QAAQ,EAAE,OAAO,EAAE,SAAS,EAAE,EAAE,GAAG,IAAI,GAAG,IAAI,EAAE,EAClD,CAAC,KAAK,EAAE,MAAM,EAAE,OAAO,EAAE,EAAE;gBACzB,IAAI,KAAK,EAAE,CAAC;oBAAC,MAAM,CAAC,KAAK,CAAC,CAAC;oBAAC,OAAO;gBAAC,CAAC;gBACrC,MAAM,UAAU,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,OAAO,CAAC;gBACxC,qDAAqD;gBACrD,OAAO,CAAC,EAAE,GAAG,EAAE,MAAM,CAAC,IAAI,EAAE,EAAE,KAAK,EAAE,EAAE,EAAE,YAAY,EAAE,CAAC,EAAE,WAAW,EAAE,UAAU,EAAE,CAAC,CAAC;YACvF,CAAC,CACF,CAAC;QACJ,CAAC,CAAC,CAAC;IACL,CAAC;IAED,KAAK,CAAC,WAAW;QACf,IAAI,CAAC;YACH,YAAY,CAAC,SAAS,EAAE,CAAC,WAAW,CAAC,EAAE,EAAE,QAAQ,EAAE,OAAO,EAAE,KAAK,EAAE,MAAM,EAAE,CAAC,CAAC;YAC7E,OAAO,IAAI,CAAC;QACd,CAAC;QAAC,MAAM,CAAC;YAAC,OAAO,KAAK,CAAC;QAAC,CAAC;IAC3B,CAAC;IAEO,KAAK,CAAC,eAAe;QAC3B,IAAI,MAAM,IAAI,CAAC,WAAW,EAAE;YAAE,OAAO;QACrC,MAAM,IAAI,KAAK,CAAC,mFAAmF,CAAC,CAAC;IACvG,CAAC;CACF"}
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import * as fs from 'node:fs';
|
|
2
2
|
import * as path from 'node:path';
|
|
3
|
-
import { getClient
|
|
3
|
+
import { getClient } from '../copilot-sdk-client.js';
|
|
4
4
|
export class CopilotSDKHarness {
|
|
5
5
|
name = 'copilot-sdk';
|
|
6
6
|
async run(options) {
|
|
@@ -8,7 +8,6 @@ export class CopilotSDKHarness {
|
|
|
8
8
|
const client = await getClient();
|
|
9
9
|
fs.mkdirSync(options.outputDir, { recursive: true });
|
|
10
10
|
// Dynamically import SDK for approveAll
|
|
11
|
-
// @ts-ignore — module may not be installed (optional dep)
|
|
12
11
|
const { approveAll } = await import('@github/copilot-sdk');
|
|
13
12
|
// Build session config
|
|
14
13
|
const sessionConfig = {
|
|
@@ -19,7 +18,7 @@ export class CopilotSDKHarness {
|
|
|
19
18
|
};
|
|
20
19
|
// Native skill loading: point skillDirectories at the skill's parent
|
|
21
20
|
if (options.skillPath) {
|
|
22
|
-
sessionConfig.skillDirectories = [options.skillPath];
|
|
21
|
+
sessionConfig.skillDirectories = [path.dirname(options.skillPath)];
|
|
23
22
|
}
|
|
24
23
|
const session = await client.createSession(sessionConfig);
|
|
25
24
|
try {
|
|
@@ -41,8 +40,8 @@ export class CopilotSDKHarness {
|
|
|
41
40
|
// Collect full transcript from session events
|
|
42
41
|
const events = await session.getMessages();
|
|
43
42
|
const transcript = buildTranscript(events);
|
|
44
|
-
//
|
|
45
|
-
const totalTokens =
|
|
43
|
+
// SDK assistant.usage events are ephemeral and not available via getMessages()
|
|
44
|
+
const totalTokens = 0;
|
|
46
45
|
const durationMs = Date.now() - startMs;
|
|
47
46
|
return {
|
|
48
47
|
raw: raw.trim(),
|
|
@@ -57,7 +56,7 @@ export class CopilotSDKHarness {
|
|
|
57
56
|
}
|
|
58
57
|
}
|
|
59
58
|
async isAvailable() {
|
|
60
|
-
return
|
|
59
|
+
return true;
|
|
61
60
|
}
|
|
62
61
|
}
|
|
63
62
|
function buildTranscript(events) {
|
|
@@ -74,7 +73,7 @@ function buildTranscript(events) {
|
|
|
74
73
|
lines.push(`[tool:start] ${event.data?.toolName ?? 'unknown'}(${JSON.stringify(event.data?.arguments ?? {})})`);
|
|
75
74
|
break;
|
|
76
75
|
case 'tool.execution_complete':
|
|
77
|
-
lines.push(`[tool:done] ${event.data?.toolName ?? 'unknown'} → ${truncate(event.data?.result ?? '', 200)}`);
|
|
76
|
+
lines.push(`[tool:done] ${event.data?.toolName ?? 'unknown'} → ${truncate(event.data?.result?.content ?? '', 200)}`);
|
|
78
77
|
break;
|
|
79
78
|
case 'skill.invoked':
|
|
80
79
|
lines.push(`[skill] ${event.data?.name ?? 'unknown'} (${event.data?.path ?? ''})`);
|
|
@@ -86,15 +85,6 @@ function buildTranscript(events) {
|
|
|
86
85
|
}
|
|
87
86
|
return lines.join('\n');
|
|
88
87
|
}
|
|
89
|
-
function extractTokenCount(events) {
|
|
90
|
-
let total = 0;
|
|
91
|
-
for (const event of events) {
|
|
92
|
-
if (event.type === 'assistant.usage') {
|
|
93
|
-
total += (event.data?.inputTokens ?? 0) + (event.data?.outputTokens ?? 0);
|
|
94
|
-
}
|
|
95
|
-
}
|
|
96
|
-
return total;
|
|
97
|
-
}
|
|
98
88
|
function truncate(str, max) {
|
|
99
89
|
return str.length > max ? str.slice(0, max) + '...' : str;
|
|
100
90
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"copilot-sdk.js","sourceRoot":"","sources":["../../../../src/adapters/harness/copilot-sdk.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,MAAM,SAAS,CAAC;AAC9B,OAAO,KAAK,IAAI,MAAM,WAAW,CAAC;AAElC,OAAO,EAAE,SAAS,EAAE,
|
|
1
|
+
{"version":3,"file":"copilot-sdk.js","sourceRoot":"","sources":["../../../../src/adapters/harness/copilot-sdk.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,MAAM,SAAS,CAAC;AAC9B,OAAO,KAAK,IAAI,MAAM,WAAW,CAAC;AAElC,OAAO,EAAE,SAAS,EAAE,MAAM,0BAA0B,CAAC;AAErD,MAAM,OAAO,iBAAiB;IACnB,IAAI,GAAG,aAAa,CAAC;IAE9B,KAAK,CAAC,GAAG,CAAC,OAKT;QACC,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAC3B,MAAM,MAAM,GAAG,MAAM,SAAS,EAAE,CAAC;QAEjC,EAAE,CAAC,SAAS,CAAC,OAAO,CAAC,SAAS,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QAErD,wCAAwC;QACxC,MAAM,EAAE,UAAU,EAAE,GAAG,MAAM,MAAM,CAAC,qBAAqB,CAAC,CAAC;QAE3D,uBAAuB;QACvB,MAAM,aAAa,GAA4B;YAC7C,KAAK,EAAE,SAAS;YAChB,mBAAmB,EAAE,UAAU;YAC/B,gBAAgB,EAAE,OAAO,CAAC,SAAS;YACnC,gBAAgB,EAAE,EAAE,OAAO,EAAE,KAAK,EAAE;SACrC,CAAC;QAEF,qEAAqE;QACrE,IAAI,OAAO,CAAC,SAAS,EAAE,CAAC;YACtB,aAAa,CAAC,gBAAgB,GAAG,CAAC,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC,CAAC;QACrE,CAAC;QAED,MAAM,OAAO,GAAG,MAAM,MAAM,CAAC,aAAa,CAAC,aAAa,CAAC,CAAC;QAE1D,IAAI,CAAC;YACH,iCAAiC;YACjC,MAAM,WAAW,GAAgE,EAAE,CAAC;YACpF,IAAI,OAAO,CAAC,KAAK,EAAE,CAAC;gBAClB,KAAK,MAAM,IAAI,IAAI,OAAO,CAAC,KAAK,EAAE,CAAC;oBACjC,oEAAoE;oBACpE,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,SAAS,EAAE,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC;oBAC/D,EAAE,CAAC,YAAY,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;oBAC5B,WAAW,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,WAAW,EAAE,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;gBACnF,CAAC;YACH,CAAC;YAED,MAAM,QAAQ,GAAG,MAAM,OAAO,CAAC,WAAW,CACxC;gBACE,MAAM,EAAE,OAAO,CAAC,MAAM;gBACtB,GAAG,CAAC,WAAW,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,WAAW,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;aACnD,EACD,OAAO,CACR,CAAC;YAEF,MAAM,GAAG,GAAG,QAAQ,EAAE,IAAI,EAAE,OAAO,IAAI,EAAE,CAAC;YAE1C,8CAA8C;YAC9C,MAAM,MAAM,GAAG,MAAM,OAAO,CAAC,WAAW,EAAE,CAAC;YAC3C,MAAM,UAAU,GAAG,eAAe,CAAC,MAAM,CAAC,CAAC;YAE3C,+EAA+E;YAC/E,MAAM,WAAW,GAAG,CAAC,CAAC;YAEtB,MAAM,UAAU,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,OAAO,CAAC;YAExC,OAAO;gBACL,GAAG,EAAE,GAAG,CAAC,IAAI,EAAE;gBACf,UAAU;gBACV,KAAK,EAAE,EAAE;gBACT,YAAY,EAAE,WAAW;gBACzB,WAAW,EAAE,UAAU;aACxB,CAAC;QACJ,CAAC;gBAAS,CAAC;YACT,MAAM,OAAO,CAAC,UAAU,EAAE,CAAC;QAC7B,CAAC;IACH,CAAC;IAED,KAAK,CAAC,WAAW;QACf,OAAO,IAAI,CAAC;IACd,CAAC;CACF;AAED,SAAS,eAAe,CAAC,MAAa;IACpC,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;QAC3B,QAAQ,KAAK,CAAC,IAAI,EAAE,CAAC;YACnB,KAAK,cAAc;gBACjB,KAAK,CAAC,IAAI,CAAC,UAAU,KAAK,CAAC,IAAI,EAAE,OAAO,IAAI,EAAE,EAAE,CAAC,CAAC;gBAClD,MAAM;YACR,KAAK,mBAAmB;gBACtB,KAAK,CAAC,IAAI,CAAC,eAAe,KAAK,CAAC,IAAI,EAAE,OAAO,IAAI,EAAE,EAAE,CAAC,CAAC;gBACvD,MAAM;YACR,KAAK,sBAAsB;gBACzB,KAAK,CAAC,IAAI,CAAC,gBAAgB,KAAK,CAAC,IAAI,EAAE,QAAQ,IAAI,SAAS,IAAI,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,IAAI,EAAE,SAAS,IAAI,EAAE,CAAC,GAAG,CAAC,CAAC;gBAChH,MAAM;YACR,KAAK,yBAAyB;gBAC5B,KAAK,CAAC,IAAI,CAAC,eAAe,KAAK,CAAC,IAAI,EAAE,QAAQ,IAAI,SAAS,MAAM,QAAQ,CAAC,KAAK,CAAC,IAAI,EAAE,MAAM,EAAE,OAAO,IAAI,EAAE,EAAE,GAAG,CAAC,EAAE,CAAC,CAAC;gBACrH,MAAM;YACR,KAAK,eAAe;gBAClB,KAAK,CAAC,IAAI,CAAC,WAAW,KAAK,CAAC,IAAI,EAAE,IAAI,IAAI,SAAS,KAAK,KAAK,CAAC,IAAI,EAAE,IAAI,IAAI,EAAE,GAAG,CAAC,CAAC;gBACnF,MAAM;YACR,KAAK,eAAe;gBAClB,KAAK,CAAC,IAAI,CAAC,WAAW,KAAK,CAAC,IAAI,EAAE,OAAO,IAAI,EAAE,EAAE,CAAC,CAAC;gBACnD,MAAM;QACV,CAAC;IACH,CAAC;IACD,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAC1B,CAAC;AAED,SAAS,QAAQ,CAAC,GAAW,EAAE,GAAW;IACxC,OAAO,GAAG,CAAC,MAAM,GAAG,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,GAAG,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC;AAC5D,CAAC"}
|
|
@@ -1,12 +1,8 @@
|
|
|
1
1
|
import { CopilotCLIHarness } from './copilot-cli.js';
|
|
2
2
|
import { CopilotSDKHarness } from './copilot-sdk.js';
|
|
3
|
-
import {
|
|
4
|
-
import { isSDKInstalled } from '../copilot-sdk-client.js';
|
|
3
|
+
import { SnapevalError } from '../../errors.js';
|
|
5
4
|
export function resolveHarness(name) {
|
|
6
5
|
if (name === 'copilot-sdk') {
|
|
7
|
-
if (!isSDKInstalled()) {
|
|
8
|
-
throw new AdapterNotAvailableError('copilot-sdk', '@github/copilot-sdk is not installed. Install with: npm install @github/copilot-sdk');
|
|
9
|
-
}
|
|
10
6
|
return new CopilotSDKHarness();
|
|
11
7
|
}
|
|
12
8
|
if (name === 'copilot-cli') {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"resolve.js","sourceRoot":"","sources":["../../../../src/adapters/harness/resolve.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,iBAAiB,EAAE,MAAM,kBAAkB,CAAC;AACrD,OAAO,EAAE,iBAAiB,EAAE,MAAM,kBAAkB,CAAC;AACrD,OAAO,EAAE,
|
|
1
|
+
{"version":3,"file":"resolve.js","sourceRoot":"","sources":["../../../../src/adapters/harness/resolve.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,iBAAiB,EAAE,MAAM,kBAAkB,CAAC;AACrD,OAAO,EAAE,iBAAiB,EAAE,MAAM,kBAAkB,CAAC;AACrD,OAAO,EAAE,aAAa,EAAE,MAAM,iBAAiB,CAAC;AAEhD,MAAM,UAAU,cAAc,CAAC,IAAY;IACzC,IAAI,IAAI,KAAK,aAAa,EAAE,CAAC;QAC3B,OAAO,IAAI,iBAAiB,EAAE,CAAC;IACjC,CAAC;IACD,IAAI,IAAI,KAAK,aAAa,EAAE,CAAC;QAC3B,OAAO,IAAI,iBAAiB,EAAE,CAAC;IACjC,CAAC;IACD,MAAM,IAAI,aAAa,CAAC,oBAAoB,IAAI,gDAAgD,CAAC,CAAC;AACpG,CAAC"}
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import type { InferenceAdapter, Message, ChatOptions } from '../../types.js';
|
|
2
2
|
export declare class CopilotSDKInference implements InferenceAdapter {
|
|
3
3
|
readonly name = "copilot-sdk";
|
|
4
|
-
chat(messages: Message[],
|
|
4
|
+
chat(messages: Message[], options?: ChatOptions): Promise<string>;
|
|
5
5
|
}
|
|
@@ -1,14 +1,16 @@
|
|
|
1
1
|
import { getClient } from '../copilot-sdk-client.js';
|
|
2
2
|
export class CopilotSDKInference {
|
|
3
3
|
name = 'copilot-sdk';
|
|
4
|
-
async chat(messages,
|
|
4
|
+
async chat(messages, options) {
|
|
5
5
|
const client = await getClient();
|
|
6
|
-
// @ts-ignore — module may not be installed (optional dep)
|
|
7
6
|
const { approveAll } = await import('@github/copilot-sdk');
|
|
8
7
|
const systemMessages = messages.filter((m) => m.role === 'system');
|
|
9
8
|
const nonSystemMessages = messages.filter((m) => m.role !== 'system');
|
|
10
9
|
const systemContent = systemMessages.map((m) => m.content).join('\n');
|
|
11
10
|
const userPrompt = nonSystemMessages.map((m) => m.content).join('\n');
|
|
11
|
+
// Note: ChatOptions (temperature, responseFormat) are not supported by the
|
|
12
|
+
// SDK's session config. The SDK controls these at the server level.
|
|
13
|
+
void options;
|
|
12
14
|
const session = await client.createSession({
|
|
13
15
|
model: 'gpt-4.1',
|
|
14
16
|
...(systemContent
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"copilot-sdk.js","sourceRoot":"","sources":["../../../../src/adapters/inference/copilot-sdk.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,SAAS,EAAE,MAAM,0BAA0B,CAAC;AAErD,MAAM,OAAO,mBAAmB;IACrB,IAAI,GAAG,aAAa,CAAC;IAE9B,KAAK,CAAC,IAAI,CAAC,QAAmB,EAAE,
|
|
1
|
+
{"version":3,"file":"copilot-sdk.js","sourceRoot":"","sources":["../../../../src/adapters/inference/copilot-sdk.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,SAAS,EAAE,MAAM,0BAA0B,CAAC;AAErD,MAAM,OAAO,mBAAmB;IACrB,IAAI,GAAG,aAAa,CAAC;IAE9B,KAAK,CAAC,IAAI,CAAC,QAAmB,EAAE,OAAqB;QACnD,MAAM,MAAM,GAAG,MAAM,SAAS,EAAE,CAAC;QAEjC,MAAM,EAAE,UAAU,EAAE,GAAG,MAAM,MAAM,CAAC,qBAAqB,CAAC,CAAC;QAE3D,MAAM,cAAc,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,QAAQ,CAAC,CAAC;QACnE,MAAM,iBAAiB,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,QAAQ,CAAC,CAAC;QACtE,MAAM,aAAa,GAAG,cAAc,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACtE,MAAM,UAAU,GAAG,iBAAiB,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAEtE,2EAA2E;QAC3E,oEAAoE;QACpE,KAAK,OAAO,CAAC;QACb,MAAM,OAAO,GAAG,MAAM,MAAM,CAAC,aAAa,CAAC;YACzC,KAAK,EAAE,SAAS;YAChB,GAAG,CAAC,aAAa;gBACf,CAAC,CAAC,EAAE,aAAa,EAAE,EAAE,OAAO,EAAE,aAAa,EAAE,EAAE;gBAC/C,CAAC,CAAC,EAAE,CAAC;YACP,mBAAmB,EAAE,UAAU;YAC/B,gBAAgB,EAAE,EAAE,OAAO,EAAE,KAAK,EAAE;SACrC,CAAC,CAAC;QAEH,IAAI,CAAC;YACH,MAAM,QAAQ,GAAG,MAAM,OAAO,CAAC,WAAW,CAAC,EAAE,MAAM,EAAE,UAAU,EAAE,CAAC,CAAC;YACnE,OAAO,CAAC,QAAQ,EAAE,IAAI,EAAE,OAAO,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC;QAChD,CAAC;gBAAS,CAAC;YACT,MAAM,OAAO,CAAC,UAAU,EAAE,CAAC;QAC7B,CAAC;IACH,CAAC;CACF"}
|
|
@@ -34,6 +34,9 @@ export class GitHubModelsInference {
|
|
|
34
34
|
throw new Error(`GitHub Models API error: ${response.status} ${response.statusText}`);
|
|
35
35
|
}
|
|
36
36
|
const data = (await response.json());
|
|
37
|
+
if (!data.choices?.length) {
|
|
38
|
+
throw new Error('GitHub Models API returned no choices');
|
|
39
|
+
}
|
|
37
40
|
return data.choices[0].message.content;
|
|
38
41
|
}
|
|
39
42
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"github-models.js","sourceRoot":"","sources":["../../../../src/adapters/inference/github-models.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,cAAc,EAAE,MAAM,iBAAiB,CAAC;AAEjD,MAAM,QAAQ,GAAG,oCAAoC,CAAC;AACtD,MAAM,UAAU,GAAG,oBAAoB,CAAC;AAExC,MAAM,OAAO,qBAAqB;IAGH;IAFpB,IAAI,GAAG,eAAe,CAAC;IAEhC,YAA6B,QAAgB,OAAO,CAAC,GAAG,CAAC,YAAY,IAAI,EAAE;QAA9C,UAAK,GAAL,KAAK,CAAyC;IAAG,CAAC;IAE/E,KAAK,CAAC,IAAI,CAAC,QAAmB,EAAE,OAAqB;QACnD,MAAM,IAAI,GAA4B;YACpC,KAAK,EAAE,UAAU;YACjB,QAAQ;SACT,CAAC;QACF,IAAI,OAAO,EAAE,WAAW,KAAK,SAAS;YAAE,IAAI,CAAC,WAAW,GAAG,OAAO,CAAC,WAAW,CAAC;QAC/E,IAAI,OAAO,EAAE,SAAS,KAAK,SAAS;YAAE,IAAI,CAAC,UAAU,GAAG,OAAO,CAAC,SAAS,CAAC;QAC1E,IAAI,OAAO,EAAE,cAAc,KAAK,MAAM,EAAE,CAAC;YACvC,IAAI,CAAC,eAAe,GAAG,EAAE,IAAI,EAAE,aAAa,EAAE,CAAC;QACjD,CAAC;QAED,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,GAAG,QAAQ,mBAAmB,EAAE;YAC3D,MAAM,EAAE,MAAM;YACd,OAAO,EAAE;gBACP,aAAa,EAAE,UAAU,IAAI,CAAC,KAAK,EAAE;gBACrC,cAAc,EAAE,kBAAkB;aACnC;YACD,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC;SAC3B,CAAC,CAAC;QAEH,IAAI,QAAQ,CAAC,MAAM,KAAK,GAAG,EAAE,CAAC;YAC5B,MAAM,IAAI,cAAc,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACtC,CAAC;QAED,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;YACjB,MAAM,IAAI,KAAK,CAAC,4BAA4B,QAAQ,CAAC,MAAM,IAAI,QAAQ,CAAC,UAAU,EAAE,CAAC,CAAC;QACxF,CAAC;QAED,MAAM,IAAI,GAAG,CAAC,MAAM,QAAQ,CAAC,IAAI,EAAE,CAElC,CAAC;QACF,OAAO,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC;IACzC,CAAC;CACF"}
|
|
1
|
+
{"version":3,"file":"github-models.js","sourceRoot":"","sources":["../../../../src/adapters/inference/github-models.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,cAAc,EAAE,MAAM,iBAAiB,CAAC;AAEjD,MAAM,QAAQ,GAAG,oCAAoC,CAAC;AACtD,MAAM,UAAU,GAAG,oBAAoB,CAAC;AAExC,MAAM,OAAO,qBAAqB;IAGH;IAFpB,IAAI,GAAG,eAAe,CAAC;IAEhC,YAA6B,QAAgB,OAAO,CAAC,GAAG,CAAC,YAAY,IAAI,EAAE;QAA9C,UAAK,GAAL,KAAK,CAAyC;IAAG,CAAC;IAE/E,KAAK,CAAC,IAAI,CAAC,QAAmB,EAAE,OAAqB;QACnD,MAAM,IAAI,GAA4B;YACpC,KAAK,EAAE,UAAU;YACjB,QAAQ;SACT,CAAC;QACF,IAAI,OAAO,EAAE,WAAW,KAAK,SAAS;YAAE,IAAI,CAAC,WAAW,GAAG,OAAO,CAAC,WAAW,CAAC;QAC/E,IAAI,OAAO,EAAE,SAAS,KAAK,SAAS;YAAE,IAAI,CAAC,UAAU,GAAG,OAAO,CAAC,SAAS,CAAC;QAC1E,IAAI,OAAO,EAAE,cAAc,KAAK,MAAM,EAAE,CAAC;YACvC,IAAI,CAAC,eAAe,GAAG,EAAE,IAAI,EAAE,aAAa,EAAE,CAAC;QACjD,CAAC;QAED,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,GAAG,QAAQ,mBAAmB,EAAE;YAC3D,MAAM,EAAE,MAAM;YACd,OAAO,EAAE;gBACP,aAAa,EAAE,UAAU,IAAI,CAAC,KAAK,EAAE;gBACrC,cAAc,EAAE,kBAAkB;aACnC;YACD,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC;SAC3B,CAAC,CAAC;QAEH,IAAI,QAAQ,CAAC,MAAM,KAAK,GAAG,EAAE,CAAC;YAC5B,MAAM,IAAI,cAAc,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACtC,CAAC;QAED,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;YACjB,MAAM,IAAI,KAAK,CAAC,4BAA4B,QAAQ,CAAC,MAAM,IAAI,QAAQ,CAAC,UAAU,EAAE,CAAC,CAAC;QACxF,CAAC;QAED,MAAM,IAAI,GAAG,CAAC,MAAM,QAAQ,CAAC,IAAI,EAAE,CAElC,CAAC;QACF,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,MAAM,EAAE,CAAC;YAC1B,MAAM,IAAI,KAAK,CAAC,uCAAuC,CAAC,CAAC;QAC3D,CAAC;QACD,OAAO,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC;IACzC,CAAC;CACF"}
|
|
@@ -1,38 +1,18 @@
|
|
|
1
|
-
import { execFileSync } from 'node:child_process';
|
|
2
1
|
import { AdapterNotAvailableError } from '../../errors.js';
|
|
3
2
|
import { GitHubModelsInference } from './github-models.js';
|
|
4
|
-
import { CopilotInference } from './copilot.js';
|
|
5
3
|
import { CopilotSDKInference } from './copilot-sdk.js';
|
|
6
|
-
import { isSDKInstalled } from '../copilot-sdk-client.js';
|
|
7
|
-
function isCopilotAvailable() {
|
|
8
|
-
try {
|
|
9
|
-
execFileSync('copilot', ['--version'], { encoding: 'utf-8', stdio: 'pipe' });
|
|
10
|
-
return true;
|
|
11
|
-
}
|
|
12
|
-
catch {
|
|
13
|
-
return false;
|
|
14
|
-
}
|
|
15
|
-
}
|
|
16
4
|
function isGitHubTokenAvailable() {
|
|
17
5
|
return Boolean(process.env.GITHUB_TOKEN);
|
|
18
6
|
}
|
|
19
7
|
export function resolveInference(preference) {
|
|
20
8
|
if (preference === 'auto') {
|
|
21
|
-
|
|
22
|
-
const tokenAvailable = isGitHubTokenAvailable();
|
|
23
|
-
if (copilotAvailable) {
|
|
24
|
-
return new CopilotInference();
|
|
25
|
-
}
|
|
26
|
-
if (tokenAvailable) {
|
|
27
|
-
return new GitHubModelsInference();
|
|
28
|
-
}
|
|
29
|
-
throw new AdapterNotAvailableError('inference', 'No inference adapter available. Install GitHub Copilot CLI (`npm install -g @github/copilot`) or set GITHUB_TOKEN.');
|
|
9
|
+
return new CopilotSDKInference();
|
|
30
10
|
}
|
|
31
11
|
if (preference === 'copilot') {
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
return new
|
|
12
|
+
throw new AdapterNotAvailableError('copilot', 'The copilot CLI inference adapter has been removed. Use --inference copilot-sdk instead.');
|
|
13
|
+
}
|
|
14
|
+
if (preference === 'copilot-sdk') {
|
|
15
|
+
return new CopilotSDKInference();
|
|
36
16
|
}
|
|
37
17
|
if (preference === 'github-models') {
|
|
38
18
|
if (!isGitHubTokenAvailable()) {
|
|
@@ -40,12 +20,6 @@ export function resolveInference(preference) {
|
|
|
40
20
|
}
|
|
41
21
|
return new GitHubModelsInference();
|
|
42
22
|
}
|
|
43
|
-
|
|
44
|
-
if (!isSDKInstalled()) {
|
|
45
|
-
throw new AdapterNotAvailableError('copilot-sdk', '@github/copilot-sdk is not installed. Install with: npm install @github/copilot-sdk');
|
|
46
|
-
}
|
|
47
|
-
return new CopilotSDKInference();
|
|
48
|
-
}
|
|
49
|
-
throw new AdapterNotAvailableError(preference, `Unknown inference adapter "${preference}". Valid options: auto, copilot, copilot-sdk, github-models.`);
|
|
23
|
+
throw new AdapterNotAvailableError(preference, `Unknown inference adapter "${preference}". Valid options: auto, copilot-sdk, github-models.`);
|
|
50
24
|
}
|
|
51
25
|
//# sourceMappingURL=resolve.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"resolve.js","sourceRoot":"","sources":["../../../../src/adapters/inference/resolve.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"resolve.js","sourceRoot":"","sources":["../../../../src/adapters/inference/resolve.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,wBAAwB,EAAE,MAAM,iBAAiB,CAAC;AAC3D,OAAO,EAAE,qBAAqB,EAAE,MAAM,oBAAoB,CAAC;AAC3D,OAAO,EAAE,mBAAmB,EAAE,MAAM,kBAAkB,CAAC;AAEvD,SAAS,sBAAsB;IAC7B,OAAO,OAAO,CAAC,OAAO,CAAC,GAAG,CAAC,YAAY,CAAC,CAAC;AAC3C,CAAC;AAED,MAAM,UAAU,gBAAgB,CAAC,UAAkB;IACjD,IAAI,UAAU,KAAK,MAAM,EAAE,CAAC;QAC1B,OAAO,IAAI,mBAAmB,EAAE,CAAC;IACnC,CAAC;IAED,IAAI,UAAU,KAAK,SAAS,EAAE,CAAC;QAC7B,MAAM,IAAI,wBAAwB,CAChC,SAAS,EACT,0FAA0F,CAC3F,CAAC;IACJ,CAAC;IAED,IAAI,UAAU,KAAK,aAAa,EAAE,CAAC;QACjC,OAAO,IAAI,mBAAmB,EAAE,CAAC;IACnC,CAAC;IAED,IAAI,UAAU,KAAK,eAAe,EAAE,CAAC;QACnC,IAAI,CAAC,sBAAsB,EAAE,EAAE,CAAC;YAC9B,MAAM,IAAI,wBAAwB,CAChC,eAAe,EACf,+CAA+C,CAChD,CAAC;QACJ,CAAC;QACD,OAAO,IAAI,qBAAqB,EAAE,CAAC;IACrC,CAAC;IAED,MAAM,IAAI,wBAAwB,CAChC,UAAU,EACV,8BAA8B,UAAU,qDAAqD,CAC9F,CAAC;AACJ,CAAC"}
|
|
@@ -165,6 +165,14 @@ export async function evalCommand(skillPath, harness, inference, options) {
|
|
|
165
165
|
},
|
|
166
166
|
};
|
|
167
167
|
fs.writeFileSync(path.join(iterationDir, 'benchmark.json'), JSON.stringify(benchmarkWithMeta, (_key, value) => typeof value === 'number' ? Math.round(value * 10000) / 10000 : value, 2));
|
|
168
|
+
// Write feedback template if requested
|
|
169
|
+
if (options.feedback) {
|
|
170
|
+
const feedback = {};
|
|
171
|
+
for (const run of evalRuns) {
|
|
172
|
+
feedback[`eval-${run.slug}`] = '';
|
|
173
|
+
}
|
|
174
|
+
fs.writeFileSync(path.join(iterationDir, 'feedback.json'), JSON.stringify(feedback, null, 2));
|
|
175
|
+
}
|
|
168
176
|
// Check threshold if set (for CI gating)
|
|
169
177
|
if (options.threshold !== undefined) {
|
|
170
178
|
const passRate = benchmark.run_summary.with_skill.pass_rate.mean;
|