skill-evals 0.0.0-stage → 0.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +22 -0
- package/LICENSE +21 -0
- package/README.md +427 -2
- package/dist/baseten.d.ts +20 -0
- package/dist/baseten.js +89 -0
- package/dist/bin.d.ts +2 -0
- package/dist/bin.js +5 -0
- package/dist/cases.d.ts +21 -0
- package/dist/cases.js +207 -0
- package/dist/checks.d.ts +126 -0
- package/dist/checks.js +117 -0
- package/dist/cli.d.ts +7 -0
- package/dist/cli.js +172 -0
- package/dist/decision.d.ts +70 -0
- package/dist/decision.js +130 -0
- package/dist/evaluate.d.ts +70 -0
- package/dist/evaluate.js +205 -0
- package/dist/exec.d.ts +20 -0
- package/dist/exec.js +82 -0
- package/dist/harnesses/claude-code.d.ts +3 -0
- package/dist/harnesses/claude-code.js +83 -0
- package/dist/harnesses/codex.d.ts +3 -0
- package/dist/harnesses/codex.js +85 -0
- package/dist/harnesses/cursor.d.ts +3 -0
- package/dist/harnesses/cursor.js +81 -0
- package/dist/harnesses/index.d.ts +34 -0
- package/dist/harnesses/index.js +101 -0
- package/dist/harnesses/opencode.d.ts +3 -0
- package/dist/harnesses/opencode.js +75 -0
- package/dist/harnesses/pi.d.ts +3 -0
- package/dist/harnesses/pi.js +69 -0
- package/dist/harnesses/shared.d.ts +105 -0
- package/dist/harnesses/shared.js +90 -0
- package/dist/index.d.ts +10 -0
- package/dist/index.js +3 -0
- package/dist/judge.d.ts +30 -0
- package/dist/judge.js +87 -0
- package/dist/probe.d.ts +16 -0
- package/dist/probe.js +113 -0
- package/dist/project.d.ts +36 -0
- package/dist/project.js +84 -0
- package/dist/report.d.ts +8 -0
- package/dist/report.js +201 -0
- package/dist/run.d.ts +86 -0
- package/dist/run.js +220 -0
- package/dist/scope.d.ts +39 -0
- package/dist/scope.js +82 -0
- package/dist/score.d.ts +71 -0
- package/dist/score.js +92 -0
- package/dist/session.d.ts +33 -0
- package/dist/session.js +146 -0
- package/dist/subject.d.ts +54 -0
- package/dist/subject.js +179 -0
- package/package.json +59 -4
package/dist/exec.js
ADDED
|
@@ -0,0 +1,82 @@
|
|
|
1
|
+
import { spawn } from "node:child_process";
|
|
2
|
+
import { onDispose } from "./scope.js";
|
|
3
|
+
const MS_PER_SECOND = 1000;
|
|
4
|
+
// How much of a failed command's standard error goes into the failure text.
|
|
5
|
+
const SHOWN_STDERR_LENGTH = 400;
|
|
6
|
+
/**
|
|
7
|
+
* Starts a command in its own process group, so a timeout or an interrupt also stops whatever the command started.
|
|
8
|
+
* @param bin - The executable.
|
|
9
|
+
* @param args - Its arguments.
|
|
10
|
+
* @param run - Where and with what to run it.
|
|
11
|
+
* @returns The child process, with its standard input already written and closed.
|
|
12
|
+
*/
|
|
13
|
+
function start(bin, args, run) {
|
|
14
|
+
const { cwd, env, stdin } = run;
|
|
15
|
+
const options = { cwd, env, detached: true };
|
|
16
|
+
if (stdin === undefined) {
|
|
17
|
+
return spawn(bin, args, { ...options, stdio: ["ignore", "pipe", "pipe"] });
|
|
18
|
+
}
|
|
19
|
+
const child = spawn(bin, args, {
|
|
20
|
+
...options,
|
|
21
|
+
stdio: ["pipe", "pipe", "pipe"],
|
|
22
|
+
});
|
|
23
|
+
// A command that exits without reading its input must not take this process down with EPIPE.
|
|
24
|
+
child.stdin.on("error", () => undefined);
|
|
25
|
+
child.stdin.end(stdin);
|
|
26
|
+
return child;
|
|
27
|
+
}
|
|
28
|
+
/**
|
|
29
|
+
* Runs a command to its end and never rejects.
|
|
30
|
+
* @param bin - The executable.
|
|
31
|
+
* @param args - Its arguments.
|
|
32
|
+
* @param options - Where and with what to run it, and for how long.
|
|
33
|
+
* @returns Everything the command wrote to standard output, and why it failed: a timeout, a non-zero exit with the
|
|
34
|
+
* end of its standard error, or a start that did not work. `failure` is null after exit code 0.
|
|
35
|
+
*/
|
|
36
|
+
export function exec(bin, args, options) {
|
|
37
|
+
const { timeoutSeconds } = options;
|
|
38
|
+
return new Promise((resolve) => {
|
|
39
|
+
const child = start(bin, args, options);
|
|
40
|
+
const kill = () => {
|
|
41
|
+
if (child.pid === undefined)
|
|
42
|
+
return;
|
|
43
|
+
try {
|
|
44
|
+
process.kill(-child.pid, "SIGKILL");
|
|
45
|
+
}
|
|
46
|
+
catch {
|
|
47
|
+
// The group already exited.
|
|
48
|
+
}
|
|
49
|
+
};
|
|
50
|
+
// Runs when the child closes too, so nothing the command left in the background outlives the run.
|
|
51
|
+
const reap = onDispose(kill);
|
|
52
|
+
let stdout = "";
|
|
53
|
+
let stderr = "";
|
|
54
|
+
let timedOut = false;
|
|
55
|
+
child.stdout.setEncoding("utf8");
|
|
56
|
+
child.stderr.setEncoding("utf8");
|
|
57
|
+
const timer = setTimeout(() => {
|
|
58
|
+
timedOut = true;
|
|
59
|
+
kill();
|
|
60
|
+
}, timeoutSeconds * MS_PER_SECOND);
|
|
61
|
+
child.stdout.on("data", (chunk) => (stdout += chunk));
|
|
62
|
+
child.stderr.on("data", (chunk) => (stderr += chunk));
|
|
63
|
+
const finish = (failure) => {
|
|
64
|
+
clearTimeout(timer);
|
|
65
|
+
reap();
|
|
66
|
+
resolve({ stdout, failure });
|
|
67
|
+
};
|
|
68
|
+
child.on("error", (error) => {
|
|
69
|
+
finish(error.message);
|
|
70
|
+
});
|
|
71
|
+
child.on("close", (code, signal) => {
|
|
72
|
+
if (timedOut)
|
|
73
|
+
finish(`timed out after ${String(timeoutSeconds)} s`);
|
|
74
|
+
else if (code === 0)
|
|
75
|
+
finish(null);
|
|
76
|
+
else {
|
|
77
|
+
const tail = stderr.trim().slice(-SHOWN_STDERR_LENGTH);
|
|
78
|
+
finish(`exit ${String(code ?? signal)}: ${tail}`);
|
|
79
|
+
}
|
|
80
|
+
});
|
|
81
|
+
});
|
|
82
|
+
}
|
|
@@ -0,0 +1,83 @@
|
|
|
1
|
+
import path from "node:path";
|
|
2
|
+
import { jsonLines, opt, placeSkills, pluginDir, toolCall } from "./shared.js";
|
|
3
|
+
const KINDS = {
|
|
4
|
+
Bash: "shell",
|
|
5
|
+
Read: "read",
|
|
6
|
+
Write: "write",
|
|
7
|
+
Edit: "edit",
|
|
8
|
+
NotebookEdit: "edit",
|
|
9
|
+
Grep: "search",
|
|
10
|
+
Glob: "search",
|
|
11
|
+
WebFetch: "web",
|
|
12
|
+
WebSearch: "web",
|
|
13
|
+
Skill: "skill",
|
|
14
|
+
Agent: "subagent",
|
|
15
|
+
Task: "subagent",
|
|
16
|
+
};
|
|
17
|
+
const claudeCode = {
|
|
18
|
+
id: "claude-code",
|
|
19
|
+
bin: "claude",
|
|
20
|
+
authFiles: [".claude/.credentials.json"],
|
|
21
|
+
invocation({ subject, home, prompt, model, maxTurns }) {
|
|
22
|
+
const configDir = path.join(home, ".claude");
|
|
23
|
+
placeSkills(subject, path.join(configDir, "skills"));
|
|
24
|
+
return {
|
|
25
|
+
// Never --bare: it switches off the hooks a plugin ships.
|
|
26
|
+
args: [
|
|
27
|
+
"-p",
|
|
28
|
+
...opt("--plugin-dir", pluginDir(subject)),
|
|
29
|
+
"--output-format",
|
|
30
|
+
"stream-json",
|
|
31
|
+
"--verbose",
|
|
32
|
+
"--permission-mode",
|
|
33
|
+
"bypassPermissions",
|
|
34
|
+
...opt("--model", model),
|
|
35
|
+
...opt("--max-turns", maxTurns),
|
|
36
|
+
"--",
|
|
37
|
+
prompt,
|
|
38
|
+
],
|
|
39
|
+
env: { CLAUDE_CONFIG_DIR: configDir },
|
|
40
|
+
};
|
|
41
|
+
},
|
|
42
|
+
parse(stdout) {
|
|
43
|
+
const events = jsonLines(stdout);
|
|
44
|
+
const init = events
|
|
45
|
+
.filter((e) => e.type === "system")
|
|
46
|
+
.find((e) => e.subtype === "init");
|
|
47
|
+
const result = events.filter((e) => e.type === "result").at(-1);
|
|
48
|
+
const usage = result?.usage;
|
|
49
|
+
const failure = result?.is_error ? result.result || result.subtype : null;
|
|
50
|
+
return {
|
|
51
|
+
finalText: result?.result ?? "",
|
|
52
|
+
toolCalls: events
|
|
53
|
+
.filter((e) => e.type === "assistant")
|
|
54
|
+
.flatMap((e) => e.message?.content ?? [])
|
|
55
|
+
.filter((block) => block.type === "tool_use")
|
|
56
|
+
.map(({ name, input }) => toolCall(KINDS, name, input)),
|
|
57
|
+
tokens: usage
|
|
58
|
+
? {
|
|
59
|
+
input: usage.input_tokens +
|
|
60
|
+
(usage.cache_read_input_tokens ?? 0) +
|
|
61
|
+
(usage.cache_creation_input_tokens ?? 0),
|
|
62
|
+
output: usage.output_tokens,
|
|
63
|
+
}
|
|
64
|
+
: null,
|
|
65
|
+
costUsd: result?.total_cost_usd ?? null,
|
|
66
|
+
loaded: {
|
|
67
|
+
paths: init?.plugins ? init.plugins.map((plugin) => plugin.path) : null,
|
|
68
|
+
// A plugin's skill is listed as `plugin:skill`.
|
|
69
|
+
skills: init?.skills ? init.skills.map(lastSegment) : null,
|
|
70
|
+
},
|
|
71
|
+
error: result ? failure : "no result event",
|
|
72
|
+
};
|
|
73
|
+
},
|
|
74
|
+
};
|
|
75
|
+
/**
|
|
76
|
+
* Drops the plugin prefix of a skill name.
|
|
77
|
+
* @param name - A skill as Claude Code lists it, `skill` or `plugin:skill`.
|
|
78
|
+
* @returns The part after the last colon.
|
|
79
|
+
*/
|
|
80
|
+
function lastSegment(name) {
|
|
81
|
+
return name.slice(name.lastIndexOf(":") + 1);
|
|
82
|
+
}
|
|
83
|
+
export default claudeCode;
|
|
@@ -0,0 +1,85 @@
|
|
|
1
|
+
import fs from "node:fs";
|
|
2
|
+
import path from "node:path";
|
|
3
|
+
import { jsonLines, opt, placeSkills, pluginDir, sumTokens, toolCall, } from "./shared.js";
|
|
4
|
+
// Thread items that are tool calls, mapped to the part of the item that is the call's input (never its output).
|
|
5
|
+
const TOOL_ITEMS = {
|
|
6
|
+
command_execution: (item) => ({ command: item.command }),
|
|
7
|
+
file_change: (item) => ({ changes: item.changes }),
|
|
8
|
+
mcp_tool_call: (item) => ({
|
|
9
|
+
server: item.server,
|
|
10
|
+
tool: item.tool,
|
|
11
|
+
arguments: item.arguments,
|
|
12
|
+
}),
|
|
13
|
+
collab_tool_call: (item) => ({ tool: item.tool, prompt: item.prompt }),
|
|
14
|
+
web_search: (item) => ({ query: item.query }),
|
|
15
|
+
};
|
|
16
|
+
const KINDS = {
|
|
17
|
+
command_execution: "shell",
|
|
18
|
+
file_change: "edit",
|
|
19
|
+
web_search: "web",
|
|
20
|
+
collab_tool_call: "subagent",
|
|
21
|
+
};
|
|
22
|
+
/**
|
|
23
|
+
* Codex has no flag that loads a plugin directory for one run, so the plugin is installed into the throwaway home first.
|
|
24
|
+
* @param dir - A plugin directory with a Codex marketplace file.
|
|
25
|
+
* @returns The `codex` argument vectors that register the marketplace and add its first plugin.
|
|
26
|
+
*/
|
|
27
|
+
function installPlugin(dir) {
|
|
28
|
+
const marketplace = JSON.parse(fs.readFileSync(path.join(dir, ".agents/plugins/marketplace.json"), "utf8"));
|
|
29
|
+
return [
|
|
30
|
+
["plugin", "marketplace", "add", dir],
|
|
31
|
+
["plugin", "add", `${marketplace.plugins[0].name}@${marketplace.name}`],
|
|
32
|
+
];
|
|
33
|
+
}
|
|
34
|
+
const codex = {
|
|
35
|
+
id: "codex",
|
|
36
|
+
bin: "codex",
|
|
37
|
+
authFiles: [".codex/auth.json"],
|
|
38
|
+
invocation({ subject, home, workspace, prompt, model }) {
|
|
39
|
+
const codexHome = path.join(home, ".codex");
|
|
40
|
+
fs.mkdirSync(codexHome, { recursive: true });
|
|
41
|
+
placeSkills(subject, path.join(home, ".agents", "skills"));
|
|
42
|
+
const plugin = pluginDir(subject);
|
|
43
|
+
return {
|
|
44
|
+
setup: plugin ? installPlugin(plugin) : [],
|
|
45
|
+
args: [
|
|
46
|
+
"exec",
|
|
47
|
+
"--json",
|
|
48
|
+
"--skip-git-repo-check",
|
|
49
|
+
"--dangerously-bypass-approvals-and-sandbox",
|
|
50
|
+
"-C",
|
|
51
|
+
workspace,
|
|
52
|
+
...opt("-m", model),
|
|
53
|
+
"--",
|
|
54
|
+
prompt,
|
|
55
|
+
],
|
|
56
|
+
env: { CODEX_HOME: codexHome },
|
|
57
|
+
};
|
|
58
|
+
},
|
|
59
|
+
parse(stdout) {
|
|
60
|
+
const events = jsonLines(stdout);
|
|
61
|
+
const items = events
|
|
62
|
+
.filter((e) => e.type === "item.completed")
|
|
63
|
+
.map((e) => e.item);
|
|
64
|
+
const usages = events
|
|
65
|
+
.filter((e) => e.type === "turn.completed")
|
|
66
|
+
.map((e) => e.usage);
|
|
67
|
+
const failed = events.find((e) => e.type === "turn.failed");
|
|
68
|
+
const unfinished = usages.length ? null : "no turn.completed event";
|
|
69
|
+
return {
|
|
70
|
+
finalText: items.filter((item) => item.type === "agent_message").at(-1)?.text ??
|
|
71
|
+
"",
|
|
72
|
+
toolCalls: items.flatMap((item) => {
|
|
73
|
+
const input = Object.hasOwn(TOOL_ITEMS, item.type)
|
|
74
|
+
? TOOL_ITEMS[item.type]
|
|
75
|
+
: undefined;
|
|
76
|
+
return input ? [toolCall(KINDS, item.type, input(item))] : [];
|
|
77
|
+
}),
|
|
78
|
+
tokens: sumTokens(usages.map((u) => ({ input: u.input_tokens, output: u.output_tokens }))),
|
|
79
|
+
costUsd: null,
|
|
80
|
+
loaded: { paths: null, skills: null },
|
|
81
|
+
error: failed ? failed.error?.message || "turn failed" : unfinished,
|
|
82
|
+
};
|
|
83
|
+
},
|
|
84
|
+
};
|
|
85
|
+
export default codex;
|
|
@@ -0,0 +1,81 @@
|
|
|
1
|
+
import path from "node:path";
|
|
2
|
+
import { jsonLines, opt, placeSkills, pluginDir, toolCall } from "./shared.js";
|
|
3
|
+
const KINDS = {
|
|
4
|
+
shell: "shell",
|
|
5
|
+
read: "read",
|
|
6
|
+
write: "write",
|
|
7
|
+
edit: "edit",
|
|
8
|
+
delete: "edit",
|
|
9
|
+
grep: "search",
|
|
10
|
+
glob: "search",
|
|
11
|
+
ls: "search",
|
|
12
|
+
semSearch: "search",
|
|
13
|
+
webSearch: "web",
|
|
14
|
+
webFetch: "web",
|
|
15
|
+
task: "subagent",
|
|
16
|
+
};
|
|
17
|
+
/**
|
|
18
|
+
* Normalizes a tool call, which Cursor nests under a key that names the tool.
|
|
19
|
+
* @param call - One entry: `<tool>ToolCall` for a built-in tool, `function` for an MCP tool with JSON-encoded arguments.
|
|
20
|
+
* @returns The call. Arguments that are not JSON stay text.
|
|
21
|
+
*/
|
|
22
|
+
function nativeCall(call) {
|
|
23
|
+
const [entry] = Object.entries(call);
|
|
24
|
+
if (!entry)
|
|
25
|
+
throw new TypeError("a tool call that names no tool");
|
|
26
|
+
const [key, body] = entry;
|
|
27
|
+
if (key !== "function") {
|
|
28
|
+
return toolCall(KINDS, key.replace(/ToolCall$/, ""), body.args);
|
|
29
|
+
}
|
|
30
|
+
try {
|
|
31
|
+
return toolCall(KINDS, body.name, JSON.parse(body.arguments));
|
|
32
|
+
}
|
|
33
|
+
catch {
|
|
34
|
+
return toolCall(KINDS, body.name, body.arguments);
|
|
35
|
+
}
|
|
36
|
+
}
|
|
37
|
+
const cursor = {
|
|
38
|
+
id: "cursor",
|
|
39
|
+
bin: "cursor-agent",
|
|
40
|
+
authFiles: [".cursor/auth.json", ".config/cursor/auth.json"],
|
|
41
|
+
invocation({ subject, home, workspace, prompt, model }) {
|
|
42
|
+
placeSkills(subject, path.join(home, ".cursor", "skills"));
|
|
43
|
+
return {
|
|
44
|
+
args: [
|
|
45
|
+
"-p",
|
|
46
|
+
"--output-format",
|
|
47
|
+
"stream-json",
|
|
48
|
+
...opt("--plugin-dir", pluginDir(subject)),
|
|
49
|
+
"--force",
|
|
50
|
+
"--trust",
|
|
51
|
+
"--workspace",
|
|
52
|
+
workspace,
|
|
53
|
+
...opt("--model", model),
|
|
54
|
+
"--",
|
|
55
|
+
prompt,
|
|
56
|
+
],
|
|
57
|
+
env: { CURSOR_CONFIG_DIR: path.join(home, ".cursor") },
|
|
58
|
+
};
|
|
59
|
+
},
|
|
60
|
+
parse(stdout) {
|
|
61
|
+
const events = jsonLines(stdout);
|
|
62
|
+
const result = events.filter((e) => e.type === "result").at(-1);
|
|
63
|
+
const lastMessage = events.filter((e) => e.type === "assistant").at(-1);
|
|
64
|
+
const failure = result?.is_error ? result.result || result.subtype : null;
|
|
65
|
+
return {
|
|
66
|
+
// The result event joins every message of the run; the last assistant event is the final one.
|
|
67
|
+
finalText: lastMessage
|
|
68
|
+
? lastMessage.message.content.map((part) => part.text ?? "").join("")
|
|
69
|
+
: (result?.result ?? ""),
|
|
70
|
+
toolCalls: events
|
|
71
|
+
.filter((e) => e.type === "tool_call")
|
|
72
|
+
.filter((e) => e.subtype === "started")
|
|
73
|
+
.map((e) => nativeCall(e.tool_call)),
|
|
74
|
+
tokens: null,
|
|
75
|
+
costUsd: null,
|
|
76
|
+
loaded: { paths: null, skills: null },
|
|
77
|
+
error: result ? failure : "no result event",
|
|
78
|
+
};
|
|
79
|
+
},
|
|
80
|
+
};
|
|
81
|
+
export default cursor;
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
import type { Harness } from "./shared.js";
|
|
2
|
+
export declare const HARNESSES: Harness[];
|
|
3
|
+
/** A harness with the executable to spawn for it. */
|
|
4
|
+
export interface Located {
|
|
5
|
+
harness: Harness;
|
|
6
|
+
bin: string;
|
|
7
|
+
}
|
|
8
|
+
type Env = Record<string, string | undefined>;
|
|
9
|
+
/**
|
|
10
|
+
* Looks for a harness on this machine.
|
|
11
|
+
* @param harness - The harness to find.
|
|
12
|
+
* @param env - Where `PATH` and the harness's `SKILL_EVALS_BIN_<ID>` override are read from.
|
|
13
|
+
* @returns The executable to spawn, or null when the harness is not installed.
|
|
14
|
+
*/
|
|
15
|
+
export declare function findBin(harness: Harness, env?: Env): string | null;
|
|
16
|
+
/**
|
|
17
|
+
* Reads a `--harness` value.
|
|
18
|
+
* @param spec - A comma-separated list of ids, or "all".
|
|
19
|
+
* @returns The harnesses the list names; every known harness when the list is absent or "all".
|
|
20
|
+
* @throws {UsageError} When the list names an unknown harness.
|
|
21
|
+
*/
|
|
22
|
+
export declare function namedHarnesses(spec: string | undefined): Harness[];
|
|
23
|
+
/**
|
|
24
|
+
* Chooses the harnesses a run uses.
|
|
25
|
+
* @param spec - A `--harness` value. Absent means the first installed harness in preference order.
|
|
26
|
+
* @param env - Where the executables are looked up.
|
|
27
|
+
* @returns The chosen harnesses, each with its executable, and why they were chosen.
|
|
28
|
+
* @throws {UsageError} When a named harness is not installed, or none is installed at all.
|
|
29
|
+
*/
|
|
30
|
+
export declare function selectHarnesses(spec: string | undefined, env?: Env): {
|
|
31
|
+
chosen: Located[];
|
|
32
|
+
reason: string;
|
|
33
|
+
};
|
|
34
|
+
export {};
|
|
@@ -0,0 +1,101 @@
|
|
|
1
|
+
import fs from "node:fs";
|
|
2
|
+
import path from "node:path";
|
|
3
|
+
import { UsageError } from "../scope.js";
|
|
4
|
+
import claudeCode from "./claude-code.js";
|
|
5
|
+
import codex from "./codex.js";
|
|
6
|
+
import cursor from "./cursor.js";
|
|
7
|
+
import opencode from "./opencode.js";
|
|
8
|
+
import pi from "./pi.js";
|
|
9
|
+
// Default-preference order: the first installed one runs when --harness is absent.
|
|
10
|
+
// Pi leads because it has the smallest install, the fastest start, and reports tokens and cost.
|
|
11
|
+
export const HARNESSES = [pi, claudeCode, cursor, opencode, codex];
|
|
12
|
+
const overrideVar = (harness) => `SKILL_EVALS_BIN_${harness.id.toUpperCase().replaceAll("-", "_")}`;
|
|
13
|
+
/**
|
|
14
|
+
* Tells whether a path can be spawned.
|
|
15
|
+
* @param file - A path, which need not exist.
|
|
16
|
+
* @returns True for a regular file that this user may execute.
|
|
17
|
+
*/
|
|
18
|
+
function isExecutable(file) {
|
|
19
|
+
try {
|
|
20
|
+
fs.accessSync(file, fs.constants.X_OK);
|
|
21
|
+
return fs.statSync(file).isFile();
|
|
22
|
+
}
|
|
23
|
+
catch {
|
|
24
|
+
return false;
|
|
25
|
+
}
|
|
26
|
+
}
|
|
27
|
+
/**
|
|
28
|
+
* Looks for a harness on this machine.
|
|
29
|
+
* @param harness - The harness to find.
|
|
30
|
+
* @param env - Where `PATH` and the harness's `SKILL_EVALS_BIN_<ID>` override are read from.
|
|
31
|
+
* @returns The executable to spawn, or null when the harness is not installed.
|
|
32
|
+
*/
|
|
33
|
+
export function findBin(harness, env = process.env) {
|
|
34
|
+
const bin = env[overrideVar(harness)] || harness.bin;
|
|
35
|
+
const candidates = bin.includes(path.sep)
|
|
36
|
+
? [path.resolve(bin)]
|
|
37
|
+
: (env["PATH"] ?? "")
|
|
38
|
+
.split(path.delimiter)
|
|
39
|
+
.filter(Boolean)
|
|
40
|
+
.map((dir) => path.join(dir, bin));
|
|
41
|
+
return candidates.find(isExecutable) ?? null;
|
|
42
|
+
}
|
|
43
|
+
/**
|
|
44
|
+
* Looks a harness up by its id.
|
|
45
|
+
* @param id - What the user typed.
|
|
46
|
+
* @returns The harness.
|
|
47
|
+
* @throws {UsageError} When no harness has the id.
|
|
48
|
+
*/
|
|
49
|
+
function byId(id) {
|
|
50
|
+
const harness = HARNESSES.find((h) => h.id === id);
|
|
51
|
+
if (!harness) {
|
|
52
|
+
throw new UsageError(`unknown harness "${id}"; known: ${HARNESSES.map((h) => h.id).join(", ")}`);
|
|
53
|
+
}
|
|
54
|
+
return harness;
|
|
55
|
+
}
|
|
56
|
+
/**
|
|
57
|
+
* Reads a `--harness` value.
|
|
58
|
+
* @param spec - A comma-separated list of ids, or "all".
|
|
59
|
+
* @returns The harnesses the list names; every known harness when the list is absent or "all".
|
|
60
|
+
* @throws {UsageError} When the list names an unknown harness.
|
|
61
|
+
*/
|
|
62
|
+
export function namedHarnesses(spec) {
|
|
63
|
+
return !spec || spec === "all"
|
|
64
|
+
? HARNESSES
|
|
65
|
+
: spec.split(",").map((id) => byId(id.trim()));
|
|
66
|
+
}
|
|
67
|
+
/**
|
|
68
|
+
* Chooses the harnesses a run uses.
|
|
69
|
+
* @param spec - A `--harness` value. Absent means the first installed harness in preference order.
|
|
70
|
+
* @param env - Where the executables are looked up.
|
|
71
|
+
* @returns The chosen harnesses, each with its executable, and why they were chosen.
|
|
72
|
+
* @throws {UsageError} When a named harness is not installed, or none is installed at all.
|
|
73
|
+
*/
|
|
74
|
+
export function selectHarnesses(spec, env = process.env) {
|
|
75
|
+
const order = HARNESSES.map((h) => h.id).join(", ");
|
|
76
|
+
const located = namedHarnesses(spec).map((harness) => ({
|
|
77
|
+
harness,
|
|
78
|
+
bin: findBin(harness, env),
|
|
79
|
+
}));
|
|
80
|
+
const installed = located.flatMap(({ harness, bin }) => bin ? [{ harness, bin }] : []);
|
|
81
|
+
if (spec && spec !== "all") {
|
|
82
|
+
const missing = located
|
|
83
|
+
.filter((entry) => !entry.bin)
|
|
84
|
+
.map(({ harness }) => `${harness.id} (no "${harness.bin}" on PATH; set ${overrideVar(harness)})`);
|
|
85
|
+
if (missing.length) {
|
|
86
|
+
throw new UsageError(`harness not installed: ${missing.join("; ")}`);
|
|
87
|
+
}
|
|
88
|
+
return { chosen: installed, reason: "named with --harness" };
|
|
89
|
+
}
|
|
90
|
+
const [first] = installed;
|
|
91
|
+
if (!first) {
|
|
92
|
+
throw new UsageError(`no harness is installed; looked for ${order}`);
|
|
93
|
+
}
|
|
94
|
+
if (spec === "all") {
|
|
95
|
+
return { chosen: installed, reason: `every installed harness of ${order}` };
|
|
96
|
+
}
|
|
97
|
+
return {
|
|
98
|
+
chosen: [first],
|
|
99
|
+
reason: `default, the first installed harness in the order ${order}`,
|
|
100
|
+
};
|
|
101
|
+
}
|
|
@@ -0,0 +1,75 @@
|
|
|
1
|
+
import fs from "node:fs";
|
|
2
|
+
import path from "node:path";
|
|
3
|
+
import { jsonLines, opt, placeSkills, pluginDir, sumTokens, toolCall, } from "./shared.js";
|
|
4
|
+
const KINDS = {
|
|
5
|
+
bash: "shell",
|
|
6
|
+
read: "read",
|
|
7
|
+
write: "write",
|
|
8
|
+
edit: "edit",
|
|
9
|
+
patch: "edit",
|
|
10
|
+
grep: "search",
|
|
11
|
+
glob: "search",
|
|
12
|
+
list: "search",
|
|
13
|
+
webfetch: "web",
|
|
14
|
+
websearch: "web",
|
|
15
|
+
skill: "skill",
|
|
16
|
+
task: "subagent",
|
|
17
|
+
};
|
|
18
|
+
const opencode = {
|
|
19
|
+
id: "opencode",
|
|
20
|
+
bin: "opencode",
|
|
21
|
+
authFiles: [".local/share/opencode/auth.json"],
|
|
22
|
+
invocation({ subject, home, workspace, prompt, model }) {
|
|
23
|
+
const configDir = path.join(home, ".config", "opencode");
|
|
24
|
+
fs.mkdirSync(configDir, { recursive: true });
|
|
25
|
+
// V1 reads "plugin" and ignores "plugins" (seen on 1.19.6); V2 is documented to read "plugins".
|
|
26
|
+
const plugin = pluginDir(subject);
|
|
27
|
+
const plugins = plugin ? [plugin] : [];
|
|
28
|
+
fs.writeFileSync(path.join(configDir, "opencode.json"), JSON.stringify({ plugin: plugins, plugins }));
|
|
29
|
+
placeSkills(subject, path.join(home, ".agents", "skills"));
|
|
30
|
+
return {
|
|
31
|
+
args: [
|
|
32
|
+
"run",
|
|
33
|
+
"--format",
|
|
34
|
+
"json",
|
|
35
|
+
"--auto",
|
|
36
|
+
"--dir",
|
|
37
|
+
workspace,
|
|
38
|
+
...opt("-m", model),
|
|
39
|
+
"--",
|
|
40
|
+
prompt,
|
|
41
|
+
],
|
|
42
|
+
env: {},
|
|
43
|
+
};
|
|
44
|
+
},
|
|
45
|
+
parse(stdout) {
|
|
46
|
+
const events = jsonLines(stdout);
|
|
47
|
+
const steps = events
|
|
48
|
+
.filter((e) => e.type === "step_finish")
|
|
49
|
+
.map((e) => e.part);
|
|
50
|
+
const failure = events.find((e) => e.type === "error")?.error;
|
|
51
|
+
const unfinished = steps.length ? null : "no step_finish event";
|
|
52
|
+
return {
|
|
53
|
+
finalText: events
|
|
54
|
+
.slice(events.map((e) => e.type).lastIndexOf("step_start") + 1)
|
|
55
|
+
.filter((e) => e.type === "text")
|
|
56
|
+
.map((e) => e.part.text)
|
|
57
|
+
.join(""),
|
|
58
|
+
toolCalls: events
|
|
59
|
+
.filter((e) => e.type === "tool_use")
|
|
60
|
+
.map((e) => toolCall(KINDS, e.part.tool, e.part.state?.input)),
|
|
61
|
+
tokens: sumTokens(steps.map(({ tokens }) => ({
|
|
62
|
+
input: tokens.input + tokens.cache.read + tokens.cache.write,
|
|
63
|
+
output: tokens.output,
|
|
64
|
+
}))),
|
|
65
|
+
costUsd: steps.length
|
|
66
|
+
? steps.reduce((total, step) => total + (step.cost ?? 0), 0)
|
|
67
|
+
: null,
|
|
68
|
+
loaded: { paths: null, skills: null },
|
|
69
|
+
error: failure
|
|
70
|
+
? failure.data?.message || failure.name || "error"
|
|
71
|
+
: unfinished,
|
|
72
|
+
};
|
|
73
|
+
},
|
|
74
|
+
};
|
|
75
|
+
export default opencode;
|
|
@@ -0,0 +1,69 @@
|
|
|
1
|
+
import path from "node:path";
|
|
2
|
+
import { jsonLines, opt, placeSkills, pluginDir, sumTokens, toolCall, } from "./shared.js";
|
|
3
|
+
const KINDS = {
|
|
4
|
+
bash: "shell",
|
|
5
|
+
read: "read",
|
|
6
|
+
write: "write",
|
|
7
|
+
edit: "edit",
|
|
8
|
+
grep: "search",
|
|
9
|
+
find: "search",
|
|
10
|
+
ls: "search",
|
|
11
|
+
};
|
|
12
|
+
const FAILED = new Set(["error", "aborted"]);
|
|
13
|
+
const pi = {
|
|
14
|
+
id: "pi",
|
|
15
|
+
bin: "pi",
|
|
16
|
+
authFiles: [".pi/agent/auth.json", ".pi/agent/models.json"],
|
|
17
|
+
invocation({ subject, home, prompt, model }) {
|
|
18
|
+
const agentDir = path.join(home, ".pi", "agent");
|
|
19
|
+
const skills = placeSkills(subject, path.join(agentDir, "skills"));
|
|
20
|
+
return {
|
|
21
|
+
args: [
|
|
22
|
+
"-p",
|
|
23
|
+
"--mode",
|
|
24
|
+
"json",
|
|
25
|
+
"--no-session",
|
|
26
|
+
// -ne, -ns and -np switch off discovery, so what -e and --skill name is all that loads.
|
|
27
|
+
"-ne",
|
|
28
|
+
"-ns",
|
|
29
|
+
"-np",
|
|
30
|
+
...opt("-e", pluginDir(subject)),
|
|
31
|
+
...skills.flatMap((dir) => ["--skill", dir]),
|
|
32
|
+
...opt("--model", model),
|
|
33
|
+
],
|
|
34
|
+
// Pi has no `--` separator and reads a leading `-` or `@` in a message argument as a flag or a file.
|
|
35
|
+
stdin: prompt,
|
|
36
|
+
env: { PI_CODING_AGENT_DIR: agentDir, PI_OFFLINE: "1" },
|
|
37
|
+
};
|
|
38
|
+
},
|
|
39
|
+
parse(stdout) {
|
|
40
|
+
const events = jsonLines(stdout);
|
|
41
|
+
const turns = events
|
|
42
|
+
.filter((e) => e.type === "turn_end")
|
|
43
|
+
.flatMap((e) => (e.message?.role === "assistant" ? [e.message] : []));
|
|
44
|
+
const last = turns.at(-1);
|
|
45
|
+
const usages = turns.flatMap((message) => message.usage ?? []);
|
|
46
|
+
const failure = last && FAILED.has(last.stopReason)
|
|
47
|
+
? last.errorMessage || last.stopReason
|
|
48
|
+
: null;
|
|
49
|
+
return {
|
|
50
|
+
finalText: (last?.content ?? [])
|
|
51
|
+
.filter((part) => part.type === "text")
|
|
52
|
+
.map((part) => part.text)
|
|
53
|
+
.join(""),
|
|
54
|
+
toolCalls: events
|
|
55
|
+
.filter((e) => e.type === "tool_execution_start")
|
|
56
|
+
.map((e) => toolCall(KINDS, e.toolName, e.args)),
|
|
57
|
+
tokens: sumTokens(usages.map((u) => ({
|
|
58
|
+
input: u.input + u.cacheRead + u.cacheWrite,
|
|
59
|
+
output: u.output,
|
|
60
|
+
}))),
|
|
61
|
+
costUsd: usages.length
|
|
62
|
+
? usages.reduce((total, u) => total + (u.cost?.total ?? 0), 0)
|
|
63
|
+
: null,
|
|
64
|
+
loaded: { paths: null, skills: null },
|
|
65
|
+
error: last ? failure : "no turn_end event",
|
|
66
|
+
};
|
|
67
|
+
},
|
|
68
|
+
};
|
|
69
|
+
export default pi;
|