skill-evals 0.0.0-stage → 0.0.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (54) hide show
  1. package/CHANGELOG.md +22 -0
  2. package/LICENSE +21 -0
  3. package/README.md +427 -2
  4. package/dist/baseten.d.ts +20 -0
  5. package/dist/baseten.js +89 -0
  6. package/dist/bin.d.ts +2 -0
  7. package/dist/bin.js +5 -0
  8. package/dist/cases.d.ts +21 -0
  9. package/dist/cases.js +207 -0
  10. package/dist/checks.d.ts +126 -0
  11. package/dist/checks.js +117 -0
  12. package/dist/cli.d.ts +7 -0
  13. package/dist/cli.js +172 -0
  14. package/dist/decision.d.ts +70 -0
  15. package/dist/decision.js +130 -0
  16. package/dist/evaluate.d.ts +70 -0
  17. package/dist/evaluate.js +205 -0
  18. package/dist/exec.d.ts +20 -0
  19. package/dist/exec.js +82 -0
  20. package/dist/harnesses/claude-code.d.ts +3 -0
  21. package/dist/harnesses/claude-code.js +83 -0
  22. package/dist/harnesses/codex.d.ts +3 -0
  23. package/dist/harnesses/codex.js +85 -0
  24. package/dist/harnesses/cursor.d.ts +3 -0
  25. package/dist/harnesses/cursor.js +81 -0
  26. package/dist/harnesses/index.d.ts +34 -0
  27. package/dist/harnesses/index.js +101 -0
  28. package/dist/harnesses/opencode.d.ts +3 -0
  29. package/dist/harnesses/opencode.js +75 -0
  30. package/dist/harnesses/pi.d.ts +3 -0
  31. package/dist/harnesses/pi.js +69 -0
  32. package/dist/harnesses/shared.d.ts +105 -0
  33. package/dist/harnesses/shared.js +90 -0
  34. package/dist/index.d.ts +10 -0
  35. package/dist/index.js +3 -0
  36. package/dist/judge.d.ts +30 -0
  37. package/dist/judge.js +87 -0
  38. package/dist/probe.d.ts +16 -0
  39. package/dist/probe.js +113 -0
  40. package/dist/project.d.ts +36 -0
  41. package/dist/project.js +84 -0
  42. package/dist/report.d.ts +8 -0
  43. package/dist/report.js +201 -0
  44. package/dist/run.d.ts +86 -0
  45. package/dist/run.js +220 -0
  46. package/dist/scope.d.ts +39 -0
  47. package/dist/scope.js +82 -0
  48. package/dist/score.d.ts +71 -0
  49. package/dist/score.js +92 -0
  50. package/dist/session.d.ts +33 -0
  51. package/dist/session.js +146 -0
  52. package/dist/subject.d.ts +54 -0
  53. package/dist/subject.js +179 -0
  54. package/package.json +59 -4
package/dist/report.js ADDED
@@ -0,0 +1,201 @@
1
+ import { describeCheck, isDecision, isJudged } from "./checks.js";
2
+ import { invalidReason } from "./run.js";
3
+ import { formatScore } from "./score.js";
4
+ import { describeSubject } from "./subject.js";
5
+ const MS_PER_SECOND = 1000;
6
+ const COST_DECIMALS = 4;
7
+ const SHOWN_REASON_LENGTH = 300;
8
+ const METRICS = [
9
+ {
10
+ title: "Duration",
11
+ value: (usage) => usage.durationMs / MS_PER_SECOND,
12
+ format: (v) => `${v.toFixed(1)} s`,
13
+ },
14
+ {
15
+ title: "Input tokens",
16
+ value: (usage) => usage.tokens?.input ?? null,
17
+ format: (v) => String(Math.round(v)),
18
+ },
19
+ {
20
+ title: "Output tokens",
21
+ value: (usage) => usage.tokens?.output ?? null,
22
+ format: (v) => String(Math.round(v)),
23
+ },
24
+ {
25
+ title: "Cost",
26
+ value: (usage) => usage.costUsd,
27
+ format: (v) => `$${v.toFixed(COST_DECIMALS)}`,
28
+ },
29
+ ];
30
+ /**
31
+ * Finds the middle of a set of numbers.
32
+ * @param values - The numbers, in any order.
33
+ * @returns The middle value, the mean of the two middle values for an even count, or null for no values.
34
+ */
35
+ function median(values) {
36
+ if (values.length === 0)
37
+ return null;
38
+ const sorted = [...values].sort((x, y) => x - y);
39
+ const middle = sorted.slice((sorted.length - 1) >> 1, (sorted.length >> 1) + 1);
40
+ return middle.reduce((sum, value) => sum + value, 0) / middle.length;
41
+ }
42
+ /**
43
+ * Shows one value for each subject, and the change from the first to the second when there are two.
44
+ * @param values - One value for each subject; null is a missing one.
45
+ * @param format - Renders a value, and the size of the change.
46
+ * @returns The cell text, "n/a" when every value is missing.
47
+ */
48
+ function versus(values, format) {
49
+ if (values.every((value) => value === null))
50
+ return "n/a";
51
+ const shown = values
52
+ .map((value) => (value === null ? "n/a" : format(value)))
53
+ .join(" / ");
54
+ const [first, second] = values;
55
+ if (first == null || second == null)
56
+ return shown;
57
+ const delta = second - first;
58
+ return `${shown} (${delta < 0 ? "-" : "+"}${format(Math.abs(delta))})`;
59
+ }
60
+ /**
61
+ * Sums up how one check went over the valid runs of one subject.
62
+ * @param check - Decides the form: a rule gives pass or fail, a judge or a decision model gives a score.
63
+ * @param index - Its place among the case's checks.
64
+ * @param records - The valid runs.
65
+ * @returns For a deterministic check the share of runs that passed it, for a judged check or a decision check its
66
+ * mean score.
67
+ */
68
+ function checkRate(check, index, records) {
69
+ if (records.length === 0)
70
+ return "n/a";
71
+ const total = records.reduce((sum, record) => sum + (record.checks[index]?.score ?? 0), 0);
72
+ return isJudged(check) || isDecision(check)
73
+ ? formatScore(total / records.length)
74
+ : `${String(total)}/${String(records.length)}`;
75
+ }
76
+ /**
77
+ * Reports what a scoring model cost, which the table leaves out.
78
+ * @param title - Names the model's role: the judge, or the decision model.
79
+ * @param judged - What the model cost each run that it scored.
80
+ * @returns The lines to add under the table; none when it scored no run. A metric that any of the runs lacks is left out.
81
+ */
82
+ function scorerLine(title, judged) {
83
+ if (judged.length === 0)
84
+ return [];
85
+ const calls = judged.reduce((total, usage) => total + usage.calls, 0);
86
+ const totals = METRICS.flatMap((metric) => {
87
+ const values = judged.flatMap((usage) => metric.value(usage) ?? []);
88
+ if (values.length < judged.length)
89
+ return [];
90
+ const total = values.reduce((sum, value) => sum + value, 0);
91
+ return [`${metric.title.toLowerCase()} ${metric.format(total)}`];
92
+ });
93
+ const parts = [`${String(calls)} calls`, ...totals];
94
+ return [
95
+ "",
96
+ `${title}, in total and not counted in the table: ${parts.join(", ")}`,
97
+ ];
98
+ }
99
+ const cell = (text) => text.replaceAll("|", "\\|");
100
+ /**
101
+ * Builds the table row of one case.
102
+ * @param caseDef - The case whose runs the row sums up.
103
+ * @param section - The scores and runs of the harness the row is for.
104
+ * @returns A markdown table row: the case score, each check's rate, and the median of each metric over the valid runs.
105
+ */
106
+ function caseRow(caseDef, section) {
107
+ const { suite, records, labels } = section;
108
+ const tag = (label) => labels.length > 1 ? `${label.toUpperCase()} ` : "";
109
+ const valid = labels.map((label) => ({
110
+ label,
111
+ runs: records.filter((record) => record.caseId === caseDef.id &&
112
+ record.subject === label &&
113
+ !invalidReason(record)),
114
+ }));
115
+ const checks = caseDef.checks.map((check, index) => {
116
+ const rates = valid.map(({ label, runs }) => tag(label) + checkRate(check, index, runs));
117
+ return `\`${describeCheck(check)}\` ${rates.join(", ")}`;
118
+ });
119
+ const scores = suite.cases.find((scored) => scored.id === caseDef.id)?.scores;
120
+ const cells = [
121
+ caseDef.id,
122
+ versus(Object.values(scores ?? {}), formatScore),
123
+ checks.join("<br>"),
124
+ ...METRICS.map((metric) => versus(valid.map(({ runs }) => median(runs.flatMap((run) => metric.value(run) ?? []))), metric.format)),
125
+ ];
126
+ return `| ${cells.map(cell).join(" | ")} |`;
127
+ }
128
+ // The case, score, and checks columns that stand before the metric columns.
129
+ const LEADING_COLUMNS = 3;
130
+ /**
131
+ * Builds the part of the report for one harness.
132
+ * @param section - The scores and runs of one harness.
133
+ * @returns A heading, the table of cases, the suite score, what the judge and the decision model cost, and the list
134
+ * of invalid runs.
135
+ */
136
+ function harnessSection(section) {
137
+ const { suite, cases, records, labels } = section;
138
+ const heading = (title) => labels.length > 1 ? `${title}, A / B (B - A)` : title;
139
+ const invalid = records.flatMap((record) => {
140
+ const reason = invalidReason(record);
141
+ if (reason === null)
142
+ return [];
143
+ const shown = reason.replace(/\s+/g, " ").slice(0, SHOWN_REASON_LENGTH);
144
+ return [
145
+ `- ${record.subject.toUpperCase()} \`${record.caseId}\` rep ${String(record.rep)}: ${shown}`,
146
+ ];
147
+ });
148
+ const metricHeadings = METRICS.map((metric) => heading(metric.title));
149
+ const rule = Array(METRICS.length + LEADING_COLUMNS).fill("---");
150
+ return [
151
+ `## ${suite.harness}`,
152
+ "",
153
+ `| Case | ${heading("Score")} | Checks | ${metricHeadings.join(" | ")} |`,
154
+ `| ${rule.join(" | ")} |`,
155
+ ...cases.map((caseDef) => caseRow(caseDef, section)),
156
+ "",
157
+ `Suite score: ${versus(Object.values(suite.suite), formatScore)}`,
158
+ ...scorerLine("Judge", records.flatMap((record) => record.judge ?? [])),
159
+ ...scorerLine("Decision model", records.flatMap((record) => record.decision ?? [])),
160
+ ...(invalid.length
161
+ ? ["", "Invalid runs, not counted above:", "", ...invalid]
162
+ : []),
163
+ ].join("\n");
164
+ }
165
+ /**
166
+ * Renders an evaluation as the markdown report that the `run` and `score` commands print.
167
+ * @param evaluation - What `evaluate` returned.
168
+ * @returns The report: a title that says whether one subject was scored or two were compared, what ran, one section
169
+ * for each harness, and the pass bars that were not reached.
170
+ */
171
+ export function renderReport(evaluation) {
172
+ const { subjects, cases, records, belowBar, reps } = evaluation;
173
+ const compared = subjects.length > 1;
174
+ const labels = subjects.map((subject) => subject.label);
175
+ const judged = cases.some((caseDef) => caseDef.checks.some(isJudged));
176
+ const sections = evaluation.suites.map((suite) => harnessSection({
177
+ suite,
178
+ cases,
179
+ records: records.filter((record) => record.harness === suite.harness),
180
+ labels,
181
+ }));
182
+ return [
183
+ compared ? "# Skill comparison" : "# Skill scores",
184
+ "",
185
+ ...subjects.map((subject) => `- ${subject.label.toUpperCase()}: ${describeSubject(subject)}`),
186
+ `- Harness: ${evaluation.harnesses.join(", ")} (${evaluation.harnessReason})`,
187
+ `- Model: ${evaluation.model ?? "the harness default"}`,
188
+ ...(judged ? [`- Judge: ${evaluation.judge}`] : []),
189
+ ...(evaluation.decisionModel === null
190
+ ? []
191
+ : [`- Decision model: ${evaluation.decisionModel}`]),
192
+ `- Reps: ${String(reps)} for each ${compared ? "subject and case" : "case"}`,
193
+ `- Output: ${evaluation.outDir}`,
194
+ "",
195
+ sections.join("\n\n"),
196
+ ...(belowBar.length
197
+ ? ["", "Below the pass bar:", "", ...belowBar.map((line) => `- ${line}`)]
198
+ : []),
199
+ "",
200
+ ].join("\n");
201
+ }
package/dist/run.d.ts ADDED
@@ -0,0 +1,86 @@
1
+ import type { Case } from "./cases.js";
2
+ import type { Check } from "./checks.js";
3
+ import type { DecisionService } from "./decision.js";
4
+ import type { Harness, Tokens, ToolCall } from "./harnesses/shared.js";
5
+ import type { Subject } from "./subject.js";
6
+ /** What a session cost. A number the harness does not report is null. */
7
+ export interface Usage {
8
+ durationMs: number;
9
+ tokens: Tokens | null;
10
+ costUsd: number | null;
11
+ }
12
+ /** A check with what it gave one run. */
13
+ export type CheckResult = Check & {
14
+ /** Null when a judged check or a decision check got no usable answer. */
15
+ score: number | null;
16
+ /** The judge's reason, or why there is no score. */
17
+ reason?: string;
18
+ /** For a decision check, the probability the model gave each allowed answer. */
19
+ probabilities?: Record<string, number>;
20
+ };
21
+ /** Everything kept about one run. */
22
+ export interface RunRecord extends Usage {
23
+ harness: string;
24
+ /** The subject's label. */
25
+ subject: string;
26
+ caseId: string;
27
+ rep: number;
28
+ ok: boolean;
29
+ error: string | null;
30
+ finalText: string;
31
+ toolCalls: ToolCall[];
32
+ skillsLoaded: string[];
33
+ /** Whether the harness reported the subject as loaded at startup; null when it reports nothing, or when there is nothing to load. */
34
+ subjectLoaded: boolean | null;
35
+ checks: CheckResult[];
36
+ /** The run score, null when the run is invalid. */
37
+ score: number | null;
38
+ /** What the judged checks cost, kept apart from the run's own numbers; null when no judge ran. */
39
+ judge: (Usage & {
40
+ calls: number;
41
+ }) | null;
42
+ /** What the decision checks cost, kept apart in the same way; null when no request was sent. */
43
+ decision: (Usage & {
44
+ calls: number;
45
+ }) | null;
46
+ }
47
+ /** The harness and model that score judged checks. */
48
+ export interface Judge {
49
+ harness: Harness;
50
+ bin: string;
51
+ model?: string | undefined;
52
+ }
53
+ /** One run to make: one case, one harness, one subject. */
54
+ export interface RunPlan {
55
+ harness: Harness;
56
+ bin: string;
57
+ subject: Subject;
58
+ caseDef: Case;
59
+ rep: number;
60
+ model?: string | undefined;
61
+ judge: Judge;
62
+ /** The model that scores decision checks; null when no case has one. */
63
+ decision: DecisionService | null;
64
+ /** Where the run's stream and final text are written. */
65
+ outDir: string;
66
+ }
67
+ /**
68
+ * Applies the one rule for "the agent loaded a skill", shared by every harness.
69
+ * @param toolCalls - Every tool call of a run.
70
+ * @returns The sorted names of the skills the agent loaded: through a call of the harness's skill tool that names it
71
+ * (any `plugin:` prefix dropped), or through any tool call whose input points at `skills/<name>/SKILL.md`.
72
+ */
73
+ export declare function skillsLoaded(toolCalls: ToolCall[]): string[];
74
+ /**
75
+ * Decides whether a run counts.
76
+ * @param record - The parts of a run record that can disqualify it.
77
+ * @returns Why the record must not be counted, or null when it is a valid run.
78
+ */
79
+ export declare function invalidReason(record: Pick<RunRecord, "error" | "subjectLoaded" | "checks">): string | null;
80
+ /**
81
+ * Makes one run: one prompt, one harness, one subject. It runs the prompt, normalizes the output, scores every
82
+ * check, and cleans up.
83
+ * @param plan - What to run and where to write its files.
84
+ * @returns The record of the run. A failed run is a record with an error, never a rejection.
85
+ */
86
+ export declare function runOnce(plan: RunPlan): Promise<RunRecord>;
package/dist/run.js ADDED
@@ -0,0 +1,220 @@
1
+ import fs from "node:fs";
2
+ import path from "node:path";
3
+ import { checkScore, isDecision, isJudged } from "./checks.js";
4
+ import { decide } from "./decision.js";
5
+ import { sumTokens } from "./harnesses/shared.js";
6
+ import { judge } from "./judge.js";
7
+ import { weightedMean } from "./score.js";
8
+ import { session } from "./session.js";
9
+ import { hasContent } from "./subject.js";
10
+ // A judge answers in one message. The second turn is room for one stray tool call before the answer.
11
+ const JUDGE_LIMITS = { maxTurns: 2, timeoutSeconds: 300 };
12
+ // A decision check that no request was sent for: only a valid run is decided, and a suite with decision checks
13
+ // does not start without a decision model.
14
+ const NOT_DECIDED = {
15
+ score: null,
16
+ reason: "not decided, because the run is invalid",
17
+ };
18
+ // The key of the decision model stays out of every harness process, the agent's and the judge's.
19
+ const withheld = (plan) => plan.decision ? [plan.decision.secret] : [];
20
+ const SKILL_FILE = /skills[\\/]+([\w-]+)[\\/]+skill\.md/gi;
21
+ /**
22
+ * Applies the one rule for "the agent loaded a skill", shared by every harness.
23
+ * @param toolCalls - Every tool call of a run.
24
+ * @returns The sorted names of the skills the agent loaded: through a call of the harness's skill tool that names it
25
+ * (any `plugin:` prefix dropped), or through any tool call whose input points at `skills/<name>/SKILL.md`.
26
+ */
27
+ export function skillsLoaded(toolCalls) {
28
+ const names = new Set();
29
+ for (const { kind, input } of toolCalls) {
30
+ if (kind === "skill") {
31
+ // Object() turns an input that is no object into one with none of the three keys.
32
+ const given = Object(input);
33
+ const named = [given["skill"], given["name"], given["id"]].find((value) => typeof value === "string");
34
+ if (named)
35
+ names.add(named.slice(named.lastIndexOf(":") + 1));
36
+ }
37
+ for (const match of JSON.stringify(input ?? null).matchAll(SKILL_FILE)) {
38
+ names.add(String(match[1]));
39
+ }
40
+ }
41
+ return [...names].sort();
42
+ }
43
+ /**
44
+ * Decides whether a run counts.
45
+ * @param record - The parts of a run record that can disqualify it.
46
+ * @returns Why the record must not be counted, or null when it is a valid run.
47
+ */
48
+ export function invalidReason(record) {
49
+ if (record.error)
50
+ return `run failed: ${record.error}`;
51
+ if (record.subjectLoaded === false) {
52
+ return "the harness did not load the subject: it is not among the plugins and skills it reported";
53
+ }
54
+ const unscored = record.checks.find((check) => check.score === null);
55
+ if (unscored) {
56
+ const kind = isDecision(unscored) ? "decision" : "judged";
57
+ return `a ${kind} check has no score: ${String(unscored.reason)}`;
58
+ }
59
+ return null;
60
+ }
61
+ /**
62
+ * Compares two paths that may reach one directory through different symlinks.
63
+ * @param x - A path, which need not exist.
64
+ * @param y - Another path.
65
+ * @returns True when both resolve to the same real path.
66
+ */
67
+ function samePath(x, y) {
68
+ const real = (p) => {
69
+ try {
70
+ return fs.realpathSync(p);
71
+ }
72
+ catch {
73
+ return path.resolve(p);
74
+ }
75
+ };
76
+ return real(x) === real(y);
77
+ }
78
+ // Subject kind -> whether the harness reported the subject as loaded, or null when it does not report that.
79
+ // A plugin is found by its directory and a skills folder by the name of every skill in it.
80
+ const REPORTED_LOADED = {
81
+ plugin: (subject, { paths }) => paths && paths.some((loadedPath) => samePath(loadedPath, subject.dir)),
82
+ skills: (subject, { skills }) => skills && subject.skills.every((name) => skills.includes(name)),
83
+ };
84
+ /**
85
+ * Adds up what the calls of a scoring model cost.
86
+ * @param replies - Every reply of the judge, or every request to the decision model, for one run.
87
+ * @returns The totals. A cost that any reply does not report stays null.
88
+ */
89
+ function totalUsage(replies) {
90
+ const costs = replies.flatMap((reply) => reply.costUsd ?? []);
91
+ return {
92
+ calls: replies.length,
93
+ durationMs: replies.reduce((total, reply) => total + reply.durationMs, 0),
94
+ tokens: sumTokens(replies.flatMap((reply) => reply.tokens ?? [])),
95
+ costUsd: costs.length === replies.length
96
+ ? costs.reduce((total, cost) => total + cost, 0)
97
+ : null,
98
+ };
99
+ }
100
+ /**
101
+ * Scores every check of a case for one run.
102
+ * @param plan - The run; its judge scores the judged checks, with nothing loaded and a prompt that carries nothing
103
+ * about the subject.
104
+ * @param outcome - What the deterministic checks observe.
105
+ * @param judgeable - False when the run is already invalid, which leaves the judged checks without a score.
106
+ * @returns The check results in case order, every reply of the judge, and every request to the decision model.
107
+ */
108
+ async function scoreChecks(plan, outcome, judgeable) {
109
+ const { prompt: task, checks: asked } = plan.caseDef;
110
+ const withhold = withheld(plan);
111
+ const ask = (judgePrompt) => session({
112
+ ...plan.judge,
113
+ subject: null,
114
+ prompt: judgePrompt,
115
+ ...JUDGE_LIMITS,
116
+ withhold,
117
+ }, ({ parsed, error, durationMs }) => ({
118
+ finalText: parsed.finalText,
119
+ error,
120
+ durationMs,
121
+ tokens: parsed.tokens,
122
+ costUsd: parsed.costUsd,
123
+ }));
124
+ // One request carries every decision check of the case.
125
+ const decisions = asked.filter(isDecision);
126
+ const decided = judgeable && plan.decision && decisions.length
127
+ ? await decide(plan.decision.model, {
128
+ task,
129
+ answer: outcome.finalText,
130
+ checks: decisions,
131
+ })
132
+ : null;
133
+ const checks = [];
134
+ const replies = [];
135
+ for (const check of asked) {
136
+ if (isDecision(check)) {
137
+ const result = decided?.results[decisions.indexOf(check)];
138
+ checks.push({ ...check, ...(result ?? NOT_DECIDED) });
139
+ }
140
+ else if (!isJudged(check)) {
141
+ checks.push({ ...check, score: checkScore(check, outcome) });
142
+ }
143
+ else if (!judgeable) {
144
+ const reason = "not judged, because the run is invalid";
145
+ checks.push({ ...check, score: null, reason });
146
+ }
147
+ else {
148
+ const verdict = await judge(ask, {
149
+ task,
150
+ answer: outcome.finalText,
151
+ rubric: check.rubric,
152
+ });
153
+ replies.push(...verdict.replies);
154
+ checks.push({ ...check, score: verdict.score, reason: verdict.reason });
155
+ }
156
+ }
157
+ return { checks, replies, requests: decided?.calls ?? [] };
158
+ }
159
+ /**
160
+ * Turns a finished session into a run record: writes the run's files, then scores every check.
161
+ * @param plan - The run that was made.
162
+ * @param ran - What the session produced; its workspace still exists.
163
+ * @returns The record, with a score unless the run is invalid.
164
+ */
165
+ async function record(plan, ran) {
166
+ const { harness, subject, caseDef, rep } = plan;
167
+ const { stdout, parsed, error, durationMs } = ran;
168
+ const loaded = skillsLoaded(parsed.toolCalls);
169
+ const stem = path.join(plan.outDir, harness.id, caseDef.id, `${subject.label}${String(rep)}`);
170
+ fs.mkdirSync(path.dirname(stem), { recursive: true });
171
+ fs.writeFileSync(`${stem}.stream.jsonl`, stdout);
172
+ fs.writeFileSync(`${stem}.final.md`, parsed.finalText);
173
+ const base = {
174
+ harness: harness.id,
175
+ subject: subject.label,
176
+ caseId: caseDef.id,
177
+ rep,
178
+ ok: !error,
179
+ error,
180
+ finalText: parsed.finalText,
181
+ toolCalls: parsed.toolCalls,
182
+ skillsLoaded: loaded,
183
+ subjectLoaded: hasContent(subject)
184
+ ? REPORTED_LOADED[subject.kind](subject, parsed.loaded)
185
+ : null,
186
+ durationMs,
187
+ tokens: parsed.tokens,
188
+ costUsd: parsed.costUsd,
189
+ };
190
+ const { checks, replies, requests } = await scoreChecks(plan, { ...parsed, skillsLoaded: loaded, workspace: ran.workspace }, invalidReason({ ...base, checks: [] }) === null);
191
+ return {
192
+ ...base,
193
+ checks,
194
+ score: invalidReason({ ...base, checks }) ? null : weightedMean(checks),
195
+ judge: replies.length ? totalUsage(replies) : null,
196
+ decision: requests.length ? totalUsage(requests) : null,
197
+ };
198
+ }
199
+ /**
200
+ * Makes one run: one prompt, one harness, one subject. It runs the prompt, normalizes the output, scores every
201
+ * check, and cleans up.
202
+ * @param plan - What to run and where to write its files.
203
+ * @returns The record of the run. A failed run is a record with an error, never a rejection.
204
+ */
205
+ export async function runOnce(plan) {
206
+ const { harness, bin, subject, caseDef, model } = plan;
207
+ const { prompt, maxTurns, timeoutSeconds, workspaceDir } = caseDef;
208
+ return session({
209
+ harness,
210
+ bin,
211
+ // To a harness, no skills and the judge's "nothing loaded" are the same thing.
212
+ subject: hasContent(subject) ? subject : null,
213
+ prompt,
214
+ model,
215
+ maxTurns,
216
+ timeoutSeconds,
217
+ workspaceDir,
218
+ withhold: withheld(plan),
219
+ }, (ran) => record(plan, ran));
220
+ }
@@ -0,0 +1,39 @@
1
+ /** A wrong command line, case file, or environment: reported without a stack trace, exit code 2. */
2
+ export declare class UsageError extends Error {
3
+ name: string;
4
+ }
5
+ /**
6
+ * Reads the message of a caught value, which the language types as unknown.
7
+ * @param error - What a `catch` clause received.
8
+ * @returns The message of an Error, and the text form of any other thrown value.
9
+ */
10
+ export declare const messageOf: (error: unknown) => string;
11
+ /**
12
+ * Stops a command before it does any work on a platform the harness runs do not support.
13
+ * @throws {UsageError} On Windows, where process groups and the harness homes work differently.
14
+ */
15
+ export declare function requireSupportedPlatform(): void;
16
+ /**
17
+ * Registers a cleanup that runs once: when the returned function is called, or when the process exits.
18
+ * A signal ends the process only while a cleanup is registered, so a program that imports this keeps its own
19
+ * signal handling the rest of the time.
20
+ * @param cleanup - Removes what must not outlive the run. It may run during process exit, so it must be synchronous.
21
+ * @returns Runs the cleanup now and unregisters it. A second call does nothing.
22
+ */
23
+ export declare function onDispose(cleanup: () => void): () => void;
24
+ /**
25
+ * Where a command's results go.
26
+ * @param requested - The directory the user asked for, created when missing. Absent means a new temp directory.
27
+ * @param cwd - What a relative `requested` resolves against.
28
+ * @returns An absolute path to a directory that exists and that nothing removes afterwards.
29
+ */
30
+ export declare function outputDir(requested: string | undefined, cwd: string): string;
31
+ /**
32
+ * Makes a temp directory whose name gives the agent no hint about why it exists.
33
+ * @param prefix - The start of the directory name; random characters follow it.
34
+ * @returns The real path of the directory, and a `dispose` that removes it and also runs at process exit.
35
+ */
36
+ export declare function tempDir(prefix: string): {
37
+ dir: string;
38
+ dispose: () => void;
39
+ };
package/dist/scope.js ADDED
@@ -0,0 +1,82 @@
1
+ import fs from "node:fs";
2
+ import os from "node:os";
3
+ import path from "node:path";
4
+ /** A wrong command line, case file, or environment: reported without a stack trace, exit code 2. */
5
+ export class UsageError extends Error {
6
+ name = "UsageError";
7
+ }
8
+ /**
9
+ * Reads the message of a caught value, which the language types as unknown.
10
+ * @param error - What a `catch` clause received.
11
+ * @returns The message of an Error, and the text form of any other thrown value.
12
+ */
13
+ export const messageOf = (error) => error instanceof Error ? error.message : String(error);
14
+ /**
15
+ * Stops a command before it does any work on a platform the harness runs do not support.
16
+ * @throws {UsageError} On Windows, where process groups and the harness homes work differently.
17
+ */
18
+ export function requireSupportedPlatform() {
19
+ if (process.platform === "win32") {
20
+ throw new UsageError("Windows is not supported; use macOS, Linux, or WSL");
21
+ }
22
+ }
23
+ // The conventional exit code of a process that a signal ended: 128 + SIGINT.
24
+ const INTERRUPTED_EXIT_CODE = 130;
25
+ const live = new Set();
26
+ process.on("exit", () => {
27
+ for (const dispose of [...live])
28
+ dispose();
29
+ });
30
+ const SIGNALS = ["SIGINT", "SIGTERM"];
31
+ const interrupted = () => process.exit(INTERRUPTED_EXIT_CODE);
32
+ /**
33
+ * Registers a cleanup that runs once: when the returned function is called, or when the process exits.
34
+ * A signal ends the process only while a cleanup is registered, so a program that imports this keeps its own
35
+ * signal handling the rest of the time.
36
+ * @param cleanup - Removes what must not outlive the run. It may run during process exit, so it must be synchronous.
37
+ * @returns Runs the cleanup now and unregisters it. A second call does nothing.
38
+ */
39
+ export function onDispose(cleanup) {
40
+ const dispose = () => {
41
+ if (!live.delete(dispose))
42
+ return;
43
+ if (live.size === 0) {
44
+ for (const signal of SIGNALS)
45
+ process.off(signal, interrupted);
46
+ }
47
+ cleanup();
48
+ };
49
+ if (live.size === 0) {
50
+ for (const signal of SIGNALS)
51
+ process.on(signal, interrupted);
52
+ }
53
+ live.add(dispose);
54
+ return dispose;
55
+ }
56
+ /**
57
+ * Where a command's results go.
58
+ * @param requested - The directory the user asked for, created when missing. Absent means a new temp directory.
59
+ * @param cwd - What a relative `requested` resolves against.
60
+ * @returns An absolute path to a directory that exists and that nothing removes afterwards.
61
+ */
62
+ export function outputDir(requested, cwd) {
63
+ if (!requested)
64
+ return fs.mkdtempSync(path.join(os.tmpdir(), "runs-"));
65
+ const dir = path.resolve(cwd, requested);
66
+ fs.mkdirSync(dir, { recursive: true });
67
+ return dir;
68
+ }
69
+ /**
70
+ * Makes a temp directory whose name gives the agent no hint about why it exists.
71
+ * @param prefix - The start of the directory name; random characters follow it.
72
+ * @returns The real path of the directory, and a `dispose` that removes it and also runs at process exit.
73
+ */
74
+ export function tempDir(prefix) {
75
+ const dir = fs.realpathSync(fs.mkdtempSync(path.join(os.tmpdir(), prefix)));
76
+ return {
77
+ dir,
78
+ dispose: onDispose(() => {
79
+ fs.rmSync(dir, { recursive: true, force: true });
80
+ }),
81
+ };
82
+ }