skill-evals 0.0.0-stage → 0.0.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (54) hide show
  1. package/CHANGELOG.md +22 -0
  2. package/LICENSE +21 -0
  3. package/README.md +427 -2
  4. package/dist/baseten.d.ts +20 -0
  5. package/dist/baseten.js +89 -0
  6. package/dist/bin.d.ts +2 -0
  7. package/dist/bin.js +5 -0
  8. package/dist/cases.d.ts +21 -0
  9. package/dist/cases.js +207 -0
  10. package/dist/checks.d.ts +126 -0
  11. package/dist/checks.js +117 -0
  12. package/dist/cli.d.ts +7 -0
  13. package/dist/cli.js +172 -0
  14. package/dist/decision.d.ts +70 -0
  15. package/dist/decision.js +130 -0
  16. package/dist/evaluate.d.ts +70 -0
  17. package/dist/evaluate.js +205 -0
  18. package/dist/exec.d.ts +20 -0
  19. package/dist/exec.js +82 -0
  20. package/dist/harnesses/claude-code.d.ts +3 -0
  21. package/dist/harnesses/claude-code.js +83 -0
  22. package/dist/harnesses/codex.d.ts +3 -0
  23. package/dist/harnesses/codex.js +85 -0
  24. package/dist/harnesses/cursor.d.ts +3 -0
  25. package/dist/harnesses/cursor.js +81 -0
  26. package/dist/harnesses/index.d.ts +34 -0
  27. package/dist/harnesses/index.js +101 -0
  28. package/dist/harnesses/opencode.d.ts +3 -0
  29. package/dist/harnesses/opencode.js +75 -0
  30. package/dist/harnesses/pi.d.ts +3 -0
  31. package/dist/harnesses/pi.js +69 -0
  32. package/dist/harnesses/shared.d.ts +105 -0
  33. package/dist/harnesses/shared.js +90 -0
  34. package/dist/index.d.ts +10 -0
  35. package/dist/index.js +3 -0
  36. package/dist/judge.d.ts +30 -0
  37. package/dist/judge.js +87 -0
  38. package/dist/probe.d.ts +16 -0
  39. package/dist/probe.js +113 -0
  40. package/dist/project.d.ts +36 -0
  41. package/dist/project.js +84 -0
  42. package/dist/report.d.ts +8 -0
  43. package/dist/report.js +201 -0
  44. package/dist/run.d.ts +86 -0
  45. package/dist/run.js +220 -0
  46. package/dist/scope.d.ts +39 -0
  47. package/dist/scope.js +82 -0
  48. package/dist/score.d.ts +71 -0
  49. package/dist/score.js +92 -0
  50. package/dist/session.d.ts +33 -0
  51. package/dist/session.js +146 -0
  52. package/dist/subject.d.ts +54 -0
  53. package/dist/subject.js +179 -0
  54. package/package.json +59 -4
package/dist/cli.js ADDED
@@ -0,0 +1,172 @@
1
+ import fs from "node:fs";
2
+ import { parseArgs } from "node:util";
3
+ import { loadCases } from "./cases.js";
4
+ import { evaluate } from "./evaluate.js";
5
+ import { HARNESSES, findBin } from "./harnesses/index.js";
6
+ import { check } from "./probe.js";
7
+ import { findProject, withConfig } from "./project.js";
8
+ import { renderReport } from "./report.js";
9
+ import { invalidReason } from "./run.js";
10
+ import { formatScore } from "./score.js";
11
+ import { UsageError, requireSupportedPlatform } from "./scope.js";
12
+ const USAGE = `Usage: skill-evals <command> [options]
13
+
14
+ run Run every case against subject A and subject B, score both, and print a comparison.
15
+ --a <subject> the baseline (default: main)
16
+ --b <subject> the change (default: the project, uncommitted changes included)
17
+ score Run every case against one subject and print its scores.
18
+ --subject <subject> (default: the project, uncommitted changes included)
19
+ run and score also take:
20
+ --harness <ids|all> comma-separated harness ids (default: the first installed one)
21
+ --case <glob> case ids to run (default: *)
22
+ --cases <dir> directory that holds the cases (default: evals/cases in the project)
23
+ --reps <n> runs for each subject and case (default: 3)
24
+ --model <id> passed to the harness (default: the harness's own default)
25
+ --min-score <0..1> exit 1 when the suite score of the subject (B for run) is lower
26
+ --judge-harness <id> harness that scores judged checks (default: the harness under test)
27
+ --judge-model <id> model that scores judged checks (default: --model)
28
+ --decision-model <id> decision model that scores decision checks (no default)
29
+ --decision-url <url> base URL of the decision model's API (default: the provider's own)
30
+ --decision-timeout <s> seconds one request to the decision model may take (default: 60)
31
+ --out <dir> where results go (default: a new temp directory)
32
+ check Run one small prompt in each installed harness and show what its adapter reports.
33
+ --subject <subject> --harness <ids|all> (default: all) --model <id> --out <dir>
34
+ list Show harnesses in default-preference order, and the cases.
35
+ --cases <dir>
36
+
37
+ A subject is what a run loads: a directory, a git ref of the project, or "none" for no skills at all.
38
+ A directory or ref with a plugin manifest is loaded as a plugin. Any other is a folder of skills.
39
+ The project is the git repository that holds the working directory, or the working directory itself.
40
+ skill-evals.config.json in the project can set a default for cases, harness, reps, model, judgeHarness,
41
+ judgeModel, decisionModel, decisionUrl, decisionTimeout, and minScore.`;
42
+ const USAGE_EXIT_CODE = 2;
43
+ const FLAGS = {
44
+ a: {},
45
+ b: {},
46
+ subject: {},
47
+ harness: {},
48
+ case: { as: "caseGlob" },
49
+ cases: {},
50
+ reps: { read: Number },
51
+ model: {},
52
+ "min-score": { as: "minScore", read: Number },
53
+ "judge-harness": { as: "judgeHarness" },
54
+ "judge-model": { as: "judgeModel" },
55
+ "decision-model": { as: "decisionModel" },
56
+ "decision-url": { as: "decisionUrl" },
57
+ "decision-timeout": { as: "decisionTimeout", read: Number },
58
+ out: { as: "outDir" },
59
+ };
60
+ const SUITE_FLAGS = [
61
+ "harness",
62
+ "case",
63
+ "cases",
64
+ "reps",
65
+ "model",
66
+ "min-score",
67
+ "judge-harness",
68
+ "judge-model",
69
+ "decision-model",
70
+ "decision-url",
71
+ "decision-timeout",
72
+ "out",
73
+ ];
74
+ /**
75
+ * Runs the suite, prints its report to standard output, and prints one line for each finished run to standard error.
76
+ * @param options - What to evaluate.
77
+ * @returns The exit code: 1 when any run is invalid or a pass bar is not reached, else 0.
78
+ */
79
+ async function suite(options) {
80
+ const onRun = (record) => {
81
+ const outcome = invalidReason(record) ?? `score ${formatScore(record.score)}`;
82
+ console.error(`${record.harness} ${record.caseId} ${record.subject}${String(record.rep)}: ${outcome}`);
83
+ };
84
+ const evaluation = await evaluate({ ...options, onRun });
85
+ console.log(renderReport(evaluation));
86
+ return evaluation.invalid.length || evaluation.belowBar.length ? 1 : 0;
87
+ }
88
+ /**
89
+ * Prints the harnesses in default-preference order, each with where it is installed, and the cases.
90
+ * @param given - The options of the command line.
91
+ * @param project - Where the config file and the default cases directory are.
92
+ * @returns The exit code, always 0: a missing cases directory is a line of the listing.
93
+ */
94
+ function list(given, project) {
95
+ const options = withConfig(project, given);
96
+ const installed = HARNESSES.map((harness) => ({
97
+ harness,
98
+ bin: findBin(harness),
99
+ }));
100
+ const first = installed.find((entry) => entry.bin);
101
+ const cases = fs.existsSync(options.cases)
102
+ ? loadCases(options.cases).map((caseDef) => `- ${caseDef.id}: ${String(caseDef.checks.length)} checks`)
103
+ : ["- none: the directory does not exist"];
104
+ console.log([
105
+ "Harnesses, in default-preference order:",
106
+ ...installed.map(({ harness, bin }) => `- ${harness.id}: ${bin ? `installed (${bin})` : `not installed ("${harness.bin}" not found)`}`),
107
+ `Default harness: ${first ? first.harness.id : "none installed"}`,
108
+ "",
109
+ `Cases in ${options.cases}:`,
110
+ ...cases,
111
+ ].join("\n"));
112
+ return 0;
113
+ }
114
+ const COMMANDS = {
115
+ run: {
116
+ handler: ({ a = "main", b, ...options }, project) => suite({ ...options, subjects: [a, b ?? project.root] }),
117
+ flags: ["a", "b", ...SUITE_FLAGS],
118
+ },
119
+ score: {
120
+ handler: ({ subject, ...options }, project) => suite({ ...options, subjects: [subject ?? project.root] }),
121
+ flags: ["subject", ...SUITE_FLAGS],
122
+ },
123
+ check: { handler: check, flags: ["subject", "harness", "model", "out"] },
124
+ list: { handler: list, flags: ["cases"] },
125
+ };
126
+ /**
127
+ * Reads the flags of a command.
128
+ * @param command - The command, which names the flags it takes.
129
+ * @param args - The command line after the command's name.
130
+ * @returns The options that are set, each under the name the code uses for it.
131
+ * @throws {TypeError} With a code that starts with `ERR_PARSE_ARGS`, for a flag the command does not take.
132
+ */
133
+ function readOptions(command, args) {
134
+ const { values } = parseArgs({
135
+ args,
136
+ options: Object.fromEntries(command.flags.map((flag) => [flag, { type: "string" }])),
137
+ });
138
+ return Object.fromEntries(Object.entries(values).map(([flag, text]) => {
139
+ const spec = FLAGS[flag];
140
+ return [spec.as ?? flag, (spec.read ?? String)(String(text))];
141
+ }));
142
+ }
143
+ const isParseArgsError = (error) => error instanceof Error &&
144
+ "code" in error &&
145
+ String(error.code).startsWith("ERR_PARSE_ARGS");
146
+ /**
147
+ * Runs one command line.
148
+ * @param argv - The command and its flags, without the node and script paths.
149
+ * @returns The exit code: 0, 1 when a run is invalid or a pass bar is not reached, or 2 with one line on standard
150
+ * error for a wrong command line, case file, config file, or platform. It never calls process.exit.
151
+ */
152
+ export async function main(argv) {
153
+ const [name, ...args] = argv;
154
+ try {
155
+ requireSupportedPlatform();
156
+ const command = name !== undefined && Object.hasOwn(COMMANDS, name)
157
+ ? COMMANDS[name]
158
+ : undefined;
159
+ if (!command) {
160
+ const problem = name ? `unknown command "${name}"` : "missing command";
161
+ throw new UsageError(`${problem}\n\n${USAGE}`);
162
+ }
163
+ const options = readOptions(command, args);
164
+ return await command.handler(options, findProject(process.cwd()));
165
+ }
166
+ catch (error) {
167
+ if (!(error instanceof UsageError || isParseArgsError(error)))
168
+ throw error;
169
+ console.error(`skill-evals: ${error.message}`);
170
+ return USAGE_EXIT_CODE;
171
+ }
172
+ }
@@ -0,0 +1,70 @@
1
+ import type { DecisionCheck } from "./checks.js";
2
+ import type { Tokens } from "./harnesses/shared.js";
3
+ /** One question to a decision model: a yes or no question, or a choice among named options. */
4
+ export type DecisionQuestion = {
5
+ kind: "yes_no";
6
+ question: string;
7
+ } | {
8
+ kind: "choice";
9
+ question: string;
10
+ options: string[];
11
+ };
12
+ /** What to decide: the material every question is about, and the questions by name. */
13
+ export interface DecisionRequest {
14
+ state: string;
15
+ questions: Record<string, DecisionQuestion>;
16
+ }
17
+ /** What a decision model returned for one request. A number the provider does not report is null. */
18
+ export interface DecisionReply {
19
+ /**
20
+ * Question name, then answer, then probability. A yes or no question has the answers "yes" and "no". The values
21
+ * come from outside this program, so they are not trusted to be numbers until `expectedValue` has read them.
22
+ */
23
+ probabilities: Record<string, Record<string, unknown> | undefined>;
24
+ tokens: Tokens | null;
25
+ costUsd: number | null;
26
+ }
27
+ /** A model that generates no text: it returns a probability for each allowed answer of each question. */
28
+ export interface DecisionModel {
29
+ /** Rejects when the request fails or the provider's reply cannot be read. */
30
+ decide: (request: DecisionRequest) => Promise<DecisionReply>;
31
+ }
32
+ /** A decision model that is ready to call, and the environment variable that holds its key, which no harness may see. */
33
+ export interface DecisionService {
34
+ model: DecisionModel;
35
+ secret: string;
36
+ }
37
+ /** What one request to the decision model cost. */
38
+ interface DecisionCall {
39
+ durationMs: number;
40
+ tokens: Tokens | null;
41
+ costUsd: number | null;
42
+ }
43
+ /** What a decision check gets from one run. */
44
+ interface DecisionResult {
45
+ /** Null when the model gave no usable probabilities. */
46
+ score: number | null;
47
+ /** The probability of each allowed answer, as the model returned it. */
48
+ probabilities?: Record<string, number>;
49
+ /** Why there is no score. */
50
+ reason?: string;
51
+ }
52
+ /**
53
+ * Scores every decision check of one run with one request, sent once more when the first answer is unusable.
54
+ * @param model - The decision model.
55
+ * @param run - What to decide about.
56
+ * @param run.task - The prompt the agent got.
57
+ * @param run.answer - The agent's final message.
58
+ * @param run.checks - The decision checks of the case; each becomes one question of the request.
59
+ * @returns One result for each check, in order, and what every request cost. After two unusable answers no check
60
+ * has a score: one question without usable probabilities fails the whole request.
61
+ */
62
+ export declare function decide(model: DecisionModel, run: {
63
+ task: string;
64
+ answer: string;
65
+ checks: DecisionCheck[];
66
+ }): Promise<{
67
+ results: DecisionResult[];
68
+ calls: DecisionCall[];
69
+ }>;
70
+ export {};
@@ -0,0 +1,130 @@
1
+ import { messageOf } from "./scope.js";
2
+ const ATTEMPTS = 2;
3
+ // A provider may round each probability before it sends it; the published runtime keeps 6 decimals.
4
+ const ROUNDING_STEP = 1e-6;
5
+ // Rounding moves a number by half a step at most.
6
+ const WORST_ROUNDING = 0.5;
7
+ // Float error in a sum of a few numbers near 1, far below one rounding step.
8
+ const FLOAT_ERROR = 1e-9;
9
+ /**
10
+ * Says how far from 1 the probabilities of the allowed answers may add up.
11
+ * @param count - The number of allowed answers.
12
+ * @returns What rounding alone can explain: half a rounding step for each answer, and float error. It grows with the
13
+ * count because three honest probabilities can add up to 0.999999 or 1.000001. It stays far below any probability
14
+ * a model keeps back on purpose, for example for an "I don't know" answer, so such a reply is still rejected.
15
+ */
16
+ const sumTolerance = (count) => count * WORST_ROUNDING * ROUNDING_STEP + FLOAT_ERROR;
17
+ /**
18
+ * Says what each answer of a decision check is worth.
19
+ * @param check - A decision check.
20
+ * @returns Its `options`; for a yes or no question, 1 for the expected answer and 0 for the other one.
21
+ */
22
+ const answerValues = (check) => check.options ?? {
23
+ yes: Number(check.expect === "yes"),
24
+ no: Number(check.expect === "no"),
25
+ };
26
+ /**
27
+ * Writes what the decision model reads.
28
+ * @param task - The prompt the agent got.
29
+ * @param answer - The agent's final message.
30
+ * @returns The two parts, delimited, and nothing that says where the answer came from.
31
+ */
32
+ const decisionState = (task, answer) => ["<task>", task, "</task>", "", "<answer>", answer, "</answer>"].join("\n");
33
+ /**
34
+ * Scores one decision check from the probabilities of its answers.
35
+ * @param values - What each allowed answer is worth.
36
+ * @param returned - What the model returned for the question.
37
+ * @returns The expected value, which is the sum of probability times value, and the probabilities it was computed from.
38
+ * @throws {Error} When an allowed answer has no probability, a probability is not a number from 0 to 1, or the
39
+ * probabilities of the allowed answers do not add up to 1 within what rounding explains. Nothing is filled in or
40
+ * scaled.
41
+ */
42
+ function expectedValue(values, returned) {
43
+ if (!returned)
44
+ throw new Error("it returned no probabilities for a question");
45
+ const probabilities = {};
46
+ let sum = 0;
47
+ let score = 0;
48
+ for (const [answer, value] of Object.entries(values)) {
49
+ const probability = Object.hasOwn(returned, answer)
50
+ ? returned[answer]
51
+ : undefined;
52
+ if (typeof probability !== "number" ||
53
+ !(probability >= 0 && probability <= 1)) {
54
+ throw new Error(`the probability of "${answer}" must be a number from 0 to 1, got ${JSON.stringify(probability)}`);
55
+ }
56
+ probabilities[answer] = probability;
57
+ sum += probability;
58
+ score += probability * value;
59
+ }
60
+ if (Math.abs(sum - 1) > sumTolerance(Object.keys(values).length)) {
61
+ throw new Error(`the probabilities of ${Object.keys(values).join(", ")} add up to ${String(sum)}, not 1`);
62
+ }
63
+ // No probability is below 0 and no value is outside 0 to 1, so the score is from 0 to the sum, and the sum may
64
+ // pass 1 by the tolerance above.
65
+ return { score: Math.min(score, 1), probabilities };
66
+ }
67
+ /**
68
+ * Sends one request and times it.
69
+ * @param model - The decision model.
70
+ * @param request - The state and the questions.
71
+ * @returns The reply, or why there is none, and what the request cost. A failed request reports no tokens.
72
+ */
73
+ async function attempt(model, request) {
74
+ const started = Date.now();
75
+ try {
76
+ const reply = await model.decide(request);
77
+ const { tokens, costUsd } = reply;
78
+ return {
79
+ reply,
80
+ call: { durationMs: Date.now() - started, tokens, costUsd },
81
+ };
82
+ }
83
+ catch (error) {
84
+ const call = {
85
+ durationMs: Date.now() - started,
86
+ tokens: null,
87
+ costUsd: null,
88
+ };
89
+ return { reply: messageOf(error), call };
90
+ }
91
+ }
92
+ /**
93
+ * Scores every decision check of one run with one request, sent once more when the first answer is unusable.
94
+ * @param model - The decision model.
95
+ * @param run - What to decide about.
96
+ * @param run.task - The prompt the agent got.
97
+ * @param run.answer - The agent's final message.
98
+ * @param run.checks - The decision checks of the case; each becomes one question of the request.
99
+ * @returns One result for each check, in order, and what every request cost. After two unusable answers no check
100
+ * has a score: one question without usable probabilities fails the whole request.
101
+ */
102
+ export async function decide(model, run) {
103
+ const names = run.checks.map((_, index) => `q${String(index + 1)}`);
104
+ const request = {
105
+ state: decisionState(run.task, run.answer),
106
+ questions: Object.fromEntries(run.checks.map(({ question, options }, index) => [
107
+ String(names[index]),
108
+ options
109
+ ? { kind: "choice", question, options: Object.keys(options) }
110
+ : { kind: "yes_no", question },
111
+ ])),
112
+ };
113
+ const calls = [];
114
+ let failure = "";
115
+ while (calls.length < ATTEMPTS) {
116
+ const { reply, call } = await attempt(model, request);
117
+ calls.push(call);
118
+ try {
119
+ if (typeof reply === "string")
120
+ throw new Error(reply);
121
+ const results = run.checks.map((check, index) => expectedValue(answerValues(check), reply.probabilities[String(names[index])]));
122
+ return { results, calls };
123
+ }
124
+ catch (error) {
125
+ failure = messageOf(error);
126
+ }
127
+ }
128
+ const reason = `the decision model gave no usable answer: ${failure}`;
129
+ return { results: run.checks.map(() => ({ score: null, reason })), calls };
130
+ }
@@ -0,0 +1,70 @@
1
+ import type { Case } from "./cases.js";
2
+ import type { RunRecord } from "./run.js";
3
+ import type { SuiteScore } from "./score.js";
4
+ import type { SubjectSummary } from "./subject.js";
5
+ /** What `evaluate` takes. An option that is not set takes its value from the project's config file. */
6
+ export interface EvaluateOptions {
7
+ /** The working directory, which sets the project and what relative paths resolve against. Default: the process's. */
8
+ cwd?: string;
9
+ /** One or two of a directory, a git ref of the project, or "none". */
10
+ subjects: string[];
11
+ /** Comma-separated harness ids, or "all". Default: the first installed harness. */
12
+ harness?: string | undefined;
13
+ /** The directory that holds the cases. Default: `evals/cases` in the project. */
14
+ cases?: string | undefined;
15
+ /** Case ids to run; only `*` is special. Default: every case. */
16
+ caseGlob?: string | undefined;
17
+ /** Runs for each subject and case. Default: 3. */
18
+ reps?: number | undefined;
19
+ /** Passed to the harness. Default: the harness's own default. */
20
+ model?: string | undefined;
21
+ /** The harness that scores judged checks. Default: the harness under test. */
22
+ judgeHarness?: string | undefined;
23
+ /** The model that scores judged checks. Default: `model`. */
24
+ judgeModel?: string | undefined;
25
+ /** The model that scores decision checks. No default: a suite with a decision check needs it. */
26
+ decisionModel?: string | undefined;
27
+ /** The base URL of the decision model's API. Default: the provider's own. */
28
+ decisionUrl?: string | undefined;
29
+ /** The seconds one request to the decision model may take, a positive integer. Default: 60. */
30
+ decisionTimeout?: number | undefined;
31
+ /** The pass bar for the suite score of the subject under test. */
32
+ minScore?: number | undefined;
33
+ /** Where results go. Default: a new temp directory. */
34
+ outDir?: string | undefined;
35
+ /** Called with each run's record as soon as the run ends. */
36
+ onRun?: ((record: RunRecord) => void) | undefined;
37
+ }
38
+ /** What `evaluate` writes to `results.json`. */
39
+ export interface Results {
40
+ subjects: SubjectSummary[];
41
+ harnesses: string[];
42
+ harnessReason: string;
43
+ model: string | null;
44
+ /** The harness and model that scored the judged checks, in words. */
45
+ judge: string;
46
+ /** The model that scored the decision checks; null when none is set. */
47
+ decisionModel: string | null;
48
+ reps: number;
49
+ suites: SuiteScore[];
50
+ /** The pass bars that were not reached. */
51
+ belowBar: string[];
52
+ /** The runs that do not count, each with its reason. */
53
+ invalid: string[];
54
+ records: RunRecord[];
55
+ }
56
+ /** What `evaluate` returns. The evaluation succeeded when `belowBar` and `invalid` are both empty. */
57
+ export interface Evaluation extends Results {
58
+ cases: Case[];
59
+ outDir: string;
60
+ }
61
+ /**
62
+ * Runs every case against each subject, scores the runs, and writes `results.json` and each run's files to the
63
+ * output directory. One subject is scored on its own. With two, the first is the baseline and the second is the one
64
+ * under test, which the pass bars apply to. It prints nothing and never calls process.exit. While it runs, SIGINT
65
+ * and SIGTERM end the process with exit code 130 after the cleanup.
66
+ * @param given - What to evaluate and how. An option that is not set takes its value from the project's config file.
67
+ * @returns The scores, every run record, and where the files are.
68
+ * @throws {UsageError} For a wrong option, case file, or config file, before any harness starts.
69
+ */
70
+ export declare function evaluate(given: EvaluateOptions): Promise<Evaluation>;
@@ -0,0 +1,205 @@
1
+ import fs from "node:fs";
2
+ import path from "node:path";
3
+ import { connect } from "./baseten.js";
4
+ import { loadCases } from "./cases.js";
5
+ import { isDecision } from "./checks.js";
6
+ import { selectHarnesses } from "./harnesses/index.js";
7
+ import { findProject, withConfig } from "./project.js";
8
+ import { invalidReason, runOnce } from "./run.js";
9
+ import { failedBars, isUnitScore, scoreSuites } from "./score.js";
10
+ import { UsageError, outputDir, requireSupportedPlatform } from "./scope.js";
11
+ import { resolveSubject } from "./subject.js";
12
+ const DEFAULT_REPS = 3;
13
+ const MS_PER_SECOND = 1000;
14
+ // The longest time limit a timer takes is 2^32 - 1 milliseconds.
15
+ const MAX_TIMEOUT_SECONDS = 4_294_967;
16
+ /**
17
+ * Pairs each subject with its label.
18
+ * @param specs - What the caller passed as `subjects`.
19
+ * @returns "subject" for one; "a" for the baseline and "b" for the subject under test for two.
20
+ * @throws {UsageError} When there are not one or two.
21
+ */
22
+ function labelled(specs) {
23
+ const [first, second, ...more] = Array.isArray(specs) ? specs : [];
24
+ if (first === undefined || more.length > 0) {
25
+ throw new UsageError("subjects must hold one or two subjects");
26
+ }
27
+ return second === undefined
28
+ ? [{ label: "subject", spec: first }]
29
+ : [
30
+ { label: "a", spec: first },
31
+ { label: "b", spec: second },
32
+ ];
33
+ }
34
+ /**
35
+ * Rejects option values that no run could use, before any harness starts.
36
+ * @param options - The options after the config file was applied.
37
+ * @returns The number of runs for each subject and case.
38
+ * @throws {UsageError} When `reps`, `minScore`, or `decisionTimeout` is out of range.
39
+ */
40
+ function requireUsable(options) {
41
+ const { reps = DEFAULT_REPS, minScore, decisionTimeout: seconds } = options;
42
+ if (!Number.isInteger(reps) || reps < 1) {
43
+ throw new UsageError("reps must be a positive integer");
44
+ }
45
+ if (minScore !== undefined && !isUnitScore(minScore)) {
46
+ throw new UsageError("minScore must be a number from 0 to 1");
47
+ }
48
+ if (seconds !== undefined &&
49
+ !(Number.isInteger(seconds) &&
50
+ seconds >= 1 &&
51
+ seconds <= MAX_TIMEOUT_SECONDS)) {
52
+ throw new UsageError(`decisionTimeout must be a positive integer of seconds, at most ${String(MAX_TIMEOUT_SECONDS)}`);
53
+ }
54
+ return reps;
55
+ }
56
+ /**
57
+ * Connects to the decision model when a case needs one, before any harness starts.
58
+ * @param cases - The cases of the suite.
59
+ * @param options - The options after the config file was applied.
60
+ * @returns The decision model, or null when no case has a decision check.
61
+ * @throws {UsageError} When a case has a decision check and the model or its API key is not set.
62
+ */
63
+ function decisionFor(cases, options) {
64
+ if (!cases.some((caseDef) => caseDef.checks.some(isDecision)))
65
+ return null;
66
+ const { decisionModel: model, decisionUrl: url } = options;
67
+ const { decisionTimeout: seconds } = options;
68
+ if (!model) {
69
+ throw new UsageError("a case has a decision check, but no decision model is set: pass --decision-model or set decisionModel in the config file");
70
+ }
71
+ const timeoutMs = seconds === undefined ? undefined : seconds * MS_PER_SECOND;
72
+ return connect({ model, url, env: process.env, timeoutMs });
73
+ }
74
+ /**
75
+ * Makes every run of a plan, one at a time.
76
+ * @param plan - What to run.
77
+ * @returns The records in the order the runs were made.
78
+ */
79
+ async function runAll(plan) {
80
+ const { cases, reps, subjects, model, outDir, decision } = plan;
81
+ const records = [];
82
+ for (const located of plan.chosen) {
83
+ const judge = plan.judgeOf(located);
84
+ for (const caseDef of cases) {
85
+ for (let rep = 1; rep <= reps; rep++) {
86
+ for (const subject of subjects) {
87
+ const run = {
88
+ ...located,
89
+ subject,
90
+ caseDef,
91
+ rep,
92
+ model,
93
+ judge,
94
+ decision,
95
+ outDir,
96
+ };
97
+ const record = await runOnce(run);
98
+ records.push(record);
99
+ plan.onRun?.(record);
100
+ }
101
+ }
102
+ }
103
+ }
104
+ return records;
105
+ }
106
+ /**
107
+ * Scores the runs and sums up what ran.
108
+ * @param finished - The runs and what they were made with.
109
+ * @returns What goes into `results.json`. The pass bars apply to the last subject, which is the one under test.
110
+ */
111
+ function summarize(finished) {
112
+ const { subjects, selection, judgeHarness, judgeModel, model } = finished;
113
+ const { cases, records } = finished;
114
+ const labels = subjects.map(({ label }) => label);
115
+ const harnesses = selection.chosen.map(({ harness }) => harness.id);
116
+ const suites = scoreSuites({ harnesses, cases, records, labels });
117
+ return {
118
+ subjects: subjects.map(({ label, spec, kind, dir, skills, commit }) => ({
119
+ label,
120
+ spec,
121
+ kind,
122
+ dir,
123
+ skills,
124
+ commit,
125
+ })),
126
+ harnesses,
127
+ harnessReason: selection.reason,
128
+ model: model ?? null,
129
+ judge: `${judgeHarness?.harness.id ?? "the harness under test"}, model ${judgeModel ?? "the harness default"}`,
130
+ decisionModel: finished.decisionModel ?? null,
131
+ reps: finished.reps,
132
+ suites,
133
+ belowBar: failedBars(suites, labels.at(-1) ?? "", finished.minScore),
134
+ invalid: records.flatMap((record) => {
135
+ const reason = invalidReason(record);
136
+ const run = `${record.harness} ${record.caseId} ${record.subject}${String(record.rep)}`;
137
+ return reason === null ? [] : [`${run}: ${reason}`];
138
+ }),
139
+ records,
140
+ };
141
+ }
142
+ /**
143
+ * Runs every case against each subject, scores the runs, and writes `results.json` and each run's files to the
144
+ * output directory. One subject is scored on its own. With two, the first is the baseline and the second is the one
145
+ * under test, which the pass bars apply to. It prints nothing and never calls process.exit. While it runs, SIGINT
146
+ * and SIGTERM end the process with exit code 130 after the cleanup.
147
+ * @param given - What to evaluate and how. An option that is not set takes its value from the project's config file.
148
+ * @returns The scores, every run record, and where the files are.
149
+ * @throws {UsageError} For a wrong option, case file, or config file, before any harness starts.
150
+ */
151
+ export async function evaluate(given) {
152
+ const { cwd = process.cwd(), subjects: specs, onRun, ...rest } = given;
153
+ requireSupportedPlatform();
154
+ const pairs = labelled(specs);
155
+ const project = findProject(cwd);
156
+ const options = withConfig(project, rest);
157
+ const { minScore, model } = options;
158
+ const reps = requireUsable(options);
159
+ const selection = selectHarnesses(options.harness);
160
+ const [judgeHarness] = options.judgeHarness
161
+ ? selectHarnesses(options.judgeHarness).chosen
162
+ : [];
163
+ const judgeModel = options.judgeModel ?? model;
164
+ const cases = loadCases(options.cases, options.caseGlob);
165
+ const decision = decisionFor(cases, options);
166
+ const subjects = [];
167
+ try {
168
+ for (const { label, spec } of pairs) {
169
+ subjects.push(resolveSubject(label, spec, project));
170
+ }
171
+ const outDir = outputDir(options.outDir, cwd);
172
+ const records = await runAll({
173
+ chosen: selection.chosen,
174
+ judgeOf: (located) => ({
175
+ ...(judgeHarness ?? located),
176
+ model: judgeModel,
177
+ }),
178
+ decision,
179
+ cases,
180
+ reps,
181
+ subjects,
182
+ model,
183
+ outDir,
184
+ onRun,
185
+ });
186
+ const results = summarize({
187
+ subjects,
188
+ selection,
189
+ judgeHarness,
190
+ judgeModel,
191
+ decisionModel: options.decisionModel,
192
+ model,
193
+ minScore,
194
+ reps,
195
+ cases,
196
+ records,
197
+ });
198
+ fs.writeFileSync(path.join(outDir, "results.json"), JSON.stringify(results, null, "\t"));
199
+ return { ...results, cases, outDir };
200
+ }
201
+ finally {
202
+ for (const subject of subjects)
203
+ subject.dispose();
204
+ }
205
+ }
package/dist/exec.d.ts ADDED
@@ -0,0 +1,20 @@
1
+ /** How to run one command. */
2
+ export interface ExecOptions {
3
+ cwd: string;
4
+ env: Record<string, string | undefined>;
5
+ timeoutSeconds: number;
6
+ /** Written to the command's standard input. Absent means the command gets no input at all. */
7
+ stdin?: string | undefined;
8
+ }
9
+ /**
10
+ * Runs a command to its end and never rejects.
11
+ * @param bin - The executable.
12
+ * @param args - Its arguments.
13
+ * @param options - Where and with what to run it, and for how long.
14
+ * @returns Everything the command wrote to standard output, and why it failed: a timeout, a non-zero exit with the
15
+ * end of its standard error, or a start that did not work. `failure` is null after exit code 0.
16
+ */
17
+ export declare function exec(bin: string, args: string[], options: ExecOptions): Promise<{
18
+ stdout: string;
19
+ failure: string | null;
20
+ }>;