skill-evals 0.0.0-stage → 0.0.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (54) hide show
  1. package/CHANGELOG.md +22 -0
  2. package/LICENSE +21 -0
  3. package/README.md +427 -2
  4. package/dist/baseten.d.ts +20 -0
  5. package/dist/baseten.js +89 -0
  6. package/dist/bin.d.ts +2 -0
  7. package/dist/bin.js +5 -0
  8. package/dist/cases.d.ts +21 -0
  9. package/dist/cases.js +207 -0
  10. package/dist/checks.d.ts +126 -0
  11. package/dist/checks.js +117 -0
  12. package/dist/cli.d.ts +7 -0
  13. package/dist/cli.js +172 -0
  14. package/dist/decision.d.ts +70 -0
  15. package/dist/decision.js +130 -0
  16. package/dist/evaluate.d.ts +70 -0
  17. package/dist/evaluate.js +205 -0
  18. package/dist/exec.d.ts +20 -0
  19. package/dist/exec.js +82 -0
  20. package/dist/harnesses/claude-code.d.ts +3 -0
  21. package/dist/harnesses/claude-code.js +83 -0
  22. package/dist/harnesses/codex.d.ts +3 -0
  23. package/dist/harnesses/codex.js +85 -0
  24. package/dist/harnesses/cursor.d.ts +3 -0
  25. package/dist/harnesses/cursor.js +81 -0
  26. package/dist/harnesses/index.d.ts +34 -0
  27. package/dist/harnesses/index.js +101 -0
  28. package/dist/harnesses/opencode.d.ts +3 -0
  29. package/dist/harnesses/opencode.js +75 -0
  30. package/dist/harnesses/pi.d.ts +3 -0
  31. package/dist/harnesses/pi.js +69 -0
  32. package/dist/harnesses/shared.d.ts +105 -0
  33. package/dist/harnesses/shared.js +90 -0
  34. package/dist/index.d.ts +10 -0
  35. package/dist/index.js +3 -0
  36. package/dist/judge.d.ts +30 -0
  37. package/dist/judge.js +87 -0
  38. package/dist/probe.d.ts +16 -0
  39. package/dist/probe.js +113 -0
  40. package/dist/project.d.ts +36 -0
  41. package/dist/project.js +84 -0
  42. package/dist/report.d.ts +8 -0
  43. package/dist/report.js +201 -0
  44. package/dist/run.d.ts +86 -0
  45. package/dist/run.js +220 -0
  46. package/dist/scope.d.ts +39 -0
  47. package/dist/scope.js +82 -0
  48. package/dist/score.d.ts +71 -0
  49. package/dist/score.js +92 -0
  50. package/dist/session.d.ts +33 -0
  51. package/dist/session.js +146 -0
  52. package/dist/subject.d.ts +54 -0
  53. package/dist/subject.js +179 -0
  54. package/package.json +59 -4
@@ -0,0 +1,89 @@
1
+ import { UsageError, messageOf } from "./scope.js";
2
+ const DEFAULT_URL = "https://inference.baseten.co";
3
+ const ENDPOINT = "/v1/systemone";
4
+ const KEY_VARIABLE = "BASETEN_API_KEY";
5
+ const DEFAULT_TIMEOUT_MS = 60_000;
6
+ const SHOWN_BODY_LENGTH = 200;
7
+ const toWire = (asked) => asked.kind === "yes_no"
8
+ ? { type: "noul", instructions: asked.question }
9
+ : {
10
+ type: "choice",
11
+ instructions: asked.question,
12
+ criteria: Object.fromEntries(asked.options.map((option) => [option, null])),
13
+ };
14
+ /**
15
+ * Reads the API's response.
16
+ * @param text - The body of a 2xx response.
17
+ * @param questions - The questions that were asked.
18
+ * @returns The probabilities of each question that the response answers in the form of its type: `noul`, the
19
+ * probability of yes, becomes "yes" and "no", and a choice keeps its `probabilities`. The API returns no cost.
20
+ * @throws {SyntaxError} When the body is not JSON.
21
+ */
22
+ function readResponse(text, questions) {
23
+ const response = Object(JSON.parse(text));
24
+ const answers = Object(response.answers);
25
+ const probabilities = {};
26
+ for (const [name, asked] of Object.entries(questions)) {
27
+ const answer = Object(answers[name]);
28
+ if (asked.kind === "choice") {
29
+ const given = answer.probabilities;
30
+ if (given !== null && typeof given === "object") {
31
+ probabilities[name] = given;
32
+ }
33
+ }
34
+ else if (typeof answer.noul === "number") {
35
+ probabilities[name] = { yes: answer.noul, no: 1 - answer.noul };
36
+ }
37
+ }
38
+ const { input_tokens: input, output_tokens: output } = Object(response.usage);
39
+ const tokens = typeof input === "number" && typeof output === "number"
40
+ ? { input, output }
41
+ : null;
42
+ return { probabilities, tokens, costUsd: null };
43
+ }
44
+ /**
45
+ * Connects to a decision model behind Baseten's System One API.
46
+ * @param connection - The model, the endpoint, and where the key is.
47
+ * @returns The model, which sends each request as one POST, and the name of the key's variable.
48
+ * @throws {UsageError} When the key's variable is not set, or the URL is not one.
49
+ */
50
+ export function connect(connection) {
51
+ const { model, url = DEFAULT_URL, env } = connection;
52
+ const { timeoutMs = DEFAULT_TIMEOUT_MS } = connection;
53
+ const key = env[KEY_VARIABLE];
54
+ if (!key) {
55
+ throw new UsageError(`a case has a decision check, but the decision model's API key is not set: set ${KEY_VARIABLE} in the environment`);
56
+ }
57
+ if (!URL.canParse(url)) {
58
+ throw new UsageError(`the decision URL is not a URL: ${url}`);
59
+ }
60
+ const endpoint = `${url.endsWith("/") ? url.slice(0, -1) : url}${ENDPOINT}`;
61
+ const decide = async (request) => {
62
+ const questions = Object.fromEntries(Object.entries(request.questions).map(([name, asked]) => [
63
+ name,
64
+ toWire(asked),
65
+ ]));
66
+ let response;
67
+ let text;
68
+ try {
69
+ response = await fetch(endpoint, {
70
+ method: "POST",
71
+ headers: {
72
+ Authorization: `Bearer ${key}`,
73
+ "Content-Type": "application/json",
74
+ },
75
+ body: JSON.stringify({ model, state: request.state, questions }),
76
+ signal: AbortSignal.timeout(timeoutMs),
77
+ });
78
+ text = await response.text();
79
+ }
80
+ catch (error) {
81
+ throw new Error(`the request to ${endpoint} failed: ${messageOf(error)}`, { cause: error });
82
+ }
83
+ if (!response.ok) {
84
+ throw new Error(`${endpoint} answered HTTP ${String(response.status)}: ${text.trim().slice(0, SHOWN_BODY_LENGTH)}`);
85
+ }
86
+ return readResponse(text, request.questions);
87
+ };
88
+ return { model: { decide }, secret: KEY_VARIABLE };
89
+ }
package/dist/bin.d.ts ADDED
@@ -0,0 +1,2 @@
1
+ #!/usr/bin/env node
2
+ export {};
package/dist/bin.js ADDED
@@ -0,0 +1,5 @@
1
+ #!/usr/bin/env node
2
+ import { main } from "./cli.js";
3
+ // The command line starts after the paths of node and of this script.
4
+ const FIRST_ARGUMENT = 2;
5
+ process.exitCode = await main(process.argv.slice(FIRST_ARGUMENT));
@@ -0,0 +1,21 @@
1
+ import type { Check } from "./checks.js";
2
+ /** One prompt and what a good run of it does. */
3
+ export interface Case {
4
+ id: string;
5
+ prompt: string;
6
+ checks: Check[];
7
+ weight: number;
8
+ minScore?: number | undefined;
9
+ maxTurns?: number | undefined;
10
+ timeoutSeconds: number;
11
+ /** A directory whose content the agent starts with, or null for an empty workspace. */
12
+ workspaceDir: string | null;
13
+ }
14
+ /**
15
+ * Loads the cases of a suite.
16
+ * @param casesDir - A directory with one subdirectory for each case.
17
+ * @param glob - Selects cases by id; only `*` is special.
18
+ * @returns Every matching case, sorted by id.
19
+ * @throws {UsageError} When the directory cannot be read, a case is invalid, or no case matches.
20
+ */
21
+ export declare function loadCases(casesDir: string, glob?: string): Case[];
package/dist/cases.js ADDED
@@ -0,0 +1,207 @@
1
+ import fs from "node:fs";
2
+ import path from "node:path";
3
+ import { CHECKS, REGEX_FIELDS, regex } from "./checks.js";
4
+ import { TOOL_KINDS } from "./harnesses/shared.js";
5
+ import { isUnitScore } from "./score.js";
6
+ import { UsageError, messageOf } from "./scope.js";
7
+ const DEFAULT_TIMEOUT_SECONDS = 600;
8
+ // A question with one allowed answer decides nothing.
9
+ const MIN_OPTIONS = 2;
10
+ /**
11
+ * Reads the weight of a case or a check.
12
+ * @param raw - The parsed case or check.
13
+ * @param where - Names the file and the check in an error message.
14
+ * @returns The weight, 1 when none is given.
15
+ * @throws {UsageError} When the weight is not a positive number.
16
+ */
17
+ function weightOf(raw, where) {
18
+ const weight = raw?.["weight"] ?? 1;
19
+ if (!(typeof weight === "number" && Number.isFinite(weight) && weight > 0)) {
20
+ throw new UsageError(`${where}: weight must be a positive number`);
21
+ }
22
+ return weight;
23
+ }
24
+ /**
25
+ * Reads the fields a check type takes.
26
+ * @param raw - The parsed check.
27
+ * @param spec - What its type requires and allows.
28
+ * @param where - Names the file and the check in an error message.
29
+ * @returns The fields that are set, in the order the type lists them.
30
+ * @throws {UsageError} When a required field is missing, any field is not a non-empty string, or none of `oneOf` is set.
31
+ */
32
+ function readFields(raw, spec, where) {
33
+ const type = String(raw["type"]);
34
+ const given = {};
35
+ const optional = (spec.optional ?? []).filter((field) => raw[field] !== undefined);
36
+ for (const field of [...spec.fields, ...optional]) {
37
+ const value = raw[field];
38
+ if (typeof value !== "string" || value === "") {
39
+ throw new UsageError(`${where}: ${type} needs a string "${field}"`);
40
+ }
41
+ given[field] = value;
42
+ }
43
+ if (spec.oneOf && !spec.oneOf.some((field) => given[field] !== undefined)) {
44
+ throw new UsageError(`${where}: ${type} needs at least one of ${spec.oneOf.map((field) => `"${field}"`).join(", ")}`);
45
+ }
46
+ return given;
47
+ }
48
+ /**
49
+ * Rejects field values that are strings but cannot be used.
50
+ * @param given - The fields of one check.
51
+ * @param where - Names the file and the check in an error message.
52
+ * @throws {UsageError} For an unknown tool kind, an invalid regular expression, or a path that leaves the workspace.
53
+ */
54
+ function requireUsable(given, where) {
55
+ const { kind, path: file, flags } = given;
56
+ if (kind !== undefined && !TOOL_KINDS.includes(kind)) {
57
+ throw new UsageError(`${where}: unknown tool kind "${kind}"; known: ${TOOL_KINDS.join(", ")}`);
58
+ }
59
+ try {
60
+ for (const field of REGEX_FIELDS) {
61
+ const pattern = given[field];
62
+ if (pattern !== undefined)
63
+ regex(pattern, flags);
64
+ }
65
+ }
66
+ catch (error) {
67
+ throw new UsageError(`${where}: ${messageOf(error)}`);
68
+ }
69
+ if (file !== undefined &&
70
+ (path.isAbsolute(file) || file.split(/[\\/]/).includes(".."))) {
71
+ throw new UsageError(`${where}: path must stay inside the workspace: ${file}`);
72
+ }
73
+ }
74
+ /**
75
+ * Reads the answers a decision check allows.
76
+ * @param raw - The parsed check.
77
+ * @param expect - Its `expect`, which is already known to be text when it is set.
78
+ * @param where - Names the file and the check in an error message.
79
+ * @returns The `options` of the check; nothing for a yes or no question, which `expect` scores.
80
+ * @throws {UsageError} Unless exactly one of `expect` and `options` is set, `expect` is "yes" or "no", and `options`
81
+ * is an object of at least two named answers, each worth a number from 0 to 1.
82
+ */
83
+ function readAnswers(raw, expect, where) {
84
+ const options = raw["options"];
85
+ if ((expect === undefined) === (options === undefined)) {
86
+ throw new UsageError(`${where}: decision needs exactly one of "expect" and "options"`);
87
+ }
88
+ if (expect !== undefined) {
89
+ if (expect === "yes" || expect === "no")
90
+ return {};
91
+ throw new UsageError(`${where}: "expect" must be "yes" or "no"`);
92
+ }
93
+ const entries = options !== null && typeof options === "object" && !Array.isArray(options)
94
+ ? Object.entries(options)
95
+ : [];
96
+ if (entries.length < MIN_OPTIONS ||
97
+ !entries.every(([label, value]) => label !== "" && isUnitScore(value))) {
98
+ throw new UsageError(`${where}: "options" must be an object of at least two named answers, each with a number from 0 to 1`);
99
+ }
100
+ return { options: Object.fromEntries(entries) };
101
+ }
102
+ /**
103
+ * Validates one check of a case file.
104
+ * @param raw - The parsed check.
105
+ * @param where - Names the file and the check in an error message.
106
+ * @returns The check, holding only the fields its type takes.
107
+ * @throws {UsageError} For an unknown type, a bad weight, or a field that is missing or unusable.
108
+ */
109
+ function parseCheck(raw, where) {
110
+ const type = raw?.["type"];
111
+ const spec = typeof type === "string" && Object.hasOwn(CHECKS, type)
112
+ ? CHECKS[type]
113
+ : undefined;
114
+ if (!raw || !spec) {
115
+ throw new UsageError(`${where}: unknown check type ${JSON.stringify(type)}; known: ${Object.keys(CHECKS).join(", ")}`);
116
+ }
117
+ const weight = weightOf(raw, where);
118
+ const given = readFields(raw, spec, where);
119
+ requireUsable(given, where);
120
+ const answers = type === "decision" ? readAnswers(raw, given.expect, where) : {};
121
+ // The steps above checked the type, the fields it requires, the tool kind, and the answers of a decision.
122
+ return { type, weight, ...given, ...answers };
123
+ }
124
+ /**
125
+ * Reads an optional count from a case file.
126
+ * @param value - The parsed value.
127
+ * @param where - Names the file in an error message.
128
+ * @param name - Names the key in an error message.
129
+ * @returns The count, or undefined when the key is absent.
130
+ * @throws {UsageError} When the value is not a positive integer.
131
+ */
132
+ function positive(value, where, name) {
133
+ if (value === undefined)
134
+ return undefined;
135
+ if (!(typeof value === "number" && Number.isInteger(value) && value > 0)) {
136
+ throw new UsageError(`${where}: ${name} must be a positive integer`);
137
+ }
138
+ return value;
139
+ }
140
+ /**
141
+ * Loads one case from its directory.
142
+ * @param dir - A directory with `prompt.md`, `case.json`, and an optional `workspace/`.
143
+ * @returns The case, named after the directory.
144
+ * @throws {UsageError} When a file is missing or does not hold a valid case.
145
+ */
146
+ function loadCase(dir) {
147
+ const id = path.basename(dir);
148
+ const file = path.join(dir, "case.json");
149
+ let prompt;
150
+ let raw;
151
+ try {
152
+ prompt = fs.readFileSync(path.join(dir, "prompt.md"), "utf8").trim();
153
+ raw = JSON.parse(fs.readFileSync(file, "utf8"));
154
+ }
155
+ catch (error) {
156
+ throw new UsageError(`case ${id}: ${messageOf(error)}`);
157
+ }
158
+ if (!prompt)
159
+ throw new UsageError(`case ${id}: prompt.md is empty`);
160
+ const checks = raw?.["checks"];
161
+ if (!Array.isArray(checks) || checks.length === 0) {
162
+ throw new UsageError(`${file}: "checks" must be a non-empty array`);
163
+ }
164
+ const minScore = raw?.["minScore"];
165
+ if (minScore !== undefined && !isUnitScore(minScore)) {
166
+ throw new UsageError(`${file}: minScore must be a number from 0 to 1`);
167
+ }
168
+ const workspaceDir = path.join(dir, "workspace");
169
+ return {
170
+ id,
171
+ prompt,
172
+ checks: checks.map((check, index) => parseCheck(check, `${file} checks[${String(index)}]`)),
173
+ weight: weightOf(raw, file),
174
+ minScore,
175
+ maxTurns: positive(raw?.["maxTurns"], file, "maxTurns"),
176
+ timeoutSeconds: positive(raw?.["timeoutSeconds"], file, "timeoutSeconds") ??
177
+ DEFAULT_TIMEOUT_SECONDS,
178
+ workspaceDir: fs.existsSync(workspaceDir) ? workspaceDir : null,
179
+ };
180
+ }
181
+ /**
182
+ * Loads the cases of a suite.
183
+ * @param casesDir - A directory with one subdirectory for each case.
184
+ * @param glob - Selects cases by id; only `*` is special.
185
+ * @returns Every matching case, sorted by id.
186
+ * @throws {UsageError} When the directory cannot be read, a case is invalid, or no case matches.
187
+ */
188
+ export function loadCases(casesDir, glob = "*") {
189
+ const literal = (part) => part.replace(/[.+?^${}()|[\]\\]/g, "\\$&");
190
+ const matcher = new RegExp(`^${glob.split("*").map(literal).join(".*")}$`);
191
+ let entries;
192
+ try {
193
+ entries = fs.readdirSync(casesDir, { withFileTypes: true });
194
+ }
195
+ catch (error) {
196
+ const { code } = error;
197
+ throw new UsageError(`cannot read the cases directory ${casesDir} (${String(code)}); create it or pass --cases`);
198
+ }
199
+ const cases = entries
200
+ .filter((entry) => entry.isDirectory() && matcher.test(entry.name))
201
+ .map((entry) => loadCase(path.join(casesDir, entry.name)))
202
+ .sort((x, y) => x.id.localeCompare(y.id));
203
+ if (cases.length === 0) {
204
+ throw new UsageError(`no case in ${casesDir} matches "${glob}"`);
205
+ }
206
+ return cases;
207
+ }
@@ -0,0 +1,126 @@
1
+ import type { ToolCall, ToolKind } from "./harnesses/shared.js";
2
+ /** The fields a check may carry beside its type and weight; which ones it takes depends on the type. */
3
+ interface CheckFields {
4
+ skill?: string;
5
+ pattern?: string;
6
+ text?: string;
7
+ path?: string;
8
+ kind?: ToolKind;
9
+ name?: string;
10
+ inputMatches?: string;
11
+ flags?: string;
12
+ rubric?: string;
13
+ question?: string;
14
+ expect?: "yes" | "no";
15
+ }
16
+ /** The name of a field a check may carry. */
17
+ export type Field = keyof CheckFields;
18
+ /** A check that a rule scores from what the run left behind. */
19
+ export type RuledCheck = CheckFields & {
20
+ type: RuledType;
21
+ weight: number;
22
+ };
23
+ /** A check that a model scores against a rubric. */
24
+ export type JudgedCheck = CheckFields & {
25
+ type: "judge";
26
+ weight: number;
27
+ rubric: string;
28
+ };
29
+ /**
30
+ * A check that a decision model scores: it returns a probability for each allowed answer, and the score is the
31
+ * expected value. Loading a case makes sure that exactly one of `expect` and `options` is set.
32
+ */
33
+ export type DecisionCheck = CheckFields & {
34
+ type: "decision";
35
+ weight: number;
36
+ question: string;
37
+ /** Each allowed answer with what it is worth, from 0 to 1. Absent for a yes or no question, which `expect` scores. */
38
+ options?: Record<string, number>;
39
+ };
40
+ /** One thing a case expects of a run. */
41
+ export type Check = RuledCheck | JudgedCheck | DecisionCheck;
42
+ /** What a deterministic check can observe. */
43
+ export interface Outcome {
44
+ finalText: string;
45
+ toolCalls: ToolCall[];
46
+ skillsLoaded: string[];
47
+ workspace: string;
48
+ }
49
+ export declare const REGEX_FIELDS: readonly ["pattern", "name", "inputMatches"];
50
+ type Rule = (check: RuledCheck, outcome: Outcome) => number;
51
+ /** A check type: the fields it requires, the fields it may take, and the rule that scores it from 0 to 1. */
52
+ export interface Spec {
53
+ fields: Field[];
54
+ optional?: Field[];
55
+ oneOf?: Field[];
56
+ rule?: Rule;
57
+ }
58
+ /**
59
+ * Compiles a pattern of a check.
60
+ * @param pattern - The source of a regular expression.
61
+ * @param flags - Its flags; none when absent.
62
+ * @returns The expression.
63
+ * @throws {SyntaxError} When the pattern or the flags are not valid.
64
+ */
65
+ export declare const regex: (pattern: string, flags?: string) => RegExp;
66
+ declare const RULED: {
67
+ skill_loaded: Spec & {
68
+ rule: Rule;
69
+ };
70
+ skill_not_loaded: Spec & {
71
+ rule: Rule;
72
+ };
73
+ output_matches: Spec & {
74
+ rule: Rule;
75
+ };
76
+ output_lacks: Spec & {
77
+ rule: Rule;
78
+ };
79
+ output_contains: Spec & {
80
+ rule: Rule;
81
+ };
82
+ output_not_contains: Spec & {
83
+ rule: Rule;
84
+ };
85
+ file_exists: Spec & {
86
+ rule: Rule;
87
+ };
88
+ file_matches: Spec & {
89
+ rule: Rule;
90
+ };
91
+ tool_called: Spec & {
92
+ rule: Rule;
93
+ };
94
+ tool_not_called: Spec & {
95
+ rule: Rule;
96
+ };
97
+ };
98
+ type RuledType = keyof typeof RULED;
99
+ export declare const CHECKS: Record<string, Spec>;
100
+ /**
101
+ * Tells the two ways of scoring a check apart.
102
+ * @param check - Any check of a case.
103
+ * @returns True when a model scores the check, false when a rule does.
104
+ */
105
+ export declare const isJudged: (check: Check) => check is JudgedCheck;
106
+ /**
107
+ * Finds the checks that a decision model scores.
108
+ * @param check - Any check of a case.
109
+ * @returns True when the score is the expected value of a decision model's probabilities.
110
+ */
111
+ export declare const isDecision: (check: Check) => check is DecisionCheck;
112
+ /**
113
+ * Scores a deterministic check.
114
+ * @param check - A check that a rule scores.
115
+ * @param outcome - What the run left behind.
116
+ * @returns 1 when the check holds, 0 when it does not.
117
+ */
118
+ export declare const checkScore: (check: RuledCheck, outcome: Outcome) => number;
119
+ /**
120
+ * Names a check in one line of a report.
121
+ * @param check - Any check of a case.
122
+ * @returns Its type and what it looks for; long text is cut short, a decision check names the answer it expects or
123
+ * its options, and a weight other than 1 is shown.
124
+ */
125
+ export declare function describeCheck(check: Check): string;
126
+ export {};
package/dist/checks.js ADDED
@@ -0,0 +1,117 @@
1
+ import fs from "node:fs";
2
+ import path from "node:path";
3
+ export const REGEX_FIELDS = ["pattern", "name", "inputMatches"];
4
+ /**
5
+ * Compiles a pattern of a check.
6
+ * @param pattern - The source of a regular expression.
7
+ * @param flags - Its flags; none when absent.
8
+ * @returns The expression.
9
+ * @throws {SyntaxError} When the pattern or the flags are not valid.
10
+ */
11
+ export const regex = (pattern, flags = "") => new RegExp(pattern, flags);
12
+ /**
13
+ * Reads the file a check names.
14
+ * @param check - A check with a path, which loading the case kept inside the workspace.
15
+ * @param outcome - Where the workspace is.
16
+ * @returns The file's text, or null when it cannot be read.
17
+ */
18
+ function fileText(check, outcome) {
19
+ try {
20
+ return fs.readFileSync(path.join(outcome.workspace, check.path), "utf8");
21
+ }
22
+ catch {
23
+ return null;
24
+ }
25
+ }
26
+ const toolCalled = (check, outcome) => outcome.toolCalls.some((call) => (check.kind === undefined || call.kind === check.kind) &&
27
+ (check.name === undefined ||
28
+ regex(check.name, check.flags).test(call.name)) &&
29
+ (check.inputMatches === undefined ||
30
+ regex(check.inputMatches, check.flags).test(JSON.stringify(call.input ?? null))));
31
+ // Loading a case checks that every field in `fields` is a string, which is what lets a test read them as set.
32
+ const holds = (fields, test, more = {}) => ({
33
+ fields,
34
+ ...more,
35
+ rule: (check, outcome) => Number(test(check, outcome)),
36
+ });
37
+ const fails = (fields, test, more) => holds(fields, (check, outcome) => !test(check, outcome), more);
38
+ const TOOL_FIELDS = {
39
+ optional: ["kind", "name", "inputMatches", "flags"],
40
+ oneOf: ["kind", "name"],
41
+ };
42
+ const FLAGS = { optional: ["flags"] };
43
+ const skillLoaded = (check, outcome) => outcome.skillsLoaded.includes(check.skill);
44
+ const outputMatches = (check, outcome) => regex(check.pattern, check.flags).test(outcome.finalText);
45
+ const outputContains = (check, outcome) => outcome.finalText.includes(check.text);
46
+ const RULED = {
47
+ skill_loaded: holds(["skill"], skillLoaded),
48
+ skill_not_loaded: fails(["skill"], skillLoaded),
49
+ output_matches: holds(["pattern"], outputMatches, FLAGS),
50
+ output_lacks: fails(["pattern"], outputMatches, FLAGS),
51
+ output_contains: holds(["text"], outputContains),
52
+ output_not_contains: fails(["text"], outputContains),
53
+ file_exists: holds(["path"], (check, outcome) => fileText(check, outcome) !== null),
54
+ file_matches: holds(["path", "pattern"], (check, outcome) => {
55
+ const text = fileText(check, outcome);
56
+ return text !== null && regex(check.pattern, check.flags).test(text);
57
+ }, FLAGS),
58
+ tool_called: holds([], toolCalled, TOOL_FIELDS),
59
+ tool_not_called: fails([], toolCalled, TOOL_FIELDS),
60
+ };
61
+ // Every check type. A type with no rule is scored by a model instead (see judge.ts and decision.ts).
62
+ export const CHECKS = {
63
+ ...RULED,
64
+ judge: { fields: ["rubric"] },
65
+ decision: { fields: ["question"], optional: ["expect"] },
66
+ };
67
+ /**
68
+ * Tells the two ways of scoring a check apart.
69
+ * @param check - Any check of a case.
70
+ * @returns True when a model scores the check, false when a rule does.
71
+ */
72
+ export const isJudged = (check) => check.type === "judge";
73
+ /**
74
+ * Finds the checks that a decision model scores.
75
+ * @param check - Any check of a case.
76
+ * @returns True when the score is the expected value of a decision model's probabilities.
77
+ */
78
+ export const isDecision = (check) => check.type === "decision";
79
+ /**
80
+ * Scores a deterministic check.
81
+ * @param check - A check that a rule scores.
82
+ * @param outcome - What the run left behind.
83
+ * @returns 1 when the check holds, 0 when it does not.
84
+ */
85
+ export const checkScore = (check, outcome) => RULED[check.type].rule(check, outcome);
86
+ const SHOWN_TEXT_LENGTH = 60;
87
+ const ELLIPSIS = "...";
88
+ /**
89
+ * Names a check in one line of a report.
90
+ * @param check - Any check of a case.
91
+ * @returns Its type and what it looks for; long text is cut short, a decision check names the answer it expects or
92
+ * its options, and a weight other than 1 is shown.
93
+ */
94
+ export function describeCheck(check) {
95
+ const quoted = [check.text, check.rubric, check.question]
96
+ .filter((text) => text !== undefined)
97
+ .map((text) => JSON.stringify(text.length > SHOWN_TEXT_LENGTH
98
+ ? `${text.slice(0, SHOWN_TEXT_LENGTH - ELLIPSIS.length)}${ELLIPSIS}`
99
+ : text));
100
+ const patterns = REGEX_FIELDS.flatMap((field) => check[field] === undefined ? [] : [`/${check[field]}/${check.flags ?? ""}`]);
101
+ const weight = check.weight === 1 ? [] : [`(weight ${String(check.weight)})`];
102
+ const answers = isDecision(check)
103
+ ? [check.expect ?? Object.keys(check.options ?? {}).join("/")]
104
+ : [];
105
+ return [
106
+ check.type,
107
+ check.skill,
108
+ check.path,
109
+ check.kind,
110
+ ...quoted,
111
+ ...answers,
112
+ ...patterns,
113
+ ...weight,
114
+ ]
115
+ .filter(Boolean)
116
+ .join(" ");
117
+ }
package/dist/cli.d.ts ADDED
@@ -0,0 +1,7 @@
1
+ /**
2
+ * Runs one command line.
3
+ * @param argv - The command and its flags, without the node and script paths.
4
+ * @returns The exit code: 0, 1 when a run is invalid or a pass bar is not reached, or 2 with one line on standard
5
+ * error for a wrong command line, case file, config file, or platform. It never calls process.exit.
6
+ */
7
+ export declare function main(argv: string[]): Promise<number>;