skill-evals 0.0.0-stage → 0.0.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (54) hide show
  1. package/CHANGELOG.md +22 -0
  2. package/LICENSE +21 -0
  3. package/README.md +427 -2
  4. package/dist/baseten.d.ts +20 -0
  5. package/dist/baseten.js +89 -0
  6. package/dist/bin.d.ts +2 -0
  7. package/dist/bin.js +5 -0
  8. package/dist/cases.d.ts +21 -0
  9. package/dist/cases.js +207 -0
  10. package/dist/checks.d.ts +126 -0
  11. package/dist/checks.js +117 -0
  12. package/dist/cli.d.ts +7 -0
  13. package/dist/cli.js +172 -0
  14. package/dist/decision.d.ts +70 -0
  15. package/dist/decision.js +130 -0
  16. package/dist/evaluate.d.ts +70 -0
  17. package/dist/evaluate.js +205 -0
  18. package/dist/exec.d.ts +20 -0
  19. package/dist/exec.js +82 -0
  20. package/dist/harnesses/claude-code.d.ts +3 -0
  21. package/dist/harnesses/claude-code.js +83 -0
  22. package/dist/harnesses/codex.d.ts +3 -0
  23. package/dist/harnesses/codex.js +85 -0
  24. package/dist/harnesses/cursor.d.ts +3 -0
  25. package/dist/harnesses/cursor.js +81 -0
  26. package/dist/harnesses/index.d.ts +34 -0
  27. package/dist/harnesses/index.js +101 -0
  28. package/dist/harnesses/opencode.d.ts +3 -0
  29. package/dist/harnesses/opencode.js +75 -0
  30. package/dist/harnesses/pi.d.ts +3 -0
  31. package/dist/harnesses/pi.js +69 -0
  32. package/dist/harnesses/shared.d.ts +105 -0
  33. package/dist/harnesses/shared.js +90 -0
  34. package/dist/index.d.ts +10 -0
  35. package/dist/index.js +3 -0
  36. package/dist/judge.d.ts +30 -0
  37. package/dist/judge.js +87 -0
  38. package/dist/probe.d.ts +16 -0
  39. package/dist/probe.js +113 -0
  40. package/dist/project.d.ts +36 -0
  41. package/dist/project.js +84 -0
  42. package/dist/report.d.ts +8 -0
  43. package/dist/report.js +201 -0
  44. package/dist/run.d.ts +86 -0
  45. package/dist/run.js +220 -0
  46. package/dist/scope.d.ts +39 -0
  47. package/dist/scope.js +82 -0
  48. package/dist/score.d.ts +71 -0
  49. package/dist/score.js +92 -0
  50. package/dist/session.d.ts +33 -0
  51. package/dist/session.js +146 -0
  52. package/dist/subject.d.ts +54 -0
  53. package/dist/subject.js +179 -0
  54. package/package.json +59 -4
@@ -0,0 +1,71 @@
1
+ /** A case's scores, keyed by subject label. */
2
+ export interface CaseScore {
3
+ id: string;
4
+ weight: number;
5
+ minScore?: number | undefined;
6
+ scores: Record<string, number | null>;
7
+ }
8
+ /** One harness's case scores, and its suite score keyed by subject label. */
9
+ export interface SuiteScore {
10
+ harness: string;
11
+ cases: CaseScore[];
12
+ suite: Record<string, number | null>;
13
+ }
14
+ /** The part of a run record that scoring reads. */
15
+ interface ScoredRun {
16
+ harness: string;
17
+ caseId: string;
18
+ subject: string;
19
+ score: number | null;
20
+ }
21
+ /** The part of a case that scoring reads. */
22
+ interface WeightedCase {
23
+ id: string;
24
+ weight: number;
25
+ minScore?: number | undefined;
26
+ }
27
+ /**
28
+ * Tells whether a value read from a file or a model is usable as a score.
29
+ * @param value - Anything; a numeric string is not a score.
30
+ * @returns True for a finite number from 0 to 1, both ends included.
31
+ */
32
+ export declare const isUnitScore: (value: unknown) => value is number;
33
+ /**
34
+ * Renders a score the way every report line shows it.
35
+ * @param score - A score, or null for a missing one.
36
+ * @returns Two decimals, or "n/a" for null.
37
+ */
38
+ export declare const formatScore: (score: number | null) => string;
39
+ /**
40
+ * Averages scores by weight, refusing to average around a gap.
41
+ * @param items - Scores with the weight each one counts for.
42
+ * @returns Null when there is nothing to average or any item has no score.
43
+ */
44
+ export declare function weightedMean(items: {
45
+ score: number | null;
46
+ weight: number;
47
+ }[]): number | null;
48
+ /**
49
+ * Scores every case and the whole suite, once for each harness and subject label.
50
+ * @param run - What ran.
51
+ * @param run.harnesses - The ids of the harnesses, in report order.
52
+ * @param run.cases - The cases, in report order.
53
+ * @param run.records - Every run; one whose score is null does not count toward its case.
54
+ * @param run.labels - The subject labels to score.
55
+ * @returns One entry for each harness. A case with no valid run has the score null, and so has its suite.
56
+ */
57
+ export declare function scoreSuites(run: {
58
+ harnesses: string[];
59
+ cases: WeightedCase[];
60
+ records: ScoredRun[];
61
+ labels: readonly string[];
62
+ }): SuiteScore[];
63
+ /**
64
+ * Lists the pass bars that the subject under test did not reach.
65
+ * @param suites - The scores of every harness.
66
+ * @param label - The label of the subject the bars apply to.
67
+ * @param minScore - The bar for the suite score; each case carries its own.
68
+ * @returns One line for each missed bar. A missing score does not reach a bar.
69
+ */
70
+ export declare function failedBars(suites: SuiteScore[], label: string, minScore: number | undefined): string[];
71
+ export {};
package/dist/score.js ADDED
@@ -0,0 +1,92 @@
1
+ // Scores are numbers from 0 to 1, or null for "no score".
2
+ // - check score: what one check gave one run
3
+ // - run score: the weighted mean of the run's check scores; an invalid run has none
4
+ // - case score: the mean of the run scores over the reps, for one harness and one subject
5
+ // - suite score: the weighted mean of the case scores, for one harness and one subject
6
+ // Absorbs float error in a mean, so three runs at 0.8 reach a bar of 0.8.
7
+ const EPSILON = 1e-9;
8
+ const SCORE_DECIMALS = 2;
9
+ /**
10
+ * Tells whether a value read from a file or a model is usable as a score.
11
+ * @param value - Anything; a numeric string is not a score.
12
+ * @returns True for a finite number from 0 to 1, both ends included.
13
+ */
14
+ export const isUnitScore = (value) => typeof value === "number" &&
15
+ Number.isFinite(value) &&
16
+ value >= 0 &&
17
+ value <= 1;
18
+ /**
19
+ * Renders a score the way every report line shows it.
20
+ * @param score - A score, or null for a missing one.
21
+ * @returns Two decimals, or "n/a" for null.
22
+ */
23
+ export const formatScore = (score) => score === null ? "n/a" : score.toFixed(SCORE_DECIMALS);
24
+ /**
25
+ * Averages scores by weight, refusing to average around a gap.
26
+ * @param items - Scores with the weight each one counts for.
27
+ * @returns Null when there is nothing to average or any item has no score.
28
+ */
29
+ export function weightedMean(items) {
30
+ let total = 0;
31
+ let weighted = 0;
32
+ for (const { score, weight } of items) {
33
+ if (score === null)
34
+ return null;
35
+ total += weight;
36
+ weighted += score * weight;
37
+ }
38
+ return items.length === 0 ? null : weighted / total;
39
+ }
40
+ const caseScore = (records) => weightedMean(records
41
+ .filter((record) => record.score !== null)
42
+ .map((record) => ({ score: record.score, weight: 1 })));
43
+ /**
44
+ * Scores every case and the whole suite, once for each harness and subject label.
45
+ * @param run - What ran.
46
+ * @param run.harnesses - The ids of the harnesses, in report order.
47
+ * @param run.cases - The cases, in report order.
48
+ * @param run.records - Every run; one whose score is null does not count toward its case.
49
+ * @param run.labels - The subject labels to score.
50
+ * @returns One entry for each harness. A case with no valid run has the score null, and so has its suite.
51
+ */
52
+ export function scoreSuites(run) {
53
+ const perLabel = (score) => Object.fromEntries(run.labels.map((label) => [label, score(label)]));
54
+ return run.harnesses.map((harness) => {
55
+ const mine = run.records.filter((record) => record.harness === harness);
56
+ const scored = run.cases.map(({ id, weight, minScore }) => ({
57
+ id,
58
+ weight,
59
+ minScore,
60
+ scores: perLabel((label) => caseScore(mine.filter((r) => r.caseId === id && r.subject === label))),
61
+ }));
62
+ return {
63
+ harness,
64
+ cases: scored,
65
+ suite: perLabel((label) => weightedMean(scored.map((c) => ({
66
+ score: c.scores[label] ?? null,
67
+ weight: c.weight,
68
+ })))),
69
+ };
70
+ });
71
+ }
72
+ // The bar, when the score does not reach it; a missing score reaches no bar.
73
+ const missed = (score, bar) => bar !== undefined && (score === null || score < bar - EPSILON) ? [bar] : [];
74
+ /**
75
+ * Lists the pass bars that the subject under test did not reach.
76
+ * @param suites - The scores of every harness.
77
+ * @param label - The label of the subject the bars apply to.
78
+ * @param minScore - The bar for the suite score; each case carries its own.
79
+ * @returns One line for each missed bar. A missing score does not reach a bar.
80
+ */
81
+ export function failedBars(suites, label, minScore) {
82
+ return suites.flatMap(({ harness, cases, suite }) => {
83
+ const suiteScore = suite[label] ?? null;
84
+ return [
85
+ ...missed(suiteScore, minScore).map((bar) => `${harness}: suite score ${formatScore(suiteScore)} does not reach --min-score ${formatScore(bar)}`),
86
+ ...cases.flatMap((c) => {
87
+ const score = c.scores[label] ?? null;
88
+ return missed(score, c.minScore).map((bar) => `${harness}: case \`${c.id}\` score ${formatScore(score)} does not reach its minScore ${formatScore(bar)}`);
89
+ }),
90
+ ];
91
+ });
92
+ }
@@ -0,0 +1,33 @@
1
+ import type { Harness, Parsed, RunInput } from "./harnesses/shared.js";
2
+ /** One prompt for one harness. */
3
+ export interface SessionInput {
4
+ harness: Harness;
5
+ /** The harness's executable. */
6
+ bin: string;
7
+ /** What the harness loads; null loads nothing. */
8
+ subject: RunInput["subject"];
9
+ prompt: string;
10
+ model?: string | undefined;
11
+ maxTurns?: number | undefined;
12
+ timeoutSeconds: number;
13
+ /** Copied into the workspace before the harness starts. */
14
+ workspaceDir?: string | null;
15
+ /** Environment variables the harness must not get, such as the key of the model that scores its answer. */
16
+ withhold?: string[] | undefined;
17
+ }
18
+ /** What a session produced, seen while its workspace still exists. */
19
+ export interface SessionResult {
20
+ stdout: string;
21
+ parsed: Parsed;
22
+ /** The harness's own message when it produced a stream, else why the process failed. */
23
+ error: string | null;
24
+ durationMs: number;
25
+ workspace: string;
26
+ }
27
+ /**
28
+ * Runs one prompt in one harness, in a fresh home and workspace that are removed afterwards.
29
+ * @param run - The harness, the prompt, and what to load.
30
+ * @param use - Reads the result while the workspace still exists.
31
+ * @returns What `use` returns.
32
+ */
33
+ export declare function session<T>(run: SessionInput, use: (result: SessionResult) => T | Promise<T>): Promise<T>;
@@ -0,0 +1,146 @@
1
+ import fs from "node:fs";
2
+ import os from "node:os";
3
+ import path from "node:path";
4
+ import { exec } from "./exec.js";
5
+ import { messageOf, tempDir } from "./scope.js";
6
+ // This tool's own variables, and the ones npm and the shell set that name the command that is running.
7
+ const NAMES_THIS_TOOL = /^(SKILL_EVALS_|npm_|_$)/;
8
+ // Login files hold secrets, so only the owner may read the copies.
9
+ const OWNER_READ_WRITE = 0o600;
10
+ const SHOWN_SETUP_WORDS = 2;
11
+ /**
12
+ * Builds the environment of the harness process.
13
+ * @param home - The throwaway home.
14
+ * @param extra - What the adapter adds.
15
+ * @param withhold - Variables of this process that the harness must not get.
16
+ * @returns This process's environment, with the home and the XDG directories moved under `home`, and without the
17
+ * withheld variables or any variable that names this tool.
18
+ */
19
+ function childEnv(home, extra, withhold) {
20
+ const inherited = Object.fromEntries(Object.entries(process.env).filter(([key]) => !NAMES_THIS_TOOL.test(key) && !withhold.includes(key)));
21
+ return {
22
+ ...inherited,
23
+ HOME: home,
24
+ XDG_CONFIG_HOME: path.join(home, ".config"),
25
+ XDG_DATA_HOME: path.join(home, ".local", "share"),
26
+ XDG_STATE_HOME: path.join(home, ".local", "state"),
27
+ XDG_CACHE_HOME: path.join(home, ".cache"),
28
+ ...extra,
29
+ };
30
+ }
31
+ /**
32
+ * Gives the harness its login in the throwaway home.
33
+ * @param authFiles - Paths relative to a home directory; one that the user's home lacks is skipped.
34
+ * @param home - The throwaway home.
35
+ */
36
+ function copyAuthFiles(authFiles, home) {
37
+ for (const relative of authFiles) {
38
+ const source = path.join(os.homedir(), relative);
39
+ if (!fs.existsSync(source))
40
+ continue;
41
+ const target = path.join(home, relative);
42
+ fs.mkdirSync(path.dirname(target), { recursive: true });
43
+ fs.copyFileSync(source, target);
44
+ fs.chmodSync(target, OWNER_READ_WRITE);
45
+ }
46
+ }
47
+ /**
48
+ * Prepares the harness and runs the prompt.
49
+ * @param session - Which harness, and how long it may take.
50
+ * @param input - What the adapter builds the command line from.
51
+ * @returns The harness's output, why it failed if it did, and how long the prompt itself took; setup is not timed.
52
+ */
53
+ async function attempt(session, input) {
54
+ const { harness, bin, timeoutSeconds } = session;
55
+ let invocation;
56
+ try {
57
+ invocation = harness.invocation(input);
58
+ }
59
+ catch (error) {
60
+ const failure = `could not prepare the run: ${messageOf(error)}`;
61
+ return { stdout: "", failure, durationMs: 0 };
62
+ }
63
+ const options = {
64
+ cwd: input.workspace,
65
+ env: childEnv(input.home, invocation.env, session.withhold ?? []),
66
+ timeoutSeconds,
67
+ };
68
+ for (const step of invocation.setup ?? []) {
69
+ const { failure } = await exec(bin, step, options);
70
+ if (failure) {
71
+ const command = step.slice(0, SHOWN_SETUP_WORDS).join(" ");
72
+ return {
73
+ stdout: "",
74
+ failure: `setup "${command}": ${failure}`,
75
+ durationMs: 0,
76
+ };
77
+ }
78
+ }
79
+ const started = Date.now();
80
+ const result = await exec(bin, invocation.args, {
81
+ ...options,
82
+ stdin: invocation.stdin,
83
+ });
84
+ return { ...result, durationMs: Date.now() - started };
85
+ }
86
+ /**
87
+ * Normalizes a harness's output.
88
+ * @param harness - The adapter that reads the stream.
89
+ * @param stdout - What the harness wrote.
90
+ * @returns What the adapter read, with an error in place of an exception for a stream the adapter cannot read.
91
+ */
92
+ function parseStream(harness, stdout) {
93
+ try {
94
+ return harness.parse(stdout);
95
+ }
96
+ catch (error) {
97
+ return {
98
+ finalText: "",
99
+ toolCalls: [],
100
+ tokens: null,
101
+ costUsd: null,
102
+ loaded: { paths: null, skills: null },
103
+ error: `the ${harness.id} adapter could not read the stream: ${messageOf(error)}`,
104
+ };
105
+ }
106
+ }
107
+ /**
108
+ * Runs one prompt in one harness, in a fresh home and workspace that are removed afterwards.
109
+ * @param run - The harness, the prompt, and what to load.
110
+ * @param use - Reads the result while the workspace still exists.
111
+ * @returns What `use` returns.
112
+ */
113
+ export async function session(run, use) {
114
+ const { harness, subject, prompt, model, maxTurns, workspaceDir } = run;
115
+ const home = tempDir("home-");
116
+ const workspace = tempDir("proj-");
117
+ try {
118
+ if (workspaceDir) {
119
+ fs.cpSync(workspaceDir, workspace.dir, { recursive: true });
120
+ }
121
+ copyAuthFiles(harness.authFiles, home.dir);
122
+ const input = {
123
+ subject,
124
+ home: home.dir,
125
+ workspace: workspace.dir,
126
+ prompt,
127
+ model,
128
+ maxTurns,
129
+ };
130
+ const { stdout, failure, durationMs } = await attempt(run, input);
131
+ const parsed = parseStream(harness, stdout);
132
+ // The harness's own message is the most specific one, but only when it produced a stream at all.
133
+ const error = (stdout.trim() && parsed.error) || failure || parsed.error || null;
134
+ return await use({
135
+ stdout,
136
+ parsed,
137
+ error,
138
+ durationMs,
139
+ workspace: workspace.dir,
140
+ });
141
+ }
142
+ finally {
143
+ home.dispose();
144
+ workspace.dispose();
145
+ }
146
+ }
@@ -0,0 +1,54 @@
1
+ import type { Project } from "./project.js";
2
+ /** A comparison has the labels "a" and "b"; a single subject is "subject". */
3
+ export type Label = "a" | "b" | "subject";
4
+ /** The part of a subject that goes into `results.json` and the report. */
5
+ export interface SubjectSummary {
6
+ label: Label;
7
+ /** What the user named: "none", a directory, or a git ref of the project. */
8
+ spec: string;
9
+ /**
10
+ * "plugin" is a directory with a harness's plugin packaging, loaded the way that harness loads a local plugin;
11
+ * "skills" is a plain folder of skills, copied to where each harness discovers skills; "none" loads nothing.
12
+ */
13
+ kind: "plugin" | "skills" | "none";
14
+ /** Null for "none". For a git ref it is a detached worktree. */
15
+ dir: string | null;
16
+ /** The names of the skills found in dir, sorted. */
17
+ skills: string[];
18
+ /** Set when spec is a git ref. */
19
+ commit: string | null;
20
+ }
21
+ /** What a run loads into the harness. */
22
+ export interface Subject extends SubjectSummary {
23
+ /** Skill name to its directory. */
24
+ skillDirs: Record<string, string>;
25
+ /** Removes the worktree of a git ref; does nothing for any other subject. */
26
+ dispose: () => void;
27
+ }
28
+ /** What only a subject that is not "none" has. */
29
+ interface WithContent {
30
+ kind: "plugin" | "skills";
31
+ dir: string;
32
+ }
33
+ /**
34
+ * Tells a subject with a directory from the "none" baseline.
35
+ * @param subject - A subject or its summary.
36
+ * @returns True for a plugin or a skills folder, which narrows `dir` to a string.
37
+ */
38
+ export declare const hasContent: <S extends Pick<SubjectSummary, "kind" | "dir">>(subject: S) => subject is S & WithContent;
39
+ /**
40
+ * Turns what the user named into something a harness can load.
41
+ * @param label - The subject's place in the comparison.
42
+ * @param spec - `none`, a directory (used as it is, uncommitted changes included), or a git ref of the project.
43
+ * @param project - What a relative directory and a ref resolve against.
44
+ * @returns The subject. The caller must call its `dispose`.
45
+ * @throws {UsageError} When spec cannot be found, or is a folder with no skill and no plugin manifest.
46
+ */
47
+ export declare function resolveSubject(label: Label, spec: string, project: Project): Subject;
48
+ /**
49
+ * Says in one phrase what a subject is and where it came from, for the head of a report.
50
+ * @param subject - A subject or its summary.
51
+ * @returns For a git ref, the ref and its short commit hash. For a directory, its path.
52
+ */
53
+ export declare function describeSubject(subject: SubjectSummary): string;
54
+ export {};
@@ -0,0 +1,179 @@
1
+ import { execFileSync } from "node:child_process";
2
+ import fs from "node:fs";
3
+ import path from "node:path";
4
+ import { UsageError, onDispose, tempDir } from "./scope.js";
5
+ /**
6
+ * Tells a subject with a directory from the "none" baseline.
7
+ * @param subject - A subject or its summary.
8
+ * @returns True for a plugin or a skills folder, which narrows `dir` to a string.
9
+ */
10
+ export const hasContent = (subject) => subject.kind !== "none";
11
+ const PLUGIN_MARKERS = [
12
+ ".claude-plugin/plugin.json",
13
+ ".cursor-plugin/plugin.json",
14
+ ".codex-plugin/plugin.json",
15
+ "plugin.json",
16
+ ".opencode/plugins",
17
+ ];
18
+ /**
19
+ * Tells whether a directory carries any harness's plugin packaging.
20
+ * @param dir - The directory to look in.
21
+ * @returns True for a plugin manifest of Claude Code, Cursor, Codex, or OpenCode, or a package.json with a `pi` key.
22
+ */
23
+ function isPlugin(dir) {
24
+ if (PLUGIN_MARKERS.some((marker) => fs.existsSync(path.join(dir, marker)))) {
25
+ return true;
26
+ }
27
+ try {
28
+ const manifest = JSON.parse(fs.readFileSync(path.join(dir, "package.json"), "utf8"));
29
+ return (typeof manifest === "object" && manifest !== null && "pi" in manifest);
30
+ }
31
+ catch {
32
+ return false;
33
+ }
34
+ }
35
+ const hasSkillFile = (dir) => fs.existsSync(path.join(dir, "SKILL.md"));
36
+ /**
37
+ * Finds every `<dir>/<name>/SKILL.md` and `<dir>/skills/<name>/SKILL.md`, and dir itself when it holds a SKILL.md.
38
+ * @param dir - The folder to search, one level deep.
39
+ * @param ownName - The skill name of dir itself.
40
+ * @returns Skill name to its directory, in name order.
41
+ * @throws {UsageError} When two skills share a name.
42
+ */
43
+ function findSkills(dir, ownName) {
44
+ const found = hasSkillFile(dir) ? [[ownName, dir]] : [];
45
+ for (const parent of [dir, path.join(dir, "skills")]) {
46
+ if (!fs.existsSync(parent))
47
+ continue;
48
+ for (const entry of fs.readdirSync(parent, { withFileTypes: true })) {
49
+ const skillDir = path.join(parent, entry.name);
50
+ if (entry.isDirectory() && hasSkillFile(skillDir)) {
51
+ found.push([entry.name, skillDir]);
52
+ }
53
+ }
54
+ }
55
+ const names = found.map(([name]) => name);
56
+ const twice = names.find((name, index) => names.indexOf(name) !== index);
57
+ if (twice)
58
+ throw new UsageError(`${dir} holds two skills named "${twice}"`);
59
+ return Object.fromEntries(found.sort(([x], [y]) => x.localeCompare(y)));
60
+ }
61
+ const git = (repoDir, ...args) => execFileSync("git", ["-C", repoDir, ...args], {
62
+ encoding: "utf8",
63
+ stdio: ["ignore", "pipe", "pipe"],
64
+ }).trim();
65
+ /**
66
+ * Checks a git ref out into a detached worktree in a temp directory.
67
+ * @param project - The repository that owns the ref.
68
+ * @param commit - The full hash to check out.
69
+ * @returns The worktree, and a `dispose` that removes it and its registration in the repository.
70
+ */
71
+ function worktree(project, commit) {
72
+ const temp = tempDir("kit-");
73
+ const dispose = onDispose(() => {
74
+ try {
75
+ git(project.root, "worktree", "remove", "--force", temp.dir);
76
+ }
77
+ catch {
78
+ // Already removed, or never registered because the add below failed.
79
+ }
80
+ temp.dispose();
81
+ });
82
+ try {
83
+ git(project.root, "worktree", "add", "--detach", temp.dir, commit);
84
+ }
85
+ catch (error) {
86
+ dispose();
87
+ throw error;
88
+ }
89
+ return { dir: temp.dir, commit, dispose };
90
+ }
91
+ /**
92
+ * Locates the files of a subject. A directory is used as it is. Anything else must be a git ref of the project.
93
+ * @param label - Names the flag in an error message.
94
+ * @param spec - A directory, relative to the working directory, or a git ref.
95
+ * @param project - Where a ref is looked up.
96
+ * @returns The directory to load from.
97
+ * @throws {UsageError} When spec is neither a directory nor a ref of the project.
98
+ */
99
+ function checkout(label, spec, project) {
100
+ const dir = path.resolve(project.cwd, spec);
101
+ if (fs.existsSync(dir) && fs.statSync(dir).isDirectory()) {
102
+ return {
103
+ dir: fs.realpathSync(dir),
104
+ commit: null,
105
+ dispose: () => undefined,
106
+ };
107
+ }
108
+ if (!project.isGitRepo) {
109
+ throw new UsageError(`--${label} ${spec}: not a directory, and it cannot be a git ref because ${project.root} is not in a git repository`);
110
+ }
111
+ let commit;
112
+ try {
113
+ commit = git(project.root, "rev-parse", "--verify", "--quiet", `${spec}^{commit}`);
114
+ }
115
+ catch {
116
+ throw new UsageError(`--${label} ${spec}: not a directory and not a git ref of ${project.root}`);
117
+ }
118
+ return worktree(project, commit);
119
+ }
120
+ /**
121
+ * Turns what the user named into something a harness can load.
122
+ * @param label - The subject's place in the comparison.
123
+ * @param spec - `none`, a directory (used as it is, uncommitted changes included), or a git ref of the project.
124
+ * @param project - What a relative directory and a ref resolve against.
125
+ * @returns The subject. The caller must call its `dispose`.
126
+ * @throws {UsageError} When spec cannot be found, or is a folder with no skill and no plugin manifest.
127
+ */
128
+ export function resolveSubject(label, spec, project) {
129
+ if (spec === "none") {
130
+ return {
131
+ label,
132
+ spec,
133
+ kind: "none",
134
+ dir: null,
135
+ skills: [],
136
+ skillDirs: {},
137
+ commit: null,
138
+ dispose: () => undefined,
139
+ };
140
+ }
141
+ const source = checkout(label, spec, project);
142
+ try {
143
+ // A worktree has a random name, so a ref's own skill takes the name of the project folder it was checked out from.
144
+ const skillDirs = findSkills(source.dir, path.basename(source.commit ? project.root : source.dir));
145
+ const kind = isPlugin(source.dir) ? "plugin" : "skills";
146
+ if (kind === "skills" && Object.keys(skillDirs).length === 0) {
147
+ throw new UsageError(`--${label} ${spec}: no skills found in ${spec}; it needs <name>/SKILL.md, skills/<name>/SKILL.md, its own SKILL.md, or a plugin manifest`);
148
+ }
149
+ return {
150
+ label,
151
+ spec,
152
+ kind,
153
+ dir: source.dir,
154
+ skills: Object.keys(skillDirs),
155
+ skillDirs,
156
+ commit: source.commit,
157
+ dispose: source.dispose,
158
+ };
159
+ }
160
+ catch (error) {
161
+ source.dispose();
162
+ throw error;
163
+ }
164
+ }
165
+ const KINDS = { plugin: "plugin", skills: "skills folder" };
166
+ const SHORT_HASH_LENGTH = 7;
167
+ /**
168
+ * Says in one phrase what a subject is and where it came from, for the head of a report.
169
+ * @param subject - A subject or its summary.
170
+ * @returns For a git ref, the ref and its short commit hash. For a directory, its path.
171
+ */
172
+ export function describeSubject(subject) {
173
+ if (!hasContent(subject))
174
+ return "no skills";
175
+ const source = subject.commit
176
+ ? `\`${subject.spec}\` at ${subject.commit.slice(0, SHORT_HASH_LENGTH)}`
177
+ : `directory \`${subject.dir}\` as it is on disk`;
178
+ return `${KINDS[subject.kind]}, ${source}`;
179
+ }
package/package.json CHANGED
@@ -1,6 +1,61 @@
1
1
  {
2
2
  "name": "skill-evals",
3
- "version": "0.0.0-stage",
4
- "stub": true,
5
- "description": "Temporary package placeholder for staged publishing"
6
- }
3
+ "version": "0.0.1",
4
+ "description": "Score how agent skills change agent behavior, from 0 to 1, in Claude Code, Codex, Cursor, OpenCode, and Pi",
5
+ "type": "module",
6
+ "license": "MIT",
7
+ "repository": {
8
+ "type": "git",
9
+ "url": "git+https://github.com/PhillipChaffee/skill-evals.git"
10
+ },
11
+ "bin": {
12
+ "skill-evals": "dist/bin.js"
13
+ },
14
+ "exports": {
15
+ ".": {
16
+ "types": "./dist/index.d.ts",
17
+ "default": "./dist/index.js"
18
+ }
19
+ },
20
+ "files": [
21
+ "dist",
22
+ "README.md",
23
+ "LICENSE",
24
+ "CHANGELOG.md"
25
+ ],
26
+ "//": "Deviations from the full-lint typescript template, each with its reason. (1) No \"private\", and name, version, description, license, repository, bin, exports, and files are set: this package is published. (2) The build and prepack scripts: Node does not strip types under node_modules, so the package ships compiled JavaScript. (3) @types/node: the sources use Node APIs, as the template's tsconfig comment instructs; 22 matches the engines floor. (4) vitest and @vitest/coverage-v8 are 5.0.3, not the template's 5.0.0, and minimatch is overridden to 11.0.1: osv-scanner fails the advisories gate on @vitest/mocker 5.0.0 (GHSA-qxg6-89mg-rqv8, GHSA-xp26-3454-mjgp) and on minimatch below 11.0.1 (GHSA-g3hv-75wg-m249), and eslint, typescript-eslint, and eslint-plugin-sonarjs all ask for minimatch ^10. Drop the override when their pins move to releases that ask for minimatch 11.",
27
+ "scripts": {
28
+ "build": "rm -rf dist && tsc -p tsconfig.build.json",
29
+ "format:check": "prettier --check .",
30
+ "knip": "knip",
31
+ "lint": "eslint .",
32
+ "lint:graph": "depcruise --validate .dependency-cruiser.cjs src",
33
+ "lint:lockfile": "lockfile-lint -p package-lock.json -t npm -a registry.npmjs.org -o \"https:\" -n -i -e false",
34
+ "prepack": "npm run build",
35
+ "test": "vitest run --coverage",
36
+ "typecheck": "tsc --noEmit"
37
+ },
38
+ "engines": {
39
+ "node": ">=22"
40
+ },
41
+ "devDependencies": {
42
+ "@eslint/js": "10.0.1",
43
+ "@types/node": "22.20.5",
44
+ "@vitest/coverage-v8": "5.0.3",
45
+ "@vitest/eslint-plugin": "1.6.27",
46
+ "dependency-cruiser": "18.3.1",
47
+ "eslint": "10.10.0",
48
+ "eslint-plugin-jsdoc": "64.3.8",
49
+ "eslint-plugin-sonarjs": "4.2.1",
50
+ "knip": "6.35.1",
51
+ "lockfile-lint": "5.0.1",
52
+ "prettier": "3.9.6",
53
+ "typescript": "6.0.3",
54
+ "typescript-eslint": "8.70.0",
55
+ "vite": "^8.0.0",
56
+ "vitest": "5.0.3"
57
+ },
58
+ "overrides": {
59
+ "minimatch": "11.0.1"
60
+ }
61
+ }