thunderjar 0.1.2 → 0.1.3-preview-eee7160
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli.js +2161 -34
- package/package.json +13 -4
- package/src/AGENTS.md +156 -0
- package/src/adapters/docker/docker.ts +117 -0
- package/src/api/declarations.ts +279 -0
- package/src/api/fragments.ts +30 -0
- package/src/api/parameter-names.d.ts +16 -0
- package/src/cli/commands/build/build.docker.test.ts +173 -0
- package/src/cli/commands/build/build.test.ts +234 -0
- package/src/cli/commands/build/build.ts +115 -0
- package/src/cli/commands/declarations.typecheck.test.ts +173 -0
- package/src/cli/commands/executions/executions.test.ts +84 -0
- package/src/cli/commands/executions/executions.ts +31 -0
- package/src/cli/commands/format.ts +33 -0
- package/src/cli/commands/generate/generate.test.ts +101 -0
- package/src/cli/commands/generate/generate.ts +18 -0
- package/src/cli/commands/generate/parameter-names.ts +131 -0
- package/src/cli/commands/init/init.test.ts +123 -0
- package/src/cli/commands/init/init.ts +189 -0
- package/src/cli/commands/options.ts +17 -0
- package/src/cli/commands/plan/plan.test.ts +168 -0
- package/src/cli/commands/plan/plan.ts +72 -0
- package/src/cli/commands/remeasure/remeasure.test.ts +536 -0
- package/src/cli/commands/remeasure/remeasure.ts +194 -0
- package/src/cli/commands/run/run.test.ts +622 -0
- package/src/cli/commands/run/run.ts +203 -0
- package/src/cli/commands/show/show.test.ts +288 -0
- package/src/cli/commands/show/show.ts +252 -0
- package/src/cli/commands/validation.test.ts +200 -0
- package/src/cli/index.test.ts +47 -0
- package/src/cli/index.ts +9 -0
- package/src/cli/program.ts +29 -0
- package/src/images/prerun-builds.ts +108 -0
- package/src/images/prerun.test.ts +243 -0
- package/src/images/prerun.ts +272 -0
- package/src/images/repository.ts +72 -0
- package/src/index.ts +24 -0
- package/src/instruments/instruments.test.ts +240 -0
- package/src/instruments/instruments.ts +205 -0
- package/src/matrix/execution.ts +122 -0
- package/src/matrix/hashing.test.ts +298 -0
- package/src/matrix/hashing.ts +131 -0
- package/src/matrix/permutations.test.ts +58 -0
- package/src/matrix/permutations.ts +50 -0
- package/src/matrix/resolve.ts +63 -0
- package/src/matrix/tags.test.ts +15 -0
- package/src/matrix/tags.ts +22 -0
- package/src/matrix/ulid.test.ts +28 -0
- package/src/matrix/ulid.ts +22 -0
- package/src/measuring/aggregate.test.ts +188 -0
- package/src/measuring/aggregate.ts +154 -0
- package/src/measuring/measure.test.ts +105 -0
- package/src/measuring/measure.ts +62 -0
- package/src/measuring/remeasure-templates/v1/isPrime.test.template.txt +4 -0
- package/src/measuring/remeasure-templates/v2/isPrime.test.template.txt +5 -0
- package/src/measuring/remeasure.test.ts +352 -0
- package/src/measuring/remeasure.ts +113 -0
- package/src/platform/fs.test.ts +89 -0
- package/src/platform/fs.ts +123 -0
- package/src/platform/path.test.ts +38 -0
- package/src/platform/path.ts +72 -0
- package/src/platform/signals.test.ts +25 -0
- package/src/platform/signals.ts +68 -0
- package/src/project/config.test.ts +68 -0
- package/src/project/config.ts +297 -0
- package/src/running/runner.test.ts +73 -0
- package/src/running/runner.ts +252 -0
- package/src/store/run-data-store.test.ts +232 -0
- package/src/store/run-data-store.ts +254 -0
- package/src/testing/cli.ts +65 -0
- package/src/testing/fake-docker.test.ts +197 -0
- package/src/testing/fake-docker.ts +104 -0
- package/src/testing/fixture-project.ts +177 -0
- package/src/testing/measurement-context.ts +5 -0
- package/src/testing/seed-repo.test.ts +10 -0
- package/src/testing/seed-repo.ts +83 -0
- package/src/testing/test-setup.ts +10 -0
|
@@ -0,0 +1,123 @@
|
|
|
1
|
+
// `thunderjar init`, through the CLI, in a copy of a checked-in fixtureTests/ scenario, because it
|
|
2
|
+
// writes into the project. Each test asserts on what it printed and the files it created or
|
|
3
|
+
// skipped. Any of these can be run by hand, in a scratch copy of the scenario:
|
|
4
|
+
//
|
|
5
|
+
// cp -r fixtureTests/scenario_6_uninitialised_project/fixtures /tmp/s6 && cd /tmp/s6 && thunderjar init
|
|
6
|
+
|
|
7
|
+
import { describe, expect, test } from "bun:test";
|
|
8
|
+
import { listEntries } from "../../../platform/fs.ts";
|
|
9
|
+
import { join } from "../../../platform/path.ts";
|
|
10
|
+
import { copyOfFixtures, scenario, thunderjar } from "../../../testing/cli.ts";
|
|
11
|
+
|
|
12
|
+
/** Every file under `.thunderjar/` in `dir`, sorted. */
|
|
13
|
+
const filesIn = (dir: string) =>
|
|
14
|
+
listEntries(join(dir, ".thunderjar"), { recursive: true })
|
|
15
|
+
.filter((entry) => !entry.isDirectory)
|
|
16
|
+
.map((entry) => entry.path)
|
|
17
|
+
.toSorted();
|
|
18
|
+
|
|
19
|
+
const read = (dir: string, rel: string) => Bun.file(join(dir, ".thunderjar", rel)).text();
|
|
20
|
+
|
|
21
|
+
/** What `init` scaffolds in any project, in the order it prints them. */
|
|
22
|
+
const SCAFFOLD = [
|
|
23
|
+
"thunderjar.config.ts",
|
|
24
|
+
"tsconfig.json",
|
|
25
|
+
"test-cases/example.ts",
|
|
26
|
+
"test-case-parameters/baseImages/node20/index.ts",
|
|
27
|
+
"test-case-parameters/codeStates/baseline/index.ts",
|
|
28
|
+
"test-case-parameters/promptSets/snerk/index.ts",
|
|
29
|
+
"test-case-parameters/harnessModels/claude-code/index.ts",
|
|
30
|
+
"test-case-parameters/harnessModels/claude-code/models.ts",
|
|
31
|
+
"test-case-parameters/harnessModels/claude-code/versions.ts",
|
|
32
|
+
"test-case-parameters/harnessModels/claude-code/collectTokenCosts.ts",
|
|
33
|
+
"test-case-parameters/tasks/is-prime/initial-prompts/plain.ts",
|
|
34
|
+
"test-case-parameters/tasks/is-prime/measurements/isPrimeTsExists.ts",
|
|
35
|
+
];
|
|
36
|
+
|
|
37
|
+
describe("scenario_6_uninitialised_project", () => {
|
|
38
|
+
const fixtures = scenario("scenario_6_uninitialised_project");
|
|
39
|
+
|
|
40
|
+
test("init: scaffolds .thunderjar/, with a nested package.json and node_modules ignored", async () => {
|
|
41
|
+
const dir = await copyOfFixtures(fixtures);
|
|
42
|
+
|
|
43
|
+
const { stdout, stderr, exitCode } = await thunderjar(dir, ["init"]);
|
|
44
|
+
|
|
45
|
+
expect(stderr).toBe("");
|
|
46
|
+
expect(exitCode).toBe(0);
|
|
47
|
+
const created = [...SCAFFOLD, "package.json", ".gitignore"];
|
|
48
|
+
expect(stdout).toBe(created.map((f) => `created .thunderjar/${f}\n`).join(""));
|
|
49
|
+
expect(filesIn(dir)).toEqual(created.toSorted());
|
|
50
|
+
expect(await read(dir, ".gitignore")).toBe("_generated/\n.data/\nnode_modules/\n");
|
|
51
|
+
expect(JSON.parse(await read(dir, "package.json"))).toEqual({
|
|
52
|
+
name: "thunderjar-config",
|
|
53
|
+
private: true,
|
|
54
|
+
type: "module",
|
|
55
|
+
dependencies: { thunderjar: "latest" },
|
|
56
|
+
});
|
|
57
|
+
expect(await read(dir, "thunderjar.config.ts")).toBe(`import { declareConfig, sqliteStore } from "thunderjar";
|
|
58
|
+
|
|
59
|
+
export default declareConfig({
|
|
60
|
+
imageStore: { repository: "ghcr.io/your-org/thunderjar-images" },
|
|
61
|
+
runDataStore: sqliteStore({ path: "./.thunderjar/.data/runs.sqlite" }),
|
|
62
|
+
});
|
|
63
|
+
`);
|
|
64
|
+
});
|
|
65
|
+
|
|
66
|
+
test("init: every TypeScript file it scaffolds resolves its thunderjar imports", async () => {
|
|
67
|
+
const dir = await copyOfFixtures(fixtures);
|
|
68
|
+
|
|
69
|
+
await thunderjar(dir, ["init"]);
|
|
70
|
+
|
|
71
|
+
for (const file of filesIn(dir).filter((f) => f.endsWith(".ts"))) {
|
|
72
|
+
await import(join(dir, ".thunderjar", file));
|
|
73
|
+
}
|
|
74
|
+
});
|
|
75
|
+
});
|
|
76
|
+
|
|
77
|
+
describe("scenario_7_uninitialised_js_project", () => {
|
|
78
|
+
const fixtures = scenario("scenario_7_uninitialised_js_project");
|
|
79
|
+
|
|
80
|
+
test("init: a project with a root package.json gets no nested one", async () => {
|
|
81
|
+
const dir = await copyOfFixtures(fixtures);
|
|
82
|
+
|
|
83
|
+
const { stdout, stderr, exitCode } = await thunderjar(dir, ["init"]);
|
|
84
|
+
|
|
85
|
+
expect(stderr).toBe("");
|
|
86
|
+
expect(exitCode).toBe(0);
|
|
87
|
+
const created = [...SCAFFOLD, ".gitignore"];
|
|
88
|
+
expect(stdout).toBe(created.map((f) => `created .thunderjar/${f}\n`).join(""));
|
|
89
|
+
expect(filesIn(dir)).toEqual(created.toSorted());
|
|
90
|
+
expect(await read(dir, ".gitignore")).toBe("_generated/\n.data/\n");
|
|
91
|
+
});
|
|
92
|
+
});
|
|
93
|
+
|
|
94
|
+
describe("scenario_1_minimal_initial_setup", () => {
|
|
95
|
+
const fixtures = scenario("scenario_1_minimal_initial_setup");
|
|
96
|
+
|
|
97
|
+
test("init: in an existing project, creates only what's missing and leaves the rest as it was", async () => {
|
|
98
|
+
const dir = await copyOfFixtures(fixtures);
|
|
99
|
+
const before = new Map(await Promise.all(filesIn(dir).map(async (f) => [f, await read(dir, f)] as const)));
|
|
100
|
+
|
|
101
|
+
const { stdout, stderr, exitCode } = await thunderjar(dir, ["init"]);
|
|
102
|
+
|
|
103
|
+
expect(stderr).toBe("");
|
|
104
|
+
expect(exitCode).toBe(0);
|
|
105
|
+
expect(stdout).toBe(`created .thunderjar/tsconfig.json
|
|
106
|
+
created .thunderjar/test-case-parameters/harnessModels/claude-code/index.ts
|
|
107
|
+
created .thunderjar/test-case-parameters/harnessModels/claude-code/models.ts
|
|
108
|
+
created .thunderjar/test-case-parameters/harnessModels/claude-code/versions.ts
|
|
109
|
+
created .thunderjar/test-case-parameters/harnessModels/claude-code/collectTokenCosts.ts
|
|
110
|
+
created .thunderjar/package.json
|
|
111
|
+
exists .thunderjar/thunderjar.config.ts
|
|
112
|
+
exists .thunderjar/test-cases/example.ts
|
|
113
|
+
exists .thunderjar/test-case-parameters/baseImages/node20/index.ts
|
|
114
|
+
exists .thunderjar/test-case-parameters/codeStates/baseline/index.ts
|
|
115
|
+
exists .thunderjar/test-case-parameters/promptSets/snerk/index.ts
|
|
116
|
+
exists .thunderjar/test-case-parameters/tasks/is-prime/initial-prompts/plain.ts
|
|
117
|
+
exists .thunderjar/test-case-parameters/tasks/is-prime/measurements/isPrimeTsExists.ts
|
|
118
|
+
exists .thunderjar/.gitignore
|
|
119
|
+
`);
|
|
120
|
+
for (const [file, content] of before) expect(await read(dir, file)).toBe(content);
|
|
121
|
+
expect(await read(dir, "test-cases/example.ts")).toContain("fake-harness-alpha@1.0.0/fake-model-small");
|
|
122
|
+
});
|
|
123
|
+
});
|
|
@@ -0,0 +1,189 @@
|
|
|
1
|
+
import { Command } from "commander";
|
|
2
|
+
import { existsSync } from "../../../platform/fs.ts";
|
|
3
|
+
import { join } from "../../../platform/path.ts";
|
|
4
|
+
|
|
5
|
+
const config = `import { declareConfig, sqliteStore } from "thunderjar";
|
|
6
|
+
|
|
7
|
+
export default declareConfig({
|
|
8
|
+
imageStore: { repository: "ghcr.io/your-org/thunderjar-images" },
|
|
9
|
+
runDataStore: sqliteStore({ path: "./.thunderjar/.data/runs.sqlite" }),
|
|
10
|
+
});
|
|
11
|
+
`;
|
|
12
|
+
|
|
13
|
+
const stub = (fn: string, body: string) => `import { ${fn} } from "thunderjar";\n\nexport default ${fn}(${body});\n`;
|
|
14
|
+
|
|
15
|
+
const baseImage = stub(
|
|
16
|
+
"declareBaseImage",
|
|
17
|
+
`{
|
|
18
|
+
applyParameter: () => \`FROM node:20-bookworm\`,
|
|
19
|
+
}`,
|
|
20
|
+
);
|
|
21
|
+
|
|
22
|
+
// Applying a parameter is up to its applyParameter. These read commits from the repository
|
|
23
|
+
// Thunderjar is run in: checkoutCommit clones the code state's commit, with no later history,
|
|
24
|
+
// into /workspace, and ctx.git.show reads one file at a commit into the build context.
|
|
25
|
+
const codeState = `import { checkoutCommit, declareCodeState } from "thunderjar";
|
|
26
|
+
|
|
27
|
+
export default declareCodeState({
|
|
28
|
+
config: { commit: "a1b2c3" },
|
|
29
|
+
applyParameter: (config, ctx) => \`\${checkoutCommit(ctx, config.commit)}\\nRUN npm ci\`,
|
|
30
|
+
});
|
|
31
|
+
`;
|
|
32
|
+
|
|
33
|
+
const promptSet = stub(
|
|
34
|
+
"declarePromptSet",
|
|
35
|
+
`{
|
|
36
|
+
config: { commit: "e4f5a6" },
|
|
37
|
+
applyParameter: (config, ctx) => \`COPY \${ctx.git.show(config.commit, "prompts/snerk.md")} /workspace/CLAUDE.md\`,
|
|
38
|
+
}`,
|
|
39
|
+
);
|
|
40
|
+
|
|
41
|
+
const harnessIndex = stub(
|
|
42
|
+
"declareHarness",
|
|
43
|
+
`{
|
|
44
|
+
applyParameter: (_config, ctx) => \`RUN npm install -g @anthropic-ai/claude-code@\${ctx.version}\`,
|
|
45
|
+
cli: (ctx) => \`claude -p "\${ctx.initialPrompt}" --model \${ctx.modelId} > \${ctx.resultPath}\`,
|
|
46
|
+
requiredEnv: ["ANTHROPIC_API_KEY"],
|
|
47
|
+
}`,
|
|
48
|
+
);
|
|
49
|
+
|
|
50
|
+
const harnessModels = stub(
|
|
51
|
+
"declareHarnessModels",
|
|
52
|
+
`{
|
|
53
|
+
"haiku-4-5": "claude-haiku-4-5-20251001",
|
|
54
|
+
}`,
|
|
55
|
+
);
|
|
56
|
+
|
|
57
|
+
const harnessVersions = stub(
|
|
58
|
+
"declareHarnessVersions",
|
|
59
|
+
`{
|
|
60
|
+
"2.1.283": { version: "2.1.283" },
|
|
61
|
+
}`,
|
|
62
|
+
);
|
|
63
|
+
|
|
64
|
+
const collectTokenCosts = stub(
|
|
65
|
+
"declareTokenCostCollector",
|
|
66
|
+
`(_raw, _ctx) => {
|
|
67
|
+
return { outcome: "unavailable", reason: "notReported" };
|
|
68
|
+
}`,
|
|
69
|
+
);
|
|
70
|
+
|
|
71
|
+
const initialPrompt = stub(
|
|
72
|
+
"declareInitialPrompt",
|
|
73
|
+
`{
|
|
74
|
+
prompt: "Create a file isPrime.ts exporting a function that tests for primality.",
|
|
75
|
+
timeoutSeconds: 600,
|
|
76
|
+
applyParameter: () => \`\`,
|
|
77
|
+
}`,
|
|
78
|
+
);
|
|
79
|
+
|
|
80
|
+
const measurement = stub(
|
|
81
|
+
"declareMeasurement",
|
|
82
|
+
`{
|
|
83
|
+
instrument: "fileCreated",
|
|
84
|
+
config: { glob: "**/isPrime.ts" },
|
|
85
|
+
aggregate: ["rate"],
|
|
86
|
+
}`,
|
|
87
|
+
);
|
|
88
|
+
|
|
89
|
+
const testCase = stub(
|
|
90
|
+
"declareTestCase",
|
|
91
|
+
`{
|
|
92
|
+
baseImage: ["node20"],
|
|
93
|
+
codeState: ["baseline"],
|
|
94
|
+
promptSet: ["snerk"],
|
|
95
|
+
harnessModel: ["claude-code@2.1.283/haiku-4-5"],
|
|
96
|
+
task: "is-prime",
|
|
97
|
+
initialPrompt: ["plain"],
|
|
98
|
+
iterations: 1,
|
|
99
|
+
}`,
|
|
100
|
+
);
|
|
101
|
+
|
|
102
|
+
// Brings _generated/parameter-names.d.ts into the editor's program alongside the files
|
|
103
|
+
// that use its names — without it they don't resolve, and nothing is checked
|
|
104
|
+
// (spec/055-declaring-test-cases.md#typed-names). `allowImportingTsExtensions` because
|
|
105
|
+
// thunderjar ships its TypeScript source.
|
|
106
|
+
const tsconfig =
|
|
107
|
+
JSON.stringify(
|
|
108
|
+
{
|
|
109
|
+
compilerOptions: {
|
|
110
|
+
target: "ESNext",
|
|
111
|
+
module: "ESNext",
|
|
112
|
+
moduleResolution: "bundler",
|
|
113
|
+
strict: true,
|
|
114
|
+
noEmit: true,
|
|
115
|
+
skipLibCheck: true,
|
|
116
|
+
allowImportingTsExtensions: true,
|
|
117
|
+
},
|
|
118
|
+
include: ["**/*.ts"],
|
|
119
|
+
// A templateTest template isn't TypeScript until it is rendered.
|
|
120
|
+
exclude: ["node_modules", ".data", "**/*.template.ts"],
|
|
121
|
+
},
|
|
122
|
+
null,
|
|
123
|
+
2,
|
|
124
|
+
) + "\n";
|
|
125
|
+
|
|
126
|
+
const P = "test-case-parameters";
|
|
127
|
+
|
|
128
|
+
/** Files `init` creates under `.thunderjar/`, keyed by relative path. */
|
|
129
|
+
function scaffold(): Record<string, string> {
|
|
130
|
+
return {
|
|
131
|
+
"thunderjar.config.ts": config,
|
|
132
|
+
"tsconfig.json": tsconfig,
|
|
133
|
+
"test-cases/example.ts": testCase,
|
|
134
|
+
[`${P}/baseImages/node20/index.ts`]: baseImage,
|
|
135
|
+
[`${P}/codeStates/baseline/index.ts`]: codeState,
|
|
136
|
+
[`${P}/promptSets/snerk/index.ts`]: promptSet,
|
|
137
|
+
[`${P}/harnessModels/claude-code/index.ts`]: harnessIndex,
|
|
138
|
+
[`${P}/harnessModels/claude-code/models.ts`]: harnessModels,
|
|
139
|
+
[`${P}/harnessModels/claude-code/versions.ts`]: harnessVersions,
|
|
140
|
+
[`${P}/harnessModels/claude-code/collectTokenCosts.ts`]: collectTokenCosts,
|
|
141
|
+
[`${P}/tasks/is-prime/initial-prompts/plain.ts`]: initialPrompt,
|
|
142
|
+
[`${P}/tasks/is-prime/measurements/isPrimeTsExists.ts`]: measurement,
|
|
143
|
+
};
|
|
144
|
+
}
|
|
145
|
+
|
|
146
|
+
export interface InitResult {
|
|
147
|
+
created: string[];
|
|
148
|
+
skipped: string[];
|
|
149
|
+
}
|
|
150
|
+
|
|
151
|
+
/**
|
|
152
|
+
* Scaffolds `<projectRoot>/.thunderjar/`. Existing files are never overwritten.
|
|
153
|
+
* A nested package.json is created only when the project has no root one.
|
|
154
|
+
*/
|
|
155
|
+
export async function init(projectRoot: string): Promise<InitResult> {
|
|
156
|
+
const root = join(projectRoot, ".thunderjar");
|
|
157
|
+
const nested = !existsSync(join(projectRoot, "package.json"));
|
|
158
|
+
|
|
159
|
+
const files = scaffold();
|
|
160
|
+
if (nested) {
|
|
161
|
+
files["package.json"] =
|
|
162
|
+
JSON.stringify(
|
|
163
|
+
{ name: "thunderjar-config", private: true, type: "module", dependencies: { thunderjar: "latest" } },
|
|
164
|
+
null,
|
|
165
|
+
2,
|
|
166
|
+
) + "\n";
|
|
167
|
+
}
|
|
168
|
+
files[".gitignore"] = ["_generated/", ".data/", ...(nested ? ["node_modules/"] : [])].join("\n") + "\n";
|
|
169
|
+
|
|
170
|
+
const result: InitResult = { created: [], skipped: [] };
|
|
171
|
+
for (const [rel, content] of Object.entries(files)) {
|
|
172
|
+
const path = join(root, rel);
|
|
173
|
+
if (existsSync(path)) {
|
|
174
|
+
result.skipped.push(rel);
|
|
175
|
+
continue;
|
|
176
|
+
}
|
|
177
|
+
await Bun.write(path, content);
|
|
178
|
+
result.created.push(rel);
|
|
179
|
+
}
|
|
180
|
+
return result;
|
|
181
|
+
}
|
|
182
|
+
|
|
183
|
+
export function createInitCommand(): Command {
|
|
184
|
+
return new Command("init").description("Create .thunderjar/ in this project.").action(async () => {
|
|
185
|
+
const { created, skipped } = await init(process.cwd());
|
|
186
|
+
for (const f of created) console.log(`created .thunderjar/${f}`);
|
|
187
|
+
for (const f of skipped) console.log(`exists .thunderjar/${f}`);
|
|
188
|
+
});
|
|
189
|
+
}
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
// Options several commands share.
|
|
2
|
+
|
|
3
|
+
import { InvalidArgumentError, Option } from "commander";
|
|
4
|
+
|
|
5
|
+
/** An option that takes a value, may be given only once, and whose value isn't another flag. */
|
|
6
|
+
function valueOption(flags: string, description: string): Option {
|
|
7
|
+
return new Option(flags, description).argParser((value: string, previous: string | undefined) => {
|
|
8
|
+
if (previous !== undefined) throw new InvalidArgumentError("It may only be given once.");
|
|
9
|
+
if (value.startsWith("-")) throw new InvalidArgumentError("It needs a value, not another flag.");
|
|
10
|
+
return value;
|
|
11
|
+
});
|
|
12
|
+
}
|
|
13
|
+
|
|
14
|
+
export const imagePrefixOption = (description: string) => valueOption("--image-prefix <name>", description);
|
|
15
|
+
|
|
16
|
+
export const dataDirOption = () =>
|
|
17
|
+
valueOption("--data-dir <dir>", "keep the run data store in this folder, in place of .thunderjar/.data/");
|
|
@@ -0,0 +1,168 @@
|
|
|
1
|
+
// `thunderjar plan`, through the CLI, in the checked-in fixtureTests/ scenarios, in place. Each
|
|
2
|
+
// asserts on stdout, against the recording fake Docker, which says which prerun images exist. Any
|
|
3
|
+
// of these can be run by hand:
|
|
4
|
+
//
|
|
5
|
+
// bun run fixture scenario_2 plan is-prime-baseline
|
|
6
|
+
|
|
7
|
+
import { describe, expect, test } from "bun:test";
|
|
8
|
+
import { fakeDocker, scenario, thunderjar } from "../../../testing/cli.ts";
|
|
9
|
+
|
|
10
|
+
const prerunTags = (stdout: string) => stdout.match(/prerun-h[0-9a-f]{8}/g);
|
|
11
|
+
|
|
12
|
+
describe("scenario_1_minimal_initial_setup", () => {
|
|
13
|
+
const fixtures = scenario("scenario_1_minimal_initial_setup");
|
|
14
|
+
|
|
15
|
+
test("plan example: a 1/1/1/1/1 test case has one permutation", async () => {
|
|
16
|
+
const { stdout, stderr, exitCode } = await thunderjar(fixtures, ["plan", "example"]);
|
|
17
|
+
|
|
18
|
+
expect(stderr).toBe("");
|
|
19
|
+
expect(exitCode).toBe(0);
|
|
20
|
+
expect(stdout).toBe(`baseImage: node20 (26d9745f)
|
|
21
|
+
codeState: baseline (bb1584cd)
|
|
22
|
+
promptSet: snerk (827aa0ce)
|
|
23
|
+
harnessModel: fake-harness-alpha@1.0.0/fake-model-small (da287975)
|
|
24
|
+
task: is-prime
|
|
25
|
+
initialPrompt: plain (9e30b385)
|
|
26
|
+
|
|
27
|
+
Matrix shape: 1/1/1/1/1 (1 permutation × 1 iteration = 1 container run)
|
|
28
|
+
|
|
29
|
+
1 node20 | baseline | snerk | fake-harness-alpha@1.0.0/fake-model-small | plain
|
|
30
|
+
prerun-h8ffdda6e (will build)
|
|
31
|
+
`);
|
|
32
|
+
});
|
|
33
|
+
});
|
|
34
|
+
|
|
35
|
+
describe("scenario_2_typical_project", () => {
|
|
36
|
+
const fixtures = scenario("scenario_2_typical_project");
|
|
37
|
+
|
|
38
|
+
// spec/100-cli-reference.md#plan, with the scenario's real hashes. The scenario is deterministic,
|
|
39
|
+
// so these are exact: a changed hash is a change to execution identity, a tripwire to investigate
|
|
40
|
+
// rather than loosen (spec/120-test-boundaries.md). Regenerate them deliberately, in the change
|
|
41
|
+
// that alters the scenario or the hashing.
|
|
42
|
+
const IS_PRIME_BASELINE = `baseImage: node20 (26d9745f)
|
|
43
|
+
codeState: baseline (bb1584cd)
|
|
44
|
+
promptSet: snerk (827aa0ce), glurk (a4ecb061)
|
|
45
|
+
harnessModel: fake-harness-alpha@1.0.0/fake-model-small (da287975), fake-harness-beta@1.0.0/fake-model-large (832747bd)
|
|
46
|
+
task: is-prime
|
|
47
|
+
initialPrompt: plain (9e30b385), terse (7bb8874f)
|
|
48
|
+
|
|
49
|
+
Matrix shape: 1/1/2/2/2 (8 permutations × 5 iterations = 40 container runs)
|
|
50
|
+
|
|
51
|
+
1 node20 | baseline | snerk | fake-harness-alpha@1.0.0/fake-model-small | plain
|
|
52
|
+
prerun-h8ffdda6e (will build)
|
|
53
|
+
|
|
54
|
+
2 node20 | baseline | snerk | fake-harness-alpha@1.0.0/fake-model-small | terse
|
|
55
|
+
prerun-hefe85dda (will build)
|
|
56
|
+
|
|
57
|
+
3 node20 | baseline | snerk | fake-harness-beta@1.0.0/fake-model-large | plain
|
|
58
|
+
prerun-h8baa2cff (will build)
|
|
59
|
+
|
|
60
|
+
4 node20 | baseline | snerk | fake-harness-beta@1.0.0/fake-model-large | terse
|
|
61
|
+
prerun-h56ea4852 (will build)
|
|
62
|
+
|
|
63
|
+
5 node20 | baseline | glurk | fake-harness-alpha@1.0.0/fake-model-small | plain
|
|
64
|
+
prerun-hfaa55eaf (will build)
|
|
65
|
+
|
|
66
|
+
6 node20 | baseline | glurk | fake-harness-alpha@1.0.0/fake-model-small | terse
|
|
67
|
+
prerun-h3e2038b8 (will build)
|
|
68
|
+
|
|
69
|
+
7 node20 | baseline | glurk | fake-harness-beta@1.0.0/fake-model-large | plain
|
|
70
|
+
prerun-h09fa95b9 (will build)
|
|
71
|
+
|
|
72
|
+
8 node20 | baseline | glurk | fake-harness-beta@1.0.0/fake-model-large | terse
|
|
73
|
+
prerun-he7379c79 (will build)`;
|
|
74
|
+
|
|
75
|
+
test("plan is-prime-baseline prints the parameters with their hashes, and each permutation's prerun image", async () => {
|
|
76
|
+
const docker = await fakeDocker();
|
|
77
|
+
const { stdout, stderr, exitCode } = await thunderjar(fixtures, ["plan", "is-prime-baseline"], docker.env);
|
|
78
|
+
|
|
79
|
+
expect(stderr).toBe("");
|
|
80
|
+
expect(exitCode).toBe(0);
|
|
81
|
+
expect(stdout).toBe(`${IS_PRIME_BASELINE}\n`);
|
|
82
|
+
});
|
|
83
|
+
|
|
84
|
+
test("plan glurk-on-node-lts: a renamed parameter keeps its hash; one whose file differs doesn't", async () => {
|
|
85
|
+
const { stdout, stderr, exitCode } = await thunderjar(fixtures, ["plan", "glurk-on-node-lts"]);
|
|
86
|
+
|
|
87
|
+
expect(stderr).toBe("");
|
|
88
|
+
expect(exitCode).toBe(0);
|
|
89
|
+
// node-lts is node20 under another name, so it has node20's hash, and glurk on it has the same
|
|
90
|
+
// prerun image as on node20 (entry 8 of is-prime-baseline). glurk-reworded is glurk with its
|
|
91
|
+
// FAKE_INSTRUCTIONS.md beside the declaration edited, so its hash and prerun image differ.
|
|
92
|
+
expect(stdout).toContain("baseImage: node-lts (26d9745f)\n");
|
|
93
|
+
expect(stdout).toContain("promptSet: glurk (a4ecb061), glurk-reworded (88256825)\n");
|
|
94
|
+
expect(prerunTags(stdout)).toEqual(["prerun-he7379c79", "prerun-h2960067a"]);
|
|
95
|
+
expect(prerunTags(IS_PRIME_BASELINE)).not.toContain("prerun-h2960067a");
|
|
96
|
+
});
|
|
97
|
+
|
|
98
|
+
test("plan with an unknown test case says so", async () => {
|
|
99
|
+
const { stdout, stderr, exitCode } = await thunderjar(fixtures, ["plan", "no-such-test-case"]);
|
|
100
|
+
|
|
101
|
+
expect(stdout).toBe("");
|
|
102
|
+
expect(stderr).toBe(
|
|
103
|
+
`error: no test case named "no-such-test-case" (expected ${fixtures}/.thunderjar/test-cases/no-such-test-case.ts)\n`,
|
|
104
|
+
);
|
|
105
|
+
expect(exitCode).toBe(1);
|
|
106
|
+
});
|
|
107
|
+
|
|
108
|
+
test("a prerun image that already exists is shown as reused; the rest as built", async () => {
|
|
109
|
+
const docker = await fakeDocker(["thunderjar:prerun-h8ffdda6e", "thunderjar:prerun-he7379c79"]);
|
|
110
|
+
const { stdout, stderr, exitCode } = await thunderjar(fixtures, ["plan", "is-prime-baseline"], docker.env);
|
|
111
|
+
|
|
112
|
+
expect(stderr).toBe("");
|
|
113
|
+
expect(exitCode).toBe(0);
|
|
114
|
+
expect(stdout.split("\n").filter((line) => line.includes("prerun-h"))).toEqual([
|
|
115
|
+
" prerun-h8ffdda6e (exists, reused)",
|
|
116
|
+
" prerun-hefe85dda (will build)",
|
|
117
|
+
" prerun-h8baa2cff (will build)",
|
|
118
|
+
" prerun-h56ea4852 (will build)",
|
|
119
|
+
" prerun-hfaa55eaf (will build)",
|
|
120
|
+
" prerun-h3e2038b8 (will build)",
|
|
121
|
+
" prerun-h09fa95b9 (will build)",
|
|
122
|
+
" prerun-he7379c79 (exists, reused)",
|
|
123
|
+
]);
|
|
124
|
+
});
|
|
125
|
+
|
|
126
|
+
test("without Docker, plan still prints, and says whether images exist is unknown", async () => {
|
|
127
|
+
const docker = await fakeDocker([], ["unreachable"]);
|
|
128
|
+
const { stdout, stderr, exitCode } = await thunderjar(fixtures, ["plan", "is-prime-baseline"], docker.env);
|
|
129
|
+
|
|
130
|
+
expect(stderr).toBe("");
|
|
131
|
+
expect(exitCode).toBe(0);
|
|
132
|
+
expect(stdout).toBe(
|
|
133
|
+
`${IS_PRIME_BASELINE.replaceAll(" (will build)", "")}\n\n` +
|
|
134
|
+
"Docker can't be reached (Cannot connect to the Docker daemon at unix:///var/run/docker.sock. Is the docker daemon running?), so whether each prerun image exists is unknown.\n",
|
|
135
|
+
);
|
|
136
|
+
});
|
|
137
|
+
|
|
138
|
+
test("plan --image-prefix looks up each prerun image under the prefix", async () => {
|
|
139
|
+
const docker = await fakeDocker(["throwaway:prerun-hefe85dda", "thunderjar:prerun-h56ea4852"]);
|
|
140
|
+
|
|
141
|
+
const { stdout, stderr, exitCode } = await thunderjar(fixtures, ["plan", "two-pairs", "--image-prefix", "throwaway"], docker.env);
|
|
142
|
+
|
|
143
|
+
expect(stderr).toBe("");
|
|
144
|
+
expect(exitCode).toBe(0);
|
|
145
|
+
// Only the image under the prefix counts; the one under the default prefix doesn't.
|
|
146
|
+
expect(stdout).toContain(" prerun-hefe85dda (exists, reused)\n");
|
|
147
|
+
expect(stdout).toContain(" prerun-h56ea4852 (will build)\n");
|
|
148
|
+
expect(docker.commands()).toEqual([
|
|
149
|
+
"docker version --format {{.Server.Version}}",
|
|
150
|
+
"docker image inspect --format {{.Id}} throwaway:prerun-hefe85dda",
|
|
151
|
+
"docker image inspect --format {{.Id}} throwaway:prerun-h56ea4852",
|
|
152
|
+
]);
|
|
153
|
+
});
|
|
154
|
+
|
|
155
|
+
test("plan with no test case, --image-prefix without a name, or an extra argument, says what's wrong, then its usage", async () => {
|
|
156
|
+
for (const [args, error] of [
|
|
157
|
+
[["plan"], "error: missing required argument 'test case'"],
|
|
158
|
+
[["plan", "two-pairs", "--image-prefix"], "error: option '--image-prefix <name>' argument missing"],
|
|
159
|
+
[["plan", "two-pairs", "extra"], "error: too many arguments for 'plan'"],
|
|
160
|
+
] as const) {
|
|
161
|
+
const { stdout, stderr, exitCode } = await thunderjar(fixtures, [...args]);
|
|
162
|
+
expect(stdout).toBe("");
|
|
163
|
+
expect(stderr).toStartWith(error);
|
|
164
|
+
expect(stderr).toContain("Usage: thunderjar plan [options] <test case>");
|
|
165
|
+
expect(exitCode).toBe(1);
|
|
166
|
+
}
|
|
167
|
+
});
|
|
168
|
+
});
|
|
@@ -0,0 +1,72 @@
|
|
|
1
|
+
// `thunderjar plan <test case>` — spec/100-cli-reference.md#plan,
|
|
2
|
+
// spec/055-declaring-test-cases.md#declaration. Lists the parameters a test case names with
|
|
3
|
+
// their parameter hashes, then its matrix shape, container run count, and each permutation
|
|
4
|
+
// with its prerun image and whether that image exists (and would be reused) or would be built.
|
|
5
|
+
// Without Docker, plan still works, and says the images' status is unknown.
|
|
6
|
+
|
|
7
|
+
import { Command } from "commander";
|
|
8
|
+
import { dockerCli, type Docker } from "../../../adapters/docker/docker.ts";
|
|
9
|
+
import { findThunderjarRoot, loadTestCase } from "../../../project/config.ts";
|
|
10
|
+
import { abbreviateHash } from "../../../matrix/hashing.ts";
|
|
11
|
+
import { matrixShape, permutationRows } from "../../../matrix/permutations.ts";
|
|
12
|
+
import { LOCAL_REPOSITORY, prerunStatuses, showPrerunStatus, unreachableNote } from "../../../images/prerun.ts";
|
|
13
|
+
import { resolveTestCase } from "../../../matrix/resolve.ts";
|
|
14
|
+
import { prerunTag } from "../../../matrix/tags.ts";
|
|
15
|
+
import { count } from "../format.ts";
|
|
16
|
+
import { imagePrefixOption } from "../options.ts";
|
|
17
|
+
|
|
18
|
+
export interface PlanOptions {
|
|
19
|
+
docker?: Docker;
|
|
20
|
+
/** The image repository prerun images are looked up in, `--image-prefix`. */
|
|
21
|
+
repository?: string;
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
/** The parameter lines, in declaration field order. `task` has no hash of its own. */
|
|
25
|
+
const LABELS = ["baseImage", "codeState", "promptSet", "harnessModel", "task", "initialPrompt"] as const;
|
|
26
|
+
|
|
27
|
+
const LABEL_WIDTH = Math.max(...LABELS.map((label) => label.length)) + 2; // + ":" + one space
|
|
28
|
+
|
|
29
|
+
export async function plan(name: string, cwd: string = process.cwd(), options: PlanOptions = {}): Promise<string> {
|
|
30
|
+
const { docker = dockerCli(), repository = LOCAL_REPOSITORY } = options;
|
|
31
|
+
const root = findThunderjarRoot(cwd);
|
|
32
|
+
const testCase = await loadTestCase(root, name);
|
|
33
|
+
const { hashes, permutations } = await resolveTestCase(root, testCase);
|
|
34
|
+
const found = await prerunStatuses(docker, repository, permutations);
|
|
35
|
+
const status = (hash: string) => ("statuses" in found ? showPrerunStatus(found.statuses.get(hash)) : "");
|
|
36
|
+
|
|
37
|
+
const parameterLines = LABELS.map((label) => {
|
|
38
|
+
const value =
|
|
39
|
+
label === "task"
|
|
40
|
+
? testCase.task
|
|
41
|
+
: testCase[label].map((n) => `${n} (${abbreviateHash(hashes[label].get(n)!)})`).join(", ");
|
|
42
|
+
return `${(label + ":").padEnd(LABEL_WIDTH)}${value}`;
|
|
43
|
+
});
|
|
44
|
+
|
|
45
|
+
const runs = permutations.length * testCase.iterations;
|
|
46
|
+
const summary =
|
|
47
|
+
`Matrix shape: ${matrixShape(testCase).join("/")} ` +
|
|
48
|
+
`(${count(permutations.length, "permutation")} × ${count(testCase.iterations, "iteration")} = ${count(runs, "container run")})`;
|
|
49
|
+
|
|
50
|
+
// Each permutation: a right-aligned number, then the five names as columns; below, its
|
|
51
|
+
// prerun image, which carries the permutation hash.
|
|
52
|
+
const rows = permutationRows(permutations);
|
|
53
|
+
const numberWidth = Math.max(2, String(permutations.length).length);
|
|
54
|
+
const entries = permutations.map(
|
|
55
|
+
(permutation, n) =>
|
|
56
|
+
`${String(n + 1).padStart(numberWidth)} ${rows[n]}\n` +
|
|
57
|
+
`${" ".repeat(numberWidth + 2)}${prerunTag(permutation.hash)}${status(permutation.hash)}`,
|
|
58
|
+
);
|
|
59
|
+
|
|
60
|
+
const note = "unreachable" in found ? ["", unreachableNote(found.unreachable)] : [];
|
|
61
|
+
return [...parameterLines, "", summary, "", entries.join("\n\n"), ...note].join("\n");
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
export function createPlanCommand(): Command {
|
|
65
|
+
return new Command("plan")
|
|
66
|
+
.description("Print what a run of a test case would do.")
|
|
67
|
+
.argument("<test case>", "the test case to plan")
|
|
68
|
+
.addOption(imagePrefixOption("the image repository prerun images are looked up in"))
|
|
69
|
+
.action(async (name: string, options: { imagePrefix?: string }) => {
|
|
70
|
+
console.log(await plan(name, process.cwd(), { repository: options.imagePrefix }));
|
|
71
|
+
});
|
|
72
|
+
}
|