thunderjar 0.1.3-preview-eee7160 → 0.1.3-preview-e09ffa1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli.js +1934 -1721
- package/package.json +14 -8
- package/src/api/checkoutCommit.ts +35 -0
- package/src/api/claudeCode.ts +74 -0
- package/src/api/declarations.ts +176 -133
- package/src/api/index.ts +44 -0
- package/src/api/parameterNames.ts +19 -0
- package/src/AGENTS.md +0 -156
- package/src/adapters/docker/docker.ts +0 -117
- package/src/api/fragments.ts +0 -30
- package/src/api/parameter-names.d.ts +0 -16
- package/src/cli/commands/build/build.docker.test.ts +0 -173
- package/src/cli/commands/build/build.test.ts +0 -234
- package/src/cli/commands/build/build.ts +0 -115
- package/src/cli/commands/declarations.typecheck.test.ts +0 -173
- package/src/cli/commands/executions/executions.test.ts +0 -84
- package/src/cli/commands/executions/executions.ts +0 -31
- package/src/cli/commands/format.ts +0 -33
- package/src/cli/commands/generate/generate.test.ts +0 -101
- package/src/cli/commands/generate/generate.ts +0 -18
- package/src/cli/commands/generate/parameter-names.ts +0 -131
- package/src/cli/commands/init/init.test.ts +0 -123
- package/src/cli/commands/init/init.ts +0 -189
- package/src/cli/commands/options.ts +0 -17
- package/src/cli/commands/plan/plan.test.ts +0 -168
- package/src/cli/commands/plan/plan.ts +0 -72
- package/src/cli/commands/remeasure/remeasure.test.ts +0 -536
- package/src/cli/commands/remeasure/remeasure.ts +0 -194
- package/src/cli/commands/run/run.test.ts +0 -622
- package/src/cli/commands/run/run.ts +0 -203
- package/src/cli/commands/show/show.test.ts +0 -288
- package/src/cli/commands/show/show.ts +0 -252
- package/src/cli/commands/validation.test.ts +0 -200
- package/src/cli/index.test.ts +0 -47
- package/src/cli/index.ts +0 -9
- package/src/cli/program.ts +0 -29
- package/src/images/prerun-builds.ts +0 -108
- package/src/images/prerun.test.ts +0 -243
- package/src/images/prerun.ts +0 -272
- package/src/images/repository.ts +0 -72
- package/src/index.ts +0 -24
- package/src/instruments/instruments.test.ts +0 -240
- package/src/instruments/instruments.ts +0 -205
- package/src/matrix/execution.ts +0 -122
- package/src/matrix/hashing.test.ts +0 -298
- package/src/matrix/hashing.ts +0 -131
- package/src/matrix/permutations.test.ts +0 -58
- package/src/matrix/permutations.ts +0 -50
- package/src/matrix/resolve.ts +0 -63
- package/src/matrix/tags.test.ts +0 -15
- package/src/matrix/tags.ts +0 -22
- package/src/matrix/ulid.test.ts +0 -28
- package/src/matrix/ulid.ts +0 -22
- package/src/measuring/aggregate.test.ts +0 -188
- package/src/measuring/aggregate.ts +0 -154
- package/src/measuring/measure.test.ts +0 -105
- package/src/measuring/measure.ts +0 -62
- package/src/measuring/remeasure-templates/v1/isPrime.test.template.txt +0 -4
- package/src/measuring/remeasure-templates/v2/isPrime.test.template.txt +0 -5
- package/src/measuring/remeasure.test.ts +0 -352
- package/src/measuring/remeasure.ts +0 -113
- package/src/platform/fs.test.ts +0 -89
- package/src/platform/fs.ts +0 -123
- package/src/platform/path.test.ts +0 -38
- package/src/platform/path.ts +0 -72
- package/src/platform/signals.test.ts +0 -25
- package/src/platform/signals.ts +0 -68
- package/src/project/config.test.ts +0 -68
- package/src/project/config.ts +0 -297
- package/src/running/runner.test.ts +0 -73
- package/src/running/runner.ts +0 -252
- package/src/store/run-data-store.test.ts +0 -232
- package/src/store/run-data-store.ts +0 -254
- package/src/testing/cli.ts +0 -65
- package/src/testing/fake-docker.test.ts +0 -197
- package/src/testing/fake-docker.ts +0 -104
- package/src/testing/fixture-project.ts +0 -177
- package/src/testing/measurement-context.ts +0 -5
- package/src/testing/seed-repo.test.ts +0 -10
- package/src/testing/seed-repo.ts +0 -83
- package/src/testing/test-setup.ts +0 -10
|
@@ -1,234 +0,0 @@
|
|
|
1
|
-
// `thunderjar build`, through the CLI, in the checked-in fixtureTests/ scenarios, in place.
|
|
2
|
-
//
|
|
3
|
-
// - `--dry-run` prints each permutation's Dockerfile, so what each image is built from is
|
|
4
|
-
// asserted on stdout, with nothing built. Any of these can be run by hand:
|
|
5
|
-
//
|
|
6
|
-
// bun run fixture scenario_2 build two-pairs --dry-run
|
|
7
|
-
//
|
|
8
|
-
// - The rest stop before an image is made, against the recording fake Docker
|
|
9
|
-
// (spec/120-test-boundaries.md#2-cli-with-recording-adapters): Docker can't be reached, a commit
|
|
10
|
-
// is missing, the project isn't in a git repository, or a build fails, which no scenario's does.
|
|
11
|
-
//
|
|
12
|
-
// What a real build makes, and which images it reuses, is in build.docker.test.ts.
|
|
13
|
-
|
|
14
|
-
import { describe, expect, test } from "bun:test";
|
|
15
|
-
import { existsSync, listDir, makeTempDir } from "../../../platform/fs.ts";
|
|
16
|
-
import { dirname, join } from "../../../platform/path.ts";
|
|
17
|
-
import { fakeDocker, scenario, thunderjar } from "../../../testing/cli.ts";
|
|
18
|
-
|
|
19
|
-
type FakeDocker = Awaited<ReturnType<typeof fakeDocker>>;
|
|
20
|
-
|
|
21
|
-
/**
|
|
22
|
-
* Runs `thunderjar build <testCase> <flags…>` in `project`, keeping build contexts under a fresh
|
|
23
|
-
* $TMPDIR, with $TMPDIR replaced by a placeholder.
|
|
24
|
-
*/
|
|
25
|
-
async function cliBuild(
|
|
26
|
-
project: string,
|
|
27
|
-
testCase: string,
|
|
28
|
-
{ docker, flags = [], env = {} }: { docker?: FakeDocker; flags?: string[]; env?: Record<string, string> } = {},
|
|
29
|
-
) {
|
|
30
|
-
docker ??= await fakeDocker();
|
|
31
|
-
const tmpDir = await makeTempDir("thunderjar-fixture-tmp-");
|
|
32
|
-
const result = await thunderjar(project, ["build", testCase, ...flags], { ...docker.env, TMPDIR: tmpDir, ...env });
|
|
33
|
-
const normalise = (text: string) => text.replaceAll(tmpDir, "$TMPDIR");
|
|
34
|
-
return {
|
|
35
|
-
exitCode: result.exitCode,
|
|
36
|
-
stdout: normalise(result.stdout),
|
|
37
|
-
stderr: normalise(result.stderr),
|
|
38
|
-
commands: docker.commands().map(normalise),
|
|
39
|
-
builds: join(tmpDir, "thunderjar", "builds"),
|
|
40
|
-
};
|
|
41
|
-
}
|
|
42
|
-
|
|
43
|
-
describe("scenario_1_minimal_initial_setup", () => {
|
|
44
|
-
const fixtures = scenario("scenario_1_minimal_initial_setup");
|
|
45
|
-
|
|
46
|
-
test("build example --dry-run: the Dockerfile copies the files beside the declarations", async () => {
|
|
47
|
-
const { stdout, stderr, exitCode } = await cliBuild(fixtures, "example", { flags: ["--dry-run"] });
|
|
48
|
-
|
|
49
|
-
expect(stderr).toBe("");
|
|
50
|
-
expect(exitCode).toBe(0);
|
|
51
|
-
expect(stdout).toBe(`Dockerfiles for example (1 permutation). Nothing is built.
|
|
52
|
-
|
|
53
|
-
1 node20 | baseline | snerk | fake-harness-alpha@1.0.0/fake-model-small | plain
|
|
54
|
-
thunderjar:prerun-h8ffdda6e
|
|
55
|
-
|
|
56
|
-
FROM node:20-bookworm
|
|
57
|
-
RUN echo "install fake-harness-alpha@1.0.0"
|
|
58
|
-
WORKDIR /workspace
|
|
59
|
-
COPY parameters/codeState/files/ /workspace/
|
|
60
|
-
RUN npm ci
|
|
61
|
-
COPY parameters/promptSet/FAKE_INSTRUCTIONS.md /workspace/FAKE_INSTRUCTIONS.md
|
|
62
|
-
`);
|
|
63
|
-
});
|
|
64
|
-
});
|
|
65
|
-
|
|
66
|
-
describe("scenario_2_typical_project", () => {
|
|
67
|
-
// Its parameters copy the files beside their declarations, so it builds in place, with no commits to read.
|
|
68
|
-
const fixtures = scenario("scenario_2_typical_project");
|
|
69
|
-
const ALPHA = "thunderjar:prerun-hefe85dda";
|
|
70
|
-
const BETA = "thunderjar:prerun-h56ea4852";
|
|
71
|
-
|
|
72
|
-
test("build two-pairs --dry-run: prints each permutation's Dockerfile, without Docker, and writes nothing", async () => {
|
|
73
|
-
const { stdout, stderr, exitCode, commands, builds } = await cliBuild(fixtures, "two-pairs", {
|
|
74
|
-
docker: await fakeDocker([], ["unreachable"]),
|
|
75
|
-
flags: ["--dry-run"],
|
|
76
|
-
});
|
|
77
|
-
|
|
78
|
-
expect(stderr).toBe("");
|
|
79
|
-
expect(exitCode).toBe(0);
|
|
80
|
-
// The tags are those `thunderjar plan two-pairs` shows. Each fragment is in order, with
|
|
81
|
-
// Thunderjar's WORKDIR before the code state: nothing else is Thunderjar's.
|
|
82
|
-
expect(stdout).toBe(`Dockerfiles for two-pairs (2 permutations). Nothing is built.
|
|
83
|
-
|
|
84
|
-
1 node20 | baseline | snerk | fake-harness-alpha@1.0.0/fake-model-small | terse
|
|
85
|
-
${ALPHA}
|
|
86
|
-
|
|
87
|
-
FROM node:20-bookworm
|
|
88
|
-
RUN echo "install fake-harness-alpha@1.0.0"
|
|
89
|
-
WORKDIR /workspace
|
|
90
|
-
COPY parameters/codeState/files/ /workspace/
|
|
91
|
-
RUN npm ci
|
|
92
|
-
COPY parameters/promptSet/FAKE_INSTRUCTIONS.md /workspace/FAKE_INSTRUCTIONS.md
|
|
93
|
-
|
|
94
|
-
2 node20 | baseline | snerk | fake-harness-beta@1.0.0/fake-model-large | terse
|
|
95
|
-
${BETA}
|
|
96
|
-
|
|
97
|
-
FROM node:20-bookworm
|
|
98
|
-
RUN echo "install fake-harness-beta@1.0.0"
|
|
99
|
-
WORKDIR /workspace
|
|
100
|
-
COPY parameters/codeState/files/ /workspace/
|
|
101
|
-
RUN npm ci
|
|
102
|
-
COPY parameters/promptSet/FAKE_INSTRUCTIONS.md /workspace/FAKE_INSTRUCTIONS.md
|
|
103
|
-
`);
|
|
104
|
-
expect(commands).toEqual([]);
|
|
105
|
-
expect(existsSync(builds)).toBe(false);
|
|
106
|
-
});
|
|
107
|
-
|
|
108
|
-
test("--dry-run prints every permutation's Dockerfile, even when its image exists", async () => {
|
|
109
|
-
const { stdout, commands } = await cliBuild(fixtures, "two-pairs", {
|
|
110
|
-
docker: await fakeDocker([ALPHA, BETA]),
|
|
111
|
-
flags: ["--dry-run"],
|
|
112
|
-
});
|
|
113
|
-
|
|
114
|
-
expect(stdout).toContain(`RUN echo "install fake-harness-alpha@1.0.0"\n`);
|
|
115
|
-
expect(stdout).toContain(`RUN echo "install fake-harness-beta@1.0.0"\n`);
|
|
116
|
-
expect(commands).toEqual([]);
|
|
117
|
-
});
|
|
118
|
-
|
|
119
|
-
test("--dry-run tags with --image-prefix", async () => {
|
|
120
|
-
const { stdout } = await cliBuild(fixtures, "two-pairs", { flags: ["--dry-run", "--image-prefix", "throwaway"] });
|
|
121
|
-
|
|
122
|
-
expect(stdout).toContain(" throwaway:prerun-hefe85dda\n");
|
|
123
|
-
expect(stdout).toContain(" throwaway:prerun-h56ea4852\n");
|
|
124
|
-
});
|
|
125
|
-
|
|
126
|
-
test("without Docker, build stops before reading anything", async () => {
|
|
127
|
-
const { stdout, stderr, exitCode, commands } = await cliBuild(fixtures, "two-pairs", {
|
|
128
|
-
docker: await fakeDocker([], ["unreachable"]),
|
|
129
|
-
});
|
|
130
|
-
|
|
131
|
-
expect(exitCode).toBe(1);
|
|
132
|
-
expect(stdout).toBe("");
|
|
133
|
-
expect(stderr).toBe(
|
|
134
|
-
"error: Docker can't be reached (Cannot connect to the Docker daemon at unix:///var/run/docker.sock. " +
|
|
135
|
-
"Is the docker daemon running?). Building prerun images needs Docker.\n",
|
|
136
|
-
);
|
|
137
|
-
expect(commands).toEqual(["docker version --format {{.Server.Version}}"]);
|
|
138
|
-
});
|
|
139
|
-
|
|
140
|
-
test("a failed build stops, and keeps its build context for inspection", async () => {
|
|
141
|
-
const { stdout, stderr, exitCode, commands, builds } = await cliBuild(fixtures, "two-pairs", {
|
|
142
|
-
docker: await fakeDocker([], ["fail-build"]),
|
|
143
|
-
});
|
|
144
|
-
|
|
145
|
-
expect(exitCode).toBe(1);
|
|
146
|
-
expect(stdout).toEndWith(" 1 node20 | baseline | snerk | fake-harness-alpha@1.0.0/fake-model-small | terse\n");
|
|
147
|
-
expect(stderr).toBe(`fake docker: the build failed
|
|
148
|
-
error: building ${ALPHA} failed (docker build exited 1).
|
|
149
|
-
Its build context is kept in $TMPDIR/thunderjar/builds/prerun-hefe85dda
|
|
150
|
-
`);
|
|
151
|
-
expect(commands.filter((c) => c.startsWith("docker build"))).toHaveLength(1);
|
|
152
|
-
expect(listDir(join(builds, "prerun-hefe85dda"))).toContain("Dockerfile");
|
|
153
|
-
});
|
|
154
|
-
|
|
155
|
-
test("build with no test case, an unknown flag, or --image-prefix without a name, says what's wrong, then its usage", async () => {
|
|
156
|
-
for (const [args, error] of [
|
|
157
|
-
[["build"], "error: missing required argument 'test case'"],
|
|
158
|
-
[["build", "two-pairs", "--dry"], "error: unknown option '--dry'"],
|
|
159
|
-
[["build", "two-pairs", "--image-prefix"], "error: option '--image-prefix <name>' argument missing"],
|
|
160
|
-
[["build", "--image-prefix", "two-pairs"], "error: missing required argument 'test case'"],
|
|
161
|
-
] as const) {
|
|
162
|
-
const { stdout, stderr, exitCode } = await thunderjar(fixtures, [...args]);
|
|
163
|
-
expect(stdout).toBe("");
|
|
164
|
-
expect(stderr).toStartWith(error);
|
|
165
|
-
expect(stderr).toContain("Usage: thunderjar build [options] <test case>");
|
|
166
|
-
expect(exitCode).toBe(1);
|
|
167
|
-
}
|
|
168
|
-
});
|
|
169
|
-
});
|
|
170
|
-
|
|
171
|
-
describe("scenario_4_git_project", () => {
|
|
172
|
-
// Its code states and prompt sets read two real commits of this repository, the ones that added
|
|
173
|
-
// and changed fixtures/project/ (tags fixture-scenario-4-first and fixture-scenario-4-second).
|
|
174
|
-
const fixtures = scenario("scenario_4_git_project");
|
|
175
|
-
const FIRST = "523984c7fd72f047265ab89ce5901605af6acb2b";
|
|
176
|
-
const SECOND = "952687db16b373d58f79f46690f9c5b0aa1aa141";
|
|
177
|
-
const PROJECT = "fixtureTests/scenario_4_git_project/fixtures/project";
|
|
178
|
-
|
|
179
|
-
test("build two-commits --dry-run: each code state checks out its own commit, each prompt set copies its file at its own", async () => {
|
|
180
|
-
const { stdout, stderr, exitCode, commands } = await cliBuild(fixtures, "two-commits", { flags: ["--dry-run"] });
|
|
181
|
-
|
|
182
|
-
expect(stderr).toBe("");
|
|
183
|
-
expect(exitCode).toBe(0);
|
|
184
|
-
expect(commands).toEqual([]);
|
|
185
|
-
const permutation = (row: string, tag: string, commit: string, promptCommit: string) => `
|
|
186
|
-
${row} | fake-harness-alpha@1.0.0/fake-model-small | plain
|
|
187
|
-
thunderjar:${tag}
|
|
188
|
-
|
|
189
|
-
FROM node:20-bookworm
|
|
190
|
-
RUN echo "install fake-harness-alpha@1.0.0"
|
|
191
|
-
WORKDIR /workspace
|
|
192
|
-
COPY git/${commit}.bundle /tmp/code.bundle
|
|
193
|
-
RUN git clone --quiet /tmp/code.bundle /workspace \\
|
|
194
|
-
&& git -C /workspace checkout --quiet --detach ${commit} \\
|
|
195
|
-
&& git -C /workspace branch --quiet --delete --force code-state \\
|
|
196
|
-
&& git -C /workspace remote remove origin \\
|
|
197
|
-
&& git -C /workspace config user.name "Thunderjar agent" \\
|
|
198
|
-
&& git -C /workspace config user.email agent@thunderjar.invalid \\
|
|
199
|
-
&& rm /tmp/code.bundle
|
|
200
|
-
RUN cd ${PROJECT} && npm ci
|
|
201
|
-
COPY git/${promptCommit}/${PROJECT}/prompts/house-style.md /workspace/HOUSE_STYLE.md
|
|
202
|
-
`;
|
|
203
|
-
expect(stdout).toBe(
|
|
204
|
-
"Dockerfiles for two-commits (4 permutations). Nothing is built.\n" +
|
|
205
|
-
permutation(" 1 node20 | first | house-style-first ", "prerun-hf803f77d", FIRST, FIRST) +
|
|
206
|
-
permutation(" 2 node20 | first | house-style-second", "prerun-h85c50e99", FIRST, SECOND) +
|
|
207
|
-
permutation(" 3 node20 | second | house-style-first ", "prerun-hc7fcb855", SECOND, FIRST) +
|
|
208
|
-
permutation(" 4 node20 | second | house-style-second", "prerun-h51cb391f", SECOND, SECOND),
|
|
209
|
-
);
|
|
210
|
-
});
|
|
211
|
-
|
|
212
|
-
test("a commit that isn't in this repository stops the build before anything is built", async () => {
|
|
213
|
-
const { stdout, stderr, exitCode, commands } = await cliBuild(fixtures, "unpushed-commit");
|
|
214
|
-
|
|
215
|
-
expect(exitCode).toBe(1);
|
|
216
|
-
expect(stdout).toBe("");
|
|
217
|
-
expect(stderr).toBe(`error: commit a1b2c3 (code state "unpushed") is not in this repository.
|
|
218
|
-
Fetch it, or check that it has been pushed.
|
|
219
|
-
`);
|
|
220
|
-
expect(commands.filter((c) => c.startsWith("docker build"))).toEqual([]);
|
|
221
|
-
});
|
|
222
|
-
|
|
223
|
-
test("a project whose parameters read commits must be in a git repository", async () => {
|
|
224
|
-
// Git doesn't look above the scenario's folder, so finds no repository for fixtures/.
|
|
225
|
-
const { stdout, stderr, exitCode, commands } = await cliBuild(fixtures, "two-commits", {
|
|
226
|
-
env: { GIT_CEILING_DIRECTORIES: dirname(fixtures) },
|
|
227
|
-
});
|
|
228
|
-
|
|
229
|
-
expect(exitCode).toBe(1);
|
|
230
|
-
expect(stdout).toBe("");
|
|
231
|
-
expect(stderr).toBe(`error: ${fixtures} is not in a git repository, but code state "first" reads commit ${FIRST} from it.\n`);
|
|
232
|
-
expect(commands.filter((c) => c.startsWith("docker build"))).toEqual([]);
|
|
233
|
-
});
|
|
234
|
-
});
|
|
@@ -1,115 +0,0 @@
|
|
|
1
|
-
// `thunderjar build <test case>` — builds each permutation's prerun image that doesn't exist
|
|
2
|
-
// yet, then stops: nothing is run and nothing is recorded. spec/080-docker-execution.md#process,
|
|
3
|
-
// as far as "Prerun image ready".
|
|
4
|
-
//
|
|
5
|
-
// What it checks before the first build, and where it keeps each build context, are
|
|
6
|
-
// src/images/prerun-builds.ts's, which `run` builds through too.
|
|
7
|
-
//
|
|
8
|
-
// `--dry-run` prints the Dockerfile each permutation's prerun image would be built from, for
|
|
9
|
-
// every permutation, whether or not its image exists. It builds nothing, writes nothing, and
|
|
10
|
-
// doesn't ask Docker anything, nor read any commit, so what it prints doesn't depend on either.
|
|
11
|
-
|
|
12
|
-
import { Command } from "commander";
|
|
13
|
-
import { join } from "../../../platform/path.ts";
|
|
14
|
-
import { dockerCli, type Docker } from "../../../adapters/docker/docker.ts";
|
|
15
|
-
import { findThunderjarRoot, loadTestCase } from "../../../project/config.ts";
|
|
16
|
-
import { permutationRows } from "../../../matrix/permutations.ts";
|
|
17
|
-
import { LOCAL_REPOSITORY, loadPrerunParameters, prerunImageRef, renderPrerun } from "../../../images/prerun.ts";
|
|
18
|
-
import { preparePrerunBuilds } from "../../../images/prerun-builds.ts";
|
|
19
|
-
import { resolveTestCase } from "../../../matrix/resolve.ts";
|
|
20
|
-
import { defaultRootDir } from "../../../running/runner.ts";
|
|
21
|
-
import { count, seconds } from "../format.ts";
|
|
22
|
-
import { imagePrefixOption } from "../options.ts";
|
|
23
|
-
|
|
24
|
-
export interface BuildOptions {
|
|
25
|
-
cwd?: string;
|
|
26
|
-
/** Receives the output a block at a time, as it is produced. */
|
|
27
|
-
log?: (text: string) => void;
|
|
28
|
-
docker?: Docker;
|
|
29
|
-
/** Print each permutation's Dockerfile instead of building, `--dry-run`. */
|
|
30
|
-
dryRun?: boolean;
|
|
31
|
-
/** The image repository prerun images are tagged in, `--image-prefix`. Tests use a throwaway one. */
|
|
32
|
-
repository?: string;
|
|
33
|
-
/** Build contexts are kept under `<rootDir>/builds/`. Defaults to `$TMPDIR/thunderjar`. */
|
|
34
|
-
rootDir?: string;
|
|
35
|
-
/** The clock for timing builds; stubbed in tests so output is deterministic. */
|
|
36
|
-
now?: () => number;
|
|
37
|
-
}
|
|
38
|
-
|
|
39
|
-
export async function build(name: string, options: BuildOptions = {}): Promise<void> {
|
|
40
|
-
const {
|
|
41
|
-
cwd = process.cwd(),
|
|
42
|
-
log = (text) => console.log(text),
|
|
43
|
-
docker = dockerCli(),
|
|
44
|
-
dryRun = false,
|
|
45
|
-
repository = LOCAL_REPOSITORY,
|
|
46
|
-
rootDir = defaultRootDir(),
|
|
47
|
-
now = Date.now,
|
|
48
|
-
} = options;
|
|
49
|
-
|
|
50
|
-
const root = findThunderjarRoot(cwd);
|
|
51
|
-
const testCase = await loadTestCase(root, name);
|
|
52
|
-
const { permutations } = await resolveTestCase(root, testCase);
|
|
53
|
-
const rows = permutationRows(permutations);
|
|
54
|
-
const numberWidth = Math.max(2, String(permutations.length).length);
|
|
55
|
-
const pad = " ".repeat(numberWidth + 2);
|
|
56
|
-
|
|
57
|
-
if (dryRun) {
|
|
58
|
-
log(`Dockerfiles for ${name} (${count(permutations.length, "permutation")}). Nothing is built.`);
|
|
59
|
-
for (const [n, permutation] of permutations.entries()) {
|
|
60
|
-
const { dockerfile } = renderPrerun(await loadPrerunParameters(root, testCase.task, permutation));
|
|
61
|
-
log("");
|
|
62
|
-
log(`${String(n + 1).padStart(numberWidth)} ${rows[n]}`);
|
|
63
|
-
log(`${pad}${prerunImageRef(repository, permutation.hash)}`);
|
|
64
|
-
log("");
|
|
65
|
-
log(dockerfile.trimEnd().replace(/^(?=.)/gm, pad));
|
|
66
|
-
}
|
|
67
|
-
return;
|
|
68
|
-
}
|
|
69
|
-
|
|
70
|
-
const builds = await preparePrerunBuilds({
|
|
71
|
-
root,
|
|
72
|
-
task: testCase.task,
|
|
73
|
-
permutations,
|
|
74
|
-
docker,
|
|
75
|
-
repository,
|
|
76
|
-
rootDir,
|
|
77
|
-
purpose: "Building prerun images",
|
|
78
|
-
});
|
|
79
|
-
const toBuild = builds.missing.size;
|
|
80
|
-
if (toBuild > 0) {
|
|
81
|
-
log(`Building ${count(toBuild, "prerun image")} for ${name} (${count(permutations.length, "permutation")})`);
|
|
82
|
-
log(`Build contexts in ${join(rootDir, "builds")}`);
|
|
83
|
-
} else {
|
|
84
|
-
log(`Every prerun image for ${name} exists (${count(permutations.length, "permutation")})`);
|
|
85
|
-
}
|
|
86
|
-
|
|
87
|
-
for (const [n, permutation] of permutations.entries()) {
|
|
88
|
-
const ref = prerunImageRef(repository, permutation.hash);
|
|
89
|
-
log("");
|
|
90
|
-
log(`${String(n + 1).padStart(numberWidth)} ${rows[n]}`);
|
|
91
|
-
if (!builds.missing.has(permutation.hash)) {
|
|
92
|
-
log(`${pad}${ref} reused`);
|
|
93
|
-
continue;
|
|
94
|
-
}
|
|
95
|
-
const started = now();
|
|
96
|
-
await builds.build(permutation.hash);
|
|
97
|
-
log(`${pad}${ref} built (${seconds(now() - started)})`);
|
|
98
|
-
}
|
|
99
|
-
|
|
100
|
-
log("");
|
|
101
|
-
log(
|
|
102
|
-
`${count(permutations.length, "prerun image")}: ${toBuild} built, ${permutations.length - toBuild} reused`,
|
|
103
|
-
);
|
|
104
|
-
}
|
|
105
|
-
|
|
106
|
-
export function createBuildCommand(): Command {
|
|
107
|
-
return new Command("build")
|
|
108
|
-
.description("Build a test case's prerun images.")
|
|
109
|
-
.argument("<test case>", "the test case whose prerun images to build")
|
|
110
|
-
.option("--dry-run", "print each permutation's Dockerfile instead of building")
|
|
111
|
-
.addOption(imagePrefixOption("the image repository prerun images are tagged in"))
|
|
112
|
-
.action(async (name: string, options: { dryRun?: boolean; imagePrefix?: string }) => {
|
|
113
|
-
await build(name, { dryRun: options.dryRun ?? false, repository: options.imagePrefix });
|
|
114
|
-
});
|
|
115
|
-
}
|
|
@@ -1,173 +0,0 @@
|
|
|
1
|
-
// spec/055-declaring-test-cases.md#typed-names: a declaration naming a parameter absent from the
|
|
2
|
-
// generated types, or whose config or aggregate doesn't fit its instrument, is a compile error,
|
|
3
|
-
// and one that fits compiles cleanly.
|
|
4
|
-
//
|
|
5
|
-
// In the checked-in fixtureTests/scenario_11_type_errors, in place: `thunderjar generate` writes
|
|
6
|
-
// its gitignored types, then real `tsc` checks the project with its own tsconfig, once. Each test
|
|
7
|
-
// asserts on what `tsc` reported for one declaration. By hand:
|
|
8
|
-
//
|
|
9
|
-
// bun run fixture 11 generate
|
|
10
|
-
// cd fixtureTests/scenario_11_type_errors/fixtures && bunx tsc -p .thunderjar
|
|
11
|
-
|
|
12
|
-
import { beforeAll, describe, expect, test } from "bun:test";
|
|
13
|
-
import { join, resolve } from "../../platform/path.ts";
|
|
14
|
-
import { scenario, thunderjar } from "../../testing/cli.ts";
|
|
15
|
-
|
|
16
|
-
const tsc = resolve(import.meta.dir, "..", "..", "..", "node_modules", ".bin", "tsc");
|
|
17
|
-
const fixtures = scenario("scenario_11_type_errors");
|
|
18
|
-
|
|
19
|
-
let exitCode: number;
|
|
20
|
-
/** Each diagnostic `tsc` printed, with its indented elaboration, keyed by its file. */
|
|
21
|
-
const reported = new Map<string, string[]>();
|
|
22
|
-
|
|
23
|
-
beforeAll(async () => {
|
|
24
|
-
const generated = await thunderjar(fixtures, ["generate"]);
|
|
25
|
-
expect(generated.exitCode).toBe(0);
|
|
26
|
-
|
|
27
|
-
const proc = Bun.spawn([tsc, "--noEmit", "-p", join(fixtures, ".thunderjar")], {
|
|
28
|
-
cwd: fixtures,
|
|
29
|
-
stdout: "pipe",
|
|
30
|
-
stderr: "pipe",
|
|
31
|
-
});
|
|
32
|
-
const [stdout, stderr] = await Promise.all([proc.stdout.text(), proc.stderr.text()]);
|
|
33
|
-
exitCode = await proc.exited;
|
|
34
|
-
for (const diagnostic of (stdout + stderr).trimEnd().split(/\n(?=\S)/)) {
|
|
35
|
-
const file = diagnostic.slice(0, diagnostic.indexOf("("));
|
|
36
|
-
reported.set(file, [...(reported.get(file) ?? []), diagnostic.slice(file.length)]);
|
|
37
|
-
}
|
|
38
|
-
}, 30000);
|
|
39
|
-
|
|
40
|
-
/** What `tsc` reported in `.thunderjar/<file>`, each diagnostic without its file name. */
|
|
41
|
-
const errorsIn = (file: string) => reported.get(`.thunderjar/${file}`);
|
|
42
|
-
|
|
43
|
-
test("only the declarations that don't fit fail to compile", () => {
|
|
44
|
-
expect(exitCode).not.toBe(0);
|
|
45
|
-
expect([...reported.keys()].sort()).toEqual(
|
|
46
|
-
[
|
|
47
|
-
"test-cases/prompt-of-another-task.ts",
|
|
48
|
-
"test-cases/unknown-model.ts",
|
|
49
|
-
"test-cases/unknown-prompt-set.ts",
|
|
50
|
-
"test-case-parameters/codeStates/reads-missing-config/index.ts",
|
|
51
|
-
"test-case-parameters/codeStates/top-level-commit/index.ts",
|
|
52
|
-
...[
|
|
53
|
-
"fileCreatedByMean",
|
|
54
|
-
"grepByRate",
|
|
55
|
-
"grepWithoutPattern",
|
|
56
|
-
"misfitConfig",
|
|
57
|
-
"templateTestByMax",
|
|
58
|
-
"templateTestWithoutTemplate",
|
|
59
|
-
"unknownInstrument",
|
|
60
|
-
"withoutAggregate",
|
|
61
|
-
].map((name) => `test-case-parameters/tasks/badly-measured/measurements/${name}.ts`),
|
|
62
|
-
]
|
|
63
|
-
.map((file) => `.thunderjar/${file}`)
|
|
64
|
-
.sort(),
|
|
65
|
-
);
|
|
66
|
-
});
|
|
67
|
-
|
|
68
|
-
describe("a test case", () => {
|
|
69
|
-
test("naming only generated names compiles cleanly", () => {
|
|
70
|
-
expect(errorsIn("test-cases/sound.ts")).toBeUndefined();
|
|
71
|
-
});
|
|
72
|
-
|
|
73
|
-
test("naming an unknown prompt set fails to compile", () => {
|
|
74
|
-
expect(errorsIn("test-cases/unknown-prompt-set.ts")).toEqual([
|
|
75
|
-
`(6,15): error TS2322: Type '"snirk"' is not assignable to type '"snerk"'.`,
|
|
76
|
-
]);
|
|
77
|
-
});
|
|
78
|
-
|
|
79
|
-
test("naming a model absent from its family fails to compile", () => {
|
|
80
|
-
expect(errorsIn("test-cases/unknown-model.ts")).toEqual([
|
|
81
|
-
`(7,18): error TS2322: Type '"fake-harness-alpha@1.0.0/fake-model-huge"' is not assignable to type '"fake-harness-alpha@1.0.0/fake-model-small"'.`,
|
|
82
|
-
]);
|
|
83
|
-
});
|
|
84
|
-
|
|
85
|
-
test("naming an initial prompt that belongs to a different task fails to compile", () => {
|
|
86
|
-
expect(errorsIn("test-cases/prompt-of-another-task.ts")).toEqual([
|
|
87
|
-
`(9,19): error TS2322: Type '"fizzy"' is not assignable to type '"plain"'.`,
|
|
88
|
-
]);
|
|
89
|
-
});
|
|
90
|
-
});
|
|
91
|
-
|
|
92
|
-
describe("a measurement", () => {
|
|
93
|
-
const measurement = (task: string, name: string) => errorsIn(`test-case-parameters/tasks/${task}/measurements/${name}.ts`);
|
|
94
|
-
const bad = (name: string) => measurement("badly-measured", name);
|
|
95
|
-
|
|
96
|
-
test("whose config and aggregate fit fileCreated, grep or templateTest compiles cleanly", () => {
|
|
97
|
-
expect(measurement("is-prime", "isPrimeTsExists")).toBeUndefined();
|
|
98
|
-
expect(measurement("is-prime", "tsIgnoreCount")).toBeUndefined();
|
|
99
|
-
expect(measurement("is-prime", "isPrimeTemplateTest")).toBeUndefined();
|
|
100
|
-
});
|
|
101
|
-
|
|
102
|
-
test("whose config doesn't fit its instrument fails to compile", () => {
|
|
103
|
-
expect(bad("misfitConfig")).toEqual([
|
|
104
|
-
`(3,74): error TS2353: Object literal may only specify known properties, and 'path' does not exist in type '{ glob: string; } | { glob: string; pattern: string; } | { subject: string; template: string; }'.`,
|
|
105
|
-
]);
|
|
106
|
-
});
|
|
107
|
-
|
|
108
|
-
test("naming an unknown instrument fails to compile", () => {
|
|
109
|
-
expect(bad("unknownInstrument")).toEqual([
|
|
110
|
-
`(3,37): error TS2820: Type '"fileCraeted"' is not assignable to type '"fileCreated" | "grep" | "templateTest"'. Did you mean '"fileCreated"'?`,
|
|
111
|
-
]);
|
|
112
|
-
});
|
|
113
|
-
|
|
114
|
-
test("without an aggregate fails to compile", () => {
|
|
115
|
-
expect(bad("withoutAggregate")).toEqual([
|
|
116
|
-
`(3,35): error TS2345: Argument of type '{ instrument: "fileCreated"; config: { glob: string; }; }' is not assignable to parameter of type 'MeasurementDeclaration'.
|
|
117
|
-
Property 'aggregate' is missing in type '{ instrument: "fileCreated"; config: { glob: string; }; }' but required in type '{ instrument: "fileCreated"; config: { glob: string; }; aggregate: ("count" | "rate")[]; }'.`,
|
|
118
|
-
]);
|
|
119
|
-
});
|
|
120
|
-
|
|
121
|
-
test("with an aggregate that doesn't fit its instrument's value type fails to compile", () => {
|
|
122
|
-
expect(bad("fileCreatedByMean")).toEqual([
|
|
123
|
-
`(3,35): error TS2345: Argument of type '{ instrument: "fileCreated"; config: { glob: string; }; aggregate: "mean"[]; }' is not assignable to parameter of type 'MeasurementDeclaration'.
|
|
124
|
-
Types of property 'aggregate' are incompatible.
|
|
125
|
-
Type '"mean"[]' is not assignable to type '("count" | "rate")[]'.
|
|
126
|
-
Type '"mean"' is not assignable to type '"count" | "rate"'.`,
|
|
127
|
-
]);
|
|
128
|
-
expect(bad("grepByRate")).toEqual([
|
|
129
|
-
`(3,35): error TS2345: Argument of type '{ instrument: "grep"; config: { glob: string; pattern: string; }; aggregate: "rate"[]; }' is not assignable to parameter of type 'MeasurementDeclaration'.
|
|
130
|
-
Types of property 'aggregate' are incompatible.
|
|
131
|
-
Type '"rate"[]' is not assignable to type '("max" | "mean" | "median" | "min" | "sum")[]'.
|
|
132
|
-
Type '"rate"' is not assignable to type '"max" | "mean" | "median" | "min" | "sum"'.`,
|
|
133
|
-
]);
|
|
134
|
-
expect(bad("templateTestByMax")).toEqual([
|
|
135
|
-
`(3,35): error TS2345: Argument of type '{ instrument: "templateTest"; config: { subject: string; template: string; }; aggregate: "max"[]; }' is not assignable to parameter of type 'MeasurementDeclaration'.
|
|
136
|
-
Types of property 'aggregate' are incompatible.
|
|
137
|
-
Type '"max"[]' is not assignable to type '("mean" | "sum")[]'.
|
|
138
|
-
Type '"max"' is not assignable to type '"mean" | "sum"'.`,
|
|
139
|
-
]);
|
|
140
|
-
});
|
|
141
|
-
|
|
142
|
-
test("whose config is missing a field fails to compile", () => {
|
|
143
|
-
expect(bad("grepWithoutPattern")).toEqual([
|
|
144
|
-
`(3,35): error TS2345: Argument of type '{ instrument: "grep"; config: { glob: string; }; aggregate: "max"[]; }' is not assignable to parameter of type 'MeasurementDeclaration'.
|
|
145
|
-
Types of property 'config' are incompatible.
|
|
146
|
-
Property 'pattern' is missing in type '{ glob: string; }' but required in type '{ glob: string; pattern: string; }'.`,
|
|
147
|
-
]);
|
|
148
|
-
expect(bad("templateTestWithoutTemplate")).toEqual([
|
|
149
|
-
`(3,65): error TS2322: Type '{ subject: string; }' is not assignable to type '{ glob: string; } | { glob: string; pattern: string; } | { subject: string; template: string; }'.
|
|
150
|
-
Property 'template' is missing in type '{ subject: string; }' but required in type '{ subject: string; template: string; }'.`,
|
|
151
|
-
]);
|
|
152
|
-
});
|
|
153
|
-
});
|
|
154
|
-
|
|
155
|
-
describe("a code state", () => {
|
|
156
|
-
const codeState = (name: string) => errorsIn(`test-case-parameters/codeStates/${name}/index.ts`);
|
|
157
|
-
|
|
158
|
-
test("whose config reaches its applyParameter typed, and whose checkoutCommit takes its ctx, compiles cleanly", () => {
|
|
159
|
-
expect(codeState("pinned")).toBeUndefined();
|
|
160
|
-
});
|
|
161
|
-
|
|
162
|
-
test("reading a property its config doesn't have fails to compile", () => {
|
|
163
|
-
expect(codeState("reads-missing-config")).toEqual([
|
|
164
|
-
`(5,58): error TS2339: Property 'sha' does not exist on type '{ commit: string; }'.`,
|
|
165
|
-
]);
|
|
166
|
-
});
|
|
167
|
-
|
|
168
|
-
test("with the old top-level commit fails to compile: it belongs in config", () => {
|
|
169
|
-
expect(codeState("top-level-commit")).toEqual([
|
|
170
|
-
`(4,3): error TS2353: Object literal may only specify known properties, and 'commit' does not exist in type 'CodeStateDeclaration<undefined>'.`,
|
|
171
|
-
]);
|
|
172
|
-
});
|
|
173
|
-
});
|
|
@@ -1,84 +0,0 @@
|
|
|
1
|
-
// `thunderjar executions`, through the CLI, in the checked-in fixtureTests/ scenarios, in place.
|
|
2
|
-
// Scenario 8 starts with a run data store of four executions of two test cases; scenario 5 has
|
|
3
|
-
// none. Each asserts on stdout. Any of these can be run by hand:
|
|
4
|
-
//
|
|
5
|
-
// bun run fixture 8 executions fails
|
|
6
|
-
|
|
7
|
-
import { describe, expect, test } from "bun:test";
|
|
8
|
-
import { join } from "../../../platform/path.ts";
|
|
9
|
-
import { scenario, thunderjar } from "../../../testing/cli.ts";
|
|
10
|
-
|
|
11
|
-
describe("scenario_5_empty_parameter_folders", () => {
|
|
12
|
-
const fixtures = scenario("scenario_5_empty_parameter_folders");
|
|
13
|
-
|
|
14
|
-
test("lists nothing before anything has run", async () => {
|
|
15
|
-
expect(await thunderjar(fixtures, ["executions"])).toEqual({
|
|
16
|
-
stdout: "No executions recorded.\n",
|
|
17
|
-
stderr: "",
|
|
18
|
-
exitCode: 0,
|
|
19
|
-
});
|
|
20
|
-
expect(await thunderjar(fixtures, ["executions", "fails"])).toEqual({
|
|
21
|
-
stdout: "No executions recorded for fails.\n",
|
|
22
|
-
stderr: "",
|
|
23
|
-
exitCode: 0,
|
|
24
|
-
});
|
|
25
|
-
});
|
|
26
|
-
|
|
27
|
-
test("--data-dir lists the executions in a store kept elsewhere", async () => {
|
|
28
|
-
const dataDir = join(scenario("scenario_8_recorded_executions"), ".thunderjar", ".data");
|
|
29
|
-
expect(await thunderjar(fixtures, ["executions", "times-out", "--data-dir", dataDir])).toEqual({
|
|
30
|
-
stdout: "01k4xb7q0c2tnvabcdefghjkmn times-out 2026-09-29 15:02\n",
|
|
31
|
-
stderr: "",
|
|
32
|
-
exitCode: 0,
|
|
33
|
-
});
|
|
34
|
-
});
|
|
35
|
-
});
|
|
36
|
-
|
|
37
|
-
describe("scenario_8_recorded_executions", () => {
|
|
38
|
-
const fixtures = scenario("scenario_8_recorded_executions");
|
|
39
|
-
|
|
40
|
-
test("executions: every test case's executions, newest first", async () => {
|
|
41
|
-
expect(await thunderjar(fixtures, ["executions"])).toEqual({
|
|
42
|
-
stdout: `01k4xc0000000000000000000a fails 2026-09-30 08:00
|
|
43
|
-
01k4xb7q0c2tnvabcdefghjkmn times-out 2026-09-29 15:02
|
|
44
|
-
01k4x9j2e8mqz3abcdefghjkmn fails 2026-09-29 14:30
|
|
45
|
-
01k4tq3h8r0000000000000000 fails 2026-09-28 09:15
|
|
46
|
-
`,
|
|
47
|
-
stderr: "",
|
|
48
|
-
exitCode: 0,
|
|
49
|
-
});
|
|
50
|
-
});
|
|
51
|
-
|
|
52
|
-
test("executions <test case>: only that test case's, newest first", async () => {
|
|
53
|
-
expect(await thunderjar(fixtures, ["executions", "fails"])).toEqual({
|
|
54
|
-
stdout: `01k4xc0000000000000000000a fails 2026-09-30 08:00
|
|
55
|
-
01k4x9j2e8mqz3abcdefghjkmn fails 2026-09-29 14:30
|
|
56
|
-
01k4tq3h8r0000000000000000 fails 2026-09-28 09:15
|
|
57
|
-
`,
|
|
58
|
-
stderr: "",
|
|
59
|
-
exitCode: 0,
|
|
60
|
-
});
|
|
61
|
-
});
|
|
62
|
-
|
|
63
|
-
test("executions <test case> with nothing recorded says so", async () => {
|
|
64
|
-
expect(await thunderjar(fixtures, ["executions", "nope"])).toEqual({
|
|
65
|
-
stdout: "No executions recorded for nope.\n",
|
|
66
|
-
stderr: "",
|
|
67
|
-
exitCode: 0,
|
|
68
|
-
});
|
|
69
|
-
});
|
|
70
|
-
|
|
71
|
-
test("says what's wrong, then its usage, for extra arguments or a flag", async () => {
|
|
72
|
-
for (const [args, error] of [
|
|
73
|
-
[["executions", "a", "b"], "error: too many arguments for 'executions'"],
|
|
74
|
-
[["executions", "--json"], "error: unknown option '--json'"],
|
|
75
|
-
[["executions", "--data-dir"], "error: option '--data-dir <dir>' argument missing"],
|
|
76
|
-
] as const) {
|
|
77
|
-
const { stdout, stderr, exitCode } = await thunderjar(fixtures, [...args]);
|
|
78
|
-
expect(stdout).toBe("");
|
|
79
|
-
expect(stderr).toStartWith(error);
|
|
80
|
-
expect(stderr).toContain("Usage: thunderjar executions [options] [test case]");
|
|
81
|
-
expect(exitCode).toBe(1);
|
|
82
|
-
}
|
|
83
|
-
});
|
|
84
|
-
});
|
|
@@ -1,31 +0,0 @@
|
|
|
1
|
-
// `thunderjar executions [<test case>]` — spec/100-cli-reference.md#executions. Lists the test
|
|
2
|
-
// case executions in the run data store, newest first: execution ID, test case, and when it
|
|
3
|
-
// started, in UTC to the minute. `--data-dir` reads a store kept outside the project.
|
|
4
|
-
|
|
5
|
-
import { Command } from "commander";
|
|
6
|
-
import { jsonFileStore, runDataStorePath } from "../../../store/run-data-store.ts";
|
|
7
|
-
import { minute } from "../format.ts";
|
|
8
|
-
import { dataDirOption } from "../options.ts";
|
|
9
|
-
|
|
10
|
-
export async function executions(
|
|
11
|
-
testCase?: string,
|
|
12
|
-
{ cwd = process.cwd(), dataDir }: { cwd?: string; dataDir?: string } = {},
|
|
13
|
-
): Promise<string> {
|
|
14
|
-
const store = jsonFileStore(runDataStorePath(cwd, dataDir));
|
|
15
|
-
const listed = await store.listExecutions(testCase);
|
|
16
|
-
if (listed.length === 0) {
|
|
17
|
-
return testCase === undefined ? "No executions recorded." : `No executions recorded for ${testCase}.`;
|
|
18
|
-
}
|
|
19
|
-
const width = Math.max(...listed.map((e) => e.testCase.length));
|
|
20
|
-
return listed.map((e) => `${e.executionId} ${e.testCase.padEnd(width)} ${minute(e.ranAt)}`).join("\n");
|
|
21
|
-
}
|
|
22
|
-
|
|
23
|
-
export function createExecutionsCommand(): Command {
|
|
24
|
-
return new Command("executions")
|
|
25
|
-
.description("List recorded executions.")
|
|
26
|
-
.argument("[test case]", "list only this test case's executions")
|
|
27
|
-
.addOption(dataDirOption())
|
|
28
|
-
.action(async (testCase: string | undefined, options: { dataDir?: string }) => {
|
|
29
|
-
console.log(await executions(testCase, { dataDir: options.dataDir }));
|
|
30
|
-
});
|
|
31
|
-
}
|