thunderjar 0.1.2 → 0.1.3-preview-eee7160
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli.js +2161 -34
- package/package.json +13 -4
- package/src/AGENTS.md +156 -0
- package/src/adapters/docker/docker.ts +117 -0
- package/src/api/declarations.ts +279 -0
- package/src/api/fragments.ts +30 -0
- package/src/api/parameter-names.d.ts +16 -0
- package/src/cli/commands/build/build.docker.test.ts +173 -0
- package/src/cli/commands/build/build.test.ts +234 -0
- package/src/cli/commands/build/build.ts +115 -0
- package/src/cli/commands/declarations.typecheck.test.ts +173 -0
- package/src/cli/commands/executions/executions.test.ts +84 -0
- package/src/cli/commands/executions/executions.ts +31 -0
- package/src/cli/commands/format.ts +33 -0
- package/src/cli/commands/generate/generate.test.ts +101 -0
- package/src/cli/commands/generate/generate.ts +18 -0
- package/src/cli/commands/generate/parameter-names.ts +131 -0
- package/src/cli/commands/init/init.test.ts +123 -0
- package/src/cli/commands/init/init.ts +189 -0
- package/src/cli/commands/options.ts +17 -0
- package/src/cli/commands/plan/plan.test.ts +168 -0
- package/src/cli/commands/plan/plan.ts +72 -0
- package/src/cli/commands/remeasure/remeasure.test.ts +536 -0
- package/src/cli/commands/remeasure/remeasure.ts +194 -0
- package/src/cli/commands/run/run.test.ts +622 -0
- package/src/cli/commands/run/run.ts +203 -0
- package/src/cli/commands/show/show.test.ts +288 -0
- package/src/cli/commands/show/show.ts +252 -0
- package/src/cli/commands/validation.test.ts +200 -0
- package/src/cli/index.test.ts +47 -0
- package/src/cli/index.ts +9 -0
- package/src/cli/program.ts +29 -0
- package/src/images/prerun-builds.ts +108 -0
- package/src/images/prerun.test.ts +243 -0
- package/src/images/prerun.ts +272 -0
- package/src/images/repository.ts +72 -0
- package/src/index.ts +24 -0
- package/src/instruments/instruments.test.ts +240 -0
- package/src/instruments/instruments.ts +205 -0
- package/src/matrix/execution.ts +122 -0
- package/src/matrix/hashing.test.ts +298 -0
- package/src/matrix/hashing.ts +131 -0
- package/src/matrix/permutations.test.ts +58 -0
- package/src/matrix/permutations.ts +50 -0
- package/src/matrix/resolve.ts +63 -0
- package/src/matrix/tags.test.ts +15 -0
- package/src/matrix/tags.ts +22 -0
- package/src/matrix/ulid.test.ts +28 -0
- package/src/matrix/ulid.ts +22 -0
- package/src/measuring/aggregate.test.ts +188 -0
- package/src/measuring/aggregate.ts +154 -0
- package/src/measuring/measure.test.ts +105 -0
- package/src/measuring/measure.ts +62 -0
- package/src/measuring/remeasure-templates/v1/isPrime.test.template.txt +4 -0
- package/src/measuring/remeasure-templates/v2/isPrime.test.template.txt +5 -0
- package/src/measuring/remeasure.test.ts +352 -0
- package/src/measuring/remeasure.ts +113 -0
- package/src/platform/fs.test.ts +89 -0
- package/src/platform/fs.ts +123 -0
- package/src/platform/path.test.ts +38 -0
- package/src/platform/path.ts +72 -0
- package/src/platform/signals.test.ts +25 -0
- package/src/platform/signals.ts +68 -0
- package/src/project/config.test.ts +68 -0
- package/src/project/config.ts +297 -0
- package/src/running/runner.test.ts +73 -0
- package/src/running/runner.ts +252 -0
- package/src/store/run-data-store.test.ts +232 -0
- package/src/store/run-data-store.ts +254 -0
- package/src/testing/cli.ts +65 -0
- package/src/testing/fake-docker.test.ts +197 -0
- package/src/testing/fake-docker.ts +104 -0
- package/src/testing/fixture-project.ts +177 -0
- package/src/testing/measurement-context.ts +5 -0
- package/src/testing/seed-repo.test.ts +10 -0
- package/src/testing/seed-repo.ts +83 -0
- package/src/testing/test-setup.ts +10 -0
|
@@ -0,0 +1,203 @@
|
|
|
1
|
+
// `thunderjar run <test case>` — spec/100-cli-reference.md#run and spec/080-docker-execution.md#process.
|
|
2
|
+
// Creates an execution ID, checks that Docker can be reached and that every commit a missing
|
|
3
|
+
// prerun image reads is there, then walks every permutation in order. A permutation's prerun
|
|
4
|
+
// image is built first if it doesn't exist, the way `build` builds it
|
|
5
|
+
// (src/images/prerun-builds.ts). Then each iteration's container run goes to a
|
|
6
|
+
// ContainerRunner, by default one that runs it in a container from the prerun image and
|
|
7
|
+
// commits it into its postrun image, both under `--image-prefix`. Each prints its exit code
|
|
8
|
+
// and duration as it finishes, then a summary. Each container run is recorded in the run data
|
|
9
|
+
// store, under `--data-dir` if given, as soon as it finishes, so the runs that finished before
|
|
10
|
+
// a crash are kept. Then the task's measurements are taken against it, through
|
|
11
|
+
// a MeasurementOpener, printed, and merged into its record — so a crash while measuring leaves
|
|
12
|
+
// it recorded with an empty Result. After each, its permutation's aggregation result is
|
|
13
|
+
// recomputed and stored, though not printed. Runs that exit non-zero are measured too, for now.
|
|
14
|
+
// `--dry-run` prints the container run each would be instead — its prerun image and whether
|
|
15
|
+
// it exists or would be built, its postrun image and the harness command — without building or
|
|
16
|
+
// running anything.
|
|
17
|
+
|
|
18
|
+
import { Command } from "commander";
|
|
19
|
+
import { join, relative } from "../../../platform/path.ts";
|
|
20
|
+
import { dockerCli, type Docker } from "../../../adapters/docker/docker.ts";
|
|
21
|
+
import { recomputeAggregationResult } from "../../../measuring/aggregate.ts";
|
|
22
|
+
import { findThunderjarRoot, loadMeasurements, loadTestCase } from "../../../project/config.ts";
|
|
23
|
+
import { planExecution, type PlannedExecution } from "../../../matrix/execution.ts";
|
|
24
|
+
import { measureContainerRun } from "../../../measuring/measure.ts";
|
|
25
|
+
import { permutationRows } from "../../../matrix/permutations.ts";
|
|
26
|
+
import { LOCAL_REPOSITORY, prerunStatuses, showPrerunStatus, unreachableNote } from "../../../images/prerun.ts";
|
|
27
|
+
import { preparePrerunBuilds } from "../../../images/prerun-builds.ts";
|
|
28
|
+
import { jsonFileStore, runDataStorePath } from "../../../store/run-data-store.ts";
|
|
29
|
+
import {
|
|
30
|
+
createDockerMeasurementOpener,
|
|
31
|
+
createDockerRunner,
|
|
32
|
+
defaultRootDir,
|
|
33
|
+
type ContainerRunner,
|
|
34
|
+
type MeasurementOpener,
|
|
35
|
+
} from "../../../running/runner.ts";
|
|
36
|
+
import { ulid } from "../../../matrix/ulid.ts";
|
|
37
|
+
import { count, duration, seconds, showOutcome } from "../format.ts";
|
|
38
|
+
import { dataDirOption, imagePrefixOption } from "../options.ts";
|
|
39
|
+
|
|
40
|
+
export interface RunOptions {
|
|
41
|
+
dryRun: boolean;
|
|
42
|
+
cwd?: string;
|
|
43
|
+
/** Receives the output a block at a time, as it is produced. */
|
|
44
|
+
log?: (text: string) => void;
|
|
45
|
+
/** Makes the execution ID; stubbed in tests so output is deterministic. */
|
|
46
|
+
newExecutionId?: () => string;
|
|
47
|
+
/** The clock for timing container runs; stubbed in tests so output is deterministic. */
|
|
48
|
+
now?: () => number;
|
|
49
|
+
/** Prerun build contexts are kept under `<rootDir>/builds/`. Defaults to `$TMPDIR/thunderjar`. */
|
|
50
|
+
rootDir?: string;
|
|
51
|
+
/** Runs each container run. Defaults to one in a container from its prerun image, through `docker`. */
|
|
52
|
+
runner?: ContainerRunner;
|
|
53
|
+
/** Gives each container run's measurement context. Defaults to one over its postrun image, through `docker`. */
|
|
54
|
+
measurementOpener?: MeasurementOpener;
|
|
55
|
+
/** Builds missing prerun images, and is asked, on a dry run, whether each exists. */
|
|
56
|
+
docker?: Docker;
|
|
57
|
+
/** The run data store goes under this folder, `--data-dir`, in place of `.thunderjar/.data/`. */
|
|
58
|
+
dataDir?: string;
|
|
59
|
+
/** The image repository prerun and postrun images are tagged in, `--image-prefix`. */
|
|
60
|
+
repository?: string;
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
const indent = (text: string, spaces: number) =>
|
|
64
|
+
text
|
|
65
|
+
.split("\n")
|
|
66
|
+
.map((line) => " ".repeat(spaces) + line)
|
|
67
|
+
.join("\n");
|
|
68
|
+
|
|
69
|
+
/** `n/total`, with `n` padded to the width of `total` so the lines line up. */
|
|
70
|
+
const ordinal = (n: number, total: number) => `${String(n).padStart(String(total).length)}/${total}`;
|
|
71
|
+
|
|
72
|
+
const shape = ({ permutations, iterations }: PlannedExecution) =>
|
|
73
|
+
`${count(permutations.length, "permutation")} × ${count(iterations, "iteration")}`;
|
|
74
|
+
|
|
75
|
+
export async function run(name: string, options: RunOptions): Promise<void> {
|
|
76
|
+
const {
|
|
77
|
+
dryRun,
|
|
78
|
+
cwd = process.cwd(),
|
|
79
|
+
log = (text) => console.log(text),
|
|
80
|
+
now = Date.now,
|
|
81
|
+
newExecutionId = () => ulid(now()),
|
|
82
|
+
rootDir = defaultRootDir(),
|
|
83
|
+
docker = dockerCli(),
|
|
84
|
+
repository = LOCAL_REPOSITORY,
|
|
85
|
+
runner = createDockerRunner(docker, repository),
|
|
86
|
+
measurementOpener = createDockerMeasurementOpener(docker, repository),
|
|
87
|
+
dataDir,
|
|
88
|
+
} = options;
|
|
89
|
+
|
|
90
|
+
const root = findThunderjarRoot(cwd);
|
|
91
|
+
const testCase = await loadTestCase(root, name);
|
|
92
|
+
const execution = await planExecution(root, testCase, newExecutionId());
|
|
93
|
+
const { executionId, permutations, iterations } = execution;
|
|
94
|
+
const rows = permutationRows(permutations.map((p) => p.permutation));
|
|
95
|
+
const permutationLine = (n: number) => `Permutation ${ordinal(n + 1, permutations.length)} ${rows[n]}`;
|
|
96
|
+
|
|
97
|
+
if (dryRun) {
|
|
98
|
+
// Each permutation: its number and five names, its prerun image, then each iteration's
|
|
99
|
+
// postrun image with the harness command below it.
|
|
100
|
+
const found = await prerunStatuses(docker, repository, permutations.map((p) => p.permutation));
|
|
101
|
+
log(`Execution ${executionId} (dry run, ${shape(execution)})`);
|
|
102
|
+
if ("unreachable" in found) log(unreachableNote(found.unreachable));
|
|
103
|
+
permutations.forEach(({ permutation, prerunTag, runs }, n) => {
|
|
104
|
+
log("");
|
|
105
|
+
log(
|
|
106
|
+
[
|
|
107
|
+
permutationLine(n),
|
|
108
|
+
` ${prerunTag}${"statuses" in found ? showPrerunStatus(found.statuses.get(permutation.hash)) : ""}`,
|
|
109
|
+
...runs.map(
|
|
110
|
+
({ iterationIndex, postrunTag, command }) =>
|
|
111
|
+
` iteration ${ordinal(iterationIndex + 1, iterations)} ${postrunTag}\n${indent(command, 4)}`,
|
|
112
|
+
),
|
|
113
|
+
].join("\n"),
|
|
114
|
+
);
|
|
115
|
+
});
|
|
116
|
+
return;
|
|
117
|
+
}
|
|
118
|
+
|
|
119
|
+
const measurements = await loadMeasurements(root, testCase.task);
|
|
120
|
+
const nameWidth = Math.max(0, ...measurements.map((m) => m.name.length));
|
|
121
|
+
const storePath = runDataStorePath(cwd, dataDir);
|
|
122
|
+
const store = jsonFileStore(storePath);
|
|
123
|
+
const builds = await preparePrerunBuilds({
|
|
124
|
+
root,
|
|
125
|
+
task: testCase.task,
|
|
126
|
+
permutations: permutations.map((p) => p.permutation),
|
|
127
|
+
docker,
|
|
128
|
+
repository,
|
|
129
|
+
rootDir,
|
|
130
|
+
purpose: "Running a test case",
|
|
131
|
+
});
|
|
132
|
+
|
|
133
|
+
log(`Execution ${executionId} (${shape(execution)})`);
|
|
134
|
+
log(`Images in ${repository}`);
|
|
135
|
+
if (builds.missing.size > 0) log(`Build contexts in ${join(rootDir, "builds")}`);
|
|
136
|
+
|
|
137
|
+
// How many container runs exited with each code. A failing agent is a result, not an error:
|
|
138
|
+
// the remaining runs still run, and every run is recorded.
|
|
139
|
+
const exitCodes = new Map<number, number>();
|
|
140
|
+
for (const [n, { permutation, parameters, harnessModel, prerunTag, runs }] of permutations.entries()) {
|
|
141
|
+
log("");
|
|
142
|
+
log(permutationLine(n));
|
|
143
|
+
if (builds.missing.has(permutation.hash)) {
|
|
144
|
+
const started = now();
|
|
145
|
+
await builds.build(permutation.hash);
|
|
146
|
+
log(` ${prerunTag} built (${seconds(now() - started)})`);
|
|
147
|
+
} else {
|
|
148
|
+
log(` ${prerunTag} reused`);
|
|
149
|
+
}
|
|
150
|
+
for (const containerRun of runs) {
|
|
151
|
+
const { startedAt, finishedAt, exitCode } = await runner.runContainer(containerRun);
|
|
152
|
+
await store.recordContainerRun({
|
|
153
|
+
executionId,
|
|
154
|
+
testCase: name,
|
|
155
|
+
task: testCase.task,
|
|
156
|
+
permutationHash: permutation.hash,
|
|
157
|
+
iteration: containerRun.iterationIndex,
|
|
158
|
+
postrunImage: containerRun.postrunTag,
|
|
159
|
+
ranAt: new Date(startedAt).toISOString(),
|
|
160
|
+
parameters,
|
|
161
|
+
harnessModel,
|
|
162
|
+
result: {},
|
|
163
|
+
execution: { startedAt, finishedAt, exitCode },
|
|
164
|
+
});
|
|
165
|
+
exitCodes.set(exitCode, (exitCodes.get(exitCode) ?? 0) + 1);
|
|
166
|
+
log(
|
|
167
|
+
` iteration ${ordinal(containerRun.iterationIndex + 1, iterations)} ${containerRun.postrunTag}` +
|
|
168
|
+
` exit ${exitCode} ${duration({ startedAt, finishedAt })}`,
|
|
169
|
+
);
|
|
170
|
+
|
|
171
|
+
if (measurements.length > 0) {
|
|
172
|
+
const open = () => measurementOpener.openMeasurementContext(containerRun.postrunTag);
|
|
173
|
+
const result = await measureContainerRun(measurements, open, now);
|
|
174
|
+
await store.recordMeasures(containerRun.postrunTag, result);
|
|
175
|
+
log(
|
|
176
|
+
Object.entries(result)
|
|
177
|
+
.map(([name, { outcome }]) => ` ${name.padEnd(nameWidth)} ${showOutcome(outcome)}`)
|
|
178
|
+
.join("\n"),
|
|
179
|
+
);
|
|
180
|
+
}
|
|
181
|
+
// With no measurements there is still a duration to aggregate.
|
|
182
|
+
await recomputeAggregationResult(store, executionId, permutation.hash, measurements);
|
|
183
|
+
}
|
|
184
|
+
}
|
|
185
|
+
|
|
186
|
+
const total = permutations.length * iterations;
|
|
187
|
+
const byCode = [...exitCodes].toSorted(([a], [b]) => a - b).map(([code, n]) => `${n} exited ${code}`);
|
|
188
|
+
log("");
|
|
189
|
+
log(`${count(total, "container run")}: ${byCode.join(", ")}`);
|
|
190
|
+
log(`Recorded in ${relative(cwd, storePath)}`);
|
|
191
|
+
}
|
|
192
|
+
|
|
193
|
+
export function createRunCommand(): Command {
|
|
194
|
+
return new Command("run")
|
|
195
|
+
.description("Run a test case.")
|
|
196
|
+
.argument("<test case>", "the test case to run")
|
|
197
|
+
.option("--dry-run", "print each container run instead of building or running anything")
|
|
198
|
+
.addOption(imagePrefixOption("the image repository prerun and postrun images are tagged in"))
|
|
199
|
+
.addOption(dataDirOption())
|
|
200
|
+
.action(async (name: string, options: { dryRun?: boolean; imagePrefix?: string; dataDir?: string }) => {
|
|
201
|
+
await run(name, { dryRun: options.dryRun ?? false, repository: options.imagePrefix, dataDir: options.dataDir });
|
|
202
|
+
});
|
|
203
|
+
}
|
|
@@ -0,0 +1,288 @@
|
|
|
1
|
+
// `thunderjar show`, through the CLI, in place in fixtureTests/scenario_8_recorded_executions,
|
|
2
|
+
// which starts with a run data store of four executions:
|
|
3
|
+
//
|
|
4
|
+
// 01k4tq3h… fails recorded before aggregation results were stored; one run exited 3
|
|
5
|
+
// and was never measured
|
|
6
|
+
// 01k4x9j2… fails readmeExists, since deleted, errored in each permutation's first iteration;
|
|
7
|
+
// the behaving agent's first iteration took isPrimeTemplateTest with an
|
|
8
|
+
// older template, so that one measure, and its permutation's row, is stale
|
|
9
|
+
// 01k4xb7q… times-out a task without measurements
|
|
10
|
+
// 01k4xc00… fails the latest, after readmeExists was deleted
|
|
11
|
+
//
|
|
12
|
+
// In `fails`, fake-harness-alpha@1.0.1 exits 3 and 1.0.0 behaves. Each asserts on stdout. Any of
|
|
13
|
+
// these can be run by hand:
|
|
14
|
+
//
|
|
15
|
+
// bun run fixture 8 show fails
|
|
16
|
+
|
|
17
|
+
import { describe, expect, test } from "bun:test";
|
|
18
|
+
import { readTextSync } from "../../../platform/fs.ts";
|
|
19
|
+
import { join } from "../../../platform/path.ts";
|
|
20
|
+
import { scenario, thunderjar } from "../../../testing/cli.ts";
|
|
21
|
+
|
|
22
|
+
const OLDEST = "01k4tq3h8r0000000000000000";
|
|
23
|
+
const ERRORED = "01k4x9j2e8mqz3abcdefghjkmn";
|
|
24
|
+
const TIMES_OUT = "01k4xb7q0c2tnvabcdefghjkmn";
|
|
25
|
+
const LATEST = "01k4xc0000000000000000000a";
|
|
26
|
+
|
|
27
|
+
/** The permutations of `fails`: the failing agent, then the one that behaves. */
|
|
28
|
+
const FAILING = "42440c48";
|
|
29
|
+
const BEHAVES = "8ffdda6e";
|
|
30
|
+
|
|
31
|
+
describe("scenario_8_recorded_executions", () => {
|
|
32
|
+
const fixtures = scenario("scenario_8_recorded_executions");
|
|
33
|
+
const show = (target: string) => thunderjar(fixtures, ["show", target]);
|
|
34
|
+
|
|
35
|
+
test("show <test case>: its latest execution, each permutation's aggregation result in the order its methods were chosen", async () => {
|
|
36
|
+
expect(await show("fails")).toEqual({
|
|
37
|
+
stdout: `Execution ${LATEST} (latest of fails)
|
|
38
|
+
Test case fails
|
|
39
|
+
Task is-prime
|
|
40
|
+
Date 2026-09-30 08:00
|
|
41
|
+
|
|
42
|
+
Permutation ${FAILING} (2 iterations)
|
|
43
|
+
node20 | baseline | snerk | fake-harness-alpha@1.0.1/fake-model-small | plain
|
|
44
|
+
|
|
45
|
+
MEASURE MEASURED AGGREGATE
|
|
46
|
+
isPrimeTemplateTest 0/2 —
|
|
47
|
+
isPrimeTsExists 2/2 rate 0.00
|
|
48
|
+
tsIgnoreCount 0/2 —
|
|
49
|
+
|
|
50
|
+
exit 2/2 exited 3
|
|
51
|
+
duration mean 0.04s, max 0.04s
|
|
52
|
+
|
|
53
|
+
Permutation ${BEHAVES} (2 iterations)
|
|
54
|
+
node20 | baseline | snerk | fake-harness-alpha@1.0.0/fake-model-small | plain
|
|
55
|
+
|
|
56
|
+
MEASURE MEASURED AGGREGATE
|
|
57
|
+
isPrimeTemplateTest 2/2 mean {passed: 3.0, failed: 2.0}, sum {passed: 6, failed: 4}
|
|
58
|
+
isPrimeTsExists 2/2 rate 1.00
|
|
59
|
+
tsIgnoreCount 2/2 median 1.0, min 1, sum 2
|
|
60
|
+
|
|
61
|
+
exit 2/2 exited 0
|
|
62
|
+
duration mean 0.04s, max 0.04s
|
|
63
|
+
`,
|
|
64
|
+
stderr: "",
|
|
65
|
+
exitCode: 0,
|
|
66
|
+
});
|
|
67
|
+
});
|
|
68
|
+
|
|
69
|
+
test("show <execution ID>: that execution, even when it isn't the latest, keeping a measurement deleted since, leaving an errored iteration out of the aggregate, and marking only the row one of whose values is stale", async () => {
|
|
70
|
+
expect(await show(ERRORED)).toEqual({
|
|
71
|
+
stdout: `Execution ${ERRORED}
|
|
72
|
+
Test case fails
|
|
73
|
+
Task is-prime
|
|
74
|
+
Date 2026-09-29 14:30
|
|
75
|
+
|
|
76
|
+
Permutation ${FAILING} (2 iterations)
|
|
77
|
+
node20 | baseline | snerk | fake-harness-alpha@1.0.1/fake-model-small | plain
|
|
78
|
+
|
|
79
|
+
MEASURE MEASURED AGGREGATE
|
|
80
|
+
isPrimeTemplateTest 0/2 —
|
|
81
|
+
isPrimeTsExists 2/2 rate 0.00
|
|
82
|
+
readmeExists 1/2 count 1, rate 1.00
|
|
83
|
+
tsIgnoreCount 0/2 —
|
|
84
|
+
|
|
85
|
+
exit 2/2 exited 3
|
|
86
|
+
duration mean 0.04s, max 0.04s
|
|
87
|
+
|
|
88
|
+
Permutation ${BEHAVES} (2 iterations)
|
|
89
|
+
node20 | baseline | snerk | fake-harness-alpha@1.0.0/fake-model-small | plain
|
|
90
|
+
|
|
91
|
+
MEASURE MEASURED AGGREGATE
|
|
92
|
+
isPrimeTemplateTest 2/2 mean {passed: 3.0, failed: 2.0}, sum {passed: 6, failed: 4} stale, run \`remeasure\`
|
|
93
|
+
isPrimeTsExists 2/2 rate 1.00
|
|
94
|
+
readmeExists 1/2 count 0, rate 0.00
|
|
95
|
+
tsIgnoreCount 2/2 median 1.0, min 1, sum 2
|
|
96
|
+
|
|
97
|
+
exit 2/2 exited 0
|
|
98
|
+
duration mean 0.04s, max 0.04s
|
|
99
|
+
`,
|
|
100
|
+
stderr: "",
|
|
101
|
+
exitCode: 0,
|
|
102
|
+
});
|
|
103
|
+
});
|
|
104
|
+
|
|
105
|
+
test("show <container run> measured with an older template: (stale) after that measure's value only, beside a current and a deleted measurement", async () => {
|
|
106
|
+
const tag = `postrun-e${ERRORED}-h${BEHAVES}-i00`;
|
|
107
|
+
expect(await show(tag)).toEqual({
|
|
108
|
+
stdout: `Container run ${tag}
|
|
109
|
+
Execution ${ERRORED} (fails)
|
|
110
|
+
Permutation ${BEHAVES}, iteration 1
|
|
111
|
+
Ran at 2026-09-29 14:30
|
|
112
|
+
|
|
113
|
+
Parameters
|
|
114
|
+
baseImage node20 26d9
|
|
115
|
+
codeState baseline bb15
|
|
116
|
+
promptSet snerk 827a
|
|
117
|
+
harnessModel fake-harness-alpha@1.0.0/fake-model-small da28
|
|
118
|
+
initialPrompt plain 9e30
|
|
119
|
+
|
|
120
|
+
Measures
|
|
121
|
+
isPrimeTemplateTest {passed: 3, failed: 2} (stale)
|
|
122
|
+
isPrimeTsExists true
|
|
123
|
+
readmeExists error (disk on fire)
|
|
124
|
+
tsIgnoreCount 1
|
|
125
|
+
|
|
126
|
+
Duration 0.04s exit code 0
|
|
127
|
+
`,
|
|
128
|
+
stderr: "",
|
|
129
|
+
exitCode: 0,
|
|
130
|
+
});
|
|
131
|
+
});
|
|
132
|
+
|
|
133
|
+
test("show <container run>: its parameters, measures, duration and exit code", async () => {
|
|
134
|
+
const tag = `postrun-e${ERRORED}-h${BEHAVES}-i01`;
|
|
135
|
+
expect(await show(tag)).toEqual({
|
|
136
|
+
stdout: `Container run ${tag}
|
|
137
|
+
Execution ${ERRORED} (fails)
|
|
138
|
+
Permutation ${BEHAVES}, iteration 2
|
|
139
|
+
Ran at 2026-09-29 14:30
|
|
140
|
+
|
|
141
|
+
Parameters
|
|
142
|
+
baseImage node20 26d9
|
|
143
|
+
codeState baseline bb15
|
|
144
|
+
promptSet snerk 827a
|
|
145
|
+
harnessModel fake-harness-alpha@1.0.0/fake-model-small da28
|
|
146
|
+
initialPrompt plain 9e30
|
|
147
|
+
|
|
148
|
+
Measures
|
|
149
|
+
isPrimeTemplateTest {passed: 3, failed: 2}
|
|
150
|
+
isPrimeTsExists true
|
|
151
|
+
readmeExists false
|
|
152
|
+
tsIgnoreCount 1
|
|
153
|
+
|
|
154
|
+
Duration 0.04s exit code 0
|
|
155
|
+
`,
|
|
156
|
+
stderr: "",
|
|
157
|
+
exitCode: 0,
|
|
158
|
+
});
|
|
159
|
+
});
|
|
160
|
+
|
|
161
|
+
test("show <container run> that failed: why each measure was skipped, and an instrument's error", async () => {
|
|
162
|
+
const tag = `postrun-e${ERRORED}-h${FAILING}-i00`;
|
|
163
|
+
expect(await show(tag)).toEqual({
|
|
164
|
+
stdout: `Container run ${tag}
|
|
165
|
+
Execution ${ERRORED} (fails)
|
|
166
|
+
Permutation ${FAILING}, iteration 1
|
|
167
|
+
Ran at 2026-09-29 14:30
|
|
168
|
+
|
|
169
|
+
Parameters
|
|
170
|
+
baseImage node20 26d9
|
|
171
|
+
codeState baseline bb15
|
|
172
|
+
promptSet snerk 827a
|
|
173
|
+
harnessModel fake-harness-alpha@1.0.1/fake-model-small c0b2
|
|
174
|
+
initialPrompt plain 9e30
|
|
175
|
+
|
|
176
|
+
Measures
|
|
177
|
+
isPrimeTemplateTest skipped (no file matches **/isPrime.ts)
|
|
178
|
+
isPrimeTsExists false
|
|
179
|
+
readmeExists error (disk on fire)
|
|
180
|
+
tsIgnoreCount skipped (no file matches **/*.ts)
|
|
181
|
+
|
|
182
|
+
Duration 0.04s exit code 3
|
|
183
|
+
`,
|
|
184
|
+
stderr: "",
|
|
185
|
+
exitCode: 0,
|
|
186
|
+
});
|
|
187
|
+
});
|
|
188
|
+
|
|
189
|
+
test("an execution recorded without aggregation results says so, and runs with different exit codes are counted by code", async () => {
|
|
190
|
+
expect(await show(OLDEST)).toEqual({
|
|
191
|
+
stdout: `Execution ${OLDEST}
|
|
192
|
+
Test case fails
|
|
193
|
+
Task is-prime
|
|
194
|
+
Date 2026-09-28 09:15
|
|
195
|
+
|
|
196
|
+
Permutation ${FAILING} (2 iterations)
|
|
197
|
+
node20 | baseline | snerk | fake-harness-alpha@1.0.1/fake-model-small | plain
|
|
198
|
+
|
|
199
|
+
no aggregation result recorded
|
|
200
|
+
|
|
201
|
+
exit 2/2 exited 3
|
|
202
|
+
|
|
203
|
+
Permutation ${BEHAVES} (2 iterations)
|
|
204
|
+
node20 | baseline | snerk | fake-harness-alpha@1.0.0/fake-model-small | plain
|
|
205
|
+
|
|
206
|
+
no aggregation result recorded
|
|
207
|
+
|
|
208
|
+
exit 1/2 exited 0, 1/2 exited 3
|
|
209
|
+
`,
|
|
210
|
+
stderr: "",
|
|
211
|
+
exitCode: 0,
|
|
212
|
+
});
|
|
213
|
+
});
|
|
214
|
+
|
|
215
|
+
test("show <container run> leaves the checked-in store byte for byte as it was", async () => {
|
|
216
|
+
const store = join(fixtures, ".thunderjar", ".data", "runs.json");
|
|
217
|
+
const before = readTextSync(store);
|
|
218
|
+
for (const iteration of ["i00", "i01"]) expect((await show(`postrun-e${ERRORED}-h${BEHAVES}-${iteration}`)).exitCode).toBe(0);
|
|
219
|
+
expect(readTextSync(store)).toBe(before);
|
|
220
|
+
});
|
|
221
|
+
|
|
222
|
+
test("show <execution> leaves the checked-in store byte for byte as it was, even with a stale value", async () => {
|
|
223
|
+
const store = join(fixtures, ".thunderjar", ".data", "runs.json");
|
|
224
|
+
const before = readTextSync(store);
|
|
225
|
+
for (const target of [ERRORED, "fails"]) expect((await show(target)).exitCode).toBe(0);
|
|
226
|
+
expect(readTextSync(store)).toBe(before);
|
|
227
|
+
});
|
|
228
|
+
|
|
229
|
+
test("a container run with no measures shows none", async () => {
|
|
230
|
+
const { stdout } = await show(`postrun-e${OLDEST}-h${BEHAVES}-i01`);
|
|
231
|
+
expect(stdout.split("\n").slice(11)).toEqual(["", "Measures", " none", "", "Duration 0.04s exit code 3", ""]);
|
|
232
|
+
});
|
|
233
|
+
|
|
234
|
+
test("a task without measurements shows none, with its exit codes and durations", async () => {
|
|
235
|
+
expect(await show("times-out")).toEqual({
|
|
236
|
+
stdout: `Execution ${TIMES_OUT} (latest of times-out)
|
|
237
|
+
Test case times-out
|
|
238
|
+
Task wait
|
|
239
|
+
Date 2026-09-29 15:02
|
|
240
|
+
|
|
241
|
+
Permutation 389190d4 (2 iterations)
|
|
242
|
+
node20 | baseline | snerk | fake-harness-alpha@1.0.2/fake-model-small | impatient
|
|
243
|
+
|
|
244
|
+
no measures
|
|
245
|
+
|
|
246
|
+
exit 2/2 exited 124
|
|
247
|
+
duration mean 0.04s, max 0.04s
|
|
248
|
+
`,
|
|
249
|
+
stderr: "",
|
|
250
|
+
exitCode: 0,
|
|
251
|
+
});
|
|
252
|
+
});
|
|
253
|
+
|
|
254
|
+
test("an unknown argument is an error that names it, with a non-zero exit", async () => {
|
|
255
|
+
for (const target of ["nope", `postrun-e${LATEST}-h00000000-i00`]) {
|
|
256
|
+
expect(await show(target)).toEqual({
|
|
257
|
+
stdout: "",
|
|
258
|
+
stderr: `${target} is not a recorded container run, execution or test case\n`,
|
|
259
|
+
exitCode: 1,
|
|
260
|
+
});
|
|
261
|
+
}
|
|
262
|
+
});
|
|
263
|
+
|
|
264
|
+
test("says what's wrong, then its usage, for no argument, extra arguments or a flag", async () => {
|
|
265
|
+
for (const [args, error] of [
|
|
266
|
+
[["show"], "error: missing required argument 'target'"],
|
|
267
|
+
[["show", "a", "b"], "error: too many arguments for 'show'"],
|
|
268
|
+
[["show", "--json"], "error: unknown option '--json'"],
|
|
269
|
+
[["show", "fails", "--data-dir"], "error: option '--data-dir <dir>' argument missing"],
|
|
270
|
+
] as const) {
|
|
271
|
+
const { stdout, stderr, exitCode } = await thunderjar(fixtures, [...args]);
|
|
272
|
+
expect(stdout).toBe("");
|
|
273
|
+
expect(stderr).toStartWith(error);
|
|
274
|
+
expect(stderr).toContain("Usage: thunderjar show [options] <target>");
|
|
275
|
+
expect(exitCode).toBe(1);
|
|
276
|
+
}
|
|
277
|
+
});
|
|
278
|
+
});
|
|
279
|
+
|
|
280
|
+
describe("scenario_5_empty_parameter_folders", () => {
|
|
281
|
+
test("show --data-dir reads a store kept elsewhere", async () => {
|
|
282
|
+
const recorded = scenario("scenario_8_recorded_executions");
|
|
283
|
+
const dataDir = join(recorded, ".thunderjar", ".data");
|
|
284
|
+
const elsewhere = await thunderjar(scenario("scenario_5_empty_parameter_folders"), ["show", TIMES_OUT, "--data-dir", dataDir]);
|
|
285
|
+
expect(elsewhere.exitCode).toBe(0);
|
|
286
|
+
expect(elsewhere).toEqual(await thunderjar(recorded, ["show", TIMES_OUT]));
|
|
287
|
+
});
|
|
288
|
+
});
|