thunderjar 0.1.2 → 0.1.3-preview-eee7160
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli.js +2161 -34
- package/package.json +13 -4
- package/src/AGENTS.md +156 -0
- package/src/adapters/docker/docker.ts +117 -0
- package/src/api/declarations.ts +279 -0
- package/src/api/fragments.ts +30 -0
- package/src/api/parameter-names.d.ts +16 -0
- package/src/cli/commands/build/build.docker.test.ts +173 -0
- package/src/cli/commands/build/build.test.ts +234 -0
- package/src/cli/commands/build/build.ts +115 -0
- package/src/cli/commands/declarations.typecheck.test.ts +173 -0
- package/src/cli/commands/executions/executions.test.ts +84 -0
- package/src/cli/commands/executions/executions.ts +31 -0
- package/src/cli/commands/format.ts +33 -0
- package/src/cli/commands/generate/generate.test.ts +101 -0
- package/src/cli/commands/generate/generate.ts +18 -0
- package/src/cli/commands/generate/parameter-names.ts +131 -0
- package/src/cli/commands/init/init.test.ts +123 -0
- package/src/cli/commands/init/init.ts +189 -0
- package/src/cli/commands/options.ts +17 -0
- package/src/cli/commands/plan/plan.test.ts +168 -0
- package/src/cli/commands/plan/plan.ts +72 -0
- package/src/cli/commands/remeasure/remeasure.test.ts +536 -0
- package/src/cli/commands/remeasure/remeasure.ts +194 -0
- package/src/cli/commands/run/run.test.ts +622 -0
- package/src/cli/commands/run/run.ts +203 -0
- package/src/cli/commands/show/show.test.ts +288 -0
- package/src/cli/commands/show/show.ts +252 -0
- package/src/cli/commands/validation.test.ts +200 -0
- package/src/cli/index.test.ts +47 -0
- package/src/cli/index.ts +9 -0
- package/src/cli/program.ts +29 -0
- package/src/images/prerun-builds.ts +108 -0
- package/src/images/prerun.test.ts +243 -0
- package/src/images/prerun.ts +272 -0
- package/src/images/repository.ts +72 -0
- package/src/index.ts +24 -0
- package/src/instruments/instruments.test.ts +240 -0
- package/src/instruments/instruments.ts +205 -0
- package/src/matrix/execution.ts +122 -0
- package/src/matrix/hashing.test.ts +298 -0
- package/src/matrix/hashing.ts +131 -0
- package/src/matrix/permutations.test.ts +58 -0
- package/src/matrix/permutations.ts +50 -0
- package/src/matrix/resolve.ts +63 -0
- package/src/matrix/tags.test.ts +15 -0
- package/src/matrix/tags.ts +22 -0
- package/src/matrix/ulid.test.ts +28 -0
- package/src/matrix/ulid.ts +22 -0
- package/src/measuring/aggregate.test.ts +188 -0
- package/src/measuring/aggregate.ts +154 -0
- package/src/measuring/measure.test.ts +105 -0
- package/src/measuring/measure.ts +62 -0
- package/src/measuring/remeasure-templates/v1/isPrime.test.template.txt +4 -0
- package/src/measuring/remeasure-templates/v2/isPrime.test.template.txt +5 -0
- package/src/measuring/remeasure.test.ts +352 -0
- package/src/measuring/remeasure.ts +113 -0
- package/src/platform/fs.test.ts +89 -0
- package/src/platform/fs.ts +123 -0
- package/src/platform/path.test.ts +38 -0
- package/src/platform/path.ts +72 -0
- package/src/platform/signals.test.ts +25 -0
- package/src/platform/signals.ts +68 -0
- package/src/project/config.test.ts +68 -0
- package/src/project/config.ts +297 -0
- package/src/running/runner.test.ts +73 -0
- package/src/running/runner.ts +252 -0
- package/src/store/run-data-store.test.ts +232 -0
- package/src/store/run-data-store.ts +254 -0
- package/src/testing/cli.ts +65 -0
- package/src/testing/fake-docker.test.ts +197 -0
- package/src/testing/fake-docker.ts +104 -0
- package/src/testing/fixture-project.ts +177 -0
- package/src/testing/measurement-context.ts +5 -0
- package/src/testing/seed-repo.test.ts +10 -0
- package/src/testing/seed-repo.ts +83 -0
- package/src/testing/test-setup.ts +10 -0
package/dist/cli.js
CHANGED
|
@@ -100,7 +100,7 @@ import { EventEmitter } from "events";
|
|
|
100
100
|
import childProcess from "child_process";
|
|
101
101
|
import path from "path";
|
|
102
102
|
import fs from "fs";
|
|
103
|
-
import
|
|
103
|
+
import process2 from "process";
|
|
104
104
|
import { stripVTControlCharacters as stripVTControlCharacters2 } from "util";
|
|
105
105
|
|
|
106
106
|
// node_modules/commander/lib/help.js
|
|
@@ -737,13 +737,13 @@ class Command extends EventEmitter {
|
|
|
737
737
|
this._showSuggestionAfterError = true;
|
|
738
738
|
this._savedState = null;
|
|
739
739
|
this._outputConfiguration = {
|
|
740
|
-
writeOut: (str) =>
|
|
741
|
-
writeErr: (str) =>
|
|
740
|
+
writeOut: (str) => process2.stdout.write(str),
|
|
741
|
+
writeErr: (str) => process2.stderr.write(str),
|
|
742
742
|
outputError: (str, write) => write(str),
|
|
743
|
-
getOutHelpWidth: () =>
|
|
744
|
-
getErrHelpWidth: () =>
|
|
745
|
-
getOutHasColors: () => useColor() ?? (
|
|
746
|
-
getErrHasColors: () => useColor() ?? (
|
|
743
|
+
getOutHelpWidth: () => process2.stdout.isTTY ? process2.stdout.columns : undefined,
|
|
744
|
+
getErrHelpWidth: () => process2.stderr.isTTY ? process2.stderr.columns : undefined,
|
|
745
|
+
getOutHasColors: () => useColor() ?? (process2.stdout.isTTY && process2.stdout.hasColors?.()),
|
|
746
|
+
getErrHasColors: () => useColor() ?? (process2.stderr.isTTY && process2.stderr.hasColors?.()),
|
|
747
747
|
stripColor: (str) => stripVTControlCharacters2(str)
|
|
748
748
|
};
|
|
749
749
|
this._hidden = false;
|
|
@@ -951,7 +951,7 @@ Expecting one of '${allowedValues.join("', '")}'`);
|
|
|
951
951
|
if (this._exitCallback) {
|
|
952
952
|
this._exitCallback(new CommanderError(exitCode, code, message));
|
|
953
953
|
}
|
|
954
|
-
|
|
954
|
+
process2.exit(exitCode);
|
|
955
955
|
}
|
|
956
956
|
action(fn) {
|
|
957
957
|
const listener = (args) => {
|
|
@@ -1143,16 +1143,16 @@ Expecting one of '${allowedValues.join("', '")}'`);
|
|
|
1143
1143
|
}
|
|
1144
1144
|
parseOptions = parseOptions || {};
|
|
1145
1145
|
if (argv === undefined && parseOptions.from === undefined) {
|
|
1146
|
-
if (
|
|
1146
|
+
if (process2.versions?.electron) {
|
|
1147
1147
|
parseOptions.from = "electron";
|
|
1148
1148
|
}
|
|
1149
|
-
const execArgv =
|
|
1149
|
+
const execArgv = process2.execArgv ?? [];
|
|
1150
1150
|
if (execArgv.includes("-e") || execArgv.includes("--eval") || execArgv.includes("-p") || execArgv.includes("--print")) {
|
|
1151
1151
|
parseOptions.from = "eval";
|
|
1152
1152
|
}
|
|
1153
1153
|
}
|
|
1154
1154
|
if (argv === undefined) {
|
|
1155
|
-
argv =
|
|
1155
|
+
argv = process2.argv;
|
|
1156
1156
|
}
|
|
1157
1157
|
this.rawArgs = argv.slice();
|
|
1158
1158
|
let userArgs;
|
|
@@ -1163,7 +1163,7 @@ Expecting one of '${allowedValues.join("', '")}'`);
|
|
|
1163
1163
|
userArgs = argv.slice(2);
|
|
1164
1164
|
break;
|
|
1165
1165
|
case "electron":
|
|
1166
|
-
if (
|
|
1166
|
+
if (process2.defaultApp) {
|
|
1167
1167
|
this._scriptPath = argv[1];
|
|
1168
1168
|
userArgs = argv.slice(2);
|
|
1169
1169
|
} else {
|
|
@@ -1277,24 +1277,24 @@ Expecting one of '${allowedValues.join("', '")}'`);
|
|
|
1277
1277
|
}
|
|
1278
1278
|
const launchWithNode = sourceExt.includes(path.extname(executableFile));
|
|
1279
1279
|
let proc;
|
|
1280
|
-
if (
|
|
1280
|
+
if (process2.platform !== "win32") {
|
|
1281
1281
|
if (launchWithNode) {
|
|
1282
1282
|
args.unshift(executableFile);
|
|
1283
|
-
args = incrementNodeInspectorPort(
|
|
1284
|
-
proc = childProcess.spawn(
|
|
1283
|
+
args = incrementNodeInspectorPort(process2.execArgv).concat(args);
|
|
1284
|
+
proc = childProcess.spawn(process2.argv[0], args, { stdio: "inherit" });
|
|
1285
1285
|
} else {
|
|
1286
1286
|
proc = childProcess.spawn(executableFile, args, { stdio: "inherit" });
|
|
1287
1287
|
}
|
|
1288
1288
|
} else {
|
|
1289
1289
|
this._checkForMissingExecutable(executableFile, executableDir, subcommand._name);
|
|
1290
1290
|
args.unshift(executableFile);
|
|
1291
|
-
args = incrementNodeInspectorPort(
|
|
1292
|
-
proc = childProcess.spawn(
|
|
1291
|
+
args = incrementNodeInspectorPort(process2.execArgv).concat(args);
|
|
1292
|
+
proc = childProcess.spawn(process2.execPath, args, { stdio: "inherit" });
|
|
1293
1293
|
}
|
|
1294
1294
|
if (!proc.killed) {
|
|
1295
1295
|
const signals = ["SIGUSR1", "SIGUSR2", "SIGTERM", "SIGINT", "SIGHUP"];
|
|
1296
1296
|
signals.forEach((signal) => {
|
|
1297
|
-
|
|
1297
|
+
process2.on(signal, () => {
|
|
1298
1298
|
if (proc.killed === false && proc.exitCode === null) {
|
|
1299
1299
|
proc.kill(signal);
|
|
1300
1300
|
}
|
|
@@ -1305,7 +1305,7 @@ Expecting one of '${allowedValues.join("', '")}'`);
|
|
|
1305
1305
|
proc.on("close", (code) => {
|
|
1306
1306
|
code = code ?? 1;
|
|
1307
1307
|
if (!exitCallback) {
|
|
1308
|
-
|
|
1308
|
+
process2.exit(code);
|
|
1309
1309
|
} else {
|
|
1310
1310
|
exitCallback(new CommanderError(code, "commander.executeSubCommandAsync", "(close)"));
|
|
1311
1311
|
}
|
|
@@ -1317,7 +1317,7 @@ Expecting one of '${allowedValues.join("', '")}'`);
|
|
|
1317
1317
|
throw new Error(`'${executableFile}' not executable`);
|
|
1318
1318
|
}
|
|
1319
1319
|
if (!exitCallback) {
|
|
1320
|
-
|
|
1320
|
+
process2.exit(1);
|
|
1321
1321
|
} else {
|
|
1322
1322
|
const wrappedError = new CommanderError(1, "commander.executeSubCommandAsync", "(error)");
|
|
1323
1323
|
wrappedError.nestedError = err;
|
|
@@ -1666,11 +1666,11 @@ Expecting one of '${allowedValues.join("', '")}'`);
|
|
|
1666
1666
|
}
|
|
1667
1667
|
_parseOptionsEnv() {
|
|
1668
1668
|
this.options.forEach((option) => {
|
|
1669
|
-
if (option.envVar && option.envVar in
|
|
1669
|
+
if (option.envVar && option.envVar in process2.env) {
|
|
1670
1670
|
const optionKey = option.attributeName();
|
|
1671
1671
|
if (this.getOptionValue(optionKey) === undefined || ["default", "config", "env"].includes(this.getOptionValueSource(optionKey))) {
|
|
1672
1672
|
if (option.required || option.optional) {
|
|
1673
|
-
this.emit(`optionEnv:${option.name()}`,
|
|
1673
|
+
this.emit(`optionEnv:${option.name()}`, process2.env[option.envVar]);
|
|
1674
1674
|
} else {
|
|
1675
1675
|
this.emit(`optionEnv:${option.name()}`);
|
|
1676
1676
|
}
|
|
@@ -1969,7 +1969,7 @@ Expecting one of '${allowedValues.join("', '")}'`);
|
|
|
1969
1969
|
}
|
|
1970
1970
|
help(contextOptions) {
|
|
1971
1971
|
this.outputHelp(contextOptions);
|
|
1972
|
-
let exitCode = Number(
|
|
1972
|
+
let exitCode = Number(process2.exitCode ?? 0);
|
|
1973
1973
|
if (exitCode === 0 && contextOptions && typeof contextOptions !== "function" && contextOptions.error) {
|
|
1974
1974
|
exitCode = 1;
|
|
1975
1975
|
}
|
|
@@ -2035,9 +2035,9 @@ function incrementNodeInspectorPort(args) {
|
|
|
2035
2035
|
});
|
|
2036
2036
|
}
|
|
2037
2037
|
function useColor() {
|
|
2038
|
-
if (
|
|
2038
|
+
if (process2.env.NO_COLOR || process2.env.FORCE_COLOR === "0" || process2.env.FORCE_COLOR === "false")
|
|
2039
2039
|
return false;
|
|
2040
|
-
if (
|
|
2040
|
+
if (process2.env.FORCE_COLOR || process2.env.CLICOLOR_FORCE !== undefined)
|
|
2041
2041
|
return true;
|
|
2042
2042
|
return;
|
|
2043
2043
|
}
|
|
@@ -2047,7 +2047,7 @@ var program = new Command;
|
|
|
2047
2047
|
// package.json
|
|
2048
2048
|
var package_default = {
|
|
2049
2049
|
name: "thunderjar",
|
|
2050
|
-
version: "0.1.
|
|
2050
|
+
version: "0.1.3-preview-eee7160",
|
|
2051
2051
|
type: "module",
|
|
2052
2052
|
description: "Thunderjar",
|
|
2053
2053
|
license: "MIT",
|
|
@@ -2058,19 +2058,27 @@ var package_default = {
|
|
|
2058
2058
|
bin: {
|
|
2059
2059
|
thunderjar: "./dist/cli.js"
|
|
2060
2060
|
},
|
|
2061
|
+
main: "./src/index.ts",
|
|
2062
|
+
types: "./src/index.ts",
|
|
2061
2063
|
files: [
|
|
2062
|
-
"dist"
|
|
2064
|
+
"dist",
|
|
2065
|
+
"src"
|
|
2063
2066
|
],
|
|
2064
2067
|
publishConfig: {
|
|
2065
2068
|
access: "public"
|
|
2066
2069
|
},
|
|
2067
2070
|
scripts: {
|
|
2068
2071
|
build: "bun build src/cli/index.ts --target bun --outfile dist/cli.js",
|
|
2069
|
-
lint: "oxlint",
|
|
2072
|
+
lint: "bun scripts/sync-oxlint-layout.ts && bun scripts/sync-oxlint-layout.ts --check && oxlint",
|
|
2070
2073
|
typecheck: "tsc --noEmit",
|
|
2071
2074
|
test: "bun test",
|
|
2075
|
+
"test:prepare": "bun scripts/prepare-tests.ts",
|
|
2076
|
+
"test:docker": "THUNDERJAR_TEST_DOCKER=1 bun test docker",
|
|
2077
|
+
fixture: "bun scripts/fixture.ts",
|
|
2078
|
+
"dev:link": "bun scripts/dev-link.ts",
|
|
2072
2079
|
"docs:dev": "vitepress dev docs",
|
|
2073
2080
|
"docs:build": "vitepress build docs",
|
|
2081
|
+
"docs:build:versions": "bun scripts/build-versioned-docs.ts",
|
|
2074
2082
|
"docs:preview": "vitepress preview docs",
|
|
2075
2083
|
changeset: "changeset",
|
|
2076
2084
|
version: "changeset version",
|
|
@@ -2084,6 +2092,7 @@ var package_default = {
|
|
|
2084
2092
|
},
|
|
2085
2093
|
devDependencies: {
|
|
2086
2094
|
"@changesets/cli": "^2.31.0",
|
|
2095
|
+
"@changesets/get-release-plan": "^4.0.16",
|
|
2087
2096
|
"@types/bun": "^1.4.3",
|
|
2088
2097
|
oxlint: "^1.88.0",
|
|
2089
2098
|
typescript: "^7.0.2",
|
|
@@ -2091,24 +2100,2142 @@ var package_default = {
|
|
|
2091
2100
|
}
|
|
2092
2101
|
};
|
|
2093
2102
|
|
|
2103
|
+
// src/platform/path.ts
|
|
2104
|
+
function isAbsolute(path2) {
|
|
2105
|
+
return path2.startsWith("/");
|
|
2106
|
+
}
|
|
2107
|
+
function normalize(path2) {
|
|
2108
|
+
if (path2 === "")
|
|
2109
|
+
return ".";
|
|
2110
|
+
const absolute = isAbsolute(path2);
|
|
2111
|
+
const trailing = path2.endsWith("/");
|
|
2112
|
+
const out = [];
|
|
2113
|
+
for (const part of path2.split("/")) {
|
|
2114
|
+
if (part === "" || part === ".")
|
|
2115
|
+
continue;
|
|
2116
|
+
if (part === "..") {
|
|
2117
|
+
if (out.length > 0 && out.at(-1) !== "..")
|
|
2118
|
+
out.pop();
|
|
2119
|
+
else if (!absolute)
|
|
2120
|
+
out.push("..");
|
|
2121
|
+
} else
|
|
2122
|
+
out.push(part);
|
|
2123
|
+
}
|
|
2124
|
+
let result = (absolute ? "/" : "") + out.join("/");
|
|
2125
|
+
if (result === "")
|
|
2126
|
+
result = absolute ? "/" : ".";
|
|
2127
|
+
if (trailing && !result.endsWith("/"))
|
|
2128
|
+
result += "/";
|
|
2129
|
+
return result;
|
|
2130
|
+
}
|
|
2131
|
+
function join(...parts) {
|
|
2132
|
+
const joined = parts.filter((part) => part !== "").join("/");
|
|
2133
|
+
return joined === "" ? "." : normalize(joined);
|
|
2134
|
+
}
|
|
2135
|
+
function resolve(...parts) {
|
|
2136
|
+
let path2 = "";
|
|
2137
|
+
for (let i = parts.length - 1;i >= 0 && !isAbsolute(path2); i--) {
|
|
2138
|
+
const part = parts[i];
|
|
2139
|
+
if (part !== "")
|
|
2140
|
+
path2 = path2 === "" ? part : `${part}/${path2}`;
|
|
2141
|
+
}
|
|
2142
|
+
if (!isAbsolute(path2))
|
|
2143
|
+
path2 = `${process.cwd()}/${path2}`;
|
|
2144
|
+
const normal = normalize(path2);
|
|
2145
|
+
return normal.length > 1 && normal.endsWith("/") ? normal.slice(0, -1) : normal;
|
|
2146
|
+
}
|
|
2147
|
+
function relative(from, to) {
|
|
2148
|
+
const a = resolve(from).split("/").filter(Boolean);
|
|
2149
|
+
const b = resolve(to).split("/").filter(Boolean);
|
|
2150
|
+
let common = 0;
|
|
2151
|
+
while (common < a.length && common < b.length && a[common] === b[common])
|
|
2152
|
+
common++;
|
|
2153
|
+
return [...Array(a.length - common).fill(".."), ...b.slice(common)].join("/");
|
|
2154
|
+
}
|
|
2155
|
+
function dirname(path2) {
|
|
2156
|
+
if (path2 === "")
|
|
2157
|
+
return ".";
|
|
2158
|
+
const trimmed = path2.length > 1 ? path2.replace(/\/+$/, "") : path2;
|
|
2159
|
+
const at = trimmed.lastIndexOf("/");
|
|
2160
|
+
if (at === -1)
|
|
2161
|
+
return ".";
|
|
2162
|
+
if (at === 0)
|
|
2163
|
+
return "/";
|
|
2164
|
+
return trimmed.slice(0, at);
|
|
2165
|
+
}
|
|
2166
|
+
function basename(path2, ext) {
|
|
2167
|
+
const trimmed = path2.length > 1 ? path2.replace(/\/+$/, "") : path2;
|
|
2168
|
+
const base = trimmed.slice(trimmed.lastIndexOf("/") + 1);
|
|
2169
|
+
return ext && base !== ext && base.endsWith(ext) ? base.slice(0, -ext.length) : base;
|
|
2170
|
+
}
|
|
2171
|
+
function extname(path2) {
|
|
2172
|
+
const base = basename(path2);
|
|
2173
|
+
const at = base.lastIndexOf(".");
|
|
2174
|
+
return at <= 0 ? "" : base.slice(at);
|
|
2175
|
+
}
|
|
2176
|
+
|
|
2177
|
+
// src/adapters/docker/docker.ts
|
|
2178
|
+
class DockerBuildError extends Error {
|
|
2179
|
+
ref;
|
|
2180
|
+
exitCode;
|
|
2181
|
+
constructor(ref, exitCode) {
|
|
2182
|
+
super(`docker build of ${ref} exited ${exitCode}`);
|
|
2183
|
+
this.ref = ref;
|
|
2184
|
+
this.exitCode = exitCode;
|
|
2185
|
+
}
|
|
2186
|
+
}
|
|
2187
|
+
|
|
2188
|
+
class DockerCommandError extends Error {
|
|
2189
|
+
args;
|
|
2190
|
+
exitCode;
|
|
2191
|
+
stderr;
|
|
2192
|
+
constructor(args, exitCode, stderr) {
|
|
2193
|
+
super(`docker ${args[0]} exited ${exitCode}${stderr.trim() ? `: ${stderr.trim().split(`
|
|
2194
|
+
`).at(-1)}` : ""}`);
|
|
2195
|
+
this.args = args;
|
|
2196
|
+
this.exitCode = exitCode;
|
|
2197
|
+
this.stderr = stderr;
|
|
2198
|
+
}
|
|
2199
|
+
}
|
|
2200
|
+
async function docker(binary, args, output) {
|
|
2201
|
+
const stdio = {
|
|
2202
|
+
collect: { stdout: "ignore", stderr: "pipe" },
|
|
2203
|
+
stderr: { stdout: 2, stderr: 2 },
|
|
2204
|
+
stdout: { stdout: "pipe", stderr: 2 }
|
|
2205
|
+
};
|
|
2206
|
+
const child = Bun.spawn([binary, ...args], { stdin: "ignore", ...stdio[output] });
|
|
2207
|
+
const text = (stream) => stream instanceof ReadableStream ? new Response(stream).text() : Promise.resolve("");
|
|
2208
|
+
const [stdout, stderr] = await Promise.all([text(child.stdout), text(child.stderr), child.exited]);
|
|
2209
|
+
return { exitCode: child.exitCode ?? 1, stdout, stderr };
|
|
2210
|
+
}
|
|
2211
|
+
function dockerCli(binary = process.env.THUNDERJAR_DOCKER || "docker") {
|
|
2212
|
+
const must = async (args) => {
|
|
2213
|
+
const { exitCode, stderr } = await docker(binary, args, "collect");
|
|
2214
|
+
if (exitCode !== 0)
|
|
2215
|
+
throw new DockerCommandError(args, exitCode, stderr);
|
|
2216
|
+
};
|
|
2217
|
+
return {
|
|
2218
|
+
async unreachable() {
|
|
2219
|
+
try {
|
|
2220
|
+
const { exitCode, stderr } = await docker(binary, ["version", "--format", "{{.Server.Version}}"], "collect");
|
|
2221
|
+
if (exitCode === 0)
|
|
2222
|
+
return;
|
|
2223
|
+
return stderr.trim().split(`
|
|
2224
|
+
`).at(-1) || `\`${binary} version\` exited ${exitCode}`;
|
|
2225
|
+
} catch (error) {
|
|
2226
|
+
if (error.code === "ENOENT")
|
|
2227
|
+
return `\`${binary}\` was not found`;
|
|
2228
|
+
throw error;
|
|
2229
|
+
}
|
|
2230
|
+
},
|
|
2231
|
+
async imageExists(ref) {
|
|
2232
|
+
return (await docker(binary, ["image", "inspect", "--format", "{{.Id}}", ref], "collect")).exitCode === 0;
|
|
2233
|
+
},
|
|
2234
|
+
async build(contextDir, ref) {
|
|
2235
|
+
const { exitCode } = await docker(binary, ["build", "--tag", ref, contextDir], "stderr");
|
|
2236
|
+
if (exitCode !== 0)
|
|
2237
|
+
throw new DockerBuildError(ref, exitCode);
|
|
2238
|
+
},
|
|
2239
|
+
async run(ref, name, script) {
|
|
2240
|
+
const { exitCode, stdout } = await docker(binary, ["run", "--name", name, "--entrypoint", "sh", ref, "-c", script], "stdout");
|
|
2241
|
+
return { exitCode, stdout };
|
|
2242
|
+
},
|
|
2243
|
+
async commit(name, ref, like) {
|
|
2244
|
+
const format = "{{json .Config.Entrypoint}}\t{{json .Config.Cmd}}";
|
|
2245
|
+
const inspected = await docker(binary, ["image", "inspect", "--format", format, like], "stdout");
|
|
2246
|
+
if (inspected.exitCode !== 0)
|
|
2247
|
+
throw new DockerCommandError(["image", "inspect", like], inspected.exitCode, "");
|
|
2248
|
+
const [entrypoint, cmd] = inspected.stdout.trim().split("\t");
|
|
2249
|
+
const changes = [`ENTRYPOINT ${entrypoint === "null" ? '[""]' : entrypoint}`, `CMD ${cmd === "null" ? '["sh"]' : cmd}`];
|
|
2250
|
+
await must(["commit", ...changes.flatMap((change) => ["--change", change]), name, ref]);
|
|
2251
|
+
},
|
|
2252
|
+
create: (ref, name) => must(["create", "--name", name, "--entrypoint", "sh", ref]),
|
|
2253
|
+
copyFrom: (name, path2, dest) => must(["cp", `${name}:${path2}`, dest]),
|
|
2254
|
+
removeContainer: (name) => must(["rm", "--force", name])
|
|
2255
|
+
};
|
|
2256
|
+
}
|
|
2257
|
+
|
|
2258
|
+
// src/platform/fs.ts
|
|
2259
|
+
var {$ } = globalThis.Bun;
|
|
2260
|
+
function tmpdir() {
|
|
2261
|
+
return (Bun.env.TMPDIR ?? "/tmp").replace(/\/+$/, "") || "/";
|
|
2262
|
+
}
|
|
2263
|
+
async function makeTempDir(prefix) {
|
|
2264
|
+
return (await $`mktemp -d ${join(tmpdir(), `${prefix}XXXXXX`)}`.quiet().text()).trim();
|
|
2265
|
+
}
|
|
2266
|
+
async function makeDir(dir) {
|
|
2267
|
+
await $`mkdir -p ${dir}`.quiet();
|
|
2268
|
+
}
|
|
2269
|
+
async function remove(path2) {
|
|
2270
|
+
await $`rm -rf ${path2}`.quiet();
|
|
2271
|
+
}
|
|
2272
|
+
async function copy(from, to) {
|
|
2273
|
+
if (await isDirectory(from)) {
|
|
2274
|
+
await makeDir(to);
|
|
2275
|
+
await $`cp -RP ${from}/. ${to}`.quiet();
|
|
2276
|
+
} else {
|
|
2277
|
+
await $`cp -P ${from} ${to}`.quiet();
|
|
2278
|
+
}
|
|
2279
|
+
}
|
|
2280
|
+
async function move(from, to) {
|
|
2281
|
+
await $`mv -f ${from} ${to}`.quiet();
|
|
2282
|
+
}
|
|
2283
|
+
async function exists(path2) {
|
|
2284
|
+
return (await $`test -e ${path2}`.quiet().nothrow()).exitCode === 0;
|
|
2285
|
+
}
|
|
2286
|
+
async function isDirectory(path2) {
|
|
2287
|
+
return (await $`test -d ${path2}`.quiet().nothrow()).exitCode === 0;
|
|
2288
|
+
}
|
|
2289
|
+
function existsSync(path2) {
|
|
2290
|
+
return Bun.spawnSync(["test", "-e", path2]).exitCode === 0;
|
|
2291
|
+
}
|
|
2292
|
+
function isDirectorySync(path2) {
|
|
2293
|
+
return Bun.spawnSync(["test", "-d", path2]).exitCode === 0;
|
|
2294
|
+
}
|
|
2295
|
+
function readBytesSync(path2) {
|
|
2296
|
+
const result = Bun.spawnSync(["cat", "--", path2], { stderr: "pipe" });
|
|
2297
|
+
if (result.exitCode !== 0)
|
|
2298
|
+
throw new Error(`can't read ${path2}: ${result.stderr.toString().trim()}`);
|
|
2299
|
+
return new Uint8Array(result.stdout);
|
|
2300
|
+
}
|
|
2301
|
+
function listFilesSync(dir) {
|
|
2302
|
+
if (!isDirectorySync(dir))
|
|
2303
|
+
throw new Error(`not a directory: ${dir}`);
|
|
2304
|
+
return [...new Bun.Glob("**").scanSync({ cwd: dir, dot: true, onlyFiles: true, followSymlinks: false })].toSorted();
|
|
2305
|
+
}
|
|
2306
|
+
function listEntries(dir, { recursive = false } = {}) {
|
|
2307
|
+
if (!isDirectorySync(dir))
|
|
2308
|
+
throw new Error(`not a directory: ${dir}`);
|
|
2309
|
+
const pattern = new Bun.Glob(recursive ? "**" : "*");
|
|
2310
|
+
const scan = (onlyFiles) => [...pattern.scanSync({ cwd: dir, dot: true, onlyFiles, followSymlinks: false })];
|
|
2311
|
+
const files = new Set(scan(true));
|
|
2312
|
+
const isRealDirectory = (path2) => Bun.spawnSync(["sh", "-c", 'test -d "$1" && test ! -L "$1"', "sh", join(dir, path2)]).exitCode === 0;
|
|
2313
|
+
return scan(false).toSorted().map((path2) => ({ path: path2, isDirectory: !files.has(path2) && isRealDirectory(path2) }));
|
|
2314
|
+
}
|
|
2315
|
+
|
|
2316
|
+
// src/api/declarations.ts
|
|
2317
|
+
var DECLARED_BY = Symbol.for("thunderjar.declaredBy");
|
|
2318
|
+
function declaredBy(value) {
|
|
2319
|
+
if ((typeof value !== "object" || value === null) && typeof value !== "function")
|
|
2320
|
+
return;
|
|
2321
|
+
const mark = value[DECLARED_BY];
|
|
2322
|
+
return typeof mark === "string" ? mark : undefined;
|
|
2323
|
+
}
|
|
2324
|
+
|
|
2325
|
+
// src/instruments/instruments.ts
|
|
2326
|
+
var aggregateMethods = {
|
|
2327
|
+
fileCreated: ["rate", "count"],
|
|
2328
|
+
grep: ["mean", "median", "min", "max", "sum"],
|
|
2329
|
+
templateTest: ["sum", "mean"]
|
|
2330
|
+
};
|
|
2331
|
+
var show = (value) => JSON.stringify(value) ?? String(value);
|
|
2332
|
+
var checkPath = (field, value, what) => typeof value === "string" && value !== "" ? [] : [[field, `must be ${what}, got ${show(value)}`]];
|
|
2333
|
+
var fileCreated = {
|
|
2334
|
+
async evaluate({ glob }, ctx) {
|
|
2335
|
+
return { outcome: "measured", data: (await ctx.glob(glob)).length > 0 };
|
|
2336
|
+
},
|
|
2337
|
+
checkConfig(config) {
|
|
2338
|
+
return checkPath("config.glob", config?.glob, "a glob pattern");
|
|
2339
|
+
}
|
|
2340
|
+
};
|
|
2341
|
+
var lines = (text) => text === "" ? [] : text.replace(/\r?\n$/, "").split(/\r?\n/);
|
|
2342
|
+
var grep = {
|
|
2343
|
+
async evaluate({ glob, pattern }, ctx) {
|
|
2344
|
+
const files = await ctx.glob(glob);
|
|
2345
|
+
if (files.length === 0)
|
|
2346
|
+
return { outcome: "skipped", reason: `no file matches ${glob}` };
|
|
2347
|
+
const regex = new RegExp(pattern);
|
|
2348
|
+
let count = 0;
|
|
2349
|
+
for (const file of files) {
|
|
2350
|
+
for (const line of lines(await ctx.readFile(file)))
|
|
2351
|
+
if (regex.test(line))
|
|
2352
|
+
count++;
|
|
2353
|
+
}
|
|
2354
|
+
return { outcome: "measured", data: count };
|
|
2355
|
+
},
|
|
2356
|
+
checkConfig(config) {
|
|
2357
|
+
const { glob, pattern } = config ?? {};
|
|
2358
|
+
const problems = checkPath("config.glob", glob, "a glob pattern");
|
|
2359
|
+
if (typeof pattern !== "string") {
|
|
2360
|
+
problems.push(["config.pattern", `must be a JavaScript regular expression, got ${show(pattern)}`]);
|
|
2361
|
+
} else {
|
|
2362
|
+
try {
|
|
2363
|
+
new RegExp(pattern);
|
|
2364
|
+
} catch (error) {
|
|
2365
|
+
problems.push(["config.pattern", `${show(pattern)} is not a valid regular expression: ${error.message}`]);
|
|
2366
|
+
}
|
|
2367
|
+
}
|
|
2368
|
+
return problems;
|
|
2369
|
+
}
|
|
2370
|
+
};
|
|
2371
|
+
var shellWord = (text) => `'${text.replaceAll("'", `'\\''`)}'`;
|
|
2372
|
+
var LOADERS = {
|
|
2373
|
+
".js": "js",
|
|
2374
|
+
".mjs": "js",
|
|
2375
|
+
".cjs": "js",
|
|
2376
|
+
".jsx": "jsx",
|
|
2377
|
+
".tsx": "tsx"
|
|
2378
|
+
};
|
|
2379
|
+
var identifier = (stem) => {
|
|
2380
|
+
const name = stem.replace(/[^\w$]/g, "_");
|
|
2381
|
+
return /^[0-9]/.test(name) ? `_${name}` : name || "subject";
|
|
2382
|
+
};
|
|
2383
|
+
|
|
2384
|
+
class CannotBind extends Error {
|
|
2385
|
+
}
|
|
2386
|
+
async function render(template, subject, ctx) {
|
|
2387
|
+
const dir = dirname(subject);
|
|
2388
|
+
const file = basename(subject);
|
|
2389
|
+
const stem = basename(file, extname(file));
|
|
2390
|
+
const loader = LOADERS[extname(file)] ?? "ts";
|
|
2391
|
+
let exports;
|
|
2392
|
+
try {
|
|
2393
|
+
exports = new Bun.Transpiler({ loader }).scan(await ctx.readFile(subject)).exports;
|
|
2394
|
+
} catch (error) {
|
|
2395
|
+
throw new CannotBind(`can't read the exports of ${subject}: ${error.message}`);
|
|
2396
|
+
}
|
|
2397
|
+
if (exports.length === 0)
|
|
2398
|
+
throw new CannotBind(`${subject} has no export to bind {{defaultExport}} to`);
|
|
2399
|
+
if (exports.length > 1) {
|
|
2400
|
+
throw new CannotBind(`${subject} has ${exports.length} exports (${exports.join(", ")}); {{defaultExport}} binds to a single one`);
|
|
2401
|
+
}
|
|
2402
|
+
let defaultExport = exports[0];
|
|
2403
|
+
let subjectPath = `./${file}`;
|
|
2404
|
+
if (defaultExport === "default") {
|
|
2405
|
+
defaultExport = identifier(stem);
|
|
2406
|
+
subjectPath = `./${stem}.templateTest.bind.ts`;
|
|
2407
|
+
await ctx.writeFile(join(dir, subjectPath), `export { default as ${defaultExport} } from "./${file}";
|
|
2408
|
+
`);
|
|
2409
|
+
}
|
|
2410
|
+
const values = { defaultExport, subjectPath };
|
|
2411
|
+
const rendered = template.replace(/\{\{\s*(\w+)\s*\}\}/g, (placeholder, name) => {
|
|
2412
|
+
if (!Object.hasOwn(values, name))
|
|
2413
|
+
throw new CannotBind(`the template has an unknown placeholder ${placeholder}`);
|
|
2414
|
+
return values[name];
|
|
2415
|
+
});
|
|
2416
|
+
const testPath = join(dir, `${stem}.templateTest.test.ts`);
|
|
2417
|
+
await ctx.writeFile(testPath, rendered);
|
|
2418
|
+
return testPath;
|
|
2419
|
+
}
|
|
2420
|
+
async function runTest(testPath, ctx) {
|
|
2421
|
+
const { stdout, stderr } = await ctx.exec(`bun test ${shellWord(`./${testPath}`)}`);
|
|
2422
|
+
const output = `${stdout}
|
|
2423
|
+
${stderr}`;
|
|
2424
|
+
const count = (label) => Number(new RegExp(`^\\s*(\\d+) ${label}$`, "m").exec(output)?.[1] ?? "0");
|
|
2425
|
+
const ran = /^Ran (\d+) tests? /m.exec(output);
|
|
2426
|
+
if (!ran || ran[1] === "0")
|
|
2427
|
+
throw new CannotBind(`bun test ${testPath} reported no results`);
|
|
2428
|
+
if (count("errors?") > 0) {
|
|
2429
|
+
const error = /^\w*error: .*$/im.exec(output)?.[0];
|
|
2430
|
+
throw new CannotBind(`bun test ${testPath} failed outside any test${error ? `: ${error}` : ""}`);
|
|
2431
|
+
}
|
|
2432
|
+
return { passed: count("pass"), failed: count("fail") };
|
|
2433
|
+
}
|
|
2434
|
+
var templateTest = {
|
|
2435
|
+
async evaluate({ subject, template }, ctx) {
|
|
2436
|
+
const subjects = await ctx.glob(subject);
|
|
2437
|
+
if (subjects.length === 0)
|
|
2438
|
+
return { outcome: "skipped", reason: `no file matches ${subject}` };
|
|
2439
|
+
const source = await Bun.file(template).text();
|
|
2440
|
+
const total = { passed: 0, failed: 0 };
|
|
2441
|
+
try {
|
|
2442
|
+
for (const path2 of subjects) {
|
|
2443
|
+
const { passed, failed } = await runTest(await render(source, path2, ctx), ctx);
|
|
2444
|
+
total.passed += passed;
|
|
2445
|
+
total.failed += failed;
|
|
2446
|
+
}
|
|
2447
|
+
} catch (error) {
|
|
2448
|
+
if (error instanceof CannotBind)
|
|
2449
|
+
return { outcome: "erroredWhileMeasuring", error };
|
|
2450
|
+
throw error;
|
|
2451
|
+
}
|
|
2452
|
+
return { outcome: "measured", data: total };
|
|
2453
|
+
},
|
|
2454
|
+
checkConfig(config) {
|
|
2455
|
+
const { subject, template } = config ?? {};
|
|
2456
|
+
return [
|
|
2457
|
+
...checkPath("config.subject", subject, "a glob pattern"),
|
|
2458
|
+
...checkPath("config.template", template, "a path relative to the measurement file")
|
|
2459
|
+
];
|
|
2460
|
+
},
|
|
2461
|
+
bundledFiles: ["template"]
|
|
2462
|
+
};
|
|
2463
|
+
var instruments = { fileCreated, grep, templateTest };
|
|
2464
|
+
var isInstrumentName = (name) => typeof name === "string" && Object.hasOwn(instruments, name);
|
|
2465
|
+
|
|
2466
|
+
// src/project/config.ts
|
|
2467
|
+
class ConfigError extends Error {
|
|
2468
|
+
}
|
|
2469
|
+
|
|
2470
|
+
class ThunderjarRootNotFoundError extends ConfigError {
|
|
2471
|
+
constructor(startDir) {
|
|
2472
|
+
super(`error: no .thunderjar/ folder found in ${startDir} or any parent directory.
|
|
2473
|
+
Run \`thunderjar init\` first.`);
|
|
2474
|
+
}
|
|
2475
|
+
}
|
|
2476
|
+
|
|
2477
|
+
class TestCaseNotFoundError extends ConfigError {
|
|
2478
|
+
constructor(name, path2) {
|
|
2479
|
+
super(`error: no test case named "${name}" (expected ${path2})`);
|
|
2480
|
+
}
|
|
2481
|
+
}
|
|
2482
|
+
|
|
2483
|
+
class InvalidConfigError extends ConfigError {
|
|
2484
|
+
problems;
|
|
2485
|
+
constructor(problems) {
|
|
2486
|
+
super(problems.map(({ file, field, message }) => `error: .thunderjar/${file}: ${field ? `${field}: ` : ""}${message}`).join(`
|
|
2487
|
+
`));
|
|
2488
|
+
this.problems = problems;
|
|
2489
|
+
}
|
|
2490
|
+
}
|
|
2491
|
+
function findThunderjarRoot(startDir = process.cwd()) {
|
|
2492
|
+
let dir = resolve(startDir);
|
|
2493
|
+
for (;; ) {
|
|
2494
|
+
const candidate = join(dir, ".thunderjar");
|
|
2495
|
+
if (existsSync(candidate))
|
|
2496
|
+
return candidate;
|
|
2497
|
+
const parent = dirname(dir);
|
|
2498
|
+
if (parent === dir)
|
|
2499
|
+
throw new ThunderjarRootNotFoundError(startDir);
|
|
2500
|
+
dir = parent;
|
|
2501
|
+
}
|
|
2502
|
+
}
|
|
2503
|
+
async function loadDeclaration(root, file, declareFn, problems) {
|
|
2504
|
+
let module;
|
|
2505
|
+
try {
|
|
2506
|
+
module = await import(join(root, file));
|
|
2507
|
+
} catch (error) {
|
|
2508
|
+
problems.push({ file, message: `could not be loaded: ${error instanceof Error ? error.message : String(error)}` });
|
|
2509
|
+
return;
|
|
2510
|
+
}
|
|
2511
|
+
if (!("default" in module) || module.default === undefined) {
|
|
2512
|
+
problems.push({ file, message: `has no default export. Export \`${declareFn}(...)\` as the default.` });
|
|
2513
|
+
return;
|
|
2514
|
+
}
|
|
2515
|
+
const by = declaredBy(module.default);
|
|
2516
|
+
if (by !== declareFn) {
|
|
2517
|
+
problems.push({
|
|
2518
|
+
file,
|
|
2519
|
+
message: by ? `default export is wrapped in ${by}, expected ${declareFn}.` : `default export is not wrapped in ${declareFn}(...).`
|
|
2520
|
+
});
|
|
2521
|
+
return;
|
|
2522
|
+
}
|
|
2523
|
+
return module.default;
|
|
2524
|
+
}
|
|
2525
|
+
var show2 = (value) => JSON.stringify(value) ?? String(value);
|
|
2526
|
+
function requireFile(root, testCaseFile, field, name, file, problems) {
|
|
2527
|
+
if (existsSync(join(root, file)))
|
|
2528
|
+
return true;
|
|
2529
|
+
problems.push({ file: testCaseFile, field, message: `${show2(name)} has no matching ${file}` });
|
|
2530
|
+
return false;
|
|
2531
|
+
}
|
|
2532
|
+
var FOLDER_PARAMETERS = [
|
|
2533
|
+
["baseImage", "baseImages", "declareBaseImage"],
|
|
2534
|
+
["codeState", "codeStates", "declareCodeState"],
|
|
2535
|
+
["promptSet", "promptSets", "declarePromptSet"]
|
|
2536
|
+
];
|
|
2537
|
+
var ARRAY_FIELDS = ["baseImage", "codeState", "promptSet", "harnessModel", "initialPrompt"];
|
|
2538
|
+
var PAIR = /^([^@/]+)@([^/]+)\/(.+)$/;
|
|
2539
|
+
function parseHarnessModelPair(pair) {
|
|
2540
|
+
const match = PAIR.exec(pair);
|
|
2541
|
+
if (!match)
|
|
2542
|
+
return;
|
|
2543
|
+
const [, family, version, model] = match;
|
|
2544
|
+
return { family, version, model };
|
|
2545
|
+
}
|
|
2546
|
+
async function collectMeasurements(root, task, problems) {
|
|
2547
|
+
const dir = `test-case-parameters/tasks/${task}/measurements`;
|
|
2548
|
+
if (!existsSync(join(root, dir)))
|
|
2549
|
+
return [];
|
|
2550
|
+
const files = [...new Bun.Glob("*.ts").scanSync({ cwd: join(root, dir) })].filter((name) => !basename(name, ".ts").includes(".")).toSorted();
|
|
2551
|
+
const measurements = [];
|
|
2552
|
+
for (const name of files) {
|
|
2553
|
+
const file = `${dir}/${name}`;
|
|
2554
|
+
const declaration = await loadDeclaration(root, file, "declareMeasurement", problems);
|
|
2555
|
+
if (!declaration)
|
|
2556
|
+
continue;
|
|
2557
|
+
const { instrument, config, aggregate } = declaration;
|
|
2558
|
+
if (!isInstrumentName(instrument)) {
|
|
2559
|
+
problems.push({
|
|
2560
|
+
file,
|
|
2561
|
+
field: "instrument",
|
|
2562
|
+
message: `${show2(instrument)} is not a known instrument. Known instruments: ${Object.keys(instruments).join(", ")}`
|
|
2563
|
+
});
|
|
2564
|
+
continue;
|
|
2565
|
+
}
|
|
2566
|
+
const before = problems.length;
|
|
2567
|
+
const { checkConfig, bundledFiles = [] } = instruments[instrument];
|
|
2568
|
+
const configProblems = checkConfig(config);
|
|
2569
|
+
for (const [field, message] of configProblems)
|
|
2570
|
+
problems.push({ file, field, message });
|
|
2571
|
+
if (configProblems.length === 0) {
|
|
2572
|
+
for (const key of bundledFiles) {
|
|
2573
|
+
const path2 = config[key];
|
|
2574
|
+
requireFile(root, file, `config.${key}`, path2, join(dir, path2), problems);
|
|
2575
|
+
}
|
|
2576
|
+
}
|
|
2577
|
+
if (!Array.isArray(aggregate)) {
|
|
2578
|
+
problems.push({ file, field: "aggregate", message: `must be an array of aggregation methods, got ${show2(aggregate)}` });
|
|
2579
|
+
} else {
|
|
2580
|
+
const menu = aggregateMethods[instrument];
|
|
2581
|
+
for (const method of aggregate) {
|
|
2582
|
+
if (menu.includes(method))
|
|
2583
|
+
continue;
|
|
2584
|
+
problems.push({
|
|
2585
|
+
file,
|
|
2586
|
+
field: "aggregate",
|
|
2587
|
+
message: `${show2(method)} is not a way ${instrument}'s values combine. Use one of: ${menu.join(", ")}`
|
|
2588
|
+
});
|
|
2589
|
+
}
|
|
2590
|
+
}
|
|
2591
|
+
if (problems.length === before) {
|
|
2592
|
+
measurements.push({ ...declaration, name: basename(name, ".ts"), dir: join(root, dir) });
|
|
2593
|
+
}
|
|
2594
|
+
}
|
|
2595
|
+
return measurements;
|
|
2596
|
+
}
|
|
2597
|
+
async function loadMeasurements(root, task) {
|
|
2598
|
+
const problems = [];
|
|
2599
|
+
const measurements = await collectMeasurements(root, task, problems);
|
|
2600
|
+
if (problems.length > 0)
|
|
2601
|
+
throw new InvalidConfigError(problems);
|
|
2602
|
+
return measurements;
|
|
2603
|
+
}
|
|
2604
|
+
async function checkTestCase(root, file, testCase, problems) {
|
|
2605
|
+
const parameters = "test-case-parameters";
|
|
2606
|
+
const names = {};
|
|
2607
|
+
for (const field of ARRAY_FIELDS) {
|
|
2608
|
+
const value = testCase[field];
|
|
2609
|
+
if (!Array.isArray(value)) {
|
|
2610
|
+
problems.push({ file, field, message: `must be an array of names, got ${show2(value)}` });
|
|
2611
|
+
} else if (value.length === 0) {
|
|
2612
|
+
problems.push({ file, field, message: "must name at least one value, got []" });
|
|
2613
|
+
} else if (value.some((v) => typeof v !== "string")) {
|
|
2614
|
+
problems.push({ file, field, message: `must be an array of names, got ${show2(value)}` });
|
|
2615
|
+
} else {
|
|
2616
|
+
names[field] = value;
|
|
2617
|
+
}
|
|
2618
|
+
}
|
|
2619
|
+
const iterations = testCase.iterations;
|
|
2620
|
+
if (typeof iterations !== "number" || !Number.isInteger(iterations) || iterations < 1) {
|
|
2621
|
+
problems.push({ file, field: "iterations", message: `must be a positive integer, got ${show2(iterations)}` });
|
|
2622
|
+
}
|
|
2623
|
+
for (const [field, folder, declareFn] of FOLDER_PARAMETERS) {
|
|
2624
|
+
for (const name of names[field] ?? []) {
|
|
2625
|
+
const index = `${parameters}/${folder}/${name}/index.ts`;
|
|
2626
|
+
if (requireFile(root, file, field, name, index, problems)) {
|
|
2627
|
+
await loadDeclaration(root, index, declareFn, problems);
|
|
2628
|
+
}
|
|
2629
|
+
}
|
|
2630
|
+
}
|
|
2631
|
+
const task = testCase.task;
|
|
2632
|
+
let taskExists = false;
|
|
2633
|
+
if (typeof task !== "string") {
|
|
2634
|
+
problems.push({ file, field: "task", message: `must be a task name, got ${show2(task)}` });
|
|
2635
|
+
} else {
|
|
2636
|
+
taskExists = requireFile(root, file, "task", task, `${parameters}/tasks/${task}/`, problems);
|
|
2637
|
+
}
|
|
2638
|
+
if (taskExists) {
|
|
2639
|
+
await collectMeasurements(root, task, problems);
|
|
2640
|
+
for (const name of names.initialPrompt ?? []) {
|
|
2641
|
+
const prompt = `${parameters}/tasks/${task}/initial-prompts/${name}.ts`;
|
|
2642
|
+
if (existsSync(join(root, prompt))) {
|
|
2643
|
+
await loadDeclaration(root, prompt, "declareInitialPrompt", problems);
|
|
2644
|
+
continue;
|
|
2645
|
+
}
|
|
2646
|
+
const owner = new Bun.Glob(`${parameters}/tasks/*/initial-prompts/${name}.ts`).scanSync({ cwd: root }).next().value;
|
|
2647
|
+
problems.push({
|
|
2648
|
+
file,
|
|
2649
|
+
field: "initialPrompt",
|
|
2650
|
+
message: owner ? `${show2(name)} belongs to task ${show2(owner.split("/")[2])}, not ${show2(task)}` : `${show2(name)} has no matching ${prompt}`
|
|
2651
|
+
});
|
|
2652
|
+
}
|
|
2653
|
+
}
|
|
2654
|
+
for (const pair of names.harnessModel ?? []) {
|
|
2655
|
+
const parsed = parseHarnessModelPair(pair);
|
|
2656
|
+
if (!parsed) {
|
|
2657
|
+
problems.push({ file, field: "harnessModel", message: `${show2(pair)} is not of the form family@version/model` });
|
|
2658
|
+
continue;
|
|
2659
|
+
}
|
|
2660
|
+
const { family, version, model } = parsed;
|
|
2661
|
+
const familyDir = `${parameters}/harnessModels/${family}`;
|
|
2662
|
+
if (!requireFile(root, file, "harnessModel", pair, `${familyDir}/`, problems))
|
|
2663
|
+
continue;
|
|
2664
|
+
const familyProblems = [];
|
|
2665
|
+
await loadDeclaration(root, `${familyDir}/index.ts`, "declareHarness", familyProblems);
|
|
2666
|
+
await loadDeclaration(root, `${familyDir}/collectTokenCosts.ts`, "declareTokenCostCollector", familyProblems);
|
|
2667
|
+
const versions = await loadDeclaration(root, `${familyDir}/versions.ts`, "declareHarnessVersions", familyProblems);
|
|
2668
|
+
const models = await loadDeclaration(root, `${familyDir}/models.ts`, "declareHarnessModels", familyProblems);
|
|
2669
|
+
for (const p of familyProblems) {
|
|
2670
|
+
if (!problems.some((q) => q.file === p.file && q.message === p.message))
|
|
2671
|
+
problems.push(p);
|
|
2672
|
+
}
|
|
2673
|
+
if (versions && !Object.hasOwn(versions, version)) {
|
|
2674
|
+
problems.push({
|
|
2675
|
+
file,
|
|
2676
|
+
field: "harnessModel",
|
|
2677
|
+
message: `${show2(pair)}: version ${show2(version)} is not in ${familyDir}/versions.ts`
|
|
2678
|
+
});
|
|
2679
|
+
}
|
|
2680
|
+
if (models && !Object.hasOwn(models, model)) {
|
|
2681
|
+
problems.push({
|
|
2682
|
+
file,
|
|
2683
|
+
field: "harnessModel",
|
|
2684
|
+
message: `${show2(pair)}: model ${show2(model)} is not in ${familyDir}/models.ts`
|
|
2685
|
+
});
|
|
2686
|
+
}
|
|
2687
|
+
}
|
|
2688
|
+
}
|
|
2689
|
+
async function loadTestCase(root, name) {
|
|
2690
|
+
const file = `test-cases/${name}.ts`;
|
|
2691
|
+
const path2 = join(root, file);
|
|
2692
|
+
if (!existsSync(path2))
|
|
2693
|
+
throw new TestCaseNotFoundError(name, path2);
|
|
2694
|
+
const problems = [];
|
|
2695
|
+
const testCase = await loadDeclaration(root, file, "declareTestCase", problems);
|
|
2696
|
+
if (testCase)
|
|
2697
|
+
await checkTestCase(root, file, testCase, problems);
|
|
2698
|
+
if (problems.length > 0)
|
|
2699
|
+
throw new InvalidConfigError(problems);
|
|
2700
|
+
return testCase;
|
|
2701
|
+
}
|
|
2702
|
+
|
|
2703
|
+
// src/matrix/permutations.ts
|
|
2704
|
+
var AXES = ["baseImage", "codeState", "promptSet", "harnessModel", "initialPrompt"];
|
|
2705
|
+
function matrixShape(testCase) {
|
|
2706
|
+
return AXES.map((axis) => testCase[axis].length);
|
|
2707
|
+
}
|
|
2708
|
+
function expandPermutations(testCase) {
|
|
2709
|
+
const permutations = [];
|
|
2710
|
+
for (const baseImage of testCase.baseImage)
|
|
2711
|
+
for (const codeState of testCase.codeState)
|
|
2712
|
+
for (const promptSet of testCase.promptSet)
|
|
2713
|
+
for (const harnessModel of testCase.harnessModel)
|
|
2714
|
+
for (const initialPrompt of testCase.initialPrompt)
|
|
2715
|
+
permutations.push({ baseImage, codeState, promptSet, harnessModel, initialPrompt });
|
|
2716
|
+
return permutations;
|
|
2717
|
+
}
|
|
2718
|
+
function permutationRows(permutations) {
|
|
2719
|
+
const cells = permutations.map((p) => AXES.map((axis) => p[axis]));
|
|
2720
|
+
const widths = AXES.map((_, i) => Math.max(...cells.map((row) => row[i].length)));
|
|
2721
|
+
return cells.map((row) => row.map((cell, i) => i === AXES.length - 1 ? cell : cell.padEnd(widths[i])).join(" | "));
|
|
2722
|
+
}
|
|
2723
|
+
|
|
2724
|
+
// src/matrix/hashing.ts
|
|
2725
|
+
var sha256 = (data) => new Bun.CryptoHasher("sha256").update(data).digest("hex");
|
|
2726
|
+
function canonical(value) {
|
|
2727
|
+
if (typeof value === "function")
|
|
2728
|
+
return JSON.stringify(value.toString());
|
|
2729
|
+
if (Array.isArray(value))
|
|
2730
|
+
return `[${value.map(canonical).join(",")}]`;
|
|
2731
|
+
if (typeof value === "object" && value !== null) {
|
|
2732
|
+
const keys = Object.keys(value).toSorted();
|
|
2733
|
+
return `{${keys.map((k) => `${JSON.stringify(k)}:${canonical(value[k])}`).join(",")}}`;
|
|
2734
|
+
}
|
|
2735
|
+
return JSON.stringify(value) ?? "null";
|
|
2736
|
+
}
|
|
2737
|
+
var hashValue = (value) => sha256(canonical(value));
|
|
2738
|
+
var abbreviateHash = (hash) => hash.slice(0, 8);
|
|
2739
|
+
function hashFolder(dir, exclude = []) {
|
|
2740
|
+
const files = listFilesSync(dir).filter((file) => !exclude.includes(file));
|
|
2741
|
+
return hashValue(files.map((file) => [file, sha256(readBytesSync(join(dir, file)))]));
|
|
2742
|
+
}
|
|
2743
|
+
var hashBaseImage = (path2) => hashFolder(path2);
|
|
2744
|
+
var hashCodeState = (path2) => hashFolder(path2);
|
|
2745
|
+
var hashPromptSet = (path2) => hashFolder(path2);
|
|
2746
|
+
var hashInitialPrompt = (path2) => sha256(readBytesSync(path2));
|
|
2747
|
+
var FAMILY_MAP_FILES = ["models.ts", "versions.ts", "collectTokenCosts.ts"];
|
|
2748
|
+
var EXECUTION_OVERRIDES = ["applyParameter", "cli", "requiredEnv"];
|
|
2749
|
+
async function loadMap(file) {
|
|
2750
|
+
const module = await import(file);
|
|
2751
|
+
return module.default ?? {};
|
|
2752
|
+
}
|
|
2753
|
+
async function hashHarnessModelPair(family, version, model) {
|
|
2754
|
+
const versions = await loadMap(join(family, "versions.ts"));
|
|
2755
|
+
const models = await loadMap(join(family, "models.ts"));
|
|
2756
|
+
if (!Object.hasOwn(versions, version))
|
|
2757
|
+
throw new Error(`version "${version}" is not in ${family}/versions.ts`);
|
|
2758
|
+
if (!Object.hasOwn(models, model))
|
|
2759
|
+
throw new Error(`model "${model}" is not in ${family}/models.ts`);
|
|
2760
|
+
const entry = versions[version];
|
|
2761
|
+
if (typeof entry.version !== "string") {
|
|
2762
|
+
throw new Error(`version "${version}" in ${family}/versions.ts has no pinned version; floating versions are not supported yet`);
|
|
2763
|
+
}
|
|
2764
|
+
const overrides = Object.fromEntries(EXECUTION_OVERRIDES.filter((k) => (k in entry)).map((k) => [k, entry[k]]));
|
|
2765
|
+
return hashValue({
|
|
2766
|
+
shared: hashFolder(family, FAMILY_MAP_FILES),
|
|
2767
|
+
version: entry.version,
|
|
2768
|
+
overrides,
|
|
2769
|
+
modelId: models[model]
|
|
2770
|
+
});
|
|
2771
|
+
}
|
|
2772
|
+
var hashPermutation = (hashes) => hashValue(hashes);
|
|
2773
|
+
function hashMeasurement({ instrument, config, dir }) {
|
|
2774
|
+
const keys = isInstrumentName(instrument) ? instruments[instrument].bundledFiles ?? [] : [];
|
|
2775
|
+
if (keys.length === 0)
|
|
2776
|
+
return hashValue({ instrument, config });
|
|
2777
|
+
if (dir === undefined)
|
|
2778
|
+
throw new Error(`a ${instrument} measurement's hash needs the folder its bundled files are in`);
|
|
2779
|
+
const paths = config;
|
|
2780
|
+
const files = Object.fromEntries(keys.map((key) => [key, sha256(readBytesSync(resolve(dir, paths[key])))]));
|
|
2781
|
+
return hashValue({ instrument, config, files });
|
|
2782
|
+
}
|
|
2783
|
+
|
|
2784
|
+
// src/matrix/tags.ts
|
|
2785
|
+
var prerunTag = (permutationHash) => `prerun-h${abbreviateHash(permutationHash)}`;
|
|
2786
|
+
var postrunTag = (executionId, permutationHash, iterationIndex) => `postrun-e${executionId}-h${abbreviateHash(permutationHash)}-i${String(iterationIndex).padStart(2, "0")}`;
|
|
2787
|
+
var imageRef = (repository, tag) => `${repository}:${tag}`;
|
|
2788
|
+
|
|
2789
|
+
// src/images/prerun.ts
|
|
2790
|
+
var LOCAL_REPOSITORY = "thunderjar";
|
|
2791
|
+
var prerunImageRef = (repository, permutationHash) => imageRef(repository, prerunTag(permutationHash));
|
|
2792
|
+
var PARAMETER_DIRS = {
|
|
2793
|
+
baseImage: "parameters/baseImage",
|
|
2794
|
+
harness: "parameters/harnessModel",
|
|
2795
|
+
codeState: "parameters/codeState",
|
|
2796
|
+
promptSet: "parameters/promptSet"
|
|
2797
|
+
};
|
|
2798
|
+
var importDefault = async (path2) => (await import(path2)).default;
|
|
2799
|
+
async function loadPrerunParameters(root, task, permutation) {
|
|
2800
|
+
const parameters = join(root, "test-case-parameters");
|
|
2801
|
+
const folder = async (dir, owner) => {
|
|
2802
|
+
const { config, applyParameter } = await importDefault(join(dir, "index.ts"));
|
|
2803
|
+
return { owner, dir, config, applyParameter };
|
|
2804
|
+
};
|
|
2805
|
+
const { family, version } = parseHarnessModelPair(permutation.harnessModel);
|
|
2806
|
+
const familyDir = join(parameters, "harnessModels", family);
|
|
2807
|
+
const harness = await importDefault(join(familyDir, "index.ts"));
|
|
2808
|
+
const entry = (await importDefault(join(familyDir, "versions.ts")))[version];
|
|
2809
|
+
if (!("version" in entry))
|
|
2810
|
+
throw new Error(`${permutation.harnessModel}: floating versions are not supported yet`);
|
|
2811
|
+
const initialPrompt = await importDefault(join(parameters, "tasks", task, "initial-prompts", `${permutation.initialPrompt}.ts`));
|
|
2812
|
+
return {
|
|
2813
|
+
baseImage: await folder(join(parameters, "baseImages", permutation.baseImage), `base image "${permutation.baseImage}"`),
|
|
2814
|
+
harness: {
|
|
2815
|
+
owner: `harness/model pair "${permutation.harnessModel}"`,
|
|
2816
|
+
dir: familyDir,
|
|
2817
|
+
exclude: FAMILY_MAP_FILES,
|
|
2818
|
+
config: harness.config,
|
|
2819
|
+
applyParameter: entry.applyParameter ?? harness.applyParameter,
|
|
2820
|
+
version: entry.version
|
|
2821
|
+
},
|
|
2822
|
+
codeState: await folder(join(parameters, "codeStates", permutation.codeState), `code state "${permutation.codeState}"`),
|
|
2823
|
+
promptSet: await folder(join(parameters, "promptSets", permutation.promptSet), `prompt set "${permutation.promptSet}"`),
|
|
2824
|
+
initialPrompt: {
|
|
2825
|
+
owner: `initial prompt "${permutation.initialPrompt}"`,
|
|
2826
|
+
config: initialPrompt.config,
|
|
2827
|
+
applyParameter: initialPrompt.applyParameter
|
|
2828
|
+
}
|
|
2829
|
+
};
|
|
2830
|
+
}
|
|
2831
|
+
var COMMIT = /^[0-9a-f]{4,64}$/i;
|
|
2832
|
+
function recordingGit(owner, needs) {
|
|
2833
|
+
const checkCommit = (method, commit) => {
|
|
2834
|
+
if (!COMMIT.test(commit)) {
|
|
2835
|
+
throw new Error(`error: ${owner}: ctx.git.${method} needs a commit hash, got ${JSON.stringify(commit)}.`);
|
|
2836
|
+
}
|
|
2837
|
+
};
|
|
2838
|
+
return {
|
|
2839
|
+
bundle(commit) {
|
|
2840
|
+
checkCommit("bundle", commit);
|
|
2841
|
+
const path2 = `git/${commit}.bundle`;
|
|
2842
|
+
needs.push({ kind: "bundle", owner, commit, path: path2 });
|
|
2843
|
+
return path2;
|
|
2844
|
+
},
|
|
2845
|
+
show(commit, file) {
|
|
2846
|
+
checkCommit("show", commit);
|
|
2847
|
+
const normal = normalize(file);
|
|
2848
|
+
if (file === "" || isAbsolute(file) || normal === "." || normal.split("/").includes("..")) {
|
|
2849
|
+
throw new Error(`error: ${owner}: ctx.git.show needs a path inside the repository, got ${JSON.stringify(file)}.`);
|
|
2850
|
+
}
|
|
2851
|
+
const path2 = `git/${commit}/${normal}`;
|
|
2852
|
+
needs.push({ kind: "file", owner, commit, file: normal, path: path2 });
|
|
2853
|
+
return path2;
|
|
2854
|
+
}
|
|
2855
|
+
};
|
|
2856
|
+
}
|
|
2857
|
+
function renderPrerun(parameters) {
|
|
2858
|
+
const git = [];
|
|
2859
|
+
const ctx = (role) => ({ git: recordingGit(parameters[role].owner, git), dir: PARAMETER_DIRS[role] });
|
|
2860
|
+
const { baseImage, harness, codeState, promptSet, initialPrompt } = parameters;
|
|
2861
|
+
const steps = [
|
|
2862
|
+
baseImage.applyParameter(baseImage.config, ctx("baseImage")),
|
|
2863
|
+
harness.applyParameter(harness.config, { ...ctx("harness"), version: harness.version }),
|
|
2864
|
+
"WORKDIR /workspace",
|
|
2865
|
+
codeState.applyParameter(codeState.config, ctx("codeState")),
|
|
2866
|
+
promptSet.applyParameter(promptSet.config, ctx("promptSet")),
|
|
2867
|
+
initialPrompt.applyParameter(initialPrompt.config, { git: recordingGit(initialPrompt.owner, git) })
|
|
2868
|
+
];
|
|
2869
|
+
return { dockerfile: `${steps.filter((step) => step.trim() !== "").join(`
|
|
2870
|
+
`)}
|
|
2871
|
+
`, git };
|
|
2872
|
+
}
|
|
2873
|
+
|
|
2874
|
+
class MissingCommitsError extends Error {
|
|
2875
|
+
}
|
|
2876
|
+
async function checkCommits(needs, repository) {
|
|
2877
|
+
const problems = new Set;
|
|
2878
|
+
const present = new Map;
|
|
2879
|
+
for (const need of needs) {
|
|
2880
|
+
if (!present.has(need.commit))
|
|
2881
|
+
present.set(need.commit, await repository.resolveCommit(need.commit) !== undefined);
|
|
2882
|
+
if (!present.get(need.commit)) {
|
|
2883
|
+
problems.add(`error: commit ${need.commit} (${need.owner}) is not in this repository.
|
|
2884
|
+
Fetch it, or check that it has been pushed.`);
|
|
2885
|
+
} else if (need.kind === "file" && await repository.readFile(need.commit, need.file) === undefined) {
|
|
2886
|
+
problems.add(`error: ${need.file} is not in commit ${need.commit} (${need.owner}).`);
|
|
2887
|
+
}
|
|
2888
|
+
}
|
|
2889
|
+
if (problems.size > 0)
|
|
2890
|
+
throw new MissingCommitsError([...problems].join(`
|
|
2891
|
+
`));
|
|
2892
|
+
}
|
|
2893
|
+
async function assembleBuildContext(parameters, repository, dir) {
|
|
2894
|
+
const { dockerfile, git } = renderPrerun(parameters);
|
|
2895
|
+
await remove(dir);
|
|
2896
|
+
await makeDir(dir);
|
|
2897
|
+
await Bun.write(join(dir, "Dockerfile"), dockerfile);
|
|
2898
|
+
for (const role of Object.keys(PARAMETER_DIRS)) {
|
|
2899
|
+
const { dir: source, exclude = [] } = parameters[role];
|
|
2900
|
+
const target = join(dir, PARAMETER_DIRS[role]);
|
|
2901
|
+
await copy(source, target);
|
|
2902
|
+
for (const path2 of exclude)
|
|
2903
|
+
await remove(join(target, path2));
|
|
2904
|
+
}
|
|
2905
|
+
const written = new Set;
|
|
2906
|
+
for (const need of git) {
|
|
2907
|
+
if (written.has(need.path))
|
|
2908
|
+
continue;
|
|
2909
|
+
written.add(need.path);
|
|
2910
|
+
if (!repository)
|
|
2911
|
+
throw new Error(`${need.owner} reads commit ${need.commit}, but there is no repository to read it from`);
|
|
2912
|
+
const dest = join(dir, need.path);
|
|
2913
|
+
await makeDir(dirname(dest));
|
|
2914
|
+
if (need.kind === "bundle") {
|
|
2915
|
+
const full = await repository.resolveCommit(need.commit);
|
|
2916
|
+
if (!full)
|
|
2917
|
+
throw new Error(`commit ${need.commit} is not in the repository`);
|
|
2918
|
+
await repository.bundle(full, dest);
|
|
2919
|
+
} else {
|
|
2920
|
+
const contents = await repository.readFile(need.commit, need.file);
|
|
2921
|
+
if (contents === undefined)
|
|
2922
|
+
throw new Error(`${need.file} is not in commit ${need.commit}`);
|
|
2923
|
+
await Bun.write(dest, contents);
|
|
2924
|
+
}
|
|
2925
|
+
}
|
|
2926
|
+
}
|
|
2927
|
+
async function prerunStatuses(docker2, repository, permutations) {
|
|
2928
|
+
const unreachable = await docker2.unreachable();
|
|
2929
|
+
if (unreachable !== undefined)
|
|
2930
|
+
return { unreachable };
|
|
2931
|
+
const statuses = new Map;
|
|
2932
|
+
for (const { hash } of permutations) {
|
|
2933
|
+
if (statuses.has(hash))
|
|
2934
|
+
continue;
|
|
2935
|
+
statuses.set(hash, await docker2.imageExists(prerunImageRef(repository, hash)) ? "exists" : "missing");
|
|
2936
|
+
}
|
|
2937
|
+
return { statuses };
|
|
2938
|
+
}
|
|
2939
|
+
var showPrerunStatus = (status) => status === "exists" ? " (exists, reused)" : status === "missing" ? " (will build)" : "";
|
|
2940
|
+
var unreachableNote = (reason) => `Docker can't be reached (${reason}), so whether each prerun image exists is unknown.`;
|
|
2941
|
+
|
|
2942
|
+
// src/images/repository.ts
|
|
2943
|
+
async function git(cwd, args) {
|
|
2944
|
+
const child = Bun.spawn(["git", ...args], { cwd, stdio: ["ignore", "pipe", "pipe"] });
|
|
2945
|
+
const [stdout, stderr] = await Promise.all([
|
|
2946
|
+
new Response(child.stdout).arrayBuffer(),
|
|
2947
|
+
new Response(child.stderr).text(),
|
|
2948
|
+
child.exited
|
|
2949
|
+
]);
|
|
2950
|
+
return { exitCode: child.exitCode ?? 1, stdout: Buffer.from(stdout), stderr };
|
|
2951
|
+
}
|
|
2952
|
+
async function gitOrThrow(cwd, args) {
|
|
2953
|
+
const result = await git(cwd, args);
|
|
2954
|
+
if (result.exitCode !== 0)
|
|
2955
|
+
throw new Error(`git ${args.join(" ")} failed in ${cwd}:
|
|
2956
|
+
${result.stderr.trim()}`);
|
|
2957
|
+
return result.stdout;
|
|
2958
|
+
}
|
|
2959
|
+
async function findRepository(dir) {
|
|
2960
|
+
const { exitCode, stdout, stderr } = await git(dir, ["rev-parse", "--path-format=absolute", "--git-common-dir"]);
|
|
2961
|
+
if (exitCode !== 0 && /not a git repository/i.test(stderr))
|
|
2962
|
+
return;
|
|
2963
|
+
if (exitCode !== 0)
|
|
2964
|
+
throw new Error(`error: git can't read the repository ${dir} is in:
|
|
2965
|
+
${stderr.trim()}`);
|
|
2966
|
+
const objects = join(stdout.toString("utf8").trim(), "objects");
|
|
2967
|
+
return {
|
|
2968
|
+
async resolveCommit(commit) {
|
|
2969
|
+
const { exitCode: exitCode2, stdout: stdout2 } = await git(dir, ["rev-parse", "--verify", "--quiet", `${commit}^{commit}`]);
|
|
2970
|
+
return exitCode2 === 0 ? stdout2.toString("utf8").trim() : undefined;
|
|
2971
|
+
},
|
|
2972
|
+
async readFile(commit, path2) {
|
|
2973
|
+
const { exitCode: exitCode2, stdout: stdout2 } = await git(dir, ["show", `${commit}:${path2}`]);
|
|
2974
|
+
return exitCode2 === 0 ? stdout2 : undefined;
|
|
2975
|
+
},
|
|
2976
|
+
async bundle(commit, dest) {
|
|
2977
|
+
const scratch = await makeTempDir("thunderjar-bundle-");
|
|
2978
|
+
try {
|
|
2979
|
+
await gitOrThrow(scratch, ["init", "--quiet", "--bare"]);
|
|
2980
|
+
await Bun.write(join(scratch, "objects", "info", "alternates"), `${objects}
|
|
2981
|
+
`);
|
|
2982
|
+
await gitOrThrow(scratch, ["update-ref", "refs/heads/code-state", commit]);
|
|
2983
|
+
await gitOrThrow(scratch, ["symbolic-ref", "HEAD", "refs/heads/code-state"]);
|
|
2984
|
+
await gitOrThrow(scratch, ["bundle", "create", "--quiet", dest, "HEAD", "code-state"]);
|
|
2985
|
+
} finally {
|
|
2986
|
+
await remove(scratch);
|
|
2987
|
+
}
|
|
2988
|
+
}
|
|
2989
|
+
};
|
|
2990
|
+
}
|
|
2991
|
+
|
|
2992
|
+
// src/images/prerun-builds.ts
|
|
2993
|
+
var buildContextDir = (rootDir, permutationHash) => join(rootDir, "builds", prerunTag(permutationHash));
|
|
2994
|
+
async function preparePrerunBuilds(options) {
|
|
2995
|
+
const { root, task, permutations, docker: docker2, repository, rootDir, purpose } = options;
|
|
2996
|
+
const found = await prerunStatuses(docker2, repository, permutations);
|
|
2997
|
+
if ("unreachable" in found) {
|
|
2998
|
+
throw new Error(`error: Docker can't be reached (${found.unreachable}). ${purpose} needs Docker.`);
|
|
2999
|
+
}
|
|
3000
|
+
const parameters = new Map;
|
|
3001
|
+
for (const permutation of permutations) {
|
|
3002
|
+
if (found.statuses.get(permutation.hash) !== "missing")
|
|
3003
|
+
continue;
|
|
3004
|
+
parameters.set(permutation.hash, await loadPrerunParameters(root, task, permutation));
|
|
3005
|
+
}
|
|
3006
|
+
const gitNeeds = [...parameters.values()].flatMap((p) => renderPrerun(p).git);
|
|
3007
|
+
let repo;
|
|
3008
|
+
if (gitNeeds.length > 0) {
|
|
3009
|
+
const projectDir = dirname(root);
|
|
3010
|
+
repo = await findRepository(projectDir);
|
|
3011
|
+
if (!repo) {
|
|
3012
|
+
const [{ owner, commit }] = gitNeeds;
|
|
3013
|
+
throw new Error(`error: ${projectDir} is not in a git repository, but ${owner} reads commit ${commit} from it.`);
|
|
3014
|
+
}
|
|
3015
|
+
await checkCommits(gitNeeds, repo);
|
|
3016
|
+
}
|
|
3017
|
+
const missing = new Set(parameters.keys());
|
|
3018
|
+
return {
|
|
3019
|
+
missing,
|
|
3020
|
+
async build(permutationHash) {
|
|
3021
|
+
const declared = parameters.get(permutationHash);
|
|
3022
|
+
if (!declared || !missing.has(permutationHash))
|
|
3023
|
+
throw new Error(`prerun image ${permutationHash} isn't missing`);
|
|
3024
|
+
const ref = prerunImageRef(repository, permutationHash);
|
|
3025
|
+
const contextDir = buildContextDir(rootDir, permutationHash);
|
|
3026
|
+
await assembleBuildContext(declared, repo, contextDir);
|
|
3027
|
+
try {
|
|
3028
|
+
await docker2.build(contextDir, ref);
|
|
3029
|
+
} catch (error) {
|
|
3030
|
+
if (!(error instanceof DockerBuildError))
|
|
3031
|
+
throw error;
|
|
3032
|
+
throw new Error(`error: building ${ref} failed (docker build exited ${error.exitCode}).
|
|
3033
|
+
Its build context is kept in ${contextDir}`);
|
|
3034
|
+
}
|
|
3035
|
+
missing.delete(permutationHash);
|
|
3036
|
+
}
|
|
3037
|
+
};
|
|
3038
|
+
}
|
|
3039
|
+
|
|
3040
|
+
// src/matrix/resolve.ts
|
|
3041
|
+
async function resolveTestCase(root, testCase) {
|
|
3042
|
+
const parameters = join(root, "test-case-parameters");
|
|
3043
|
+
const hashParameter = {
|
|
3044
|
+
baseImage: (n) => hashBaseImage(join(parameters, "baseImages", n)),
|
|
3045
|
+
codeState: (n) => hashCodeState(join(parameters, "codeStates", n)),
|
|
3046
|
+
promptSet: (n) => hashPromptSet(join(parameters, "promptSets", n)),
|
|
3047
|
+
harnessModel: (n) => {
|
|
3048
|
+
const { family, version, model } = parseHarnessModelPair(n);
|
|
3049
|
+
return hashHarnessModelPair(join(parameters, "harnessModels", family), version, model);
|
|
3050
|
+
},
|
|
3051
|
+
initialPrompt: (n) => hashInitialPrompt(join(parameters, "tasks", testCase.task, "initial-prompts", `${n}.ts`))
|
|
3052
|
+
};
|
|
3053
|
+
const hashes = {};
|
|
3054
|
+
for (const axis of AXES) {
|
|
3055
|
+
hashes[axis] = new Map;
|
|
3056
|
+
for (const n of testCase[axis])
|
|
3057
|
+
hashes[axis].set(n, await hashParameter[axis](n));
|
|
3058
|
+
}
|
|
3059
|
+
const permutations = expandPermutations(testCase).map((permutation) => ({
|
|
3060
|
+
...permutation,
|
|
3061
|
+
hash: hashPermutation({
|
|
3062
|
+
baseImage: hashes.baseImage.get(permutation.baseImage),
|
|
3063
|
+
codeState: hashes.codeState.get(permutation.codeState),
|
|
3064
|
+
promptSet: hashes.promptSet.get(permutation.promptSet),
|
|
3065
|
+
harnessModel: hashes.harnessModel.get(permutation.harnessModel),
|
|
3066
|
+
initialPrompt: hashes.initialPrompt.get(permutation.initialPrompt)
|
|
3067
|
+
})
|
|
3068
|
+
}));
|
|
3069
|
+
return { hashes, permutations };
|
|
3070
|
+
}
|
|
3071
|
+
|
|
3072
|
+
// src/platform/signals.ts
|
|
3073
|
+
var COMMON = {
|
|
3074
|
+
SIGHUP: 1,
|
|
3075
|
+
SIGINT: 2,
|
|
3076
|
+
SIGQUIT: 3,
|
|
3077
|
+
SIGILL: 4,
|
|
3078
|
+
SIGTRAP: 5,
|
|
3079
|
+
SIGABRT: 6,
|
|
3080
|
+
SIGIOT: 6,
|
|
3081
|
+
SIGFPE: 8,
|
|
3082
|
+
SIGKILL: 9,
|
|
3083
|
+
SIGSEGV: 11,
|
|
3084
|
+
SIGPIPE: 13,
|
|
3085
|
+
SIGALRM: 14,
|
|
3086
|
+
SIGTERM: 15,
|
|
3087
|
+
SIGXCPU: 24,
|
|
3088
|
+
SIGXFSZ: 25,
|
|
3089
|
+
SIGVTALRM: 26,
|
|
3090
|
+
SIGPROF: 27,
|
|
3091
|
+
SIGWINCH: 28
|
|
3092
|
+
};
|
|
3093
|
+
var LINUX = {
|
|
3094
|
+
SIGBUS: 7,
|
|
3095
|
+
SIGUSR1: 10,
|
|
3096
|
+
SIGUSR2: 12,
|
|
3097
|
+
SIGSTKFLT: 16,
|
|
3098
|
+
SIGCHLD: 17,
|
|
3099
|
+
SIGCONT: 18,
|
|
3100
|
+
SIGSTOP: 19,
|
|
3101
|
+
SIGTSTP: 20,
|
|
3102
|
+
SIGTTIN: 21,
|
|
3103
|
+
SIGTTOU: 22,
|
|
3104
|
+
SIGURG: 23,
|
|
3105
|
+
SIGIO: 29,
|
|
3106
|
+
SIGPOLL: 29,
|
|
3107
|
+
SIGPWR: 30,
|
|
3108
|
+
SIGSYS: 31
|
|
3109
|
+
};
|
|
3110
|
+
var BSD = {
|
|
3111
|
+
SIGEMT: 7,
|
|
3112
|
+
SIGBUS: 10,
|
|
3113
|
+
SIGSYS: 12,
|
|
3114
|
+
SIGURG: 16,
|
|
3115
|
+
SIGSTOP: 17,
|
|
3116
|
+
SIGTSTP: 18,
|
|
3117
|
+
SIGCONT: 19,
|
|
3118
|
+
SIGCHLD: 20,
|
|
3119
|
+
SIGTTIN: 21,
|
|
3120
|
+
SIGTTOU: 22,
|
|
3121
|
+
SIGIO: 23,
|
|
3122
|
+
SIGINFO: 29,
|
|
3123
|
+
SIGUSR1: 30,
|
|
3124
|
+
SIGUSR2: 31
|
|
3125
|
+
};
|
|
3126
|
+
var SIGNALS = { ...COMMON, ...process.platform === "linux" ? LINUX : BSD };
|
|
3127
|
+
function signalNumber(signal) {
|
|
3128
|
+
if (typeof signal === "number")
|
|
3129
|
+
return signal;
|
|
3130
|
+
const number = SIGNALS[signal];
|
|
3131
|
+
if (number === undefined)
|
|
3132
|
+
throw new Error(`unknown signal: ${signal}`);
|
|
3133
|
+
return number;
|
|
3134
|
+
}
|
|
3135
|
+
|
|
3136
|
+
// src/running/runner.ts
|
|
3137
|
+
var defaultRootDir = () => join(tmpdir(), "thunderjar");
|
|
3138
|
+
var exitCodeOf = (proc) => proc.exitCode ?? 128 + signalNumber(proc.signalCode);
|
|
3139
|
+
var shellQuote = (text) => `'${text.replaceAll("'", `'\\''`)}'`;
|
|
3140
|
+
var containerWrapper = (command, timeoutSeconds) => [
|
|
3141
|
+
"mkdir -p /thunderjar",
|
|
3142
|
+
"start=$(date +%s%3N)",
|
|
3143
|
+
`timeout -k 5 ${timeoutSeconds} sh -c ${shellQuote(command)} >&2`,
|
|
3144
|
+
"code=$?",
|
|
3145
|
+
"end=$(date +%s%3N)",
|
|
3146
|
+
`echo "{\\"startedAt\\":$start,\\"finishedAt\\":$end,\\"exitCode\\":$code}" > /thunderjar/run.json`,
|
|
3147
|
+
"cat /thunderjar/run.json"
|
|
3148
|
+
].join(`
|
|
3149
|
+
`);
|
|
3150
|
+
function createDockerRunner(docker2, repository) {
|
|
3151
|
+
return {
|
|
3152
|
+
async runContainer(run) {
|
|
3153
|
+
const prerun = imageRef(repository, run.prerunTag);
|
|
3154
|
+
const name = `thunderjar-${run.postrunTag}-${crypto.randomUUID().slice(0, 8)}`;
|
|
3155
|
+
try {
|
|
3156
|
+
const { exitCode, stdout } = await docker2.run(prerun, name, containerWrapper(run.command, run.timeoutSeconds));
|
|
3157
|
+
const runJson = stdout.trim().split(`
|
|
3158
|
+
`).at(-1) ?? "";
|
|
3159
|
+
if (exitCode !== 0 || !runJson.startsWith("{")) {
|
|
3160
|
+
throw new Error(`error: running a container from ${prerun} failed (docker run exited ${exitCode}).`);
|
|
3161
|
+
}
|
|
3162
|
+
await docker2.commit(name, imageRef(repository, run.postrunTag), prerun);
|
|
3163
|
+
return JSON.parse(runJson);
|
|
3164
|
+
} finally {
|
|
3165
|
+
await docker2.removeContainer(name).catch(() => {});
|
|
3166
|
+
}
|
|
3167
|
+
}
|
|
3168
|
+
};
|
|
3169
|
+
}
|
|
3170
|
+
async function exec(command, cwd) {
|
|
3171
|
+
const child = Bun.spawn(["sh", "-c", command], { cwd, stdio: ["ignore", "pipe", "pipe"] });
|
|
3172
|
+
const [stdout, stderr] = await Promise.all([new Response(child.stdout).text(), new Response(child.stderr).text(), child.exited]);
|
|
3173
|
+
return { stdout, stderr, exitCode: exitCodeOf(child) };
|
|
3174
|
+
}
|
|
3175
|
+
function createDockerMeasurementOpener(docker2, repository) {
|
|
3176
|
+
return {
|
|
3177
|
+
async openMeasurementContext(postrunTag2) {
|
|
3178
|
+
const scratch = await makeTempDir("thunderjar-measure-");
|
|
3179
|
+
const workdir = join(scratch, "workspace");
|
|
3180
|
+
const name = `thunderjar-measure-${crypto.randomUUID().slice(0, 8)}`;
|
|
3181
|
+
try {
|
|
3182
|
+
await makeDir(workdir);
|
|
3183
|
+
await docker2.create(imageRef(repository, postrunTag2), name);
|
|
3184
|
+
await docker2.copyFrom(name, "/workspace/.", workdir);
|
|
3185
|
+
const resultPath = join(scratch, "result.json");
|
|
3186
|
+
await docker2.copyFrom(name, "/thunderjar/result.json", resultPath).catch(() => {});
|
|
3187
|
+
const harnessResult = await exists(resultPath) ? await Bun.file(resultPath).text() : "";
|
|
3188
|
+
return scratchContext(workdir, harnessResult, scratch);
|
|
3189
|
+
} catch (error) {
|
|
3190
|
+
await remove(scratch);
|
|
3191
|
+
throw error;
|
|
3192
|
+
} finally {
|
|
3193
|
+
await docker2.removeContainer(name).catch(() => {});
|
|
3194
|
+
}
|
|
3195
|
+
}
|
|
3196
|
+
};
|
|
3197
|
+
}
|
|
3198
|
+
function scratchContext(workdir, harnessResult, scratch) {
|
|
3199
|
+
return {
|
|
3200
|
+
workdir,
|
|
3201
|
+
harnessResult,
|
|
3202
|
+
exec: (command) => exec(command, workdir),
|
|
3203
|
+
readFile: (path2) => Bun.file(resolve(workdir, path2)).text(),
|
|
3204
|
+
async writeFile(path2, contents) {
|
|
3205
|
+
await Bun.write(resolve(workdir, path2), contents);
|
|
3206
|
+
},
|
|
3207
|
+
glob: async (pattern) => (await Array.fromAsync(new Bun.Glob(pattern).scan({ cwd: workdir, dot: true }))).toSorted(),
|
|
3208
|
+
close: () => remove(scratch)
|
|
3209
|
+
};
|
|
3210
|
+
}
|
|
3211
|
+
|
|
3212
|
+
// src/cli/commands/format.ts
|
|
3213
|
+
var count = (n, noun) => `${n} ${noun}${n === 1 ? "" : "s"}`;
|
|
3214
|
+
var showValue = (value) => typeof value === "object" && value !== null && !Array.isArray(value) ? `{${Object.entries(value).map(([key, v]) => `${key}: ${showValue(v)}`).join(", ")}}` : JSON.stringify(value) ?? String(value);
|
|
3215
|
+
var showOutcome = (outcome) => outcome.outcome === "measured" ? showValue(outcome.data) : outcome.outcome === "skipped" ? "skipped" : `erroredWhileMeasuring: ${outcome.error.message}`;
|
|
3216
|
+
var seconds = (ms) => `${(ms / 1000).toFixed(2)}s`;
|
|
3217
|
+
var duration = ({ startedAt, finishedAt }) => seconds(finishedAt - startedAt);
|
|
3218
|
+
var minute = (iso) => iso.slice(0, 16).replace("T", " ");
|
|
3219
|
+
|
|
3220
|
+
// src/cli/commands/options.ts
|
|
3221
|
+
function valueOption(flags, description) {
|
|
3222
|
+
return new Option(flags, description).argParser((value, previous) => {
|
|
3223
|
+
if (previous !== undefined)
|
|
3224
|
+
throw new InvalidArgumentError("It may only be given once.");
|
|
3225
|
+
if (value.startsWith("-"))
|
|
3226
|
+
throw new InvalidArgumentError("It needs a value, not another flag.");
|
|
3227
|
+
return value;
|
|
3228
|
+
});
|
|
3229
|
+
}
|
|
3230
|
+
var imagePrefixOption = (description) => valueOption("--image-prefix <name>", description);
|
|
3231
|
+
var dataDirOption = () => valueOption("--data-dir <dir>", "keep the run data store in this folder, in place of .thunderjar/.data/");
|
|
3232
|
+
|
|
3233
|
+
// src/cli/commands/build/build.ts
|
|
3234
|
+
async function build(name, options = {}) {
|
|
3235
|
+
const {
|
|
3236
|
+
cwd = process.cwd(),
|
|
3237
|
+
log = (text) => console.log(text),
|
|
3238
|
+
docker: docker2 = dockerCli(),
|
|
3239
|
+
dryRun = false,
|
|
3240
|
+
repository = LOCAL_REPOSITORY,
|
|
3241
|
+
rootDir = defaultRootDir(),
|
|
3242
|
+
now = Date.now
|
|
3243
|
+
} = options;
|
|
3244
|
+
const root = findThunderjarRoot(cwd);
|
|
3245
|
+
const testCase = await loadTestCase(root, name);
|
|
3246
|
+
const { permutations } = await resolveTestCase(root, testCase);
|
|
3247
|
+
const rows = permutationRows(permutations);
|
|
3248
|
+
const numberWidth = Math.max(2, String(permutations.length).length);
|
|
3249
|
+
const pad = " ".repeat(numberWidth + 2);
|
|
3250
|
+
if (dryRun) {
|
|
3251
|
+
log(`Dockerfiles for ${name} (${count(permutations.length, "permutation")}). Nothing is built.`);
|
|
3252
|
+
for (const [n, permutation] of permutations.entries()) {
|
|
3253
|
+
const { dockerfile } = renderPrerun(await loadPrerunParameters(root, testCase.task, permutation));
|
|
3254
|
+
log("");
|
|
3255
|
+
log(`${String(n + 1).padStart(numberWidth)} ${rows[n]}`);
|
|
3256
|
+
log(`${pad}${prerunImageRef(repository, permutation.hash)}`);
|
|
3257
|
+
log("");
|
|
3258
|
+
log(dockerfile.trimEnd().replace(/^(?=.)/gm, pad));
|
|
3259
|
+
}
|
|
3260
|
+
return;
|
|
3261
|
+
}
|
|
3262
|
+
const builds = await preparePrerunBuilds({
|
|
3263
|
+
root,
|
|
3264
|
+
task: testCase.task,
|
|
3265
|
+
permutations,
|
|
3266
|
+
docker: docker2,
|
|
3267
|
+
repository,
|
|
3268
|
+
rootDir,
|
|
3269
|
+
purpose: "Building prerun images"
|
|
3270
|
+
});
|
|
3271
|
+
const toBuild = builds.missing.size;
|
|
3272
|
+
if (toBuild > 0) {
|
|
3273
|
+
log(`Building ${count(toBuild, "prerun image")} for ${name} (${count(permutations.length, "permutation")})`);
|
|
3274
|
+
log(`Build contexts in ${join(rootDir, "builds")}`);
|
|
3275
|
+
} else {
|
|
3276
|
+
log(`Every prerun image for ${name} exists (${count(permutations.length, "permutation")})`);
|
|
3277
|
+
}
|
|
3278
|
+
for (const [n, permutation] of permutations.entries()) {
|
|
3279
|
+
const ref = prerunImageRef(repository, permutation.hash);
|
|
3280
|
+
log("");
|
|
3281
|
+
log(`${String(n + 1).padStart(numberWidth)} ${rows[n]}`);
|
|
3282
|
+
if (!builds.missing.has(permutation.hash)) {
|
|
3283
|
+
log(`${pad}${ref} reused`);
|
|
3284
|
+
continue;
|
|
3285
|
+
}
|
|
3286
|
+
const started = now();
|
|
3287
|
+
await builds.build(permutation.hash);
|
|
3288
|
+
log(`${pad}${ref} built (${seconds(now() - started)})`);
|
|
3289
|
+
}
|
|
3290
|
+
log("");
|
|
3291
|
+
log(`${count(permutations.length, "prerun image")}: ${toBuild} built, ${permutations.length - toBuild} reused`);
|
|
3292
|
+
}
|
|
3293
|
+
function createBuildCommand() {
|
|
3294
|
+
return new Command("build").description("Build a test case's prerun images.").argument("<test case>", "the test case whose prerun images to build").option("--dry-run", "print each permutation's Dockerfile instead of building").addOption(imagePrefixOption("the image repository prerun images are tagged in")).action(async (name, options) => {
|
|
3295
|
+
await build(name, { dryRun: options.dryRun ?? false, repository: options.imagePrefix });
|
|
3296
|
+
});
|
|
3297
|
+
}
|
|
3298
|
+
|
|
3299
|
+
// src/store/run-data-store.ts
|
|
3300
|
+
var runDataStorePath = (cwd, dataDir) => join(dataDir === undefined ? join(findThunderjarRoot(cwd), ".data") : resolve(cwd, dataDir), "runs.json");
|
|
3301
|
+
function summarizeExecutions(records) {
|
|
3302
|
+
const byId = new Map;
|
|
3303
|
+
for (const { executionId, testCase, ranAt } of records) {
|
|
3304
|
+
const summary = byId.get(executionId);
|
|
3305
|
+
if (!summary)
|
|
3306
|
+
byId.set(executionId, { executionId, testCase, ranAt, containerRuns: 1 });
|
|
3307
|
+
else {
|
|
3308
|
+
summary.containerRuns++;
|
|
3309
|
+
if (ranAt < summary.ranAt)
|
|
3310
|
+
summary.ranAt = ranAt;
|
|
3311
|
+
}
|
|
3312
|
+
}
|
|
3313
|
+
const newestFirst = (a, b) => a < b ? 1 : a > b ? -1 : 0;
|
|
3314
|
+
return [...byId.values()].toSorted((a, b) => newestFirst(a.ranAt, b.ranAt) || newestFirst(a.executionId, b.executionId));
|
|
3315
|
+
}
|
|
3316
|
+
function inExecutionOrder(records) {
|
|
3317
|
+
const place = matrixPlaces(records);
|
|
3318
|
+
return records.toSorted((a, b) => place.get(a.permutationHash) - place.get(b.permutationHash) || a.iteration - b.iteration);
|
|
3319
|
+
}
|
|
3320
|
+
function matrixPlaces(records) {
|
|
3321
|
+
const place = new Map;
|
|
3322
|
+
for (const { permutationHash } of records)
|
|
3323
|
+
if (!place.has(permutationHash))
|
|
3324
|
+
place.set(permutationHash, place.size);
|
|
3325
|
+
return place;
|
|
3326
|
+
}
|
|
3327
|
+
function jsonFileStore(path2) {
|
|
3328
|
+
const read = async () => {
|
|
3329
|
+
let text;
|
|
3330
|
+
try {
|
|
3331
|
+
text = await Bun.file(path2).text();
|
|
3332
|
+
} catch (error) {
|
|
3333
|
+
if (error.code === "ENOENT")
|
|
3334
|
+
return { containerRuns: [] };
|
|
3335
|
+
throw error;
|
|
3336
|
+
}
|
|
3337
|
+
return JSON.parse(text);
|
|
3338
|
+
};
|
|
3339
|
+
const write = async (data) => {
|
|
3340
|
+
await makeDir(dirname(path2));
|
|
3341
|
+
const temp = `${path2}.${crypto.randomUUID()}.tmp`;
|
|
3342
|
+
await Bun.write(temp, `${JSON.stringify(data, null, 2)}
|
|
3343
|
+
`);
|
|
3344
|
+
await move(temp, path2);
|
|
3345
|
+
};
|
|
3346
|
+
return {
|
|
3347
|
+
async recordContainerRun(record) {
|
|
3348
|
+
const data = await read();
|
|
3349
|
+
data.containerRuns.push(record);
|
|
3350
|
+
await write(data);
|
|
3351
|
+
},
|
|
3352
|
+
async recordMeasures(postrunImage, measures) {
|
|
3353
|
+
const data = await read();
|
|
3354
|
+
const record = data.containerRuns.find((r) => r.postrunImage === postrunImage);
|
|
3355
|
+
if (!record)
|
|
3356
|
+
throw new Error(`no container run recorded for ${postrunImage}`);
|
|
3357
|
+
record.result = { ...record.result, ...measures };
|
|
3358
|
+
await write(data);
|
|
3359
|
+
},
|
|
3360
|
+
async listExecutions(testCase) {
|
|
3361
|
+
const { containerRuns } = await read();
|
|
3362
|
+
return summarizeExecutions(testCase === undefined ? containerRuns : containerRuns.filter((r) => r.testCase === testCase));
|
|
3363
|
+
},
|
|
3364
|
+
async getExecution(executionId) {
|
|
3365
|
+
const { containerRuns } = await read();
|
|
3366
|
+
return inExecutionOrder(containerRuns.filter((r) => r.executionId === executionId));
|
|
3367
|
+
},
|
|
3368
|
+
async getContainerRun(postrunImage) {
|
|
3369
|
+
const { containerRuns } = await read();
|
|
3370
|
+
return containerRuns.find((r) => r.postrunImage === postrunImage);
|
|
3371
|
+
},
|
|
3372
|
+
async recordAggregationResult(result) {
|
|
3373
|
+
const data = await read();
|
|
3374
|
+
const results = data.aggregationResults ??= [];
|
|
3375
|
+
const same = results.findIndex((r) => r.executionId === result.executionId && r.permutationHash === result.permutationHash);
|
|
3376
|
+
if (same === -1)
|
|
3377
|
+
results.push(result);
|
|
3378
|
+
else
|
|
3379
|
+
results[same] = result;
|
|
3380
|
+
await write(data);
|
|
3381
|
+
},
|
|
3382
|
+
async getAggregationResults(executionId) {
|
|
3383
|
+
const { containerRuns, aggregationResults = [] } = await read();
|
|
3384
|
+
const place = matrixPlaces(containerRuns.filter((r) => r.executionId === executionId));
|
|
3385
|
+
const placeOf = ({ permutationHash }) => place.get(permutationHash) ?? place.size;
|
|
3386
|
+
return aggregationResults.filter((r) => r.executionId === executionId).toSorted((a, b) => placeOf(a) - placeOf(b));
|
|
3387
|
+
}
|
|
3388
|
+
};
|
|
3389
|
+
}
|
|
3390
|
+
|
|
3391
|
+
// src/cli/commands/executions/executions.ts
|
|
3392
|
+
async function executions(testCase, { cwd = process.cwd(), dataDir } = {}) {
|
|
3393
|
+
const store = jsonFileStore(runDataStorePath(cwd, dataDir));
|
|
3394
|
+
const listed = await store.listExecutions(testCase);
|
|
3395
|
+
if (listed.length === 0) {
|
|
3396
|
+
return testCase === undefined ? "No executions recorded." : `No executions recorded for ${testCase}.`;
|
|
3397
|
+
}
|
|
3398
|
+
const width = Math.max(...listed.map((e) => e.testCase.length));
|
|
3399
|
+
return listed.map((e) => `${e.executionId} ${e.testCase.padEnd(width)} ${minute(e.ranAt)}`).join(`
|
|
3400
|
+
`);
|
|
3401
|
+
}
|
|
3402
|
+
function createExecutionsCommand() {
|
|
3403
|
+
return new Command("executions").description("List recorded executions.").argument("[test case]", "list only this test case's executions").addOption(dataDirOption()).action(async (testCase, options) => {
|
|
3404
|
+
console.log(await executions(testCase, { dataDir: options.dataDir }));
|
|
3405
|
+
});
|
|
3406
|
+
}
|
|
3407
|
+
|
|
3408
|
+
// src/cli/commands/generate/parameter-names.ts
|
|
3409
|
+
async function listDirNames(dir) {
|
|
3410
|
+
try {
|
|
3411
|
+
return listEntries(dir).filter((entry) => entry.isDirectory).map((entry) => entry.path).sort();
|
|
3412
|
+
} catch {
|
|
3413
|
+
return [];
|
|
3414
|
+
}
|
|
3415
|
+
}
|
|
3416
|
+
async function listFileStems(dir) {
|
|
3417
|
+
try {
|
|
3418
|
+
return listEntries(dir).filter((entry) => !entry.isDirectory && entry.path.endsWith(".ts")).map((entry) => entry.path.slice(0, -".ts".length)).sort();
|
|
3419
|
+
} catch {
|
|
3420
|
+
return [];
|
|
3421
|
+
}
|
|
3422
|
+
}
|
|
3423
|
+
async function loadFamilyNames(root, family, problems) {
|
|
3424
|
+
const familyDir = `test-case-parameters/harnessModels/${family}`;
|
|
3425
|
+
const models = await loadDeclaration(root, `${familyDir}/models.ts`, "declareHarnessModels", problems);
|
|
3426
|
+
const versions = await loadDeclaration(root, `${familyDir}/versions.ts`, "declareHarnessVersions", problems);
|
|
3427
|
+
return {
|
|
3428
|
+
models: Object.keys(models ?? {}).sort(),
|
|
3429
|
+
versions: Object.keys(versions ?? {}).sort()
|
|
3430
|
+
};
|
|
3431
|
+
}
|
|
3432
|
+
async function collectParameterNames(root) {
|
|
3433
|
+
const parameters = join(root, "test-case-parameters");
|
|
3434
|
+
const [baseImages, codeStates, promptSets, taskNames, familyNames] = await Promise.all([
|
|
3435
|
+
listDirNames(join(parameters, "baseImages")),
|
|
3436
|
+
listDirNames(join(parameters, "codeStates")),
|
|
3437
|
+
listDirNames(join(parameters, "promptSets")),
|
|
3438
|
+
listDirNames(join(parameters, "tasks")),
|
|
3439
|
+
listDirNames(join(parameters, "harnessModels"))
|
|
3440
|
+
]);
|
|
3441
|
+
const tasks = {};
|
|
3442
|
+
for (const task of taskNames) {
|
|
3443
|
+
tasks[task] = await listFileStems(join(parameters, "tasks", task, "initial-prompts"));
|
|
3444
|
+
}
|
|
3445
|
+
const harnessFamilies = {};
|
|
3446
|
+
const problems = [];
|
|
3447
|
+
for (const family of familyNames) {
|
|
3448
|
+
harnessFamilies[family] = await loadFamilyNames(root, family, problems);
|
|
3449
|
+
}
|
|
3450
|
+
if (problems.length > 0)
|
|
3451
|
+
throw new InvalidConfigError(problems);
|
|
3452
|
+
return { baseImages, codeStates, promptSets, tasks, harnessFamilies };
|
|
3453
|
+
}
|
|
3454
|
+
function literalUnion(names) {
|
|
3455
|
+
return names.length === 0 ? "never" : names.map((name) => JSON.stringify(name)).join(" | ");
|
|
3456
|
+
}
|
|
3457
|
+
function renderParameterNames(names) {
|
|
3458
|
+
const lines2 = [
|
|
3459
|
+
"// _generated/parameter-names.d.ts \u2014 generated by `thunderjar generate`, not edited",
|
|
3460
|
+
"",
|
|
3461
|
+
`type BaseImageName = ${literalUnion(names.baseImages)};`,
|
|
3462
|
+
`type CodeStateName = ${literalUnion(names.codeStates)};`,
|
|
3463
|
+
`type PromptSetName = ${literalUnion(names.promptSets)};`,
|
|
3464
|
+
`type TaskName = ${literalUnion(Object.keys(names.tasks))};`
|
|
3465
|
+
];
|
|
3466
|
+
const tasks = Object.entries(names.tasks);
|
|
3467
|
+
if (tasks.length === 0) {
|
|
3468
|
+
lines2.push("type InitialPromptNames = Record<string, never>;");
|
|
3469
|
+
} else {
|
|
3470
|
+
lines2.push("type InitialPromptNames = {");
|
|
3471
|
+
for (const [task, prompts] of tasks) {
|
|
3472
|
+
lines2.push(` ${JSON.stringify(task)}: ${literalUnion(prompts)};`);
|
|
3473
|
+
}
|
|
3474
|
+
lines2.push("};");
|
|
3475
|
+
}
|
|
3476
|
+
const pairs = Object.entries(names.harnessFamilies).filter(([, { versions, models }]) => versions.length > 0 && models.length > 0).map(([family, { versions, models }]) => "`" + family + "@${" + literalUnion(versions) + "}/${" + literalUnion(models) + "}`");
|
|
3477
|
+
if (pairs.length === 0) {
|
|
3478
|
+
lines2.push("type HarnessModelName = never;");
|
|
3479
|
+
} else if (pairs.length === 1) {
|
|
3480
|
+
lines2.push(`type HarnessModelName = ${pairs[0]};`);
|
|
3481
|
+
} else {
|
|
3482
|
+
lines2.push("type HarnessModelName =");
|
|
3483
|
+
lines2.push(...pairs.map((pair) => ` | ${pair}`));
|
|
3484
|
+
lines2[lines2.length - 1] += ";";
|
|
3485
|
+
}
|
|
3486
|
+
lines2.push("");
|
|
3487
|
+
return lines2.join(`
|
|
3488
|
+
`);
|
|
3489
|
+
}
|
|
3490
|
+
async function writeGeneratedTypes(root) {
|
|
3491
|
+
const names = await collectParameterNames(root);
|
|
3492
|
+
const content = renderParameterNames(names);
|
|
3493
|
+
const path2 = join(root, "_generated", "parameter-names.d.ts");
|
|
3494
|
+
await Bun.write(path2, content);
|
|
3495
|
+
return { path: path2, content };
|
|
3496
|
+
}
|
|
3497
|
+
|
|
2094
3498
|
// src/cli/commands/generate/generate.ts
|
|
3499
|
+
async function generate(cwd = process.cwd()) {
|
|
3500
|
+
const root = findThunderjarRoot(cwd);
|
|
3501
|
+
const { path: path2 } = await writeGeneratedTypes(root);
|
|
3502
|
+
return { path: path2 };
|
|
3503
|
+
}
|
|
2095
3504
|
function createGenerateCommand() {
|
|
2096
|
-
return new Command("generate").description("
|
|
2097
|
-
|
|
3505
|
+
return new Command("generate").description("Write the parameter-name types for this project.").action(async () => {
|
|
3506
|
+
const { path: path2 } = await generate();
|
|
3507
|
+
console.log(`generated ${path2}`);
|
|
2098
3508
|
});
|
|
2099
3509
|
}
|
|
2100
3510
|
|
|
2101
3511
|
// src/cli/commands/init/init.ts
|
|
3512
|
+
var config = `import { declareConfig, sqliteStore } from "thunderjar";
|
|
3513
|
+
|
|
3514
|
+
export default declareConfig({
|
|
3515
|
+
imageStore: { repository: "ghcr.io/your-org/thunderjar-images" },
|
|
3516
|
+
runDataStore: sqliteStore({ path: "./.thunderjar/.data/runs.sqlite" }),
|
|
3517
|
+
});
|
|
3518
|
+
`;
|
|
3519
|
+
var stub = (fn, body) => `import { ${fn} } from "thunderjar";
|
|
3520
|
+
|
|
3521
|
+
export default ${fn}(${body});
|
|
3522
|
+
`;
|
|
3523
|
+
var baseImage = stub("declareBaseImage", `{
|
|
3524
|
+
applyParameter: () => \`FROM node:20-bookworm\`,
|
|
3525
|
+
}`);
|
|
3526
|
+
var codeState = `import { checkoutCommit, declareCodeState } from "thunderjar";
|
|
3527
|
+
|
|
3528
|
+
export default declareCodeState({
|
|
3529
|
+
config: { commit: "a1b2c3" },
|
|
3530
|
+
applyParameter: (config, ctx) => \`\${checkoutCommit(ctx, config.commit)}\\nRUN npm ci\`,
|
|
3531
|
+
});
|
|
3532
|
+
`;
|
|
3533
|
+
var promptSet = stub("declarePromptSet", `{
|
|
3534
|
+
config: { commit: "e4f5a6" },
|
|
3535
|
+
applyParameter: (config, ctx) => \`COPY \${ctx.git.show(config.commit, "prompts/snerk.md")} /workspace/CLAUDE.md\`,
|
|
3536
|
+
}`);
|
|
3537
|
+
var harnessIndex = stub("declareHarness", `{
|
|
3538
|
+
applyParameter: (_config, ctx) => \`RUN npm install -g @anthropic-ai/claude-code@\${ctx.version}\`,
|
|
3539
|
+
cli: (ctx) => \`claude -p "\${ctx.initialPrompt}" --model \${ctx.modelId} > \${ctx.resultPath}\`,
|
|
3540
|
+
requiredEnv: ["ANTHROPIC_API_KEY"],
|
|
3541
|
+
}`);
|
|
3542
|
+
var harnessModels = stub("declareHarnessModels", `{
|
|
3543
|
+
"haiku-4-5": "claude-haiku-4-5-20251001",
|
|
3544
|
+
}`);
|
|
3545
|
+
var harnessVersions = stub("declareHarnessVersions", `{
|
|
3546
|
+
"2.1.283": { version: "2.1.283" },
|
|
3547
|
+
}`);
|
|
3548
|
+
var collectTokenCosts = stub("declareTokenCostCollector", `(_raw, _ctx) => {
|
|
3549
|
+
return { outcome: "unavailable", reason: "notReported" };
|
|
3550
|
+
}`);
|
|
3551
|
+
var initialPrompt = stub("declareInitialPrompt", `{
|
|
3552
|
+
prompt: "Create a file isPrime.ts exporting a function that tests for primality.",
|
|
3553
|
+
timeoutSeconds: 600,
|
|
3554
|
+
applyParameter: () => \`\`,
|
|
3555
|
+
}`);
|
|
3556
|
+
var measurement = stub("declareMeasurement", `{
|
|
3557
|
+
instrument: "fileCreated",
|
|
3558
|
+
config: { glob: "**/isPrime.ts" },
|
|
3559
|
+
aggregate: ["rate"],
|
|
3560
|
+
}`);
|
|
3561
|
+
var testCase = stub("declareTestCase", `{
|
|
3562
|
+
baseImage: ["node20"],
|
|
3563
|
+
codeState: ["baseline"],
|
|
3564
|
+
promptSet: ["snerk"],
|
|
3565
|
+
harnessModel: ["claude-code@2.1.283/haiku-4-5"],
|
|
3566
|
+
task: "is-prime",
|
|
3567
|
+
initialPrompt: ["plain"],
|
|
3568
|
+
iterations: 1,
|
|
3569
|
+
}`);
|
|
3570
|
+
var tsconfig = JSON.stringify({
|
|
3571
|
+
compilerOptions: {
|
|
3572
|
+
target: "ESNext",
|
|
3573
|
+
module: "ESNext",
|
|
3574
|
+
moduleResolution: "bundler",
|
|
3575
|
+
strict: true,
|
|
3576
|
+
noEmit: true,
|
|
3577
|
+
skipLibCheck: true,
|
|
3578
|
+
allowImportingTsExtensions: true
|
|
3579
|
+
},
|
|
3580
|
+
include: ["**/*.ts"],
|
|
3581
|
+
exclude: ["node_modules", ".data", "**/*.template.ts"]
|
|
3582
|
+
}, null, 2) + `
|
|
3583
|
+
`;
|
|
3584
|
+
var P = "test-case-parameters";
|
|
3585
|
+
function scaffold() {
|
|
3586
|
+
return {
|
|
3587
|
+
"thunderjar.config.ts": config,
|
|
3588
|
+
"tsconfig.json": tsconfig,
|
|
3589
|
+
"test-cases/example.ts": testCase,
|
|
3590
|
+
[`${P}/baseImages/node20/index.ts`]: baseImage,
|
|
3591
|
+
[`${P}/codeStates/baseline/index.ts`]: codeState,
|
|
3592
|
+
[`${P}/promptSets/snerk/index.ts`]: promptSet,
|
|
3593
|
+
[`${P}/harnessModels/claude-code/index.ts`]: harnessIndex,
|
|
3594
|
+
[`${P}/harnessModels/claude-code/models.ts`]: harnessModels,
|
|
3595
|
+
[`${P}/harnessModels/claude-code/versions.ts`]: harnessVersions,
|
|
3596
|
+
[`${P}/harnessModels/claude-code/collectTokenCosts.ts`]: collectTokenCosts,
|
|
3597
|
+
[`${P}/tasks/is-prime/initial-prompts/plain.ts`]: initialPrompt,
|
|
3598
|
+
[`${P}/tasks/is-prime/measurements/isPrimeTsExists.ts`]: measurement
|
|
3599
|
+
};
|
|
3600
|
+
}
|
|
3601
|
+
async function init(projectRoot) {
|
|
3602
|
+
const root = join(projectRoot, ".thunderjar");
|
|
3603
|
+
const nested = !existsSync(join(projectRoot, "package.json"));
|
|
3604
|
+
const files = scaffold();
|
|
3605
|
+
if (nested) {
|
|
3606
|
+
files["package.json"] = JSON.stringify({ name: "thunderjar-config", private: true, type: "module", dependencies: { thunderjar: "latest" } }, null, 2) + `
|
|
3607
|
+
`;
|
|
3608
|
+
}
|
|
3609
|
+
files[".gitignore"] = ["_generated/", ".data/", ...nested ? ["node_modules/"] : []].join(`
|
|
3610
|
+
`) + `
|
|
3611
|
+
`;
|
|
3612
|
+
const result = { created: [], skipped: [] };
|
|
3613
|
+
for (const [rel, content] of Object.entries(files)) {
|
|
3614
|
+
const path2 = join(root, rel);
|
|
3615
|
+
if (existsSync(path2)) {
|
|
3616
|
+
result.skipped.push(rel);
|
|
3617
|
+
continue;
|
|
3618
|
+
}
|
|
3619
|
+
await Bun.write(path2, content);
|
|
3620
|
+
result.created.push(rel);
|
|
3621
|
+
}
|
|
3622
|
+
return result;
|
|
3623
|
+
}
|
|
2102
3624
|
function createInitCommand() {
|
|
2103
|
-
return new Command("init").description("
|
|
2104
|
-
|
|
3625
|
+
return new Command("init").description("Create .thunderjar/ in this project.").action(async () => {
|
|
3626
|
+
const { created, skipped } = await init(process.cwd());
|
|
3627
|
+
for (const f of created)
|
|
3628
|
+
console.log(`created .thunderjar/${f}`);
|
|
3629
|
+
for (const f of skipped)
|
|
3630
|
+
console.log(`exists .thunderjar/${f}`);
|
|
3631
|
+
});
|
|
3632
|
+
}
|
|
3633
|
+
|
|
3634
|
+
// src/cli/commands/plan/plan.ts
|
|
3635
|
+
var LABELS = ["baseImage", "codeState", "promptSet", "harnessModel", "task", "initialPrompt"];
|
|
3636
|
+
var LABEL_WIDTH = Math.max(...LABELS.map((label) => label.length)) + 2;
|
|
3637
|
+
async function plan(name, cwd = process.cwd(), options = {}) {
|
|
3638
|
+
const { docker: docker2 = dockerCli(), repository = LOCAL_REPOSITORY } = options;
|
|
3639
|
+
const root = findThunderjarRoot(cwd);
|
|
3640
|
+
const testCase2 = await loadTestCase(root, name);
|
|
3641
|
+
const { hashes, permutations } = await resolveTestCase(root, testCase2);
|
|
3642
|
+
const found = await prerunStatuses(docker2, repository, permutations);
|
|
3643
|
+
const status = (hash) => ("statuses" in found) ? showPrerunStatus(found.statuses.get(hash)) : "";
|
|
3644
|
+
const parameterLines = LABELS.map((label) => {
|
|
3645
|
+
const value = label === "task" ? testCase2.task : testCase2[label].map((n) => `${n} (${abbreviateHash(hashes[label].get(n))})`).join(", ");
|
|
3646
|
+
return `${(label + ":").padEnd(LABEL_WIDTH)}${value}`;
|
|
3647
|
+
});
|
|
3648
|
+
const runs = permutations.length * testCase2.iterations;
|
|
3649
|
+
const summary = `Matrix shape: ${matrixShape(testCase2).join("/")} ` + `(${count(permutations.length, "permutation")} \xD7 ${count(testCase2.iterations, "iteration")} = ${count(runs, "container run")})`;
|
|
3650
|
+
const rows = permutationRows(permutations);
|
|
3651
|
+
const numberWidth = Math.max(2, String(permutations.length).length);
|
|
3652
|
+
const entries = permutations.map((permutation, n) => `${String(n + 1).padStart(numberWidth)} ${rows[n]}
|
|
3653
|
+
` + `${" ".repeat(numberWidth + 2)}${prerunTag(permutation.hash)}${status(permutation.hash)}`);
|
|
3654
|
+
const note = "unreachable" in found ? ["", unreachableNote(found.unreachable)] : [];
|
|
3655
|
+
return [...parameterLines, "", summary, "", entries.join(`
|
|
3656
|
+
|
|
3657
|
+
`), ...note].join(`
|
|
3658
|
+
`);
|
|
3659
|
+
}
|
|
3660
|
+
function createPlanCommand() {
|
|
3661
|
+
return new Command("plan").description("Print what a run of a test case would do.").argument("<test case>", "the test case to plan").addOption(imagePrefixOption("the image repository prerun images are looked up in")).action(async (name, options) => {
|
|
3662
|
+
console.log(await plan(name, process.cwd(), { repository: options.imagePrefix }));
|
|
3663
|
+
});
|
|
3664
|
+
}
|
|
3665
|
+
|
|
3666
|
+
// src/measuring/aggregate.ts
|
|
3667
|
+
var sum = (xs) => xs.reduce((a, b) => a + b, 0);
|
|
3668
|
+
var mean = (xs) => sum(xs) / xs.length;
|
|
3669
|
+
var median = (xs) => {
|
|
3670
|
+
const sorted = xs.toSorted((a, b) => a - b);
|
|
3671
|
+
const mid = sorted.length >> 1;
|
|
3672
|
+
return sorted.length % 2 === 1 ? sorted[mid] : (sorted[mid - 1] + sorted[mid]) / 2;
|
|
3673
|
+
};
|
|
3674
|
+
var byKey = (records, combine) => {
|
|
3675
|
+
const keys = [...new Set(records.flatMap((r) => Object.keys(r)))];
|
|
3676
|
+
return Object.fromEntries(keys.map((key) => [key, combine(records.map((r) => r[key] ?? 0))]));
|
|
3677
|
+
};
|
|
3678
|
+
var ofBooleans = {
|
|
3679
|
+
rate: (bs) => bs.filter(Boolean).length / bs.length,
|
|
3680
|
+
count: (bs) => bs.filter(Boolean).length
|
|
3681
|
+
};
|
|
3682
|
+
var ofNumbers = {
|
|
3683
|
+
mean,
|
|
3684
|
+
median,
|
|
3685
|
+
min: (xs) => Math.min(...xs),
|
|
3686
|
+
max: (xs) => Math.max(...xs),
|
|
3687
|
+
sum
|
|
3688
|
+
};
|
|
3689
|
+
var ofRecords = {
|
|
3690
|
+
sum: (rs) => byKey(rs, sum),
|
|
3691
|
+
mean: (rs) => byKey(rs, mean)
|
|
3692
|
+
};
|
|
3693
|
+
var isRecordOfNumbers = (value) => typeof value === "object" && value !== null && !Array.isArray(value) && Object.values(value).every((v) => typeof v === "number");
|
|
3694
|
+
function menuFor(values) {
|
|
3695
|
+
if (values.every((v) => typeof v === "boolean"))
|
|
3696
|
+
return ["booleans", ofBooleans];
|
|
3697
|
+
if (values.every((v) => typeof v === "number"))
|
|
3698
|
+
return ["numbers", ofNumbers];
|
|
3699
|
+
if (values.every(isRecordOfNumbers))
|
|
3700
|
+
return ["records of numbers", ofRecords];
|
|
3701
|
+
return;
|
|
3702
|
+
}
|
|
3703
|
+
function combine(values, methods) {
|
|
3704
|
+
if (values.length === 0)
|
|
3705
|
+
return {};
|
|
3706
|
+
const menu = menuFor(values);
|
|
3707
|
+
if (!menu)
|
|
3708
|
+
throw new Error(`can't combine ${JSON.stringify(values)}: not all booleans, numbers or records of numbers`);
|
|
3709
|
+
const [kind, menuMethods] = menu;
|
|
3710
|
+
return Object.fromEntries(methods.map((method) => {
|
|
3711
|
+
const apply = menuMethods[method];
|
|
3712
|
+
if (!apply)
|
|
3713
|
+
throw new Error(`"${method}" doesn't combine ${kind}`);
|
|
3714
|
+
return [method, apply(values)];
|
|
3715
|
+
}));
|
|
3716
|
+
}
|
|
3717
|
+
function aggregatePermutation(records, measurements) {
|
|
3718
|
+
const [first] = records;
|
|
3719
|
+
if (!first)
|
|
3720
|
+
throw new Error("no container runs to aggregate");
|
|
3721
|
+
const { executionId, permutationHash } = first;
|
|
3722
|
+
if (records.some((r) => r.executionId !== executionId || r.permutationHash !== permutationHash)) {
|
|
3723
|
+
throw new Error("container runs to aggregate must be of one permutation of one execution");
|
|
3724
|
+
}
|
|
3725
|
+
const measures = {};
|
|
3726
|
+
for (const { name, aggregate } of measurements) {
|
|
3727
|
+
const values = records.flatMap(({ result }) => {
|
|
3728
|
+
const outcome = result[name]?.outcome;
|
|
3729
|
+
return outcome?.outcome === "measured" ? [outcome.data] : [];
|
|
3730
|
+
});
|
|
3731
|
+
measures[name] = { measured: values.length, ...combine(values, aggregate) };
|
|
3732
|
+
}
|
|
3733
|
+
const durations = records.map(({ execution }) => execution.finishedAt - execution.startedAt);
|
|
3734
|
+
return {
|
|
3735
|
+
executionId,
|
|
3736
|
+
permutationHash,
|
|
3737
|
+
iterations: records.length,
|
|
3738
|
+
measures,
|
|
3739
|
+
duration: { mean: mean(durations), max: Math.max(...durations) }
|
|
3740
|
+
};
|
|
3741
|
+
}
|
|
3742
|
+
async function recomputeAggregationResult(store, executionId, permutationHash, measurements) {
|
|
3743
|
+
const records = (await store.getExecution(executionId)).filter((r) => r.permutationHash === permutationHash);
|
|
3744
|
+
const result = aggregatePermutation(records, measurements);
|
|
3745
|
+
await store.recordAggregationResult(result);
|
|
3746
|
+
return result;
|
|
3747
|
+
}
|
|
3748
|
+
|
|
3749
|
+
// src/measuring/measure.ts
|
|
3750
|
+
var message = (error) => error instanceof Error ? error.message : String(error);
|
|
3751
|
+
var recordable = (result) => result.outcome === "erroredWhileMeasuring" ? { outcome: "erroredWhileMeasuring", error: { message: message(result.error) } } : result;
|
|
3752
|
+
async function evaluate({ instrument, config: config2, dir }, ctx) {
|
|
3753
|
+
try {
|
|
3754
|
+
if (!isInstrumentName(instrument))
|
|
3755
|
+
throw new Error(`unknown instrument ${JSON.stringify(instrument)}`);
|
|
3756
|
+
const chosen = instruments[instrument];
|
|
3757
|
+
const fields = config2;
|
|
3758
|
+
const bundled = Object.fromEntries((chosen.bundledFiles ?? []).map((key) => [key, resolve(dir, fields[key])]));
|
|
3759
|
+
return recordable(await chosen.evaluate({ ...fields, ...bundled }, ctx));
|
|
3760
|
+
} catch (error) {
|
|
3761
|
+
return { outcome: "erroredWhileMeasuring", error: { message: message(error) } };
|
|
3762
|
+
}
|
|
3763
|
+
}
|
|
3764
|
+
async function measureContainerRun(measurements, open, now = Date.now) {
|
|
3765
|
+
const result = {};
|
|
3766
|
+
for (const measurement2 of measurements) {
|
|
3767
|
+
const ctx = await open();
|
|
3768
|
+
let outcome;
|
|
3769
|
+
try {
|
|
3770
|
+
outcome = await evaluate(measurement2, ctx);
|
|
3771
|
+
} finally {
|
|
3772
|
+
await ctx.close();
|
|
3773
|
+
}
|
|
3774
|
+
result[measurement2.name] = {
|
|
3775
|
+
instrument: measurement2.instrument,
|
|
3776
|
+
contentHash: hashMeasurement(measurement2),
|
|
3777
|
+
measuredAt: new Date(now()).toISOString(),
|
|
3778
|
+
outcome
|
|
3779
|
+
};
|
|
3780
|
+
}
|
|
3781
|
+
return result;
|
|
3782
|
+
}
|
|
3783
|
+
|
|
3784
|
+
// src/measuring/remeasure.ts
|
|
3785
|
+
function measureStatus(recorded, measurement2) {
|
|
3786
|
+
if (recorded === undefined || recorded.outcome.outcome === "erroredWhileMeasuring")
|
|
3787
|
+
return "missing";
|
|
3788
|
+
return recorded.contentHash === hashMeasurement(measurement2) ? "current" : "stale";
|
|
3789
|
+
}
|
|
3790
|
+
function planRemeasure(records, measurements) {
|
|
3791
|
+
const runs = records.flatMap(({ executionId, postrunImage, result }) => {
|
|
3792
|
+
const measurementNames = measurements.filter((measurement2) => measureStatus(result[measurement2.name], measurement2) !== "current").map(({ name }) => name).toSorted();
|
|
3793
|
+
return measurementNames.length === 0 ? [] : [{ executionId, postrunImage, measurementNames }];
|
|
3794
|
+
});
|
|
3795
|
+
return { runs, executions: summarizeExecutions2(records, measurements) };
|
|
3796
|
+
}
|
|
3797
|
+
function summarizeExecutions2(records, measurements) {
|
|
3798
|
+
const resultsByExecution = new Map;
|
|
3799
|
+
for (const { executionId, result } of records) {
|
|
3800
|
+
resultsByExecution.set(executionId, [...resultsByExecution.get(executionId) ?? [], result]);
|
|
3801
|
+
}
|
|
3802
|
+
const sorted = measurements.toSorted((a, b) => a.name < b.name ? -1 : 1);
|
|
3803
|
+
const declared = new Set(measurements.map(({ name }) => name));
|
|
3804
|
+
return [...resultsByExecution].map(([executionId, results]) => {
|
|
3805
|
+
const summaries = sorted.map((measurement2) => {
|
|
3806
|
+
const statuses = results.map((result) => measureStatus(result[measurement2.name], measurement2));
|
|
3807
|
+
const status = statuses.includes("missing") ? "missing" : statuses.includes("stale") ? "stale" : "current";
|
|
3808
|
+
return { name: measurement2.name, contentHash: hashMeasurement(measurement2), status };
|
|
3809
|
+
});
|
|
3810
|
+
const storedMeasures = new Map;
|
|
3811
|
+
for (const name of results.flatMap((result) => Object.keys(result)).filter((name2) => !declared.has(name2))) {
|
|
3812
|
+
storedMeasures.set(name, (storedMeasures.get(name) ?? 0) + 1);
|
|
3813
|
+
}
|
|
3814
|
+
const deleted = [...storedMeasures].toSorted(([a], [b]) => a < b ? -1 : 1).map(([name, count2]) => ({ name, storedMeasures: count2 }));
|
|
3815
|
+
return { executionId, measurements: summaries, deleted };
|
|
3816
|
+
});
|
|
3817
|
+
}
|
|
3818
|
+
|
|
3819
|
+
// src/cli/commands/remeasure/remeasure.ts
|
|
3820
|
+
var footer = (updated, recomputed) => [
|
|
3821
|
+
`Updated ${count(updated, "container run")}. Recomputed ${count(recomputed, "aggregation result")}.`,
|
|
3822
|
+
"The agent was not rerun."
|
|
3823
|
+
];
|
|
3824
|
+
var NOTHING_TO_DO = "Already current, nothing to do.";
|
|
3825
|
+
function measuredLines(measured) {
|
|
3826
|
+
const taken = measured.flatMap(({ record, result }) => Object.entries(result).map((entry) => ({ record, entry })));
|
|
3827
|
+
const width = Math.max(...taken.map(({ entry: [name] }) => name.length));
|
|
3828
|
+
return taken.map(({ record: { permutationHash, iteration }, entry: [name, { outcome }] }) => ` ${abbreviateHash(permutationHash)} i${String(iteration).padStart(2, "0")} ${name.padEnd(width)} ${showOutcome(outcome)}`);
|
|
3829
|
+
}
|
|
3830
|
+
function executionReport(task, containerRuns, { executionId, measurements, deleted }, measured, recomputed) {
|
|
3831
|
+
const rows = [
|
|
3832
|
+
...measurements.map(({ name, contentHash, status }) => [
|
|
3833
|
+
name,
|
|
3834
|
+
status === "current" ? "up to date" : `new / changed (${abbreviateHash(contentHash)})`
|
|
3835
|
+
]),
|
|
3836
|
+
...deleted.map(({ name, storedMeasures }) => [
|
|
3837
|
+
name,
|
|
3838
|
+
`deleted from task; ${count(storedMeasures, "stored measure")} kept, excluded from aggregates`
|
|
3839
|
+
])
|
|
3840
|
+
];
|
|
3841
|
+
const width = Math.max(...rows.map(([name]) => name.length));
|
|
3842
|
+
return [
|
|
3843
|
+
`Execution ${executionId} (${count(containerRuns, "container run")})`,
|
|
3844
|
+
`Current measurements for task ${task}:`,
|
|
3845
|
+
...rows.map(([name, status]) => ` ${name.padEnd(width)} ${status}`),
|
|
3846
|
+
"",
|
|
3847
|
+
...measured.length === 0 ? [NOTHING_TO_DO] : ["Measuring from each postrun image:", ...measuredLines(measured), ""],
|
|
3848
|
+
...footer(measured.length, recomputed)
|
|
3849
|
+
].join(`
|
|
3850
|
+
`);
|
|
3851
|
+
}
|
|
3852
|
+
function testCaseReport(testCase2, executions2, updated, recomputed) {
|
|
3853
|
+
const rows = executions2.map(({ executionId, containerRuns, ranAt, plan: plan2 }) => ({
|
|
3854
|
+
executionId,
|
|
3855
|
+
runs: count(containerRuns, "run"),
|
|
3856
|
+
day: ranAt.slice(0, 10),
|
|
3857
|
+
current: plan2.runs.length === 0
|
|
3858
|
+
}));
|
|
3859
|
+
const width = Math.max(...rows.map(({ runs }) => runs.length));
|
|
3860
|
+
const total = executions2.reduce((sum2, { containerRuns }) => sum2 + containerRuns, 0);
|
|
3861
|
+
return [
|
|
3862
|
+
`${testCase2} has ${count(executions2.length, "execution")} (${count(total, "container run")})`,
|
|
3863
|
+
...rows.map(({ executionId, runs, day, current }) => ` ${executionId} ${runs.padEnd(width)} ${day}${current ? " (already current, nothing to do)" : ""}`),
|
|
3864
|
+
"",
|
|
3865
|
+
updated === 0 ? NOTHING_TO_DO : `Measuring ${count(updated, "container run")}... done`,
|
|
3866
|
+
...footer(updated, recomputed)
|
|
3867
|
+
].join(`
|
|
3868
|
+
`);
|
|
3869
|
+
}
|
|
3870
|
+
async function remeasureExecution(cwd, store, opener, executionId) {
|
|
3871
|
+
const records = await store.getExecution(executionId);
|
|
3872
|
+
const measurements = await loadMeasurements(findThunderjarRoot(cwd), records[0].task);
|
|
3873
|
+
const plan2 = planRemeasure(records, measurements);
|
|
3874
|
+
const measured = [];
|
|
3875
|
+
for (const { postrunImage, measurementNames } of plan2.runs) {
|
|
3876
|
+
const record = records.find((r) => r.postrunImage === postrunImage);
|
|
3877
|
+
const planned = measurementNames.map((name) => measurements.find((m) => m.name === name));
|
|
3878
|
+
const result = await measureContainerRun(planned, () => opener.openMeasurementContext(record.postrunImage));
|
|
3879
|
+
await store.recordMeasures(record.postrunImage, result);
|
|
3880
|
+
measured.push({ record, result });
|
|
3881
|
+
}
|
|
3882
|
+
const permutationHashes = new Set(records.map(({ permutationHash }) => permutationHash));
|
|
3883
|
+
for (const permutationHash of permutationHashes) {
|
|
3884
|
+
await recomputeAggregationResult(store, executionId, permutationHash, measurements);
|
|
3885
|
+
}
|
|
3886
|
+
return { records, plan: plan2, measured, recomputed: permutationHashes.size };
|
|
3887
|
+
}
|
|
3888
|
+
async function remeasure(target, { cwd = process.cwd(), dataDir } = {}) {
|
|
3889
|
+
const store = jsonFileStore(runDataStorePath(cwd, dataDir));
|
|
3890
|
+
const opener = createDockerMeasurementOpener(dockerCli(), LOCAL_REPOSITORY);
|
|
3891
|
+
if ((await store.getExecution(target)).length > 0) {
|
|
3892
|
+
const { records, plan: plan2, measured, recomputed } = await remeasureExecution(cwd, store, opener, target);
|
|
3893
|
+
return executionReport(records[0].task, records.length, plan2.executions[0], measured, recomputed);
|
|
3894
|
+
}
|
|
3895
|
+
const executions2 = await store.listExecutions(target);
|
|
3896
|
+
if (executions2.length > 0) {
|
|
3897
|
+
const planned = [];
|
|
3898
|
+
let updated = 0;
|
|
3899
|
+
let recomputed = 0;
|
|
3900
|
+
for (const execution of executions2) {
|
|
3901
|
+
const { plan: plan2, measured, recomputed: ofExecution } = await remeasureExecution(cwd, store, opener, execution.executionId);
|
|
3902
|
+
planned.push({ ...execution, plan: plan2 });
|
|
3903
|
+
updated += measured.length;
|
|
3904
|
+
recomputed += ofExecution;
|
|
3905
|
+
}
|
|
3906
|
+
return testCaseReport(target, planned, updated, recomputed);
|
|
3907
|
+
}
|
|
3908
|
+
throw new Error(`${target} is not a recorded execution or test case`);
|
|
3909
|
+
}
|
|
3910
|
+
function createRemeasureCommand() {
|
|
3911
|
+
return new Command("remeasure").description("Apply the current measurements to past runs.").argument("<target>", "an execution ID, or a test case, meaning every execution of it").addOption(dataDirOption()).action(async (target, options) => {
|
|
3912
|
+
console.log(await remeasure(target, { dataDir: options.dataDir }));
|
|
3913
|
+
});
|
|
3914
|
+
}
|
|
3915
|
+
|
|
3916
|
+
// src/matrix/execution.ts
|
|
3917
|
+
var RESULT_PATH = "/thunderjar/result.json";
|
|
3918
|
+
var importDefault2 = async (path2) => (await import(path2)).default;
|
|
3919
|
+
async function renderCommand(root, task, permutation) {
|
|
3920
|
+
const parameters = join(root, "test-case-parameters");
|
|
3921
|
+
const { family, version, model } = parseHarnessModelPair(permutation.harnessModel);
|
|
3922
|
+
const familyDir = join(parameters, "harnessModels", family);
|
|
3923
|
+
const harness = await importDefault2(join(familyDir, "index.ts"));
|
|
3924
|
+
const versions = await importDefault2(join(familyDir, "versions.ts"));
|
|
3925
|
+
const models = await importDefault2(join(familyDir, "models.ts"));
|
|
3926
|
+
const initialPrompt2 = await importDefault2(join(parameters, "tasks", task, "initial-prompts", `${permutation.initialPrompt}.ts`));
|
|
3927
|
+
const entry = versions[version];
|
|
3928
|
+
if (!("version" in entry)) {
|
|
3929
|
+
throw new Error(`${permutation.harnessModel}: floating versions are not supported yet`);
|
|
3930
|
+
}
|
|
3931
|
+
const cli = entry.cli ?? harness.cli;
|
|
3932
|
+
const ctx = { version: entry.version, modelId: models[model], initialPrompt: initialPrompt2.prompt };
|
|
3933
|
+
return {
|
|
3934
|
+
renderCommand: (resultPath) => cli({ ...ctx, resultPath }),
|
|
3935
|
+
timeoutSeconds: initialPrompt2.timeoutSeconds,
|
|
3936
|
+
harnessModel: { family, version: ctx.version, modelId: ctx.modelId }
|
|
3937
|
+
};
|
|
3938
|
+
}
|
|
3939
|
+
async function planExecution(root, testCase2, executionId) {
|
|
3940
|
+
const { hashes, permutations } = await resolveTestCase(root, testCase2);
|
|
3941
|
+
const planned = [];
|
|
3942
|
+
for (const permutation of permutations) {
|
|
3943
|
+
const { renderCommand: render2, timeoutSeconds, harnessModel } = await renderCommand(root, testCase2.task, permutation);
|
|
3944
|
+
const command = render2(RESULT_PATH);
|
|
3945
|
+
const parameters = Object.fromEntries(AXES.map((axis) => [axis, { name: permutation[axis], parameterHash: hashes[axis].get(permutation[axis]) }]));
|
|
3946
|
+
planned.push({
|
|
3947
|
+
permutation,
|
|
3948
|
+
parameters,
|
|
3949
|
+
harnessModel,
|
|
3950
|
+
prerunTag: prerunTag(permutation.hash),
|
|
3951
|
+
runs: Array.from({ length: testCase2.iterations }, (_, iterationIndex) => ({
|
|
3952
|
+
executionId,
|
|
3953
|
+
iterationIndex,
|
|
3954
|
+
prerunTag: prerunTag(permutation.hash),
|
|
3955
|
+
postrunTag: postrunTag(executionId, permutation.hash, iterationIndex),
|
|
3956
|
+
command,
|
|
3957
|
+
renderCommand: render2,
|
|
3958
|
+
timeoutSeconds
|
|
3959
|
+
}))
|
|
3960
|
+
});
|
|
3961
|
+
}
|
|
3962
|
+
return { executionId, iterations: testCase2.iterations, permutations: planned };
|
|
3963
|
+
}
|
|
3964
|
+
|
|
3965
|
+
// src/matrix/ulid.ts
|
|
3966
|
+
var ALPHABET = "0123456789abcdefghjkmnpqrstvwxyz";
|
|
3967
|
+
function ulid(now = Date.now(), random = crypto.getRandomValues(new Uint8Array(10))) {
|
|
3968
|
+
if (!Number.isInteger(now) || now < 0 || now >= 2 ** 48)
|
|
3969
|
+
throw new RangeError(`ULID time out of range: ${now}`);
|
|
3970
|
+
if (random.length !== 10)
|
|
3971
|
+
throw new RangeError(`ULID randomness must be 10 bytes, got ${random.length}`);
|
|
3972
|
+
let time = "";
|
|
3973
|
+
for (let t = now, i = 0;i < 10; i++, t = Math.floor(t / 32))
|
|
3974
|
+
time = ALPHABET[t % 32] + time;
|
|
3975
|
+
let bits = 0n;
|
|
3976
|
+
for (const byte of random)
|
|
3977
|
+
bits = bits << 8n | BigInt(byte);
|
|
3978
|
+
let rest = "";
|
|
3979
|
+
for (let i = 0;i < 16; i++, bits >>= 5n)
|
|
3980
|
+
rest = ALPHABET[Number(bits & 31n)] + rest;
|
|
3981
|
+
return time + rest;
|
|
3982
|
+
}
|
|
3983
|
+
|
|
3984
|
+
// src/cli/commands/run/run.ts
|
|
3985
|
+
var indent = (text, spaces) => text.split(`
|
|
3986
|
+
`).map((line) => " ".repeat(spaces) + line).join(`
|
|
3987
|
+
`);
|
|
3988
|
+
var ordinal = (n, total) => `${String(n).padStart(String(total).length)}/${total}`;
|
|
3989
|
+
var shape = ({ permutations, iterations }) => `${count(permutations.length, "permutation")} \xD7 ${count(iterations, "iteration")}`;
|
|
3990
|
+
async function run(name, options) {
|
|
3991
|
+
const {
|
|
3992
|
+
dryRun,
|
|
3993
|
+
cwd = process.cwd(),
|
|
3994
|
+
log = (text) => console.log(text),
|
|
3995
|
+
now = Date.now,
|
|
3996
|
+
newExecutionId = () => ulid(now()),
|
|
3997
|
+
rootDir = defaultRootDir(),
|
|
3998
|
+
docker: docker2 = dockerCli(),
|
|
3999
|
+
repository = LOCAL_REPOSITORY,
|
|
4000
|
+
runner = createDockerRunner(docker2, repository),
|
|
4001
|
+
measurementOpener = createDockerMeasurementOpener(docker2, repository),
|
|
4002
|
+
dataDir
|
|
4003
|
+
} = options;
|
|
4004
|
+
const root = findThunderjarRoot(cwd);
|
|
4005
|
+
const testCase2 = await loadTestCase(root, name);
|
|
4006
|
+
const execution = await planExecution(root, testCase2, newExecutionId());
|
|
4007
|
+
const { executionId, permutations, iterations } = execution;
|
|
4008
|
+
const rows = permutationRows(permutations.map((p) => p.permutation));
|
|
4009
|
+
const permutationLine = (n) => `Permutation ${ordinal(n + 1, permutations.length)} ${rows[n]}`;
|
|
4010
|
+
if (dryRun) {
|
|
4011
|
+
const found = await prerunStatuses(docker2, repository, permutations.map((p) => p.permutation));
|
|
4012
|
+
log(`Execution ${executionId} (dry run, ${shape(execution)})`);
|
|
4013
|
+
if ("unreachable" in found)
|
|
4014
|
+
log(unreachableNote(found.unreachable));
|
|
4015
|
+
permutations.forEach(({ permutation, prerunTag: prerunTag2, runs }, n) => {
|
|
4016
|
+
log("");
|
|
4017
|
+
log([
|
|
4018
|
+
permutationLine(n),
|
|
4019
|
+
` ${prerunTag2}${"statuses" in found ? showPrerunStatus(found.statuses.get(permutation.hash)) : ""}`,
|
|
4020
|
+
...runs.map(({ iterationIndex, postrunTag: postrunTag2, command }) => ` iteration ${ordinal(iterationIndex + 1, iterations)} ${postrunTag2}
|
|
4021
|
+
${indent(command, 4)}`)
|
|
4022
|
+
].join(`
|
|
4023
|
+
`));
|
|
4024
|
+
});
|
|
4025
|
+
return;
|
|
4026
|
+
}
|
|
4027
|
+
const measurements = await loadMeasurements(root, testCase2.task);
|
|
4028
|
+
const nameWidth = Math.max(0, ...measurements.map((m) => m.name.length));
|
|
4029
|
+
const storePath = runDataStorePath(cwd, dataDir);
|
|
4030
|
+
const store = jsonFileStore(storePath);
|
|
4031
|
+
const builds = await preparePrerunBuilds({
|
|
4032
|
+
root,
|
|
4033
|
+
task: testCase2.task,
|
|
4034
|
+
permutations: permutations.map((p) => p.permutation),
|
|
4035
|
+
docker: docker2,
|
|
4036
|
+
repository,
|
|
4037
|
+
rootDir,
|
|
4038
|
+
purpose: "Running a test case"
|
|
4039
|
+
});
|
|
4040
|
+
log(`Execution ${executionId} (${shape(execution)})`);
|
|
4041
|
+
log(`Images in ${repository}`);
|
|
4042
|
+
if (builds.missing.size > 0)
|
|
4043
|
+
log(`Build contexts in ${join(rootDir, "builds")}`);
|
|
4044
|
+
const exitCodes = new Map;
|
|
4045
|
+
for (const [n, { permutation, parameters, harnessModel, prerunTag: prerunTag2, runs }] of permutations.entries()) {
|
|
4046
|
+
log("");
|
|
4047
|
+
log(permutationLine(n));
|
|
4048
|
+
if (builds.missing.has(permutation.hash)) {
|
|
4049
|
+
const started = now();
|
|
4050
|
+
await builds.build(permutation.hash);
|
|
4051
|
+
log(` ${prerunTag2} built (${seconds(now() - started)})`);
|
|
4052
|
+
} else {
|
|
4053
|
+
log(` ${prerunTag2} reused`);
|
|
4054
|
+
}
|
|
4055
|
+
for (const containerRun of runs) {
|
|
4056
|
+
const { startedAt, finishedAt, exitCode } = await runner.runContainer(containerRun);
|
|
4057
|
+
await store.recordContainerRun({
|
|
4058
|
+
executionId,
|
|
4059
|
+
testCase: name,
|
|
4060
|
+
task: testCase2.task,
|
|
4061
|
+
permutationHash: permutation.hash,
|
|
4062
|
+
iteration: containerRun.iterationIndex,
|
|
4063
|
+
postrunImage: containerRun.postrunTag,
|
|
4064
|
+
ranAt: new Date(startedAt).toISOString(),
|
|
4065
|
+
parameters,
|
|
4066
|
+
harnessModel,
|
|
4067
|
+
result: {},
|
|
4068
|
+
execution: { startedAt, finishedAt, exitCode }
|
|
4069
|
+
});
|
|
4070
|
+
exitCodes.set(exitCode, (exitCodes.get(exitCode) ?? 0) + 1);
|
|
4071
|
+
log(` iteration ${ordinal(containerRun.iterationIndex + 1, iterations)} ${containerRun.postrunTag}` + ` exit ${exitCode} ${duration({ startedAt, finishedAt })}`);
|
|
4072
|
+
if (measurements.length > 0) {
|
|
4073
|
+
const open = () => measurementOpener.openMeasurementContext(containerRun.postrunTag);
|
|
4074
|
+
const result = await measureContainerRun(measurements, open, now);
|
|
4075
|
+
await store.recordMeasures(containerRun.postrunTag, result);
|
|
4076
|
+
log(Object.entries(result).map(([name2, { outcome }]) => ` ${name2.padEnd(nameWidth)} ${showOutcome(outcome)}`).join(`
|
|
4077
|
+
`));
|
|
4078
|
+
}
|
|
4079
|
+
await recomputeAggregationResult(store, executionId, permutation.hash, measurements);
|
|
4080
|
+
}
|
|
4081
|
+
}
|
|
4082
|
+
const total = permutations.length * iterations;
|
|
4083
|
+
const byCode = [...exitCodes].toSorted(([a], [b]) => a - b).map(([code, n]) => `${n} exited ${code}`);
|
|
4084
|
+
log("");
|
|
4085
|
+
log(`${count(total, "container run")}: ${byCode.join(", ")}`);
|
|
4086
|
+
log(`Recorded in ${relative(cwd, storePath)}`);
|
|
4087
|
+
}
|
|
4088
|
+
function createRunCommand() {
|
|
4089
|
+
return new Command("run").description("Run a test case.").argument("<test case>", "the test case to run").option("--dry-run", "print each container run instead of building or running anything").addOption(imagePrefixOption("the image repository prerun and postrun images are tagged in")).addOption(dataDirOption()).action(async (name, options) => {
|
|
4090
|
+
await run(name, { dryRun: options.dryRun ?? false, repository: options.imagePrefix, dataDir: options.dataDir });
|
|
4091
|
+
});
|
|
4092
|
+
}
|
|
4093
|
+
|
|
4094
|
+
// src/cli/commands/show/show.ts
|
|
4095
|
+
var labelled = (lines2) => {
|
|
4096
|
+
const width = Math.max(...lines2.map(([label]) => label.length));
|
|
4097
|
+
return lines2.map(([label, value]) => `${label.padEnd(width)} ${value}`);
|
|
4098
|
+
};
|
|
4099
|
+
function table(rows) {
|
|
4100
|
+
const widths = rows[0].map((_, i) => Math.max(...rows.map((row) => row[i]?.length ?? 0)));
|
|
4101
|
+
return rows.map((row) => row.length === 0 ? "" : ` ${row.map((cell, i) => i === row.length - 1 ? cell : cell.padEnd(widths[i])).join(" ")}`);
|
|
4102
|
+
}
|
|
4103
|
+
var permutationOf = ({ parameters }) => Object.fromEntries(AXES.map((axis) => [axis, parameters[axis].name]));
|
|
4104
|
+
var PLACES = { rate: 2, mean: 1, median: 1 };
|
|
4105
|
+
function aggregatedValue(method, value) {
|
|
4106
|
+
const places = PLACES[method];
|
|
4107
|
+
const number = (n) => places === undefined ? showValue(n) : n.toFixed(places);
|
|
4108
|
+
if (typeof value === "number")
|
|
4109
|
+
return number(value);
|
|
4110
|
+
return `{${Object.entries(value).map(([key, n]) => `${key}: ${number(n)}`).join(", ")}}`;
|
|
4111
|
+
}
|
|
4112
|
+
function aggregateCell({ measured, ...methods }) {
|
|
4113
|
+
if (measured === 0)
|
|
4114
|
+
return "\u2014";
|
|
4115
|
+
return Object.entries(methods).map(([method, value]) => `${method} ${aggregatedValue(method, value)}`).join(", ");
|
|
4116
|
+
}
|
|
4117
|
+
function exits(runs) {
|
|
4118
|
+
const byCode = new Map;
|
|
4119
|
+
for (const { execution } of runs)
|
|
4120
|
+
byCode.set(execution.exitCode, (byCode.get(execution.exitCode) ?? 0) + 1);
|
|
4121
|
+
return [...byCode].toSorted(([a], [b]) => a - b).map(([code, n]) => `${n}/${runs.length} exited ${code}`).join(", ");
|
|
4122
|
+
}
|
|
4123
|
+
var summary = (label, value) => ` ${label.padEnd(11)}${value}`;
|
|
4124
|
+
function staleness(measurements) {
|
|
4125
|
+
const currentHashes = new Map(measurements.map((measurement2) => [measurement2.name, hashMeasurement(measurement2)]));
|
|
4126
|
+
return (name, { contentHash }) => {
|
|
4127
|
+
const current = currentHashes.get(name);
|
|
4128
|
+
return current !== undefined && current !== contentHash;
|
|
4129
|
+
};
|
|
4130
|
+
}
|
|
4131
|
+
function aggregationTable({ iterations, measures }, runs, isStale) {
|
|
4132
|
+
const names = Object.keys(measures).toSorted();
|
|
4133
|
+
if (names.length === 0)
|
|
4134
|
+
return [" no measures"];
|
|
4135
|
+
const staleRow = (name) => runs.some(({ result }) => {
|
|
4136
|
+
const measure = result[name];
|
|
4137
|
+
return measure?.outcome.outcome === "measured" && isStale(name, measure);
|
|
4138
|
+
});
|
|
4139
|
+
return table([
|
|
4140
|
+
["MEASURE", "MEASURED", "AGGREGATE"],
|
|
4141
|
+
...names.map((name) => [
|
|
4142
|
+
name,
|
|
4143
|
+
`${measures[name].measured}/${iterations}`,
|
|
4144
|
+
`${aggregateCell(measures[name])}${staleRow(name) ? " stale, run `remeasure`" : ""}`
|
|
4145
|
+
])
|
|
4146
|
+
]);
|
|
4147
|
+
}
|
|
4148
|
+
var detailed = (outcome) => outcome.outcome === "measured" ? showValue(outcome.data) : outcome.outcome === "skipped" ? `skipped${outcome.reason === undefined ? "" : ` (${outcome.reason})`}` : `error (${outcome.error.message})`;
|
|
4149
|
+
function byPermutation(records) {
|
|
4150
|
+
const groups = new Map;
|
|
4151
|
+
for (const record of records) {
|
|
4152
|
+
const group = groups.get(record.permutationHash);
|
|
4153
|
+
if (group)
|
|
4154
|
+
group.push(record);
|
|
4155
|
+
else
|
|
4156
|
+
groups.set(record.permutationHash, [record]);
|
|
4157
|
+
}
|
|
4158
|
+
return [...groups.values()];
|
|
4159
|
+
}
|
|
4160
|
+
function showExecution(records, aggregations, measurements, latestOf) {
|
|
4161
|
+
const [{ executionId, testCase: testCase2, task }] = records;
|
|
4162
|
+
const ranAt = records.map((r) => r.ranAt).toSorted()[0];
|
|
4163
|
+
const lines2 = labelled([
|
|
4164
|
+
["Execution", `${executionId}${latestOf === undefined ? "" : ` (latest of ${latestOf})`}`],
|
|
4165
|
+
["Test case", testCase2],
|
|
4166
|
+
["Task", task],
|
|
4167
|
+
["Date", minute(ranAt)]
|
|
4168
|
+
]);
|
|
4169
|
+
const permutations = byPermutation(records);
|
|
4170
|
+
const rows = permutationRows(permutations.map(([first]) => permutationOf(first)));
|
|
4171
|
+
const byHash = new Map(aggregations.map((a) => [a.permutationHash, a]));
|
|
4172
|
+
const isStale = staleness(measurements);
|
|
4173
|
+
permutations.forEach((runs, n) => {
|
|
4174
|
+
const aggregation = byHash.get(runs[0].permutationHash);
|
|
4175
|
+
lines2.push("", `Permutation ${abbreviateHash(runs[0].permutationHash)} (${count(runs.length, "iteration")})`, ` ${rows[n]}`, "", ...aggregation ? aggregationTable(aggregation, runs, isStale) : [" no aggregation result recorded"], "", summary("exit", exits(runs)), ...aggregation ? [summary("duration", `mean ${seconds(aggregation.duration.mean)}, max ${seconds(aggregation.duration.max)}`)] : []);
|
|
4176
|
+
});
|
|
4177
|
+
return lines2.join(`
|
|
4178
|
+
`);
|
|
4179
|
+
}
|
|
4180
|
+
function showContainerRun(record, measurements) {
|
|
4181
|
+
const { postrunImage, executionId, testCase: testCase2, permutationHash, iteration, ranAt, parameters, result, execution } = record;
|
|
4182
|
+
const names = Object.keys(result).toSorted();
|
|
4183
|
+
const isStale = staleness(measurements);
|
|
4184
|
+
const cell = (name) => `${detailed(result[name].outcome)}${isStale(name, result[name]) ? " (stale)" : ""}`;
|
|
4185
|
+
return [
|
|
4186
|
+
...labelled([
|
|
4187
|
+
["Container run", postrunImage],
|
|
4188
|
+
["Execution", `${executionId} (${testCase2})`],
|
|
4189
|
+
["Permutation", `${abbreviateHash(permutationHash)}, iteration ${iteration + 1}`],
|
|
4190
|
+
["Ran at", minute(ranAt)]
|
|
4191
|
+
]),
|
|
4192
|
+
"",
|
|
4193
|
+
"Parameters",
|
|
4194
|
+
...table(AXES.map((axis) => [axis, parameters[axis].name, parameters[axis].parameterHash.slice(0, 4)])),
|
|
4195
|
+
"",
|
|
4196
|
+
"Measures",
|
|
4197
|
+
...names.length === 0 ? [" none"] : table(names.map((name) => [name, cell(name)])),
|
|
4198
|
+
"",
|
|
4199
|
+
`Duration ${duration(execution)} exit code ${execution.exitCode}`
|
|
4200
|
+
].join(`
|
|
4201
|
+
`);
|
|
4202
|
+
}
|
|
4203
|
+
async function show3(target, { cwd = process.cwd(), dataDir } = {}) {
|
|
4204
|
+
const store = jsonFileStore(runDataStorePath(cwd, dataDir));
|
|
4205
|
+
const todays = (task) => loadMeasurements(findThunderjarRoot(cwd), task);
|
|
4206
|
+
const containerRun = await store.getContainerRun(target);
|
|
4207
|
+
if (containerRun)
|
|
4208
|
+
return showContainerRun(containerRun, await todays(containerRun.task));
|
|
4209
|
+
const execution = await store.getExecution(target);
|
|
4210
|
+
if (execution.length > 0) {
|
|
4211
|
+
return showExecution(execution, await store.getAggregationResults(target), await todays(execution[0].task));
|
|
4212
|
+
}
|
|
4213
|
+
const [latest] = await store.listExecutions(target);
|
|
4214
|
+
if (latest) {
|
|
4215
|
+
const { executionId } = latest;
|
|
4216
|
+
const records = await store.getExecution(executionId);
|
|
4217
|
+
return showExecution(records, await store.getAggregationResults(executionId), await todays(records[0].task), target);
|
|
4218
|
+
}
|
|
4219
|
+
throw new Error(`${target} is not a recorded container run, execution or test case`);
|
|
4220
|
+
}
|
|
4221
|
+
function createShowCommand() {
|
|
4222
|
+
return new Command("show").description("Show a test case, an execution or a container run.").argument("<target>", "a container run's postrun tag, an execution ID, or a test case, meaning its latest execution").addOption(dataDirOption()).action(async (target, options) => {
|
|
4223
|
+
console.log(await show3(target, { dataDir: options.dataDir }));
|
|
2105
4224
|
});
|
|
2106
4225
|
}
|
|
2107
4226
|
|
|
2108
4227
|
// src/cli/program.ts
|
|
2109
4228
|
function createProgram() {
|
|
2110
|
-
|
|
4229
|
+
const program2 = new Command().name("thunderjar").description(package_default.description).version(package_default.version).showHelpAfterError().addCommand(createInitCommand()).addCommand(createGenerateCommand()).addCommand(createPlanCommand()).addCommand(createBuildCommand()).addCommand(createRunCommand()).addCommand(createExecutionsCommand()).addCommand(createShowCommand()).addCommand(createRemeasureCommand());
|
|
4230
|
+
for (const command of program2.commands)
|
|
4231
|
+
command.showHelpAfterError();
|
|
4232
|
+
return program2;
|
|
2111
4233
|
}
|
|
2112
4234
|
|
|
2113
4235
|
// src/cli/index.ts
|
|
2114
|
-
|
|
4236
|
+
try {
|
|
4237
|
+
await createProgram().parseAsync();
|
|
4238
|
+
} catch (error) {
|
|
4239
|
+
console.error(error instanceof Error ? error.message : error);
|
|
4240
|
+
process.exitCode = 1;
|
|
4241
|
+
}
|