thunderjar 0.1.2 → 0.1.3-preview-eee7160

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "thunderjar",
-  "version": "0.1.2",
+  "version": "0.1.3-preview-eee7160",
   "type": "module",
   "description": "Thunderjar",
   "license": "MIT",
@@ -11,19 +11,27 @@
   "bin": {
     "thunderjar": "./dist/cli.js"
   },
+  "main": "./src/index.ts",
+  "types": "./src/index.ts",
   "files": [
-    "dist"
+    "dist",
+    "src"
   ],
   "publishConfig": {
     "access": "public"
   },
   "scripts": {
     "build": "bun build src/cli/index.ts --target bun --outfile dist/cli.js",
-    "lint": "oxlint",
+    "lint": "bun scripts/sync-oxlint-layout.ts && bun scripts/sync-oxlint-layout.ts --check && oxlint",
     "typecheck": "tsc --noEmit",
     "test": "bun test",
+    "test:prepare": "bun scripts/prepare-tests.ts",
+    "test:docker": "THUNDERJAR_TEST_DOCKER=1 bun test docker",
+    "fixture": "bun scripts/fixture.ts",
+    "dev:link": "bun scripts/dev-link.ts",
     "docs:dev": "vitepress dev docs",
     "docs:build": "vitepress build docs",
+    "docs:build:versions": "bun scripts/build-versioned-docs.ts",
     "docs:preview": "vitepress preview docs",
     "changeset": "changeset",
     "version": "changeset version",
@@ -37,9 +45,10 @@
   },
   "devDependencies": {
     "@changesets/cli": "^2.31.0",
+    "@changesets/get-release-plan": "^4.0.16",
     "@types/bun": "^1.4.3",
     "oxlint": "^1.88.0",
     "typescript": "^7.0.2",
     "vitepress": "^2.0.0-alpha.18"
   }
-}
+}

package/src/AGENTS.md

@@ -0,0 +1,156 @@
+# src/
+
+Thunderjar's source, in layers. Each layer imports only from the layers below it, and the lint enforces it: `scripts/layout.ts` lists the layers in order, and `bun run lint` fails an import that points upward or a file in a folder the layout doesn't know.
+
+## Layers
+
+```mermaid
+flowchart TD
+    cli --> commands["cli/commands"]
+    commands --> images & running & measuring & store
+    images --> matrix
+    running --> matrix
+    measuring --> running & store & matrix
+    store --> matrix
+    matrix --> project & instruments
+    project --> instruments
+    instruments --> api
+```
+
+The diagram shows the imports that exist today. What the lint allows is any layer lower in this order, lowest first:
+
+```
+api → instruments → project → matrix → store → running → images → measuring → cli/commands → cli
+```
+
+Every layer may also import `platform`, `adapters/docker` and `testing`.
+
+## Folders
+
+```
+src/
+  index.ts          the `thunderjar` package: re-exports api/
+  api/
+  instruments/
+  project/
+  matrix/
+  store/
+  running/
+  images/
+  measuring/
+  cli/
+    index.ts
+    program.ts
+    commands/
+      format.ts
+      <command>/
+  platform/
+  adapters/docker/
+  testing/
+```
+
+- `index.ts`
+  - The package entry users import `thunderjar` from. It only re-exports `api/`.
+- `api/`
+  - What users' configuration files import.
+  - `declarations.ts`: the `declareX()` functions. At runtime each returns its argument; the types give `config` and `ctx` their shape.
+  - `fragments.ts`: ready-made Dockerfile fragments for `applyParameter`, such as `checkoutCommit`.
+  - `parameter-names.d.ts`: fallback types for parameter names. A real project sees its `_generated/` file instead.
+- `instruments/`
+  - The built-in measuring instruments (`fileCreated`, `grep`, `templateTest`), and the aggregate methods that fit each one's values.
+- `project/`
+  - `config.ts`: finds `.thunderjar/`, and loads and checks a test case and its measurements against the files on disk. Configuration errors come from here.
+- `matrix/`
+  - Working out what to run. Nothing here runs anything.
+  - `permutations.ts`: a test case's cross product, in a stable order.
+  - `hashing.ts`: parameter and permutation hashes, from content, never names.
+  - `resolve.ts`: every permutation of a test case, with its hash.
+  - `execution.ts`: one execution laid out, with each container run's tags and harness command.
+  - `tags.ts`: prerun and postrun image tags. `ulid.ts`: execution IDs.
+- `store/`
+  - `run-data-store.ts`: where executions, container runs and aggregation results are kept. It's an interface; for now the only store is a JSON file.
+- `running/`
+  - `runner.ts`: runs one container run in a container from its prerun image, committing it into its postrun image, and opens the measurement context that instruments read a run through. A host runner stands in for Docker in tests of `run`.
+- `images/`
+  - `prerun.ts`: renders a permutation's prerun Dockerfile from its parameters' fragments, checks the commits it reads, and writes its build context.
+  - `prerun-builds.ts`: builds a test case's missing prerun images, for both `build` and `run`, after checking everything the builds need.
+  - `repository.ts`: read-only access to the user's git repository.
+- `measuring/`
+  - `measure.ts`: applies a task's measurements to one container run.
+  - `aggregate.ts`: combines a permutation's iterations into its aggregation result.
+  - `remeasure.ts`: for a backfill, whether a stored measure is missing, stale or current, which measurements each container run needs, and each execution's measurement statuses and deleted names.
+  - `remeasure-templates/`: two versions of a bundled template, static test data for `remeasure.test.ts`.
+- `cli/commands/`
+  - One folder per command, holding the command and its tests, e.g.
+    ```
+    cli/commands/build/
+      build.ts
+      build.test.ts
+      build.docker.test.ts
+    ```
+  - `<command>.ts` exports the command's function, e.g. `build()`, and `createBuildCommand()`, which declares its arguments and options as a Commander `Command` and calls that function.
+  - A command's tests drive the CLI against a checked-in `fixtureTests/` scenario, unchanged, and assert only on what the command produced. Every expected result lives in the test. See "Fixture tests" in the root `AGENTS.md`.
+  - `format.ts`: how commands print what they share, such as counts and durations.
+  - Tests that cut across commands sit directly in `cli/commands/`: configuration errors (`validation.test.ts`) and typed parameter names (`declarations.typecheck.test.ts`).
+- `cli/`
+  - `program.ts`: `createProgram()`, the Commander program, which adds every command's `createXCommand()`.
+  - `index.ts`: the `thunderjar` entry point. It parses the arguments with `createProgram()`.
+- `platform/`
+  - Stands in for `node:path`, `node:fs` and `node:os`, which this repository doesn't use (see "Use Bun" in the root `AGENTS.md`). It imports nothing else from `src/`.
+  - `path.ts`: POSIX path functions, such as `join`, `resolve` and `relative`.
+  - `fs.ts`: file helpers on Bun's APIs and shell, such as `makeTempDir`, `remove` and `listEntries`. To read or write a file, use `Bun.file` and `Bun.write` directly.
+- `adapters/docker/`
+  - The real Docker, behind the seam tests swap out. It imports nothing else from `src/` but `platform/`.
+- `testing/`
+  - Test helpers, shared by tests in every layer.
+  - `cli.ts`: runs the CLI in a `fixtureTests/` scenario, the way `bun run fixture` does. `scenario()` names a scenario's project; `copyOfFixtures()` copies it for `init` only, the one command that writes into the project; a real `run` passes `--data-dir` instead.
+  - `fake-docker.ts`: the recording fake Docker, so a test can say which images exist, and which postrun images' saved files `docker create` and `docker cp` read. It never runs a container.
+  - `measurement-context.ts`: the host measurement context, for instrument tests.
+  - `test-setup.ts`, which every `bun test` preloads.
+  - NOT ENDORSED: `fixture-project.ts` and `seed-repo.ts` generate throwaway projects. Older tests still use them while they move to scenarios; new tests don't.
+
+## Rules
+
+### Never import upward
+
+✅ **Do** import only from a layer below, or from `platform`, `adapters/docker` or `testing`.
+
+❌ **Don't** import from a layer above, even for one function. Move the function down instead.
+
+**Why:** the layers are what keep each part testable on its own, and readable without the parts above it. One upward import makes two layers depend on each other.
+
+e.g. `config.ts` checks a measurement's aggregate methods. It used to import them from `measuring/aggregate.ts`, above it, so the list moved down into `instruments/`, which both use.
+
+### Put a new module in the lowest layer that can hold it
+
+✅ **Do** pick the layer by what the module needs to import: the lowest layer whose imports suffice.
+
+❌ **Don't** put it in `cli/commands/` because a command is its first user.
+
+**Why:** a module placed too high can't be used by the layers that will later need it, and the next person moves it.
+
+### Keep tests beside their code
+
+✅ **Do** put `<module>.test.ts` in the same folder as `<module>.ts`, and real-Docker tests in `<module>.docker.test.ts`.
+
+❌ **Don't** gather tests in a separate tree.
+
+**Why:** a module and its tests change together, and the folder shows at a glance what is tested.
+
+A test of a lower layer that needs something from a higher one gets it through `testing/`, e.g. `instruments.test.ts` gets the host measurement context from `testing/measurement-context.ts`. That keeps `testing/` the one place where tests reach across layers.
+
+### Share between commands through format.ts
+
+✅ **Do** put anything two commands print alike in `cli/commands/format.ts`, or lower if it isn't about printing.
+
+❌ **Don't** import one command from another in a command's own code.
+
+**Why:** commands are siblings. If one imports another, changing how `run` prints breaks `show`.
+
+The lint can't tell a command's code from its tests, so it allows command folders to import each other. A test shouldn't need to: one that needs, say, a recorded execution to `show` starts from a scenario with a checked-in run data store instead of running `run`.
+
+### Adding a layer or a command
+
+- A command: add a folder under `cli/commands/` whose `<command>.ts` exports `createXCommand()`, add its name to `COMMANDS` in `scripts/layout.ts`, and add the command to `createProgram()` in `cli/program.ts`.
+- A layer: add it to `LAYERS` in `scripts/layout.ts`, in order, then update the diagram and this file.
+- Then run `bun run lint`, which regenerates the rules in `.oxlintrc.json`.

package/src/adapters/docker/docker.ts

@@ -0,0 +1,117 @@
+// The Docker seam — spec/120-test-boundaries.md#principles. Everything Thunderjar asks of
+// Docker goes through a `Docker`; the real one runs the `docker` CLI. Tests swap the binary,
+// not the code path: `THUNDERJAR_DOCKER` names the executable to run in place of `docker`, such
+// as fixtureTests/fake-docker.ts, which records the commands it is given
+// (spec/120-test-boundaries.md#2-cli-with-recording-adapters).
+//
+// Thunderjar builds prerun images, runs a container run in a container from one and commits it
+// into a postrun image, and copies files out of a postrun image to measure it.
+
+export interface Docker {
+  /** Why the Docker daemon can't be reached, or undefined when it can. */
+  unreachable(): Promise<string | undefined>;
+  imageExists(ref: string): Promise<boolean>;
+  /**
+   * Builds the Dockerfile in `contextDir` and tags the image `ref`. Docker's own output goes to
+   * stderr. Throws if the build fails.
+   */
+  build(contextDir: string, ref: string): Promise<void>;
+  /**
+   * Runs `script` with `sh -c` in a new container named `name` from the image `ref`, and
+   * leaves the container once it stops. Returns its stdout; its stderr goes to stderr.
+   */
+  run(ref: string, name: string, script: string): Promise<{ exitCode: number; stdout: string }>;
+  /**
+   * Commits the stopped container `name` into the image `ref`, with the entrypoint and command
+   * of the image `like` (the one it ran from), so starting it doesn't run `run`'s script again.
+   */
+  commit(name: string, ref: string, like: string): Promise<void>;
+  /** Creates, without starting it, a container named `name` from the image `ref`. */
+  create(ref: string, name: string): Promise<void>;
+  /** Copies `path` out of the container `name` to `dest` on the host, as `docker cp` does. */
+  copyFrom(name: string, path: string, dest: string): Promise<void>;
+  /** Removes the container `name`, stopped or not. */
+  removeContainer(name: string): Promise<void>;
+}
+
+export class DockerBuildError extends Error {
+  constructor(
+    readonly ref: string,
+    readonly exitCode: number,
+  ) {
+    super(`docker build of ${ref} exited ${exitCode}`);
+  }
+}
+
+/** A docker command that exited non-zero, with what it wrote to stderr. */
+export class DockerCommandError extends Error {
+  constructor(
+    readonly args: string[],
+    readonly exitCode: number,
+    readonly stderr: string,
+  ) {
+    super(`docker ${args[0]} exited ${exitCode}${stderr.trim() ? `: ${stderr.trim().split("\n").at(-1)}` : ""}`);
+  }
+}
+
+/**
+ * Runs `binary` with `args`. "collect" discards stdout and collects stderr; "stderr" passes all
+ * of its output to our stderr, keeping stdout for Thunderjar's own; "stdout" collects stdout and
+ * passes stderr to ours.
+ */
+async function docker(binary: string, args: string[], output: "collect" | "stderr" | "stdout") {
+  const stdio: Record<typeof output, { stdout: "ignore" | "pipe" | number; stderr: "pipe" | number }> = {
+    collect: { stdout: "ignore", stderr: "pipe" },
+    stderr: { stdout: 2, stderr: 2 },
+    stdout: { stdout: "pipe", stderr: 2 },
+  };
+  const child = Bun.spawn([binary, ...args], { stdin: "ignore", ...stdio[output] });
+  const text = (stream: unknown) => (stream instanceof ReadableStream ? new Response(stream).text() : Promise.resolve(""));
+  const [stdout, stderr] = await Promise.all([text(child.stdout), text(child.stderr), child.exited]);
+  return { exitCode: child.exitCode ?? 1, stdout, stderr };
+}
+
+/** A Docker that runs the `docker` CLI, or `binary` in its place. */
+export function dockerCli(binary: string = process.env.THUNDERJAR_DOCKER || "docker"): Docker {
+  /** Runs a command whose output isn't needed, throwing if it fails. */
+  const must = async (args: string[]) => {
+    const { exitCode, stderr } = await docker(binary, args, "collect");
+    if (exitCode !== 0) throw new DockerCommandError(args, exitCode, stderr);
+  };
+  return {
+    async unreachable() {
+      try {
+        const { exitCode, stderr } = await docker(binary, ["version", "--format", "{{.Server.Version}}"], "collect");
+        if (exitCode === 0) return undefined;
+        return stderr.trim().split("\n").at(-1) || `\`${binary} version\` exited ${exitCode}`;
+      } catch (error) {
+        if ((error as NodeJS.ErrnoException).code === "ENOENT") return `\`${binary}\` was not found`;
+        throw error;
+      }
+    },
+    async imageExists(ref) {
+      return (await docker(binary, ["image", "inspect", "--format", "{{.Id}}", ref], "collect")).exitCode === 0;
+    },
+    async build(contextDir, ref) {
+      const { exitCode } = await docker(binary, ["build", "--tag", ref, contextDir], "stderr");
+      if (exitCode !== 0) throw new DockerBuildError(ref, exitCode);
+    },
+    async run(ref, name, script) {
+      // The image's own entrypoint isn't run: the script is all that runs.
+      const { exitCode, stdout } = await docker(binary, ["run", "--name", name, "--entrypoint", "sh", ref, "-c", script], "stdout");
+      return { exitCode, stdout };
+    },
+    async commit(name, ref, like) {
+      const format = "{{json .Config.Entrypoint}}\t{{json .Config.Cmd}}";
+      const inspected = await docker(binary, ["image", "inspect", "--format", format, like], "stdout");
+      if (inspected.exitCode !== 0) throw new DockerCommandError(["image", "inspect", like], inspected.exitCode, "");
+      const [entrypoint, cmd] = inspected.stdout.trim().split("\t");
+      // `ENTRYPOINT []` is ignored; `[""]` clears it.
+      const changes = [`ENTRYPOINT ${entrypoint === "null" ? '[""]' : entrypoint}`, `CMD ${cmd === "null" ? '["sh"]' : cmd}`];
+      await must(["commit", ...changes.flatMap((change) => ["--change", change]), name, ref]);
+    },
+    create: (ref, name) => must(["create", "--name", name, "--entrypoint", "sh", ref]),
+    copyFrom: (name, path, dest) => must(["cp", `${name}:${path}`, dest]),
+    removeContainer: (name) => must(["rm", "--force", name]),
+  };
+}

package/src/api/declarations.ts

@@ -0,0 +1,279 @@
+// The `declareX()` functions a Thunderjar configuration file's default export is
+// wrapped in — see spec/022-coding-conventions.md#declarex-functions-not-bare-exports.
+// Each is the identity function on its argument; the point is the parameter type, which
+// gives every `ctx` its shape with nothing written at the call site.
+//
+// `BaseImageName`, `CodeStateName`, `PromptSetName`, `TaskName`, `InitialPromptNames` and
+// `HarnessModelName` are ambient globals with no import — see src/api/parameter-names.d.ts.
+
+// Each declareX() marks what it returns with its own name, so the config loader can tell
+// a wrapped default export from a bare one. Symbol.for, so the mark is recognised even
+// when the user's project resolves a different copy of this module.
+const DECLARED_BY = Symbol.for("thunderjar.declaredBy");
+
+function declared<T extends object>(declareFn: string, value: T): T {
+  if ((typeof value === "object" && value !== null) || typeof value === "function") {
+    Object.defineProperty(value, DECLARED_BY, { value: declareFn, configurable: true });
+  }
+  return value;
+}
+
+/** The name of the declareX() function `value` was returned from, if any. */
+export function declaredBy(value: unknown): string | undefined {
+  if ((typeof value !== "object" || value === null) && typeof value !== "function") return undefined;
+  const mark = (value as { [DECLARED_BY]?: unknown })[DECLARED_BY];
+  return typeof mark === "string" ? mark : undefined;
+}
+
+// --- applyParameter: spec/060-test-case-parameters.md#declarations-and-applyparameter ---
+//
+// Each parameter's `applyParameter(config, ctx)` returns the Dockerfile fragment that applies
+// it. `config` is the declaration's own `config`, as written (undefined if it has none); the
+// declaration file is hashed, so it is part of the parameter hash.
+
+/**
+ * Puts things from the repository Thunderjar is run in into the build context, and returns
+ * the path to COPY them from. A commit named here must be in that repository: it is checked
+ * before anything is built.
+ */
+export interface GitBuildContext {
+  /** A git bundle holding `commit` and its ancestors, and nothing else, with one branch, `code-state`. */
+  bundle(commit: string): string;
+  /** The file at `path` in `commit`. */
+  show(commit: string, path: string): string;
+}
+
+/** What every fragment gets. */
+interface FragmentCtx {
+  resolvedHash?: string;
+  git: GitBuildContext;
+}
+
+/**
+ * What a fragment gets when its parameter is a folder: `dir` is that folder's path in the
+ * build context, so files bundled beside the declaration can be copied in —
+ * `COPY ${ctx.dir}/files/ /workspace/`. It names the parameter's role, not its folder, so
+ * renaming the folder changes nothing.
+ */
+export interface ApplyParameterCtx extends FragmentCtx {
+  dir: string;
+}
+
+// spec/060-test-case-parameters.md#controlling-the-parameter-hash
+interface HashOverrides {
+  additionalHash?: () => string;
+  determineHash?: () => string;
+}
+
+/** A parameter declaration's `config`, and the `applyParameter` that receives it. */
+interface Applies<C, Ctx> {
+  config?: C;
+  applyParameter: (config: C, ctx: Ctx) => string;
+}
+
+// --- Base image: spec/061-base-image.md ---
+
+interface BaseImageDeclaration<C> extends HashOverrides, Applies<C, ApplyParameterCtx> {}
+
+export function declareBaseImage<C = undefined>(declaration: BaseImageDeclaration<C>): BaseImageDeclaration<C> {
+  return declared("declareBaseImage", declaration);
+}
+
+// --- Code state: spec/062-code-state.md ---
+
+interface CodeStateDeclaration<C> extends HashOverrides, Applies<C, ApplyParameterCtx> {}
+
+export function declareCodeState<C = undefined>(declaration: CodeStateDeclaration<C>): CodeStateDeclaration<C> {
+  return declared("declareCodeState", declaration);
+}
+
+// --- Prompt set: spec/063-prompt-set.md ---
+
+interface PromptSetDeclaration<C> extends HashOverrides, Applies<C, ApplyParameterCtx> {}
+
+export function declarePromptSet<C = undefined>(declaration: PromptSetDeclaration<C>): PromptSetDeclaration<C> {
+  return declared("declarePromptSet", declaration);
+}
+
+// --- Harness/model pair: spec/064-harness.md ---
+
+/** `dir` is the family folder, less models.ts, versions.ts and collectTokenCosts.ts. */
+export interface HarnessApplyCtx extends ApplyParameterCtx {
+  version: string;
+}
+
+export interface HarnessCliCtx {
+  version: string;
+  modelId: string;
+  initialPrompt: string;
+  resultPath: string;
+}
+
+interface HarnessExecutionOverrides {
+  /** Replaces the family's for this version. Gets the family's `config`, which versions.ts can't type. */
+  applyParameter?: (config: unknown, ctx: HarnessApplyCtx) => string;
+  cli?: (ctx: HarnessCliCtx) => string;
+  requiredEnv?: string[];
+}
+
+export interface HarnessDeclaration<C = unknown> extends Applies<C, HarnessApplyCtx> {
+  cli: (ctx: HarnessCliCtx) => string;
+  requiredEnv?: string[];
+}
+
+export function declareHarness<C = undefined>(declaration: HarnessDeclaration<C>): HarnessDeclaration<C> {
+  return declared("declareHarness", declaration);
+}
+
+export function declareHarnessModels(models: Record<string, string>): Record<string, string> {
+  return declared("declareHarnessModels", models);
+}
+
+export type HarnessVersionEntry = ({ version: string } | { determineHash: () => string }) & HarnessExecutionOverrides;
+
+export function declareHarnessVersions(
+  versions: Record<string, HarnessVersionEntry>,
+): Record<string, HarnessVersionEntry> {
+  return declared("declareHarnessVersions", versions);
+}
+
+// spec/087-collecting-token-costs.md#normalising-collecttokencosts
+interface CollectorContext {
+  version: string;
+  modelId: string;
+}
+
+interface ModelTokenCosts {
+  model: string;
+  inputTokens: number;
+  outputTokens: number;
+  cacheCreationTokens: number;
+  cacheReadTokens: number;
+  costUsd?: number;
+  costBasis?: "list" | "managed" | "unknown";
+}
+
+type TokenCosts =
+  | { outcome: "collected"; models: ModelTokenCosts[]; apiDurationMs?: number }
+  | { outcome: "unavailable"; reason: "absent" | "unparseable" | "notReported" };
+
+type TokenCostCollector = (raw: string, ctx: CollectorContext) => TokenCosts;
+
+export function declareTokenCostCollector(collector: TokenCostCollector): TokenCostCollector {
+  return declared("declareTokenCostCollector", collector);
+}
+
+// --- Initial prompt: spec/066-task.md#initial-prompts ---
+
+/**
+ * An initial prompt is one file, not a folder, so its fragment has no `ctx.dir`: nothing beside
+ * it is part of its hash.
+ */
+export interface InitialPromptDeclaration<C = unknown> extends HashOverrides, Applies<C, FragmentCtx> {
+  prompt: string;
+  timeoutSeconds: number;
+}
+
+export function declareInitialPrompt<C = undefined>(declaration: InitialPromptDeclaration<C>): InitialPromptDeclaration<C> {
+  return declared("declareInitialPrompt", declaration);
+}
+
+// --- Measurement: spec/060-test-case-parameters.md#measuring-instruments--measurements ---
+
+/** What an instrument's `evaluate` gives for one measurement of one container run. */
+export type MeasurementResult<T> =
+  | { outcome: "measured"; data: T }
+  | { outcome: "skipped"; reason?: string } // e.g. no output existed to check; `show` prints the reason
+  | { outcome: "erroredWhileMeasuring"; error: Error }; // the instrument itself threw
+
+/**
+ * What an instrument runs against: the postrun state of one container run —
+ * spec/060-test-case-parameters.md#measurement-context. Relative paths and glob patterns
+ * resolve in `workdir`.
+ */
+export interface MeasurementContext {
+  exec(command: string): Promise<{ stdout: string; stderr: string; exitCode: number }>;
+  readFile(path: string): Promise<string>;
+  writeFile(path: string, contents: string): Promise<void>;
+  /** Files matching `pattern`, relative to `workdir`, sorted. */
+  glob(pattern: string): Promise<string[]>;
+  /** Raw contents of the file the harness's `cli` wrote. */
+  harnessResult: string;
+  workdir: string;
+}
+
+/**
+ * The built-in instruments' config and value types, by name. src/instruments/instruments.ts implements
+ * each against exactly these types.
+ */
+export interface InstrumentTypes {
+  /** Did the agent leave a file matching `glob`? */
+  fileCreated: { config: { glob: string }; value: boolean };
+  /** How many lines, across the files matching `glob`, match `pattern`, a JavaScript regex? */
+  grep: { config: { glob: string; pattern: string }; value: number };
+  /**
+   * How many tests pass and fail when `template`, a path relative to the measurement file, is
+   * rendered against each file matching `subject` and run with `bun test`?
+   */
+  templateTest: { config: { subject: string; template: string }; value: { passed: number; failed: number } };
+}
+
+export type InstrumentName = keyof InstrumentTypes;
+
+/** The ways a value type's measures can combine — spec/085-test-case-results.md#how-a-measures-values-combine. */
+export type AggregateFor<T> = [T] extends [boolean]
+  ? "rate" | "count"
+  : [T] extends [number]
+    ? "mean" | "median" | "min" | "max" | "sum"
+    : [T] extends [Record<string, number>]
+      ? "sum" | "mean"
+      : never;
+
+/** An instrument plus its config, typed by the instrument: a `config` that doesn't fit it is a type error. */
+export type MeasurementDeclaration = {
+  [I in InstrumentName]: {
+    instrument: I;
+    config: InstrumentTypes[I]["config"];
+    /** How this measurement's values combine across iterations — see src/measuring/aggregate.ts. */
+    aggregate: AggregateFor<InstrumentTypes[I]["value"]>[];
+  };
+}[InstrumentName];
+
+export function declareMeasurement(declaration: MeasurementDeclaration): MeasurementDeclaration {
+  return declared("declareMeasurement", declaration);
+}
+
+// --- Test case: spec/055-declaring-test-cases.md ---
+
+export interface TestCaseDeclaration<T extends TaskName = TaskName> {
+  baseImage: BaseImageName[];
+  codeState: CodeStateName[];
+  promptSet: PromptSetName[];
+  harnessModel: HarnessModelName[];
+  task: T;
+  initialPrompt: Array<T extends keyof InitialPromptNames ? InitialPromptNames[T] : never>;
+  iterations: number;
+}
+
+export function declareTestCase<T extends TaskName>(declaration: TestCaseDeclaration<T>): TestCaseDeclaration<T> {
+  return declared("declareTestCase", declaration);
+}
+
+// --- Global config: spec/050-setup-and-configure.md#thunderjarconfigts ---
+
+interface SqliteStoreOptions {
+  path: string;
+}
+
+export function sqliteStore(options: SqliteStoreOptions): { kind: "sqlite" } & SqliteStoreOptions {
+  return { kind: "sqlite", ...options };
+}
+
+interface ThunderjarConfigDeclaration {
+  imageStore: { repository: string };
+  runDataStore: ReturnType<typeof sqliteStore>;
+}
+
+export function declareConfig(config: ThunderjarConfigDeclaration): ThunderjarConfigDeclaration {
+  return declared("declareConfig", config);
+}

package/src/api/fragments.ts

@@ -0,0 +1,30 @@
+// Dockerfile fragments a parameter's `applyParameter` can use, exported from `thunderjar`.
+
+import type { GitBuildContext } from "./declarations.ts";
+
+/**
+ * Checks `commit` out into /workspace as a real repository — spec/062-code-state.md#how-the-code-gets-into-the-image:
+ * cloned from a bundle of the commit and its ancestors only (so `git log --all` shows the agent
+ * nothing after it), detached, with no remote, and a git identity so the agent can commit.
+ *
+ * ```ts
+ * export default declareCodeState({
+ *   config: { commit: "a1b2c3" },
+ *   applyParameter: (config, ctx) => `${checkoutCommit(ctx, config.commit)}\nRUN npm ci`,
+ * });
+ * ```
+ */
+export function checkoutCommit(ctx: { git: GitBuildContext }, commit: string): string {
+  return [
+    `COPY ${ctx.git.bundle(commit)} /tmp/code.bundle`,
+    [
+      "RUN git clone --quiet /tmp/code.bundle /workspace",
+      `git -C /workspace checkout --quiet --detach ${commit}`,
+      "git -C /workspace branch --quiet --delete --force code-state",
+      "git -C /workspace remote remove origin",
+      `git -C /workspace config user.name "Thunderjar agent"`,
+      "git -C /workspace config user.email agent@thunderjar.invalid",
+      "rm /tmp/code.bundle",
+    ].join(" \\\n  && "),
+  ].join("\n");
+}

package/src/api/parameter-names.d.ts

@@ -0,0 +1,16 @@
+// Fallback ambient types for the parameter names a test case selects.
+//
+// `thunderjar generate` writes a project-specific `_generated/parameter-names.d.ts`
+// declaring these same global names as literal unions (see
+// spec/055-declaring-test-cases.md#typed-names). A real project's own TypeScript
+// program sees only that generated file, never this one — this file exists solely so
+// that this package's own declarations (which reference these names directly, with no
+// import) type-check on their own, and so a program that hasn't run `generate` yet
+// still compiles, permissively.
+
+type BaseImageName = string;
+type CodeStateName = string;
+type PromptSetName = string;
+type TaskName = string;
+type InitialPromptNames = Record<string, string>;
+type HarnessModelName = string;