thunderjar 0.1.3-preview-eee7160 → 0.1.3-preview-e09ffa1

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "thunderjar",
-  "version": "0.1.3-preview-eee7160",
+  "version": "0.1.3-preview-e09ffa1",
   "type": "module",
   "description": "Thunderjar",
   "license": "MIT",
@@ -11,24 +11,30 @@
   "bin": {
     "thunderjar": "./dist/cli.js"
   },
-  "main": "./src/index.ts",
-  "types": "./src/index.ts",
+  "main": "./src/api/index.ts",
+  "types": "./src/api/index.ts",
+  "exports": {
+    ".": "./src/api/index.ts"
+  },
   "files": [
     "dist",
-    "src"
+    "src/api",
+    "!src/**/*.test.ts",
+    "!src/**/*.result.json"
   ],
   "publishConfig": {
     "access": "public"
   },
   "scripts": {
     "build": "bun build src/cli/index.ts --target bun --outfile dist/cli.js",
-    "lint": "bun scripts/sync-oxlint-layout.ts && bun scripts/sync-oxlint-layout.ts --check && oxlint",
+    "lint": "bun scripts/lint-layers.ts --check && oxlint",
+    "lint:generate": "bun scripts/lint-layers.ts",
     "typecheck": "tsc --noEmit",
     "test": "bun test",
-    "test:prepare": "bun scripts/prepare-tests.ts",
-    "test:docker": "THUNDERJAR_TEST_DOCKER=1 bun test docker",
+    "test:docker": "THUNDERJAR_DOCKER_TESTS=1 bun test --timeout 600000 .docker.test",
+    "test:agent": "THUNDERJAR_AGENT_TESTS=1 bun test --timeout 900000 .agent.test",
+    "test:package": "bun scripts/package-smoke-test.ts",
     "fixture": "bun scripts/fixture.ts",
-    "dev:link": "bun scripts/dev-link.ts",
     "docs:dev": "vitepress dev docs",
     "docs:build": "vitepress build docs",
     "docs:build:versions": "bun scripts/build-versioned-docs.ts",

package/src/api/checkoutCommit.ts

@@ -0,0 +1,35 @@
+import type { ApplyParameterCtx } from "./declarations.ts";
+
+// How checkoutCommit tells Thunderjar which commit to check out: a code state's ctx carries a
+// function under this key. Symbol.for, so it is found even when the user's project resolves a
+// different copy of this module.
+export const CHECKOUT_COMMIT = Symbol.for("thunderjar.checkoutCommit");
+
+/** A code state's ctx, as Thunderjar makes it. */
+export type CheckoutCtx = ApplyParameterCtx & { [CHECKOUT_COMMIT]?: (commit: string) => void };
+
+/**
+ * Checks `commit`, a hash, out into `/workspace` as a real repository. For a code state's
+ * `applyParameter` only.
+ *
+ * Thunderjar bundles the commit and its ancestors, and nothing after it, into the build context.
+ * Straight after the base image's fragment, the prerun build clones it to `/workspace`, checks
+ * the commit out detached, removes the remote, and sets a git identity so the agent can commit.
+ * Those lines are Thunderjar's, so this returns an empty fragment, for the code state's own
+ * lines to follow:
+ *
+ * ```ts
+ * export default declareCodeState({
+ *   config: { commit: "a1b2c3" },
+ *   applyParameter: (config, ctx) => `${checkoutCommit(ctx, config.commit)}\nRUN npm ci`,
+ * });
+ * ```
+ *
+ * The commit must already be in the repository the project is in, or nothing is built.
+ */
+export function checkoutCommit(ctx: ApplyParameterCtx, commit: string): string {
+  const checkout = (ctx as CheckoutCtx)[CHECKOUT_COMMIT];
+  if (!checkout) throw new Error("checkoutCommit is only for a code state's applyParameter.");
+  checkout(commit);
+  return "";
+}

package/src/api/claudeCode.ts

@@ -0,0 +1,74 @@
+// Reading what `claude -p --output-format json` writes: one JSON object, with the reply as
+// `result`, and the tokens and cost of each model it used as `modelUsage`. For a harness's
+// `collectReply` and `collectTokenCosts`, so a Claude Code harness needn't parse it itself.
+//
+// `modelUsage` is read, not the flat `usage`: `usage` counts only the top-level agent loop, so it
+// leaves out what sub-agents spent. Its keys are camelCase, where `usage`'s are snake_case.
+import type { ModelTokenCosts, TokenCosts } from "./declarations.ts";
+
+type ModelUsage = {
+  inputTokens?: unknown;
+  outputTokens?: unknown;
+  cacheReadInputTokens?: unknown;
+  cacheCreationInputTokens?: unknown;
+  costUSD?: unknown;
+  costBasis?: unknown;
+};
+
+type ClaudeCodeResult = { type?: unknown; result?: unknown; modelUsage?: unknown; duration_api_ms?: unknown };
+
+const COST_BASES = ["list", "managed", "unknown"] as const;
+
+const count = (value: unknown) => (typeof value === "number" && Number.isFinite(value) && value >= 0 ? value : undefined);
+
+/** `raw` parsed as Claude Code's result, or undefined if it isn't one. */
+function parse(raw: string): ClaudeCodeResult | undefined {
+  let parsed: unknown;
+  try {
+    parsed = JSON.parse(raw);
+  } catch {
+    return undefined;
+  }
+  return typeof parsed === "object" && parsed !== null && (parsed as ClaudeCodeResult).type === "result"
+    ? (parsed as ClaudeCodeResult)
+    : undefined;
+}
+
+/** The agent's final reply, from the result file Claude Code wrote. Throws if it isn't one. */
+export function claudeCodeReply(raw: string): string {
+  const result = parse(raw)?.result;
+  if (typeof result !== "string") throw new Error("not a result of `claude -p --output-format json`");
+  return result;
+}
+
+/**
+ * The tokens and cost of each model, from the result file Claude Code wrote. `unparseable` if it
+ * isn't one, and `notReported` if it counts no tokens at all, as a crashed session's does. Its
+ * cost is its own estimate, from a price table bundled with it: `costBasis` says whether that
+ * table knew the model.
+ */
+export function claudeCodeTokenCosts(raw: string): TokenCosts {
+  const result = parse(raw);
+  if (!result || typeof result.modelUsage !== "object" || result.modelUsage === null) return { outcome: "unavailable", reason: "unparseable" };
+  const models: ModelTokenCosts[] = [];
+  for (const [model, usage] of Object.entries(result.modelUsage as Record<string, ModelUsage>)) {
+    const tokens = [usage.inputTokens, usage.outputTokens, usage.cacheCreationInputTokens, usage.cacheReadInputTokens].map(count);
+    if (tokens.some((n) => n === undefined)) return { outcome: "unavailable", reason: "unparseable" };
+    const [inputTokens, outputTokens, cacheCreationTokens, cacheReadTokens] = tokens as number[];
+    const costUsd = count(usage.costUSD);
+    const costBasis = COST_BASES.find((basis) => basis === usage.costBasis);
+    models.push({
+      model,
+      inputTokens: inputTokens as number,
+      outputTokens: outputTokens as number,
+      cacheCreationTokens: cacheCreationTokens as number,
+      cacheReadTokens: cacheReadTokens as number,
+      ...(costUsd !== undefined && { costUsd }),
+      ...(costBasis !== undefined && { costBasis }),
+    });
+  }
+  const total = models.reduce((sum, m) => sum + m.inputTokens + m.outputTokens + m.cacheCreationTokens + m.cacheReadTokens, 0);
+  if (total === 0) return { outcome: "unavailable", reason: "notReported" };
+  const apiDurationMs = count(result.duration_api_ms);
+  return { outcome: "collected", models, ...(apiDurationMs !== undefined && { apiDurationMs }) };
+}

package/src/api/declarations.ts

@@ -1,14 +1,17 @@
-// The `declareX()` functions a Thunderjar configuration file's default export is
-// wrapped in — see spec/022-coding-conventions.md#declarex-functions-not-bare-exports.
-// Each is the identity function on its argument; the point is the parameter type, which
-// gives every `ctx` its shape with nothing written at the call site.
-//
-// `BaseImageName`, `CodeStateName`, `PromptSetName`, `TaskName`, `InitialPromptNames` and
-// `HarnessModelName` are ambient globals with no import — see src/api/parameter-names.d.ts.
-
-// Each declareX() marks what it returns with its own name, so the config loader can tell
-// a wrapped default export from a bare one. Symbol.for, so the mark is recognised even
-// when the user's project resolves a different copy of this module.
+// The `declareX()` functions that a `.thunderjar/` file's default export is wrapped in. Each
+// returns what it is given: the point is the type of its argument.
+import type {
+  BaseImageNames,
+  CodeStateNames,
+  HarnessModelNames,
+  InitialPromptNamesByTask,
+  PromptSetNames,
+  TaskNames,
+} from "./parameterNames.ts";
+
+// Each declareX() marks what it returns with its own name, so the config loader can tell a wrapped
+// default export from a bare one. Symbol.for, so the mark is recognised even when the user's
+// project resolves a different copy of this module.
 const DECLARED_BY = Symbol.for("thunderjar.declaredBy");
 
 function declared<T extends object>(declareFn: string, value: T): T {
@@ -25,202 +28,244 @@ export function declaredBy(value: unknown): string | undefined {
   return typeof mark === "string" ? mark : undefined;
 }
 
-// --- applyParameter: spec/060-test-case-parameters.md#declarations-and-applyparameter ---
+// --- applyParameter ---
 //
-// Each parameter's `applyParameter(config, ctx)` returns the Dockerfile fragment that applies
-// it. `config` is the declaration's own `config`, as written (undefined if it has none); the
-// declaration file is hashed, so it is part of the parameter hash.
+// Each parameter's `applyParameter(config, ctx)` returns the Dockerfile fragment that applies it
+// to the prerun image; see src/execution/prerunDockerfile.ts for the order they are built in.
+// `config` is the declaration's own `config`, as written (undefined if it has none).
 
 /**
- * Puts things from the repository Thunderjar is run in into the build context, and returns
- * the path to COPY them from. A commit named here must be in that repository: it is checked
- * before anything is built.
+ * Reads the repository the project is in, without fetching. Each commit is a hash, full or
+ * abbreviated, and must already be in the repository, or nothing is built.
  */
-export interface GitBuildContext {
-  /** A git bundle holding `commit` and its ancestors, and nothing else, with one branch, `code-state`. */
-  bundle(commit: string): string;
-  /** The file at `path` in `commit`. */
+export type GitCtx = {
+  /**
+   * Puts `path`'s content at `commit` into the build context, and returns where it is there, to
+   * COPY: `COPY ${ctx.git.show("e4f5a6", "prompts/snerk.md")} /workspace/CLAUDE.md`. `path` is
+   * relative to the repository's root.
+   */
   show(commit: string, path: string): string;
-}
+};
 
 /** What every fragment gets. */
-interface FragmentCtx {
-  resolvedHash?: string;
-  git: GitBuildContext;
-}
+export type FragmentCtx = {
+  git: GitCtx;
+};
 
 /**
- * What a fragment gets when its parameter is a folder: `dir` is that folder's path in the
- * build context, so files bundled beside the declaration can be copied in —
- * `COPY ${ctx.dir}/files/ /workspace/`. It names the parameter's role, not its folder, so
- * renaming the folder changes nothing.
+ * What a fragment gets when its parameter is a folder. `dir` is that folder's path in the build
+ * context, so files kept beside the declaration can be copied in: `COPY ${ctx.dir}/files/ /workspace/`.
+ * It names the parameter's role, not its folder, so renaming the folder changes nothing.
  */
-export interface ApplyParameterCtx extends FragmentCtx {
+export type ApplyParameterCtx = FragmentCtx & {
   dir: string;
-}
-
-// spec/060-test-case-parameters.md#controlling-the-parameter-hash
-interface HashOverrides {
-  additionalHash?: () => string;
-  determineHash?: () => string;
-}
+};
 
-/** A parameter declaration's `config`, and the `applyParameter` that receives it. */
-interface Applies<C, Ctx> {
+/**
+ * A parameter declaration's `config`, and the `applyParameter` that receives it. A method, so a
+ * declaration with any `config` is still a `XDeclaration<unknown>` once loaded.
+ */
+type Applies<C, Ctx> = {
   config?: C;
-  applyParameter: (config: C, ctx: Ctx) => string;
-}
+  applyParameter?(config: C, ctx: Ctx): string;
+};
 
-// --- Base image: spec/061-base-image.md ---
-
-interface BaseImageDeclaration<C> extends HashOverrides, Applies<C, ApplyParameterCtx> {}
+/** A base image's fragment is required: it carries the prerun image's `FROM`. */
+export type BaseImageDeclaration<C = unknown> = Applies<C, ApplyParameterCtx> & {
+  applyParameter(config: C, ctx: ApplyParameterCtx): string;
+  /**
+   * More `docker run` options for the agent's containers, for agents that need more of the
+   * kernel than Docker allows by default, e.g. Claude Code's bubblewrap sandbox, which needs user
+   * namespaces: `["--security-opt", "seccomp=unconfined", "--security-opt", "apparmor=unconfined"]`.
+   * Each option and value is its own string. Ask for the least you need: `--privileged` works, but
+   * gives the agent the whole machine.
+   */
+  runOptions?: string[];
+};
 
 export function declareBaseImage<C = undefined>(declaration: BaseImageDeclaration<C>): BaseImageDeclaration<C> {
   return declared("declareBaseImage", declaration);
 }
 
-// --- Code state: spec/062-code-state.md ---
-
-interface CodeStateDeclaration<C> extends HashOverrides, Applies<C, ApplyParameterCtx> {}
+export type CodeStateDeclaration<C = unknown> = Applies<C, ApplyParameterCtx>;
 
 export function declareCodeState<C = undefined>(declaration: CodeStateDeclaration<C>): CodeStateDeclaration<C> {
   return declared("declareCodeState", declaration);
 }
 
-// --- Prompt set: spec/063-prompt-set.md ---
-
-interface PromptSetDeclaration<C> extends HashOverrides, Applies<C, ApplyParameterCtx> {}
+export type PromptSetDeclaration<C = unknown> = Applies<C, ApplyParameterCtx>;
 
 export function declarePromptSet<C = undefined>(declaration: PromptSetDeclaration<C>): PromptSetDeclaration<C> {
   return declared("declarePromptSet", declaration);
 }
 
-// --- Harness/model pair: spec/064-harness.md ---
-
-/** `dir` is the family folder, less models.ts, versions.ts and collectTokenCosts.ts. */
-export interface HarnessApplyCtx extends ApplyParameterCtx {
+/** `dir` is the harness's folder; `version` is the pair's version, as its `versions.ts` entry gives it. */
+export type HarnessApplyCtx = ApplyParameterCtx & {
   version: string;
-}
+};
 
-export interface HarnessCliCtx {
+/**
+ * What a harness's `cli` gets, to build the command that runs the agent. An initial prompt of
+ * several turns runs `cli` once per turn, in the same container, one after another.
+ */
+export type HarnessCliCtx = {
   version: string;
   modelId: string;
+  /**
+   * The turn's prompt text. Pasted into a shell command it breaks on the first quote, `$` or
+   * backtick, so read `promptPath` instead.
+   */
   initialPrompt: string;
+  /**
+   * A file in the container holding the turn's prompt, exactly, with no newline added: pass it to
+   * the agent as `"$(cat ${ctx.promptPath})"`, or as a file if it takes one.
+   */
+  promptPath: string;
+  /** Where the agent's own report of the turn goes. Each turn has its own. */
   resultPath: string;
-}
-
-interface HarnessExecutionOverrides {
-  /** Replaces the family's for this version. Gets the family's `config`, which versions.ts can't type. */
-  applyParameter?: (config: unknown, ctx: HarnessApplyCtx) => string;
-  cli?: (ctx: HarnessCliCtx) => string;
-  requiredEnv?: string[];
-}
+  /** Which turn this is, counting from 0. A single prompt is turn 0. */
+  turn: number;
+  /**
+   * A fresh UUID for each container run, the same for each of its turns: start the agent's
+   * session with it on turn 0, and resume it on later turns, e.g. `--session-id`, then `--resume`.
+   */
+  sessionId: string;
+};
 
-export interface HarnessDeclaration<C = unknown> extends Applies<C, HarnessApplyCtx> {
+export type HarnessDeclaration<C = unknown> = Applies<C, HarnessApplyCtx> & {
   cli: (ctx: HarnessCliCtx) => string;
+  /**
+   * The agent's final reply, read from the text of the result file its `cli` left, for
+   * instruments such as `replyMatches` to measure. Without it, a run has no reply.
+   */
+  collectReply?: (raw: string) => string;
+  /**
+   * The names of the environment variables the agent needs, e.g. its API key. Each is passed into
+   * this harness's containers only, as `docker run -e NAME`, so its value comes from the shell that
+   * runs Thunderjar, and is never written down. `plan` and `run` stop if one is unset.
+   */
   requiredEnv?: string[];
-}
+};
 
 export function declareHarness<C = undefined>(declaration: HarnessDeclaration<C>): HarnessDeclaration<C> {
   return declared("declareHarness", declaration);
 }
 
-export function declareHarnessModels(models: Record<string, string>): Record<string, string> {
+/** A harness's models: the name a test case uses, to the ID the agent is given. */
+export type HarnessModelsDeclaration = Record<string, string>;
+
+export function declareHarnessModels(models: HarnessModelsDeclaration): HarnessModelsDeclaration {
   return declared("declareHarnessModels", models);
 }
 
-export type HarnessVersionEntry = ({ version: string } | { determineHash: () => string }) & HarnessExecutionOverrides;
+/** A harness's versions, by the name a test case uses. */
+export type HarnessVersionsDeclaration = Record<string, { version: string }>;
 
-export function declareHarnessVersions(
-  versions: Record<string, HarnessVersionEntry>,
-): Record<string, HarnessVersionEntry> {
+export function declareHarnessVersions(versions: HarnessVersionsDeclaration): HarnessVersionsDeclaration {
   return declared("declareHarnessVersions", versions);
 }
 
-// spec/087-collecting-token-costs.md#normalising-collecttokencosts
-interface CollectorContext {
-  version: string;
-  modelId: string;
-}
-
-interface ModelTokenCosts {
+/** One model's tokens, in four buckets priced differently, and its cost if the harness reports one. */
+export type ModelTokenCosts = {
   model: string;
   inputTokens: number;
   outputTokens: number;
   cacheCreationTokens: number;
   cacheReadTokens: number;
+  /** The harness's own estimate, never computed by Thunderjar. Absent if it reports none. */
   costUsd?: number;
+  /** How far to trust `costUsd`: `unknown` when the harness's price table didn't know the model. */
   costBasis?: "list" | "managed" | "unknown";
-}
+};
 
-type TokenCosts =
-  | { outcome: "collected"; models: ModelTokenCosts[]; apiDurationMs?: number }
+/**
+ * A container run's token costs, per model, or why there are none: no result file (`absent`), one
+ * that couldn't be read (`unparseable`), or one that counted nothing (`notReported`).
+ */
+export type TokenCosts =
+  | { outcome: "collected"; models: ModelTokenCosts[]; /** Time spent waiting on the model, if the harness reports it. */ apiDurationMs?: number }
   | { outcome: "unavailable"; reason: "absent" | "unparseable" | "notReported" };
 
-type TokenCostCollector = (raw: string, ctx: CollectorContext) => TokenCosts;
+/**
+ * Reads the token costs from what the harness's `cli` wrote to its result path. Thunderjar calls
+ * it only for a result file that exists, once per turn, and sums them. A throw is `unparseable`.
+ */
+export type TokenCostCollector = (raw: string, ctx: { version: string; modelId: string }) => TokenCosts;
 
 export function declareTokenCostCollector(collector: TokenCostCollector): TokenCostCollector {
   return declared("declareTokenCostCollector", collector);
 }
 
-// --- Initial prompt: spec/066-task.md#initial-prompts ---
-
 /**
- * An initial prompt is one file, not a folder, so its fragment has no `ctx.dir`: nothing beside
- * it is part of its hash.
+ * An initial prompt is one file, not a folder, so its fragment has no `ctx.dir`. It is used when
+ * the harness runs, so its fragment is usually left out.
+ *
+ * It's a single `prompt`, or a list of `turns`, each run by the harness's `cli` in turn, in the
+ * same container and agent session, so a later turn can ask about an earlier one. Its
+ * `timeoutSeconds` covers every turn together.
  */
-export interface InitialPromptDeclaration<C = unknown> extends HashOverrides, Applies<C, FragmentCtx> {
-  prompt: string;
+export type InitialPromptDeclaration<C = unknown> = Applies<C, FragmentCtx> & {
   timeoutSeconds: number;
-}
+} & ({ prompt: string; turns?: never } | { turns: string[]; prompt?: never });
 
 export function declareInitialPrompt<C = undefined>(declaration: InitialPromptDeclaration<C>): InitialPromptDeclaration<C> {
   return declared("declareInitialPrompt", declaration);
 }
 
-// --- Measurement: spec/060-test-case-parameters.md#measuring-instruments--measurements ---
+// --- Measurements ---
 
-/** What an instrument's `evaluate` gives for one measurement of one container run. */
+/** What an instrument gives for one measurement of one container run. */
 export type MeasurementResult<T> =
   | { outcome: "measured"; data: T }
-  | { outcome: "skipped"; reason?: string } // e.g. no output existed to check; `show` prints the reason
-  | { outcome: "erroredWhileMeasuring"; error: Error }; // the instrument itself threw
+  /** Nothing to measure, e.g. no file matched. */
+  | { outcome: "skipped"; reason?: string }
+  /** The instrument itself failed. */
+  | { outcome: "erroredWhileMeasuring"; error: Error };
 
 /**
- * What an instrument runs against: the postrun state of one container run —
- * spec/060-test-case-parameters.md#measurement-context. Relative paths and glob patterns
- * resolve in `workdir`.
+ * What an instrument runs against: a measurement container, started from a container run's
+ * postrun image, and discarded once the measure is taken. Relative paths, glob patterns and
+ * commands resolve in `workdir`.
  */
-export interface MeasurementContext {
+export type MeasurementContext = {
   exec(command: string): Promise<{ stdout: string; stderr: string; exitCode: number }>;
   readFile(path: string): Promise<string>;
   writeFile(path: string, contents: string): Promise<void>;
-  /** Files matching `pattern`, relative to `workdir`, sorted. */
+  /** The files matching `pattern`, relative to `workdir`, sorted. */
   glob(pattern: string): Promise<string[]>;
-  /** Raw contents of the file the harness's `cli` wrote. */
+  /** The text of the result file the harness's `cli` left, or "" if it left none. */
   harnessResult: string;
+  /** The agent's final reply, from its harness's `collectReply`, or why there's none. */
+  reply: Reply;
+  /** Its reply to each turn, in order: one, unless its initial prompt has several turns. */
+  replies: Reply[];
   workdir: string;
-}
+};
 
-/**
- * The built-in instruments' config and value types, by name. src/instruments/instruments.ts implements
- * each against exactly these types.
- */
-export interface InstrumentTypes {
+/** The agent's final reply, as its harness's `collectReply` read it from the result file, or why there's none. */
+export type Reply = { outcome: "collected"; text: string } | { outcome: "unavailable"; reason: string };
+
+/** Each built-in instrument's config and value, by name. */
+export type InstrumentTypes = {
   /** Did the agent leave a file matching `glob`? */
   fileCreated: { config: { glob: string }; value: boolean };
-  /** How many lines, across the files matching `glob`, match `pattern`, a JavaScript regex? */
+  /** How many lines, across the files matching `glob`, match `pattern`, a JavaScript regular expression? */
   grep: { config: { glob: string; pattern: string }; value: number };
   /**
-   * How many tests pass and fail when `template`, a path relative to the measurement file, is
+   * How many tests pass and fail when `template`, a path relative to the measurement's file, is
    * rendered against each file matching `subject` and run with `bun test`?
    */
   templateTest: { config: { subject: string; template: string }; value: { passed: number; failed: number } };
-}
+  /**
+   * Does the agent's reply match `pattern`, a JavaScript regular expression? Its reply to `turn`,
+   * counting from 1, of an initial prompt of several turns, or by default its last.
+   */
+  replyMatches: { config: { pattern: string; turn?: number }; value: boolean };
+};
 
 export type InstrumentName = keyof InstrumentTypes;
 
-/** The ways a value type's measures can combine — spec/085-test-case-results.md#how-a-measures-values-combine. */
+/** The ways values of type `T` can combine across iterations. */
 export type AggregateFor<T> = [T] extends [boolean]
   ? "rate" | "count"
   : [T] extends [number]
@@ -229,12 +274,12 @@ export type AggregateFor<T> = [T] extends [boolean]
       ? "sum" | "mean"
       : never;
 
-/** An instrument plus its config, typed by the instrument: a `config` that doesn't fit it is a type error. */
+/** An instrument and its config: a `config` that doesn't fit the instrument is a type error. */
 export type MeasurementDeclaration = {
   [I in InstrumentName]: {
     instrument: I;
     config: InstrumentTypes[I]["config"];
-    /** How this measurement's values combine across iterations — see src/measuring/aggregate.ts. */
+    /** How its values combine across iterations, e.g. `rate`. */
     aggregate: AggregateFor<InstrumentTypes[I]["value"]>[];
   };
 }[InstrumentName];
@@ -243,37 +288,35 @@ export function declareMeasurement(declaration: MeasurementDeclaration): Measure
   return declared("declareMeasurement", declaration);
 }
 
-// --- Test case: spec/055-declaring-test-cases.md ---
-
-export interface TestCaseDeclaration<T extends TaskName = TaskName> {
-  baseImage: BaseImageName[];
-  codeState: CodeStateName[];
-  promptSet: PromptSetName[];
-  harnessModel: HarnessModelName[];
+/**
+ * Every name must be declared under `test-case-parameters/`, and each initial prompt must be one of
+ * `task`'s, once `thunderjar generate` has run.
+ */
+export type TestCaseDeclaration<T extends TaskNames = TaskNames> = {
+  baseImage: BaseImageNames[];
+  codeState: CodeStateNames[];
+  promptSet: PromptSetNames[];
+  harnessModel: HarnessModelNames[];
   task: T;
-  initialPrompt: Array<T extends keyof InitialPromptNames ? InitialPromptNames[T] : never>;
+  initialPrompt: (T extends keyof InitialPromptNamesByTask ? InitialPromptNamesByTask[T] : never)[];
   iterations: number;
-}
+};
 
-export function declareTestCase<T extends TaskName>(declaration: TestCaseDeclaration<T>): TestCaseDeclaration<T> {
+export function declareTestCase<T extends TaskNames>(declaration: TestCaseDeclaration<T>): TestCaseDeclaration<T> {
   return declared("declareTestCase", declaration);
 }
 
-// --- Global config: spec/050-setup-and-configure.md#thunderjarconfigts ---
+export type SqliteStore = { kind: "sqlite"; path: string };
 
-interface SqliteStoreOptions {
-  path: string;
-}
-
-export function sqliteStore(options: SqliteStoreOptions): { kind: "sqlite" } & SqliteStoreOptions {
+export function sqliteStore(options: { path: string }): SqliteStore {
   return { kind: "sqlite", ...options };
 }
 
-interface ThunderjarConfigDeclaration {
+export type ConfigDeclaration = {
   imageStore: { repository: string };
-  runDataStore: ReturnType<typeof sqliteStore>;
-}
+  runDataStore: SqliteStore;
+};
 
-export function declareConfig(config: ThunderjarConfigDeclaration): ThunderjarConfigDeclaration {
+export function declareConfig(config: ConfigDeclaration): ConfigDeclaration {
   return declared("declareConfig", config);
 }

package/src/api/index.ts

@@ -0,0 +1,44 @@
+// The `thunderjar` library: what a project's `.thunderjar/` files import.
+export { checkoutCommit } from "./checkoutCommit.ts";
+export { claudeCodeReply, claudeCodeTokenCosts } from "./claudeCode.ts";
+export {
+  declareBaseImage,
+  declareCodeState,
+  declareConfig,
+  declareHarness,
+  declareHarnessModels,
+  declareHarnessVersions,
+  declareInitialPrompt,
+  declareMeasurement,
+  declarePromptSet,
+  declareTestCase,
+  declareTokenCostCollector,
+  sqliteStore,
+} from "./declarations.ts";
+export type {
+  AggregateFor,
+  ApplyParameterCtx,
+  BaseImageDeclaration,
+  CodeStateDeclaration,
+  ConfigDeclaration,
+  FragmentCtx,
+  GitCtx,
+  HarnessApplyCtx,
+  HarnessCliCtx,
+  HarnessDeclaration,
+  HarnessModelsDeclaration,
+  HarnessVersionsDeclaration,
+  InitialPromptDeclaration,
+  InstrumentName,
+  InstrumentTypes,
+  MeasurementContext,
+  MeasurementDeclaration,
+  MeasurementResult,
+  ModelTokenCosts,
+  PromptSetDeclaration,
+  Reply,
+  SqliteStore,
+  TestCaseDeclaration,
+  TokenCostCollector,
+  TokenCosts,
+} from "./declarations.ts";