thunderjar 0.1.3-preview-388d791 → 0.1.3-preview-e09ffa1

package/dist/cli.js

@@ -2047,7 +2047,7 @@ var program = new Command;
 // package.json
 var package_default = {
   name: "thunderjar",
-  version: "0.1.3-preview-388d791",
+  version: "0.1.3-preview-e09ffa1",
   type: "module",
   description: "Thunderjar",
   license: "MIT",
@@ -2066,7 +2066,8 @@ var package_default = {
   files: [
     "dist",
     "src/api",
-    "!src/**/*.test.ts"
+    "!src/**/*.test.ts",
+    "!src/**/*.result.json"
   ],
   publishConfig: {
     access: "public"
@@ -2077,7 +2078,9 @@ var package_default = {
     "lint:generate": "bun scripts/lint-layers.ts",
     typecheck: "tsc --noEmit",
     test: "bun test",
-    "test:docker": "THUNDERJAR_DOCKER_TESTS=1 bun test .docker.test",
+    "test:docker": "THUNDERJAR_DOCKER_TESTS=1 bun test --timeout 600000 .docker.test",
+    "test:agent": "THUNDERJAR_AGENT_TESTS=1 bun test --timeout 900000 .agent.test",
+    "test:package": "bun scripts/package-smoke-test.ts",
     fixture: "bun scripts/fixture.ts",
     "docs:dev": "vitepress dev docs",
     "docs:build": "vitepress build docs",
@@ -2329,19 +2332,38 @@ var grep = {
   },
   checkConfig(config) {
     const { glob, pattern } = config ?? {};
-    const problems = checkString("config.glob", glob, "a glob pattern");
-    if (typeof pattern !== "string") {
-      problems.push(["config.pattern", `must be a JavaScript regular expression, got ${show(pattern)}`]);
-    } else {
-      try {
-        new RegExp(pattern);
-      } catch (error) {
-        const reason = error.message.replace(/^Invalid regular expression: /, "");
-        problems.push(["config.pattern", `${show(pattern)} is not a valid regular expression: ${reason}`]);
-      }
+    return [...checkString("config.glob", glob, "a glob pattern"), ...checkPattern(pattern)];
+  }
+};
+function checkPattern(pattern) {
+  if (typeof pattern !== "string")
+    return [["config.pattern", `must be a JavaScript regular expression, got ${show(pattern)}`]];
+  try {
+    new RegExp(pattern);
+    return [];
+  } catch (error) {
+    const reason = error.message.replace(/^Invalid regular expression: /, "");
+    return [["config.pattern", `${show(pattern)} is not a valid regular expression: ${reason}`]];
+  }
+}
+var replyMatches = {
+  async evaluate({ pattern, turn }, ctx) {
+    const reply = turn === undefined ? ctx.reply : ctx.replies[turn - 1];
+    if (!reply)
+      return { outcome: "skipped", reason: `no turn ${turn}: the initial prompt has ${ctx.replies.length}` };
+    if (reply.outcome !== "collected")
+      return { outcome: "skipped", reason: `no reply: ${reply.reason}` };
+    return { outcome: "measured", data: new RegExp(pattern).test(reply.text) };
+  },
+  checkConfig(config) {
+    const { pattern, turn } = config ?? {};
+    const problems = checkPattern(pattern);
+    if (turn !== undefined && (typeof turn !== "number" || !Number.isInteger(turn) || turn < 1)) {
+      problems.push(["config.turn", `must be a turn, counting from 1, got ${show(turn)}`]);
     }
     return problems;
-  }
+  },
+  usesContainer: false
 };
 var shellWord = (text) => `'${text.replaceAll("'", `'\\''`)}'`;
 var dirname = (path2) => path2.includes("/") ? path2.slice(0, path2.lastIndexOf("/")) : ".";
@@ -2436,11 +2458,12 @@ var templateTest = {
   },
   bundledFiles: ["template"]
 };
-var instruments = { fileCreated, grep, templateTest };
+var instruments = { fileCreated, grep, templateTest, replyMatches };
 var aggregateMethods = {
   fileCreated: ["rate", "count"],
   grep: ["mean", "median", "min", "max", "sum"],
-  templateTest: ["sum", "mean"]
+  templateTest: ["sum", "mean"],
+  replyMatches: ["rate", "count"]
 };
 var isInstrumentName = (name) => typeof name === "string" && Object.hasOwn(instruments, name);
 
@@ -2516,8 +2539,57 @@ async function loadMeasurements(root, task) {
   return measurements;
 }
 
-// src/config/testCase.ts
+// src/config/requiredEnv.ts
+var NAME = /^[A-Za-z_][A-Za-z0-9_]*$/;
 var show3 = (value) => JSON.stringify(value) ?? String(value);
+function requiredEnvProblem(requiredEnv) {
+  if (requiredEnv === undefined)
+    return;
+  if (!Array.isArray(requiredEnv))
+    return `must be an array of environment variable names, got ${show3(requiredEnv)}`;
+  const bad = requiredEnv.filter((name) => typeof name !== "string" || !NAME.test(name));
+  if (bad.length > 0)
+    return `${bad.map(show3).join(", ")} ${bad.length === 1 ? "is not an environment variable name" : "are not environment variable names"}`;
+  return;
+}
+var requiredEnvOf = (harness) => harness.requiredEnv ?? [];
+async function checkRequiredEnv(root, testCase, env = Bun.env) {
+  const harnesses = [...new Set(testCase.harnessModel.map((pair) => parsePair(pair).harness))];
+  const problems = [];
+  const missing = [];
+  for (const name of harnesses) {
+    const harness = await loadDeclaration(root, `${PARAMETERS}/harnessModels/${name}/index.ts`, "declareHarness", problems);
+    for (const variable of harness ? requiredEnvOf(harness) : []) {
+      if (!env[variable])
+        missing.push(`${name} needs ${variable}, which is not set.`);
+    }
+  }
+  if (problems.length > 0)
+    throw problemsError(problems);
+  if (missing.length === 0)
+    return;
+  const hint = `Export ${missing.length === 1 ? "it" : "them"} in the shell that runs thunderjar.`;
+  throw new ConfigError(...missing.slice(0, -1), `${missing.at(-1)}
+${hint}`);
+}
+
+// src/config/runOptions.ts
+var show4 = (value) => JSON.stringify(value) ?? String(value);
+var RESERVED = ["--name", "--pull", "--entrypoint", "--rm", "--detach", "-d"];
+function runOptionsProblem(runOptions) {
+  if (runOptions === undefined)
+    return;
+  if (!Array.isArray(runOptions) || runOptions.some((option) => typeof option !== "string")) {
+    return `must be an array of docker run options and their values, each a string, got ${show4(runOptions)}`;
+  }
+  const reserved = runOptions.filter((option) => RESERVED.includes(option.split("=")[0]));
+  if (reserved.length > 0)
+    return `${reserved.map(show4).join(", ")} can't be given: Thunderjar sets ${reserved.length === 1 ? "it" : "them"} itself`;
+  return;
+}
+
+// src/config/testCase.ts
+var show5 = (value) => JSON.stringify(value) ?? String(value);
 var PARAMETER_FOLDERS = [
   ["baseImages", "base images"],
   ["codeStates", "code states"],
@@ -2558,7 +2630,7 @@ class TestCaseChecker {
   async requires(field, name, path2) {
     if (await existsIn(this.root, path2))
       return true;
-    this.problem(field, `${show3(name)} has no matching ${FOLDER}/${path2}`);
+    this.problem(field, `${show5(name)} has no matching ${FOLDER}/${path2}`);
     return false;
   }
   arrays(testCase) {
@@ -2566,7 +2638,7 @@ class TestCaseChecker {
     for (const field of ARRAY_FIELDS) {
       const value = testCase[field];
       if (!Array.isArray(value) || value.some((name) => typeof name !== "string")) {
-        this.problem(field, `must be an array of names, got ${show3(value)}`);
+        this.problem(field, `must be an array of names, got ${show5(value)}`);
       } else if (value.length === 0) {
         this.problem(field, "must name at least one value, got []");
       } else {
@@ -2577,7 +2649,7 @@ class TestCaseChecker {
   }
   checkIterations(iterations) {
     if (typeof iterations !== "number" || !Number.isInteger(iterations) || iterations < 1) {
-      this.problem("iterations", `must be a positive integer, got ${show3(iterations)}`);
+      this.problem("iterations", `must be a positive integer, got ${show5(iterations)}`);
     }
   }
   async checkFolderFields(names) {
@@ -2585,14 +2657,17 @@ class TestCaseChecker {
       for (const name of names[field] ?? []) {
         const index = `${PARAMETERS}/${folder}/${name}/index.ts`;
         if (await this.requires(field, name, index)) {
-          await loadDeclaration(this.root, index, declareFn, this.problems);
+          const declaration = await loadDeclaration(this.root, index, declareFn, this.problems);
+          const optionsProblem = field === "baseImage" && declaration && runOptionsProblem(declaration.runOptions);
+          if (optionsProblem)
+            this.problems.push({ file: index, field: "runOptions", message: optionsProblem });
         }
       }
     }
   }
   async checkTask(task, initialPrompts) {
     if (typeof task !== "string") {
-      this.problem("task", `must be a task name, got ${show3(task)}`);
+      this.problem("task", `must be a task name, got ${show5(task)}`);
       return;
     }
     if (!await this.requires("task", task, `${PARAMETERS}/tasks/${task}/`))
@@ -2601,11 +2676,29 @@ class TestCaseChecker {
     for (const name of initialPrompts) {
       const prompt = `${PARAMETERS}/tasks/${task}/initial-prompts/${name}.ts`;
       if (await existsIn(this.root, prompt)) {
-        await loadDeclaration(this.root, prompt, "declareInitialPrompt", this.problems);
+        const declaration = await loadDeclaration(this.root, prompt, "declareInitialPrompt", this.problems);
+        const timeout = declaration?.timeoutSeconds;
+        if (declaration && (typeof timeout !== "number" || !Number.isFinite(timeout) || timeout <= 0)) {
+          this.problems.push({ file: prompt, field: "timeoutSeconds", message: `must be a number of seconds above 0, got ${show5(timeout)}` });
+        }
+        if (declaration)
+          this.checkPrompt(prompt, declaration);
         continue;
       }
       const owner = new Bun.Glob(`${PARAMETERS}/tasks/*/initial-prompts/${name}.ts`).scanSync({ cwd: this.root }).next().value;
-      this.problem("initialPrompt", owner ? `${show3(name)} belongs to task ${show3(owner.split("/")[2])}, not ${show3(task)}` : `${show3(name)} has no matching ${FOLDER}/${prompt}`);
+      this.problem("initialPrompt", owner ? `${show5(name)} belongs to task ${show5(owner.split("/")[2])}, not ${show5(task)}` : `${show5(name)} has no matching ${FOLDER}/${prompt}`);
+    }
+  }
+  checkPrompt(file, { prompt, turns }) {
+    const problem = (field, message) => this.problems.push({ file, field, message });
+    if (prompt !== undefined && turns !== undefined) {
+      problem("turns", "can't be given beside prompt. Give a single prompt, or a list of turns.");
+    } else if (turns !== undefined) {
+      if (!Array.isArray(turns) || turns.length === 0 || turns.some((turn) => typeof turn !== "string")) {
+        problem("turns", `must be a list of at least one prompt, got ${show5(turns)}`);
+      }
+    } else if (typeof prompt !== "string") {
+      problem("prompt", `must be the prompt's text, or give a list of turns instead, got ${show5(prompt)}`);
     }
   }
   async harness(name) {
@@ -2613,7 +2706,13 @@ class TestCaseChecker {
     if (loaded)
       return loaded;
     const dir = `${PARAMETERS}/harnessModels/${name}`;
-    await loadDeclaration(this.root, `${dir}/index.ts`, "declareHarness", this.problems);
+    const declaration = await loadDeclaration(this.root, `${dir}/index.ts`, "declareHarness", this.problems);
+    const envProblem = declaration && requiredEnvProblem(declaration.requiredEnv);
+    if (envProblem)
+      this.problems.push({ file: `${dir}/index.ts`, field: "requiredEnv", message: envProblem });
+    if (declaration && declaration.collectReply !== undefined && typeof declaration.collectReply !== "function") {
+      this.problems.push({ file: `${dir}/index.ts`, field: "collectReply", message: `must be a function of the result file's text, got ${show5(declaration.collectReply)}` });
+    }
     await loadDeclaration(this.root, `${dir}/collectTokenCosts.ts`, "declareTokenCostCollector", this.problems);
     const harness = {
       versions: await loadDeclaration(this.root, `${dir}/versions.ts`, "declareHarnessVersions", this.problems),
@@ -2626,7 +2725,7 @@ class TestCaseChecker {
     for (const pair of pairs) {
       const match = HARNESS_MODEL.exec(pair);
       if (!match) {
-        this.problem("harnessModel", `${show3(pair)} is not of the form harness@version/model`);
+        this.problem("harnessModel", `${show5(pair)} is not of the form harness@version/model`);
         continue;
       }
       const [, name, version, model] = match;
@@ -2635,10 +2734,10 @@ class TestCaseChecker {
         continue;
       const { versions, models } = await this.harness(name);
       if (versions && !Object.hasOwn(versions, version)) {
-        this.problem("harnessModel", `${show3(pair)}: version ${show3(version)} is not in ${FOLDER}/${dir}/versions.ts`);
+        this.problem("harnessModel", `${show5(pair)}: version ${show5(version)} is not in ${FOLDER}/${dir}/versions.ts`);
       }
       if (models && !Object.hasOwn(models, model)) {
-        this.problem("harnessModel", `${show3(pair)}: model ${show3(model)} is not in ${FOLDER}/${dir}/models.ts`);
+        this.problem("harnessModel", `${show5(pair)}: model ${show5(model)} is not in ${FOLDER}/${dir}/models.ts`);
       }
     }
   }
@@ -2703,8 +2802,27 @@ async function mustRun(args) {
   }
   return result.stdout.toString();
 }
-async function runContainer(image, name, script) {
-  await mustRun(["run", "--name", name, "--pull", "never", "--entrypoint", "sh", image, "-c", script]);
+async function runContainer(image, name, script, { env = [], runOptions = [], timeoutSeconds } = {}) {
+  const binary = dockerBinary();
+  const passed = env.flatMap((variable) => ["-e", variable]);
+  const args = ["run", "--name", name, "--pull", "never", ...passed, ...runOptions, "--entrypoint", "sh", image, "-c", script];
+  const child = Bun.spawn([binary, ...args], { env: { ...Bun.env }, stdin: "ignore", stdout: "ignore", stderr: "pipe" });
+  let killing;
+  const timer = timeoutSeconds === undefined ? undefined : setTimeout(() => {
+    const killedAt2 = Date.now();
+    killing = mustRun(["kill", name]).then(() => killedAt2, () => {
+      return;
+    });
+  }, timeoutSeconds * 1000);
+  const [exitCode, stderr] = await Promise.all([child.exited, new Response(child.stderr).text()]);
+  clearTimeout(timer);
+  const killedAt = await killing;
+  if (killedAt !== undefined)
+    return { timedOut: true, killedAt };
+  if (exitCode !== 0) {
+    throw new DockerContainerError(reasonOf(stderr) || `\`${binary} run\` exited ${exitCode}`);
+  }
+  return { timedOut: false };
 }
 async function readContainerFile(name, path2) {
   const dir = `${Bun.env.TMPDIR || "/tmp"}/thunderjar-cp-${crypto.randomUUID()}`;
@@ -3185,6 +3303,24 @@ function displayPath(cwd, path2) {
 }
 
 // src/cli/format.ts
+function agentEnd(execution, harnessResult) {
+  if (execution.timedOut)
+    return `timed out after ${execution.timeoutSeconds}s`;
+  const ended = execution.exitCode === 0 ? "done" : `exited ${execution.exitCode}`;
+  return harnessResult === null ? `${ended}, no result file` : ended;
+}
+var endedCleanly = (execution, harnessResult) => !execution.timedOut && execution.exitCode === 0 && harnessResult !== null;
+var thousands = (n) => Math.round(n).toLocaleString("en-US");
+var tokensOf = ({ inputTokens, outputTokens, cacheCreationTokens, cacheReadTokens }) => inputTokens + outputTokens + cacheCreationTokens + cacheReadTokens;
+var totalTokens = (costs) => costs?.outcome === "collected" ? costs.models.reduce((sum, model) => sum + tokensOf(model), 0) : undefined;
+function knownCost(costs) {
+  if (costs?.outcome !== "collected" || costs.models.length === 0)
+    return;
+  if (costs.models.some(({ costUsd, costBasis }) => costUsd === undefined || costBasis === "unknown"))
+    return;
+  return costs.models.reduce((sum, { costUsd }) => sum + costUsd, 0);
+}
+var dollars = (usd) => usd > 0 && usd < 0.0005 ? "<$0.001" : `$${usd.toFixed(3)}`;
 var plural = (n, noun) => `${n} ${noun}${n === 1 ? "" : "s"}`;
 var minute = (iso) => new Date(iso).toISOString().slice(0, 16).replace("T", " ");
 
@@ -3354,20 +3490,33 @@ var testCase = declaration("declareTestCase", `{
 var baseImage = declaration("declareBaseImage", `{
   applyParameter: () => \`FROM node:20-bookworm\`,
 }`);
-var harness = declaration("declareHarness", `{
-  applyParameter: (_config, ctx) => \`RUN npm install -g @anthropic-ai/claude-code@\${ctx.version}\`,
-  cli: (ctx) => \`claude -p "\${ctx.initialPrompt}" --model \${ctx.modelId} > \${ctx.resultPath}\`,
+var harness = `import { claudeCodeReply, declareHarness } from "thunderjar";
+
+// Claude Code, installed as root, then run as the image's non-root \`node\` user: it won't skip its
+// permission prompts as root, and nobody is there to answer them.
+export default declareHarness({
+  applyParameter: (_config, ctx) => \`RUN npm install -g @anthropic-ai/claude-code@\${ctx.version}
+RUN mkdir -p /thunderjar && chown -R node:node /workspace /thunderjar
+USER node\`,
+  // The prompt is read from its file, so quotes and newlines reach the agent intact. Turn 0 starts
+  // the session, and any later turn resumes it.
+  cli: (ctx) =>
+    \`claude -p "$(cat \${ctx.promptPath})" --model \${ctx.modelId} --output-format json --dangerously-skip-permissions\` +
+    \` \${ctx.turn === 0 ? "--session-id" : "--resume"} \${ctx.sessionId} > \${ctx.resultPath}\`,
+  collectReply: claudeCodeReply,
   requiredEnv: ["ANTHROPIC_API_KEY"],
-}`);
+});
+`;
 var harnessModels = declaration("declareHarnessModels", `{
   "haiku-4-5": "claude-haiku-4-5-20251001",
 }`);
 var harnessVersions = declaration("declareHarnessVersions", `{
   "2.1.283": { version: "2.1.283" },
 }`);
-var collectTokenCosts = declaration("declareTokenCostCollector", `(_raw, _ctx) => {
-  return { outcome: "unavailable", reason: "notReported" };
-}`);
+var collectTokenCosts = `import { claudeCodeTokenCosts, declareTokenCostCollector } from "thunderjar";
+
+export default declareTokenCostCollector((raw) => claudeCodeTokenCosts(raw));
+`;
 var initialPrompt = declaration("declareInitialPrompt", `{
   prompt: "Create a file isPrime.ts exporting a function that tests for primality.",
   timeoutSeconds: 600,
@@ -3477,6 +3626,7 @@ async function imageStatuses(permutations, imagePrefix) {
 async function plan(cwd, name, imagePrefix) {
   const root = await findThunderjarRoot(cwd);
   const testCase2 = await loadTestCase(root, name);
+  await checkRequiredEnv(root, testCase2);
   const permutations = await hashPermutations(root, testCase2);
   const { statuses, unavailable } = await imageStatuses(permutations, imagePrefix);
   return {
@@ -3497,42 +3647,69 @@ function createPlanCommand() {
 
 // src/execution/planExecution.ts
 var RESULT_PATH = "/thunderjar/result.json";
-async function harnessCommand(root, task, permutation, problems) {
+var PROMPT_PATH = "/thunderjar/prompt.txt";
+function turnPaths(turn, count) {
+  if (count === 1)
+    return { promptPath: PROMPT_PATH, resultPath: RESULT_PATH };
+  return { promptPath: `/thunderjar/turns/${turn + 1}/prompt.txt`, resultPath: `/thunderjar/turns/${turn + 1}/result.json` };
+}
+var promptsOf = (initialPrompt2) => initialPrompt2.turns ?? [initialPrompt2.prompt];
+async function loadHarness(root, task, permutation, problems) {
   const { harness: name, version, model } = parsePair(permutation.harnessModel);
   const dir = `${PARAMETERS}/harnessModels/${name}`;
-  const [harness2, versions, models, initialPrompt2] = await Promise.all([
+  const [baseImage2, harness2, collectTokenCosts2, versions, models, initialPrompt2] = await Promise.all([
+    loadDeclaration(root, `${PARAMETERS}/baseImages/${permutation.baseImage}/index.ts`, "declareBaseImage", problems),
     loadDeclaration(root, `${dir}/index.ts`, "declareHarness", problems),
+    loadDeclaration(root, `${dir}/collectTokenCosts.ts`, "declareTokenCostCollector", problems),
     loadDeclaration(root, `${dir}/versions.ts`, "declareHarnessVersions", problems),
     loadDeclaration(root, `${dir}/models.ts`, "declareHarnessModels", problems),
     loadDeclaration(root, `${PARAMETERS}/tasks/${task}/initial-prompts/${permutation.initialPrompt}.ts`, "declareInitialPrompt", problems)
   ]);
-  if (!harness2 || !versions || !models || !initialPrompt2)
+  if (!baseImage2 || !harness2 || !collectTokenCosts2 || !versions || !models || !initialPrompt2)
     return;
-  return harness2.cli({
-    version: versions[version].version,
-    modelId: models[model],
-    initialPrompt: initialPrompt2.prompt,
-    resultPath: RESULT_PATH
+  const pinned = versions[version].version;
+  return { baseImage: baseImage2, harness: harness2, collectTokenCosts: collectTokenCosts2, version: pinned, modelId: models[model], initialPrompt: initialPrompt2 };
+}
+function planTurns({ harness: harness2, version, modelId, initialPrompt: initialPrompt2 }, sessionId) {
+  const prompts = promptsOf(initialPrompt2);
+  return prompts.map((prompt, turn) => {
+    const paths = turnPaths(turn, prompts.length);
+    const command = harness2.cli({ version, modelId, initialPrompt: prompt, ...paths, turn, sessionId });
+    return { prompt, ...paths, command };
   });
 }
 async function planExecution(root, testCase2, executionId, imagePrefix = DEFAULT_IMAGE_PREFIX) {
   const hashed = await hashPermutations(root, testCase2);
   const problems = [];
-  const commands = await Promise.all(hashed.map((permutation) => harnessCommand(root, testCase2.task, permutation, problems)));
+  const loaded = await Promise.all(hashed.map((permutation) => loadHarness(root, testCase2.task, permutation, problems)));
   if (problems.length > 0)
     throw problemsError(problems);
   return {
     executionId,
-    permutations: hashed.map((permutation, p) => ({
-      number: p + 1,
-      permutation,
-      prerunImage: prerunImage(permutation.hash, imagePrefix),
-      iterations: Array.from({ length: testCase2.iterations }, (_, i) => ({
-        number: i + 1,
-        postrunImage: postrunImage(executionId, p + 1, i + 1, imagePrefix),
-        command: commands[p]
-      }))
-    }))
+    permutations: hashed.map((permutation, p) => {
+      const harnessed = loaded[p];
+      return {
+        number: p + 1,
+        permutation,
+        prerunImage: prerunImage(permutation.hash, imagePrefix),
+        env: requiredEnvOf(harnessed.harness),
+        runOptions: harnessed.baseImage.runOptions ?? [],
+        timeoutSeconds: harnessed.initialPrompt.timeoutSeconds,
+        harness: harnessed.harness,
+        collectTokenCosts: harnessed.collectTokenCosts,
+        version: harnessed.version,
+        modelId: harnessed.modelId,
+        iterations: Array.from({ length: testCase2.iterations }, (_, i) => {
+          const sessionId = crypto.randomUUID();
+          return {
+            number: i + 1,
+            postrunImage: postrunImage(executionId, p + 1, i + 1, imagePrefix),
+            sessionId,
+            turns: planTurns(harnessed, sessionId)
+          };
+        })
+      };
+    })
   };
 }
 
@@ -3549,37 +3726,69 @@ var NOW_MS_FUNCTION = [
   "}"
 ].join(`
 `);
-var executionWrapper = (command) => [
-  "mkdir -p /thunderjar",
-  NOW_MS_FUNCTION,
-  "start=$(now_ms)",
-  `sh -c ${shellQuote(command)}`,
-  "code=$?",
-  "end=$(now_ms)",
-  `echo "{\\"startedAt\\":$start,\\"finishedAt\\":$end,\\"exitCode\\":$code}" > ${RUN_PATH}`
-].join(`
+function executionWrapper(turns) {
+  const folders = [...new Set(["/thunderjar", ...turns.map(({ promptPath }) => promptPath.slice(0, promptPath.lastIndexOf("/")))])];
+  const many = turns.length > 1;
+  const runTurn = ({ command }, i) => {
+    const run = [
+      "turn_start=$(now_ms)",
+      `sh -c ${shellQuote(command)}`,
+      "code=$?",
+      ...many ? [`turns="$turns${i === 0 ? "" : ","}{\\"startedAt\\":$turn_start,\\"finishedAt\\":$(now_ms),\\"exitCode\\":$code}"`] : []
+    ];
+    return i === 0 ? run : ["if [ $code -eq 0 ]; then", ...run.map((line) => `  ${line}`), "fi"];
+  };
+  return [
+    `mkdir -p ${folders.join(" ")}`,
+    ...turns.map(({ prompt, promptPath }) => `printf '%s' ${shellQuote(prompt)} > ${promptPath}`),
+    NOW_MS_FUNCTION,
+    "start=$(now_ms)",
+    `echo "{\\"startedAt\\":$start}" > ${RUN_PATH}`,
+    ...many ? ['turns=""'] : [],
+    ...turns.flatMap(runTurn),
+    "end=$(now_ms)",
+    `echo "{\\"startedAt\\":$start,\\"finishedAt\\":$end,\\"exitCode\\":$code${many ? ",\\\"turns\\\":[$turns]" : ""}}" > ${RUN_PATH}`
+  ].join(`
 `);
+}
 var isEpochMs = (value) => Number.isInteger(value) && value >= 1000000000000 && value < 10000000000000;
-function parseExecution(text) {
+var isTurn = (value) => {
+  const { startedAt, finishedAt, exitCode } = value ?? {};
+  return isEpochMs(startedAt) && isEpochMs(finishedAt) && typeof exitCode === "number";
+};
+function parseExecution(text, timeoutSeconds, killedAt) {
   let parsed;
   try {
     parsed = text === undefined ? undefined : JSON.parse(text);
   } catch {
     parsed = undefined;
   }
-  const { startedAt, finishedAt, exitCode } = parsed ?? {};
-  if (!isEpochMs(startedAt) || !isEpochMs(finishedAt) || typeof exitCode !== "number") {
-    throw new DockerContainerError(`The execution wrapper left no valid ${RUN_PATH}: ${text === undefined ? "there is none" : text.trim()}`);
-  }
-  return { startedAt, finishedAt, exitCode };
+  const { startedAt, finishedAt, exitCode, turns } = parsed ?? {};
+  const invalid = () => new DockerContainerError(`The execution wrapper left no valid ${RUN_PATH}: ${text === undefined ? "there is none" : text.trim()}`);
+  if (!isEpochMs(startedAt))
+    throw invalid();
+  if (killedAt !== undefined)
+    return { startedAt, finishedAt: killedAt, exitCode: null, timedOut: true, timeoutSeconds };
+  if (!isEpochMs(finishedAt) || typeof exitCode !== "number")
+    throw invalid();
+  if (turns !== undefined && (!Array.isArray(turns) || !turns.every(isTurn)))
+    throw invalid();
+  return { startedAt, finishedAt, exitCode, timedOut: false, timeoutSeconds, ...turns && { turns } };
 }
 async function runInContainer(run) {
   try {
-    await runContainer(run.prerunImage, run.container, executionWrapper(run.command));
-    const execution = parseExecution(await readContainerFile(run.container, RUN_PATH));
-    const harnessResult = await readContainerFile(run.container, RESULT_PATH) ?? null;
+    const stop = await runContainer(run.prerunImage, run.container, executionWrapper(run.turns), {
+      env: run.env,
+      runOptions: run.runOptions,
+      timeoutSeconds: run.timeoutSeconds
+    });
+    const killedAt = stop.timedOut ? stop.killedAt : undefined;
+    const execution = parseExecution(await readContainerFile(run.container, RUN_PATH), run.timeoutSeconds, killedAt);
+    const harnessResults = [];
+    for (const { resultPath } of run.turns)
+      harnessResults.push(await readContainerFile(run.container, resultPath) ?? null);
     await commitContainer(run.container, run.postrunImage, run.prerunImage);
-    return { execution, harnessResult };
+    return { execution, harnessResults };
   } finally {
     await removeContainer(run.container).catch(() => {});
   }
@@ -3592,7 +3801,7 @@ async function measureContainerRun(measurements, open, now = () => new Date) {
   const result = {};
   for (const { name, declaration: declaration2, config: config2, contentHash } of measurements) {
     const instrument = instruments[declaration2.instrument];
-    const ctx = await open();
+    const ctx = await open(instrument.usesContainer !== false);
     let outcome;
     try {
       outcome = recordable(await instrument.evaluate(config2, ctx));
@@ -3608,7 +3817,23 @@ async function measureContainerRun(measurements, open, now = () => new Date) {
 
 // src/execution/measurementContext.ts
 var WORKDIR = "/workspace";
-async function openMeasurementContext(postrunImage2, container, harnessResult) {
+function recordedContext({ harnessResult, reply, replies }) {
+  const noContainer = async () => {
+    throw new Error("this instrument has no measurement container: it reads only what was recorded of the run");
+  };
+  return {
+    workdir: WORKDIR,
+    harnessResult,
+    reply,
+    replies,
+    exec: noContainer,
+    readFile: noContainer,
+    writeFile: noContainer,
+    glob: noContainer,
+    close: async () => {}
+  };
+}
+async function openMeasurementContext(postrunImage2, container, { harnessResult, reply, replies }) {
   try {
     await startContainer(postrunImage2, container);
   } catch (error) {
@@ -3625,6 +3850,8 @@ async function openMeasurementContext(postrunImage2, container, harnessResult) {
   return {
     workdir: WORKDIR,
     harnessResult,
+    reply,
+    replies,
     exec: (command) => run(["sh", "-c", command]),
     readFile: (path2) => mustRun2(`can't read ${path2}`, ["cat", "--", path2]),
     async writeFile(path2, contents) {
@@ -3640,8 +3867,82 @@ async function openMeasurementContext(postrunImage2, container, harnessResult) {
   };
 }
 
+// src/execution/reply.ts
+var message2 = (error) => error instanceof Error ? error.message : String(error);
+function collectReply(collect, harnessResult) {
+  if (harnessResult === null)
+    return { outcome: "unavailable", reason: "the agent left no result file" };
+  if (!collect)
+    return { outcome: "unavailable", reason: "its harness has no collectReply" };
+  let text;
+  try {
+    text = collect(harnessResult);
+  } catch (error) {
+    return { outcome: "unavailable", reason: `collectReply threw: ${message2(error)}` };
+  }
+  if (typeof text !== "string")
+    return { outcome: "unavailable", reason: `collectReply returned ${typeof text}, not a string` };
+  return { outcome: "collected", text };
+}
+
+// src/execution/tokenCosts.ts
+var isCount = (value) => typeof value === "number" && Number.isFinite(value) && value >= 0;
+function isTokenCosts(value) {
+  const costs = value;
+  if (typeof costs !== "object" || costs === null)
+    return false;
+  if (costs.outcome === "unavailable")
+    return ["absent", "unparseable", "notReported"].includes(costs.reason);
+  if (costs.outcome !== "collected" || !Array.isArray(costs.models))
+    return false;
+  if (costs.apiDurationMs !== undefined && !isCount(costs.apiDurationMs))
+    return false;
+  return costs.models.every((model) => {
+    const counts = [model.inputTokens, model.outputTokens, model.cacheCreationTokens, model.cacheReadTokens];
+    return typeof model.model === "string" && counts.every(isCount) && (model.costUsd === undefined || isCount(model.costUsd));
+  });
+}
+function collectOne(collect, raw, ctx) {
+  if (raw === null)
+    return { outcome: "unavailable", reason: "absent" };
+  try {
+    const costs = collect(raw, ctx);
+    return isTokenCosts(costs) ? costs : { outcome: "unavailable", reason: "unparseable" };
+  } catch {
+    return { outcome: "unavailable", reason: "unparseable" };
+  }
+}
+function add(a, b) {
+  const costUsd = a.costUsd !== undefined && b.costUsd !== undefined ? a.costUsd + b.costUsd : undefined;
+  const costBasis = a.costBasis === b.costBasis ? a.costBasis : "unknown";
+  return {
+    model: a.model,
+    inputTokens: a.inputTokens + b.inputTokens,
+    outputTokens: a.outputTokens + b.outputTokens,
+    cacheCreationTokens: a.cacheCreationTokens + b.cacheCreationTokens,
+    cacheReadTokens: a.cacheReadTokens + b.cacheReadTokens,
+    ...costUsd !== undefined && { costUsd },
+    ...costBasis !== undefined && { costBasis }
+  };
+}
+function collectTokenCosts2(collect, harnessResults, ctx) {
+  const all = harnessResults.map((raw) => collectOne(collect, raw, ctx));
+  const collected = all.filter((costs) => costs.outcome === "collected");
+  if (collected.length === 0)
+    return all[0] ?? { outcome: "unavailable", reason: "absent" };
+  const byModel = new Map;
+  for (const model of collected.flatMap((costs) => costs.models)) {
+    const before = byModel.get(model.model);
+    byModel.set(model.model, before ? add(before, model) : model);
+  }
+  const durations = collected.map((costs) => costs.apiDurationMs);
+  const apiDurationMs = durations.every((ms) => ms !== undefined) ? durations.reduce((a, b) => a + b, 0) : undefined;
+  return { outcome: "collected", models: [...byModel.values()], ...apiDurationMs !== undefined && { apiDurationMs } };
+}
+
 // src/execution/runExecution.ts
-function containerRunRecord({ plan: plan2, name, testCase: testCase2 }, { number: permutationNumber, permutation }, iteration, execution, harnessResult) {
+function containerRunRecord({ plan: plan2, name, testCase: testCase2 }, { number: permutationNumber, permutation }, iteration, execution, turns, tokenCosts) {
+  const last = turns.at(-1);
   const { harness: harness2, version, model } = parsePair(permutation.harnessModel);
   return {
     executionId: plan2.executionId,
@@ -3655,7 +3956,11 @@ function containerRunRecord({ plan: plan2, name, testCase: testCase2 }, { number
     parameters: Object.fromEntries(AXES.map((axis) => [axis, { name: permutation[axis], parameterHash: permutation.parameterHashes[axis] }])),
     harnessModel: { harness: harness2, version, model, modelId: permutation.modelId },
     execution,
-    harnessResult,
+    harnessResult: last.harnessResult,
+    reply: last.reply,
+    tokenCosts,
+    sessionId: iteration.sessionId,
+    ...turns.length > 1 && { turns },
     result: {}
   };
 }
@@ -3681,21 +3986,31 @@ async function runExecution(run, store, hooks = {}) {
       for (const iteration of permutation.iterations) {
         hooks.onRunning?.(permutation, iteration);
         const container = containerName(plan2.executionId, permutation.number, iteration.number, imagePrefix);
-        const { execution, harnessResult } = await runInContainer({
+        const { execution, harnessResults } = await runInContainer({
           prerunImage: permutation.prerunImage,
           postrunImage: iteration.postrunImage,
-          command: iteration.command,
+          turns: iteration.turns,
+          env: permutation.env,
+          runOptions: permutation.runOptions,
+          timeoutSeconds: permutation.timeoutSeconds,
           container
         });
-        const record = containerRunRecord(run, permutation, iteration, execution, harnessResult);
+        const turns = harnessResults.map((harnessResult2) => ({ harnessResult: harnessResult2, reply: collectReply(permutation.harness.collectReply, harnessResult2) }));
+        const tokenCosts = collectTokenCosts2(permutation.collectTokenCosts, harnessResults, {
+          version: permutation.version,
+          modelId: permutation.modelId
+        });
+        const record = containerRunRecord(run, permutation, iteration, execution, turns, tokenCosts);
+        const { harnessResult, reply } = record;
         await store.recordContainerRun(record);
         if (execution.exitCode !== 0)
           status = "failed";
-        hooks.onRan?.(permutation, iteration, execution);
+        hooks.onRan?.(permutation, iteration, record);
         if (measurements.length === 0)
           continue;
         let opened = 0;
-        const open = () => openMeasurementContext(iteration.postrunImage, `${container}-measure-${++opened}`, harnessResult ?? "");
+        const recorded = { harnessResult: harnessResult ?? "", reply, replies: turns.map((turn) => turn.reply) };
+        const open = async (usesContainer) => usesContainer ? openMeasurementContext(iteration.postrunImage, `${container}-measure-${++opened}`, recorded) : recordedContext(recorded);
         const result = await measureContainerRun(measurements, open);
         await store.recordContainerRun({ ...record, result });
         hooks.onMeasured?.(permutation, iteration, result);
@@ -3731,15 +4046,22 @@ function ulid(now = Date.now(), random = crypto.getRandomValues(new Uint8Array(1
 }
 
 // src/cli/commands/run/run.ts
-var indent = (text, by) => text.split(`
-`).map((line) => by + line).join(`
-`);
+var FIELD_INDENT = "    ";
+var LABEL_WIDTH2 = 10;
+var field = (label, value) => `${FIELD_INDENT}${`${label}:`.padEnd(LABEL_WIDTH2)}${value.split(`
+`).join(`
+${FIELD_INDENT}${" ".repeat(LABEL_WIDTH2)}`)}`;
 function renderExecution({ executionId, permutations }) {
   const blocks = permutations.map((permutation) => [
     `Permutation ${permutation.number}: prerun image ${permutation.prerunImage}`,
     ...permutation.iterations.flatMap((iteration) => [
       `  Iteration ${iteration.number}: postrun image ${iteration.postrunImage}`,
-      indent(iteration.command, "    ")
+      ...permutation.env.length > 0 ? [field("env", permutation.env.join(", "))] : [],
+      ...permutation.runOptions.length > 0 ? [field("docker", permutation.runOptions.join(" "))] : [],
+      ...iteration.turns.length === 1 ? [field("prompt", iteration.turns[0].prompt), field("command", iteration.turns[0].command)] : [
+        field("session", iteration.sessionId),
+        ...iteration.turns.flatMap(({ prompt, command }, t) => [field(`prompt ${t + 1}`, prompt), field(`turn ${t + 1}`, command)])
+      ]
     ])
   ].join(`
 `));
@@ -3751,6 +4073,7 @@ async function dryRun2(cwd, name, imagePrefix) {
   const root = await findThunderjarRoot(cwd);
   const executionId = ulid();
   const testCase2 = await loadTestCase(root, name);
+  await checkRequiredEnv(root, testCase2);
   return renderExecution(await planExecution(root, testCase2, executionId, imagePrefix));
 }
 var ordinal = (n, total) => `${String(n).padStart(String(total).length)}/${total}`;
@@ -3767,6 +4090,7 @@ function label(testCase2, { permutation }) {
 async function run(cwd, name, imagePrefix, dataDir) {
   const root = await findThunderjarRoot(cwd);
   const testCase2 = await loadTestCase(root, name);
+  await checkRequiredEnv(root, testCase2);
   const plan2 = await planExecution(root, testCase2, ulid(), imagePrefix);
   const store = dataDirOf(cwd, root, dataDir);
   const permutations = plan2.permutations.length;
@@ -3789,9 +4113,13 @@ async function run(cwd, name, imagePrefix, dataDir) {
         lineOpen = true;
         process.stdout.write(` ${ordinal(permutation.number, permutations)}  iteration ${ordinal(iteration.number, testCase2.iterations)}  ${label(testCase2, permutation)}  ... `);
       },
-      onRan: (_permutation, _iteration, { startedAt, finishedAt, exitCode }) => {
+      onRan: (_permutation, _iteration, { execution, harnessResult, tokenCosts }) => {
         lineOpen = false;
-        console.log(exitCode === 0 ? `done (${duration(finishedAt - startedAt)})` : `failed (exit ${exitCode})`);
+        const ended = agentEnd(execution, harnessResult);
+        const cost = knownCost(tokenCosts);
+        const tokens = totalTokens(tokenCosts);
+        const spent = cost !== undefined ? `, ${dollars(cost)}` : tokens !== undefined ? `, ${thousands(tokens)} tokens` : "";
+        console.log(execution.timedOut ? ended : `${ended} (${duration(execution.finishedAt - execution.startedAt)}${spent})`);
       },
       onMeasured: (permutation, _iteration, result) => {
         console.log(`${" ".repeat(ordinal(permutation.number, permutations).length + 3)}${showMeasures(result)}`);
@@ -3818,7 +4146,7 @@ async function run(cwd, name, imagePrefix, dataDir) {
   }
 }
 function createRunCommand() {
-  return new Command("run").description("Run a test case: one execution, with a new ID. Builds any missing prerun image, then runs each iteration of each permutation in a container from its prerun image, commits it into its postrun image, takes the task's measurements of it, each in a container from its postrun image, and records it in the run data store.").argument("<test case>", `the name of a test case, declared in ${FOLDER}/test-cases/<test case>.ts`).option("--dry-run", "print the images and harness commands the execution would use, without running anything").addOption(imagePrefixOption()).addOption(dataDirOption()).action(reportingConfigErrors(async (name, options) => {
+  return new Command("run").description("Run a test case: one execution, with a new ID. Builds any missing prerun image, then runs each iteration of each permutation in a container from its prerun image, commits it into its postrun image, takes the task's measurements of it, each in a container from its postrun image, and records it in the run data store.").argument("<test case>", `the name of a test case, declared in ${FOLDER}/test-cases/<test case>.ts`).option("--dry-run", "print the images, environment variables, prompts and harness commands the execution would use, without running anything").addOption(imagePrefixOption()).addOption(dataDirOption()).action(reportingConfigErrors(async (name, options) => {
     if (options.dryRun)
       console.log(await dryRun2(process.cwd(), name, options.imagePrefix));
     else
@@ -3924,6 +4252,33 @@ function duration2(ms) {
   return `${Math.floor(seconds / 60)}:${String(seconds % 60).padStart(2, "0")}`;
 }
 var runName = ({ permutation, iteration }) => `${permutation}-${iteration}`;
+var notCollected = (costs) => costs === undefined ? "not recorded" : costs.outcome === "unavailable" ? `not reported  (${costs.reason})` : "";
+function costColumns(runs) {
+  const mean2 = (xs) => xs.reduce((a, b) => a + b, 0) / xs.length;
+  const tokens = runs.flatMap(({ tokenCosts }) => totalTokens(tokenCosts) ?? []);
+  const costs = runs.flatMap(({ tokenCosts }) => knownCost(tokenCosts) ?? []);
+  const star = (n) => n < runs.length ? "*" : "";
+  return [
+    tokens.length === 0 ? "not reported" : `${thousands(mean2(tokens))}${star(tokens.length)}`,
+    costs.length === 0 ? "\u2014" : `${dollars(mean2(costs))}${star(costs.length)}`
+  ];
+}
+function tokenCostLines(costs) {
+  if (costs.outcome === "unavailable")
+    return [`  none  (${costs.reason})`];
+  const cost = ({ costUsd, costBasis }) => costUsd === undefined ? "not reported" : costBasis === "unknown" ? `not reported  (its price table doesn't know the model)` : dollars(costUsd);
+  return columns([
+    ["model", "input", "output", "cache write", "cache read", "cost"],
+    ...costs.models.map((model) => [
+      model.model,
+      thousands(model.inputTokens),
+      thousands(model.outputTokens),
+      thousands(model.cacheCreationTokens),
+      thousands(model.cacheReadTokens),
+      cost(model)
+    ])
+  ]);
+}
 function varyingAxes(runs) {
   const varying = AXES.filter((axis) => new Set(runs.map(({ parameters }) => parameters[axis].name)).size > 1);
   return varying.length > 0 ? varying : [...AXES];
@@ -3932,6 +4287,7 @@ function permutationTable(execution, runs, measurements) {
   const axes = varyingAxes(runs);
   const methodColumns = measurements.flatMap(({ name, declaration: declaration2 }) => declaration2.aggregate.map((method, i) => ({ name, method, first: i === 0 })));
   const numberWidth = String(execution.matrix.permutations).length;
+  const withCosts = runs.some(({ tokenCosts }) => tokenCosts !== undefined);
   const rows = [];
   const leftOut = [];
   const incompatible = new Map;
@@ -3955,6 +4311,13 @@ function permutationTable(execution, runs, measurements) {
           leftOut.push([runName(run2), name, showOutcome2(outcome)]);
       }
     }
+    const costs = withCosts ? costColumns(own) : [];
+    if (withCosts) {
+      for (const run2 of own) {
+        if (run2.tokenCosts?.outcome !== "collected")
+          leftOut.push([runName(run2), "tokens", notCollected(run2.tokenCosts)]);
+      }
+    }
     rows.push([
       number,
       axes.map((axis) => first.parameters[axis].name).join(" | "),
@@ -3967,10 +4330,11 @@ function permutationTable(execution, runs, measurements) {
         if (value === undefined)
           return "\u2014";
         return `${showAggregate(method, value, measured)}${measured < own.length ? "*" : ""}`;
-      })
+      }),
+      ...costs
     ]);
   }
-  const header = ["", axes.join(" | "), "runs", ...methodColumns.map(({ method }) => method)];
+  const header = ["", axes.join(" | "), "runs", ...methodColumns.map(({ method }) => method), ...withCosts ? ["tokens (mean)", "cost (mean)"] : []];
   const widths = header.map((cell, i) => Math.max(cell.length, ...rows.map((row) => row[i]?.length ?? 0)));
   for (const { name, declaration: declaration2 } of measurements) {
     const start = 3 + methodColumns.findIndex((column2) => column2.name === name);
@@ -3983,6 +4347,10 @@ function permutationTable(execution, runs, measurements) {
   const column = (i) => 1 + widths.slice(0, i).reduce((a, b) => a + b + GAP.length, 0);
   const names = methodColumns.reduce((text, { name, first }, i) => first ? `${text.padEnd(column(3 + i))}${name}` : text, "");
   const lines2 = [...measurements.length > 0 ? [names] : [], line(header), ...rows.map(line)];
+  const unclean = runs.filter(({ execution: execution2, harnessResult }) => !endedCleanly(execution2, harnessResult));
+  if (unclean.length > 0) {
+    lines2.push("", "! Agents that didn't end cleanly:", ...columns(unclean.map((run2) => [runName(run2), agentEnd(run2.execution, run2.harnessResult)])));
+  }
   if (leftOut.length > 0)
     lines2.push("", "* Left out of the aggregate:", ...columns(leftOut));
   if (incompatible.size > 0) {
@@ -4005,8 +4373,15 @@ function renderExecution2(execution, runs, measurements) {
   return [...header, "", ...body].join(`
 `);
 }
+var showReply = (reply, label2 = "") => {
+  const lines2 = reply.outcome === "collected" ? reply.text.split(`
+`) : [`none  (${reply.reason})`];
+  return lines2.map((line, i) => `  ${i === 0 ? label2 : " ".repeat(label2.length)}${line}`.trimEnd());
+};
+var showReplies = ({ reply, turns }) => turns ? turns.flatMap((turn, i) => showReply(turn.reply, `Turn ${i + 1}:  `)) : showReply(reply);
 function renderContainerRun(record) {
-  const { executionId, testCase: testCase2, permutation, permutationHash, iteration, postrunImage: postrunImage2, ranAt, parameters, harnessModel, execution, result } = record;
+  const { executionId, testCase: testCase2, permutation, permutationHash, iteration, postrunImage: postrunImage2, ranAt, parameters, harnessModel, execution, harnessResult, reply, result } = record;
+  const turnsRan = execution.turns === undefined ? [] : [["Turns", `${execution.turns.length} of ${record.turns?.length ?? "?"} ran`]];
   const names = Object.keys(result).toSorted();
   return [
     ...labelled([
@@ -4024,19 +4399,23 @@ function renderContainerRun(record) {
       ["Harness", `${harnessModel.harness} ${harnessModel.version}`],
       ["Model", `${harnessModel.model}  (${harnessModel.modelId})`],
       ["Duration", duration2(execution.finishedAt - execution.startedAt)],
-      ["Exit code", String(execution.exitCode)]
+      ["Exit", execution.timedOut ? `timed out after ${execution.timeoutSeconds}s` : String(execution.exitCode)],
+      ...turnsRan,
+      ["Result file", harnessResult === null ? "none" : plural(new TextEncoder().encode(harnessResult).length, "byte")]
     ]),
     "",
+    ...reply === undefined ? [] : ["Reply", ...showReplies(record), ""],
+    ...record.tokenCosts === undefined ? [] : ["Token costs", ...tokenCostLines(record.tokenCosts), ""],
     "Measures",
     ...names.length === 0 ? ["  none"] : columns(names.map((name) => [name, showOutcome2(result[name].outcome)]))
   ].join(`
 `);
 }
-function fail(message2) {
-  console.error(`Error: ${message2}`);
+function fail(message3) {
+  console.error(`Error: ${message3}`);
   process.exitCode = 1;
 }
-async function show4(cwd, executionId, containerRun, dataDir) {
+async function show6(cwd, executionId, containerRun, dataDir) {
   const root = containerRun === undefined || dataDir === undefined ? await findThunderjarRoot(cwd) : undefined;
   const store = fileStore(dataDir === undefined ? dataDirOf(cwd, root) : resolvePath(cwd, dataDir));
   let execution;
@@ -4062,7 +4441,7 @@ async function show4(cwd, executionId, containerRun, dataDir) {
 }
 function createShowCommand() {
   return new Command("show").description("Show an execution's recorded results: a row per permutation, with each measure aggregated across its iterations by its measurement's methods. Given a container run, show that one iteration instead: its parameters, how the agent ran, and each measure.").argument("<execution>", "the execution's ID, as listed by `thunderjar executions`").argument("[container run]", "one of its container runs, as `<permutation>-<iteration>`, e.g. `1-1`, or its postrun image").addOption(dataDirOption()).action(reportingConfigErrors(async (executionId, containerRun, options) => {
-    await show4(process.cwd(), executionId, containerRun, options.dataDir);
+    await show6(process.cwd(), executionId, containerRun, options.dataDir);
   }));
 }
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "thunderjar",
-  "version": "0.1.3-preview-388d791",
+  "version": "0.1.3-preview-e09ffa1",
   "type": "module",
   "description": "Thunderjar",
   "license": "MIT",
@@ -19,7 +19,8 @@
   "files": [
     "dist",
     "src/api",
-    "!src/**/*.test.ts"
+    "!src/**/*.test.ts",
+    "!src/**/*.result.json"
   ],
   "publishConfig": {
     "access": "public"
@@ -30,7 +31,9 @@
     "lint:generate": "bun scripts/lint-layers.ts",
     "typecheck": "tsc --noEmit",
     "test": "bun test",
-    "test:docker": "THUNDERJAR_DOCKER_TESTS=1 bun test .docker.test",
+    "test:docker": "THUNDERJAR_DOCKER_TESTS=1 bun test --timeout 600000 .docker.test",
+    "test:agent": "THUNDERJAR_AGENT_TESTS=1 bun test --timeout 900000 .agent.test",
+    "test:package": "bun scripts/package-smoke-test.ts",
     "fixture": "bun scripts/fixture.ts",
     "docs:dev": "vitepress dev docs",
     "docs:build": "vitepress build docs",

package/src/api/claudeCode.ts

@@ -0,0 +1,74 @@
+// Reading what `claude -p --output-format json` writes: one JSON object, with the reply as
+// `result`, and the tokens and cost of each model it used as `modelUsage`. For a harness's
+// `collectReply` and `collectTokenCosts`, so a Claude Code harness needn't parse it itself.
+//
+// `modelUsage` is read, not the flat `usage`: `usage` counts only the top-level agent loop, so it
+// leaves out what sub-agents spent. Its keys are camelCase, where `usage`'s are snake_case.
+import type { ModelTokenCosts, TokenCosts } from "./declarations.ts";
+
+type ModelUsage = {
+  inputTokens?: unknown;
+  outputTokens?: unknown;
+  cacheReadInputTokens?: unknown;
+  cacheCreationInputTokens?: unknown;
+  costUSD?: unknown;
+  costBasis?: unknown;
+};
+
+type ClaudeCodeResult = { type?: unknown; result?: unknown; modelUsage?: unknown; duration_api_ms?: unknown };
+
+const COST_BASES = ["list", "managed", "unknown"] as const;
+
+const count = (value: unknown) => (typeof value === "number" && Number.isFinite(value) && value >= 0 ? value : undefined);
+
+/** `raw` parsed as Claude Code's result, or undefined if it isn't one. */
+function parse(raw: string): ClaudeCodeResult | undefined {
+  let parsed: unknown;
+  try {
+    parsed = JSON.parse(raw);
+  } catch {
+    return undefined;
+  }
+  return typeof parsed === "object" && parsed !== null && (parsed as ClaudeCodeResult).type === "result"
+    ? (parsed as ClaudeCodeResult)
+    : undefined;
+}
+
+/** The agent's final reply, from the result file Claude Code wrote. Throws if it isn't one. */
+export function claudeCodeReply(raw: string): string {
+  const result = parse(raw)?.result;
+  if (typeof result !== "string") throw new Error("not a result of `claude -p --output-format json`");
+  return result;
+}
+
+/**
+ * The tokens and cost of each model, from the result file Claude Code wrote. `unparseable` if it
+ * isn't one, and `notReported` if it counts no tokens at all, as a crashed session's does. Its
+ * cost is its own estimate, from a price table bundled with it: `costBasis` says whether that
+ * table knew the model.
+ */
+export function claudeCodeTokenCosts(raw: string): TokenCosts {
+  const result = parse(raw);
+  if (!result || typeof result.modelUsage !== "object" || result.modelUsage === null) return { outcome: "unavailable", reason: "unparseable" };
+  const models: ModelTokenCosts[] = [];
+  for (const [model, usage] of Object.entries(result.modelUsage as Record<string, ModelUsage>)) {
+    const tokens = [usage.inputTokens, usage.outputTokens, usage.cacheCreationInputTokens, usage.cacheReadInputTokens].map(count);
+    if (tokens.some((n) => n === undefined)) return { outcome: "unavailable", reason: "unparseable" };
+    const [inputTokens, outputTokens, cacheCreationTokens, cacheReadTokens] = tokens as number[];
+    const costUsd = count(usage.costUSD);
+    const costBasis = COST_BASES.find((basis) => basis === usage.costBasis);
+    models.push({
+      model,
+      inputTokens: inputTokens as number,
+      outputTokens: outputTokens as number,
+      cacheCreationTokens: cacheCreationTokens as number,
+      cacheReadTokens: cacheReadTokens as number,
+      ...(costUsd !== undefined && { costUsd }),
+      ...(costBasis !== undefined && { costBasis }),
+    });
+  }
+  const total = models.reduce((sum, m) => sum + m.inputTokens + m.outputTokens + m.cacheCreationTokens + m.cacheReadTokens, 0);
+  if (total === 0) return { outcome: "unavailable", reason: "notReported" };
+  const apiDurationMs = count(result.duration_api_ms);
+  return { outcome: "collected", models, ...(apiDurationMs !== undefined && { apiDurationMs }) };
+}

package/src/api/declarations.ts

@@ -73,6 +73,14 @@ type Applies<C, Ctx> = {
 /** A base image's fragment is required: it carries the prerun image's `FROM`. */
 export type BaseImageDeclaration<C = unknown> = Applies<C, ApplyParameterCtx> & {
   applyParameter(config: C, ctx: ApplyParameterCtx): string;
+  /**
+   * More `docker run` options for the agent's containers, for agents that need more of the
+   * kernel than Docker allows by default, e.g. Claude Code's bubblewrap sandbox, which needs user
+   * namespaces: `["--security-opt", "seccomp=unconfined", "--security-opt", "apparmor=unconfined"]`.
+   * Each option and value is its own string. Ask for the least you need: `--privileged` works, but
+   * gives the agent the whole machine.
+   */
+  runOptions?: string[];
 };
 
 export function declareBaseImage<C = undefined>(declaration: BaseImageDeclaration<C>): BaseImageDeclaration<C> {
@@ -96,18 +104,46 @@ export type HarnessApplyCtx = ApplyParameterCtx & {
   version: string;
 };
 
-/** What a harness's `cli` gets, to build the command that runs the agent. */
+/**
+ * What a harness's `cli` gets, to build the command that runs the agent. An initial prompt of
+ * several turns runs `cli` once per turn, in the same container, one after another.
+ */
 export type HarnessCliCtx = {
   version: string;
   modelId: string;
+  /**
+   * The turn's prompt text. Pasted into a shell command it breaks on the first quote, `$` or
+   * backtick, so read `promptPath` instead.
+   */
   initialPrompt: string;
-  /** Where the agent's own report of the run goes. */
+  /**
+   * A file in the container holding the turn's prompt, exactly, with no newline added: pass it to
+   * the agent as `"$(cat ${ctx.promptPath})"`, or as a file if it takes one.
+   */
+  promptPath: string;
+  /** Where the agent's own report of the turn goes. Each turn has its own. */
   resultPath: string;
+  /** Which turn this is, counting from 0. A single prompt is turn 0. */
+  turn: number;
+  /**
+   * A fresh UUID for each container run, the same for each of its turns: start the agent's
+   * session with it on turn 0, and resume it on later turns, e.g. `--session-id`, then `--resume`.
+   */
+  sessionId: string;
 };
 
 export type HarnessDeclaration<C = unknown> = Applies<C, HarnessApplyCtx> & {
   cli: (ctx: HarnessCliCtx) => string;
-  /** Environment variables the agent needs, e.g. its API key. */
+  /**
+   * The agent's final reply, read from the text of the result file its `cli` left, for
+   * instruments such as `replyMatches` to measure. Without it, a run has no reply.
+   */
+  collectReply?: (raw: string) => string;
+  /**
+   * The names of the environment variables the agent needs, e.g. its API key. Each is passed into
+   * this harness's containers only, as `docker run -e NAME`, so its value comes from the shell that
+   * runs Thunderjar, and is never written down. `plan` and `run` stop if one is unset.
+   */
   requiredEnv?: string[];
 };
 
@@ -129,20 +165,31 @@ export function declareHarnessVersions(versions: HarnessVersionsDeclaration): Ha
   return declared("declareHarnessVersions", versions);
 }
 
+/** One model's tokens, in four buckets priced differently, and its cost if the harness reports one. */
 export type ModelTokenCosts = {
   model: string;
   inputTokens: number;
   outputTokens: number;
   cacheCreationTokens: number;
   cacheReadTokens: number;
+  /** The harness's own estimate, never computed by Thunderjar. Absent if it reports none. */
   costUsd?: number;
+  /** How far to trust `costUsd`: `unknown` when the harness's price table didn't know the model. */
+  costBasis?: "list" | "managed" | "unknown";
 };
 
+/**
+ * A container run's token costs, per model, or why there are none: no result file (`absent`), one
+ * that couldn't be read (`unparseable`), or one that counted nothing (`notReported`).
+ */
 export type TokenCosts =
-  | { outcome: "collected"; models: ModelTokenCosts[] }
+  | { outcome: "collected"; models: ModelTokenCosts[]; /** Time spent waiting on the model, if the harness reports it. */ apiDurationMs?: number }
   | { outcome: "unavailable"; reason: "absent" | "unparseable" | "notReported" };
 
-/** Reads the token costs from what the harness's `cli` wrote to its result path. */
+/**
+ * Reads the token costs from what the harness's `cli` wrote to its result path. Thunderjar calls
+ * it only for a result file that exists, once per turn, and sums them. A throw is `unparseable`.
+ */
 export type TokenCostCollector = (raw: string, ctx: { version: string; modelId: string }) => TokenCosts;
 
 export function declareTokenCostCollector(collector: TokenCostCollector): TokenCostCollector {
@@ -152,11 +199,14 @@ export function declareTokenCostCollector(collector: TokenCostCollector): TokenC
 /**
  * An initial prompt is one file, not a folder, so its fragment has no `ctx.dir`. It is used when
  * the harness runs, so its fragment is usually left out.
+ *
+ * It's a single `prompt`, or a list of `turns`, each run by the harness's `cli` in turn, in the
+ * same container and agent session, so a later turn can ask about an earlier one. Its
+ * `timeoutSeconds` covers every turn together.
  */
 export type InitialPromptDeclaration<C = unknown> = Applies<C, FragmentCtx> & {
-  prompt: string;
   timeoutSeconds: number;
-};
+} & ({ prompt: string; turns?: never } | { turns: string[]; prompt?: never });
 
 export function declareInitialPrompt<C = undefined>(declaration: InitialPromptDeclaration<C>): InitialPromptDeclaration<C> {
   return declared("declareInitialPrompt", declaration);
@@ -185,9 +235,16 @@ export type MeasurementContext = {
   glob(pattern: string): Promise<string[]>;
   /** The text of the result file the harness's `cli` left, or "" if it left none. */
   harnessResult: string;
+  /** The agent's final reply, from its harness's `collectReply`, or why there's none. */
+  reply: Reply;
+  /** Its reply to each turn, in order: one, unless its initial prompt has several turns. */
+  replies: Reply[];
   workdir: string;
 };
 
+/** The agent's final reply, as its harness's `collectReply` read it from the result file, or why there's none. */
+export type Reply = { outcome: "collected"; text: string } | { outcome: "unavailable"; reason: string };
+
 /** Each built-in instrument's config and value, by name. */
 export type InstrumentTypes = {
   /** Did the agent leave a file matching `glob`? */
@@ -199,6 +256,11 @@ export type InstrumentTypes = {
    * rendered against each file matching `subject` and run with `bun test`?
    */
   templateTest: { config: { subject: string; template: string }; value: { passed: number; failed: number } };
+  /**
+   * Does the agent's reply match `pattern`, a JavaScript regular expression? Its reply to `turn`,
+   * counting from 1, of an initial prompt of several turns, or by default its last.
+   */
+  replyMatches: { config: { pattern: string; turn?: number }; value: boolean };
 };
 
 export type InstrumentName = keyof InstrumentTypes;

package/src/api/index.ts

@@ -1,5 +1,6 @@
 // The `thunderjar` library: what a project's `.thunderjar/` files import.
 export { checkoutCommit } from "./checkoutCommit.ts";
+export { claudeCodeReply, claudeCodeTokenCosts } from "./claudeCode.ts";
 export {
   declareBaseImage,
   declareCodeState,
@@ -33,7 +34,9 @@ export type {
   MeasurementContext,
   MeasurementDeclaration,
   MeasurementResult,
+  ModelTokenCosts,
   PromptSetDeclaration,
+  Reply,
   SqliteStore,
   TestCaseDeclaration,
   TokenCostCollector,