testronaut 1.10.2 → 1.12.0

package/experiments/buildExperimentPublicationPackage.js

@@ -0,0 +1,219 @@
+#!/usr/bin/env node
+
+import crypto from 'node:crypto';
+import fs from 'node:fs';
+import path from 'node:path';
+import { spawnSync } from 'node:child_process';
+import { fileURLToPath } from 'node:url';
+import { BENCHMARK_TARGETS } from './optimizationBenchmark.js';
+import { CROSS_PROVIDER_CONDITIONS } from './crossProviderBenchmark.js';
+
+const HERE = path.dirname(fileURLToPath(import.meta.url));
+const PUBLICATION_ID = 'jev-optimization-experiments-2026-10';
+const RESULTS = path.join(HERE, 'results');
+const OUTPUT = path.join(RESULTS, PUBLICATION_ID);
+const PACKAGE = path.join(OUTPUT, 'data-package');
+
+function csvEscape(value) {
+  const text = typeof value === 'object' && value !== null ? JSON.stringify(value) : String(value ?? '');
+  return /[",\n]/.test(text) ? `"${text.replaceAll('"', '""')}"` : text;
+}
+
+function writeCsv(file, columns, rows) {
+  fs.writeFileSync(file, `${columns.join(',')}\n${rows.map(row => columns.map(column => csvEscape(row[column])).join(',')).join('\n')}\n`);
+}
+
+function mean(values) { return values.reduce((sum, value) => sum + value, 0) / values.length; }
+function round(value, digits = 1) { return Number(value.toFixed(digits)); }
+function sha256(file) { return crypto.createHash('sha256').update(fs.readFileSync(file)).digest('hex'); }
+
+function readCrossProvider() {
+  const stateFile = path.join(RESULTS, 'cross-provider-pilot-v1/state.json');
+  const state = JSON.parse(fs.readFileSync(stateFile, 'utf8'));
+  const runs = Object.values(state.runs).map(run => {
+    if (run.status !== 'completed' || !run.reportPath || !run.metrics) throw new Error(`Incomplete cross-provider run: ${run.key}`);
+    const report = JSON.parse(fs.readFileSync(run.reportPath, 'utf8'));
+    const selectorUsage = run.selection?.evaluation?.usage || {};
+    return {
+      experimentId: state.experimentId,
+      key: run.key,
+      block: run.block,
+      condition: run.id,
+      mission: run.unit,
+      candidateModels: run.candidates.map(item => `${item.provider}/${item.model}`).join('|'),
+      selectedProvider: run.selection.selected.provider,
+      selectedModel: run.selection.selected.model,
+      selectedBy: run.selection.selectedBy,
+      selectionScore: run.selection.score,
+      strictPass: run.metrics.failed === 0,
+      phasePassed: run.metrics.passed,
+      phaseFailed: run.metrics.failed,
+      turns: run.metrics.turns,
+      modelTokens: run.metrics.tokens,
+      jevInputTokens: run.metrics.jevInputTokens,
+      jevOutputTokens: run.metrics.jevOutputTokens,
+      selectorInputTokens: Number(selectorUsage.input_tokens || 0),
+      selectorOutputTokens: Number(selectorUsage.output_tokens || 0),
+      durationSeconds: (new Date(report.endTime) - new Date(report.startTime)) / 1000,
+      runId: report.runId,
+      cliVersion: report.cli?.version,
+    };
+  });
+  const conditions = CROSS_PROVIDER_CONDITIONS.map(condition => {
+    const values = runs.filter(run => run.condition === condition.id);
+    return {
+      condition: condition.id,
+      type: condition.id.startsWith('c') ? 'fixed-control' : 'cross-provider-treatment',
+      candidates: condition.candidates.map(item => `${item.provider}/${item.model}`).join('|'),
+      n: values.length,
+      strictPassed: values.filter(run => run.strictPass).length,
+      strictPassRate: round(values.filter(run => run.strictPass).length / values.length, 4),
+      meanModelTokens: round(mean(values.map(run => run.modelTokens)), 0),
+      meanJevTokens: round(mean(values.map(run => run.jevInputTokens + run.jevOutputTokens)), 0),
+      meanSelectorTokens: round(mean(values.map(run => run.selectorInputTokens + run.selectorOutputTokens)), 0),
+      meanDurationSeconds: round(mean(values.map(run => run.durationSeconds)), 1),
+      selections: Object.fromEntries([...new Set(values.map(run => run.selectedModel))].map(model => [model, values.filter(run => run.selectedModel === model).length])),
+    };
+  });
+  return { state, runs, conditions };
+}
+
+function crossTable(conditions) {
+  const rows = conditions.map(row => `| ${row.condition} | ${row.type} | ${row.strictPassed}/${row.n} (${Math.round(row.strictPassRate * 100)}%) | ${row.meanModelTokens.toLocaleString()} | ${row.meanJevTokens.toLocaleString()} | ${row.meanSelectorTokens.toLocaleString()} | ${row.meanDurationSeconds}s |`);
+  return ['| Condition | Design | Strict pass | Mean model tokens | Mean in-mission Jev tokens | Mean selector tokens | Mean duration |', '|---|---|---:|---:|---:|---:|---:|', ...rows].join('\n');
+}
+
+function optimizationMatrix() {
+  return Object.fromEntries(Object.entries(BENCHMARK_TARGETS).map(([target, value]) => [target, {
+    provider: value.provider,
+    missions: value.missions,
+    conditions: value.conditions,
+  }]));
+}
+
+function reportMarkdown(cross) {
+  return `# Jev optimization experiments: guardrails and model routing
+
+Publication package: \`${PUBLICATION_ID}\`  
+Generated: ${new Date().toISOString()}  
+CLI baseline: Testronaut 1.11.0
+
+## Executive summary
+
+Two blocked experiments evaluated Jev-assisted browser-testing optimization. The first contained **70 aggregate runs and 210 mission-file attempts** across OpenAI and Gemini workloads. The second contained **120 isolated mission executions** testing fixed-model controls and mission-boundary cross-provider selection. All conditions used five randomized blocks.
+
+The most defensible result is that **Jev guardrails are useful without model switching**. Guarded GPT-5.5 completed 15/15 OpenAI mission-file attempts and used 13.7% fewer model tokens than the unguarded Terra control, with a paired 95% interval of -19.8% to -7.5%. Guarded Terra also improved strict completion and nominally reduced tokens. On Gemini, guardrails preserved the already-perfect reliability of Gemini 3.8 Flash with a smaller nominal token reduction.
+
+Same-provider routing is promising but not yet broadly proven. Gemini 3.8 → 3.7 retained 15/15 strict passes with a nominal 5.1% token reduction and similar duration. OpenAI routing coverage was below one routed turn per aggregate run, making attribution underpowered.
+
+Mission-boundary cross-provider routing did **not** reduce token volume. The strongest fixed control, guarded GPT-5.5, achieved 15/15 strict passes at 87,422 mean model tokens per isolated mission. GPT-5.5 ↔ Gemini 3.5 Flash-Lite also achieved 15/15, but required 104,897 mean model tokens. Jev's extra selector call averaged only about 621–724 tokens, so selector overhead was not the cause; the model choices themselves produced longer executions.
+
+## Experiment 1: optimization ladder
+
+| Provider | Condition | Strict pass | Mean model tokens | Paired token delta vs control | Mean duration |
+|---|---|---:|---:|---:|---:|
+| OpenAI | Terra control | 5/15 (33%) | 300,073 | baseline | 293s |
+| OpenAI | Guarded GPT-5.5 | **15/15 (100%)** | **258,635** | **-13.7% [-19.8, -7.5]** | 416s |
+| OpenAI | Guarded Terra | 8/15 (53%) | 275,114 | -8.2% [-18.0, 1.7] | 328s |
+| OpenAI | Terra → Luna | 11/15 (73%) | 296,299 | -1.1% [-12.1, 9.9] | 376s |
+| Gemini | 3.8 control | **15/15 (100%)** | 491,428 | baseline | 529s |
+| Gemini | Guarded 3.5 Flash-Lite | **15/15 (100%)** | **427,766** | -12.0% [-34.7, 10.8] | 582s |
+| Gemini | 3.8 → 3.7 | **15/15 (100%)** | 461,256 | -5.1% [-19.7, 9.5] | 538s |
+
+The table highlights decision-relevant conditions; complete condition, run, and mission tables are included in \`data/optimization/\`.
+
+## Experiment 2: cross-provider mission-boundary routing
+
+${crossTable(cross.conditions)}
+
+Jev chose a Gemini candidate for 50 of 60 treatment missions. That bias was not aligned with the observed workload: GPT-5.5 was the only fixed model with perfect strict completion, and it used the fewest mean model tokens. The file-transfer mission was particularly discriminating: GPT-5.5 passed 5/5, Terra and Gemini 3.8 passed 0/5, and Gemini 3.5 Flash-Lite passed 3/5 in their fixed controls.
+
+## Product implications
+
+1. Stabilize guardrail-only operation: completion/evidence checks, fail-open behavior, primary-model continuity, and explicit telemetry.
+2. Keep execution-model routing beta and opt-in. Limit a run to one primary and one compatible same-provider secondary.
+3. Treat Gemini 3.8 → 3.7 as a conservative candidate for a larger confirmation study, not a universal default.
+4. Do not ship zero-shot cross-provider selection as an optimization default.
+5. Next test history-informed routing using versioned per-model reliability, tokens, latency, and price, with a reliability floor and cold-start fallback.
+
+## Methodology and interpretation
+
+- Both experiments used five randomized blocks (\`n=5\` per condition design).
+- Strict success required every expected phase in a mission file to exist and pass.
+- Experiment 1 aggregate runs each invoked three mission files. Experiment 2 intentionally isolated each mission to make mission-boundary selection attributable.
+- Model tokens and Jev tokens are reported separately. Token counts are not price-weighted spend.
+- Experiment 1 paired intervals use a two-sided 95% Student-t interval with four degrees of freedom.
+- Six initial Gemini 3.8 requests in experiment 2 returned provider HTTP 500 errors. The resumable benchmark reran those incomplete units using their persisted selections; the final dataset contains 120 completed reports.
+- Results describe these repositories, missions, model versions, dates, and provider behavior. They are not universal model rankings.
+
+## Data and privacy
+
+This public package contains normalized measurements, matrices, methodology, and checksums. It intentionally excludes raw browser events, DOM content, screenshots, HTML reports, environment files, credentials, absolute report paths, and verbose logs. The private source reports remain the audit source for the normalized rows.
+`;
+}
+
+function walk(root) {
+  return fs.readdirSync(root, { withFileTypes: true }).flatMap(entry => {
+    const full = path.join(root, entry.name);
+    return entry.isDirectory() ? walk(full) : [full];
+  });
+}
+
+export function buildPublicationPackage() {
+  const optimizationPackage = path.join(RESULTS, 'optimization-pilot-v1/data-package/data');
+  if (!fs.existsSync(optimizationPackage)) throw new Error('Build optimization-pilot-v1 package first with npm run benchmark:package');
+  const cross = readCrossProvider();
+  fs.mkdirSync(path.join(PACKAGE, 'data/optimization'), { recursive: true });
+  fs.mkdirSync(path.join(PACKAGE, 'data/cross-provider'), { recursive: true });
+
+  for (const name of ['conditions.csv', 'missions.csv', 'runs.csv', 'matrix.json']) {
+    fs.copyFileSync(path.join(optimizationPackage, name), path.join(PACKAGE, `data/optimization/${name}`));
+  }
+  const crossColumns = Object.keys(cross.runs[0]);
+  writeCsv(path.join(PACKAGE, 'data/cross-provider/runs.csv'), crossColumns, cross.runs);
+  writeCsv(path.join(PACKAGE, 'data/cross-provider/conditions.csv'), Object.keys(cross.conditions[0]), cross.conditions);
+  fs.writeFileSync(path.join(PACKAGE, 'data/cross-provider/matrix.json'), `${JSON.stringify({
+    experimentId: cross.state.experimentId,
+    repetitions: cross.state.repetitions,
+    conditions: CROSS_PROVIDER_CONDITIONS,
+    missions: [...new Set(cross.runs.map(run => run.mission))],
+  }, null, 2)}\n`);
+  fs.writeFileSync(path.join(PACKAGE, 'data/combined-matrix.json'), `${JSON.stringify({ optimization: optimizationMatrix(), crossProvider: CROSS_PROVIDER_CONDITIONS }, null, 2)}\n`);
+
+  const report = reportMarkdown(cross);
+  fs.writeFileSync(path.join(PACKAGE, 'REPORT.md'), report);
+  fs.writeFileSync(path.join(PACKAGE, 'README.md'), `# Jev optimization experiment publication package\n\nStart with [REPORT.md](REPORT.md). The normalized, publication-safe datasets are in \`data/optimization/\` and \`data/cross-provider/\`. See [DATA_DICTIONARY.md](DATA_DICTIONARY.md) for field definitions and \`SHA256SUMS\` for integrity.\n`);
+  fs.writeFileSync(path.join(PACKAGE, 'DATA_DICTIONARY.md'), `# Data dictionary\n\n## Optimization ladder\n\n- \`runs.csv\`: one aggregate CLI run (three mission files) per row.\n- \`missions.csv\`: one expected mission-file outcome per aggregate run.\n- \`conditions.csv\`: condition-level means, paired deltas, intervals, routing, and completion-gate counts.\n- \`matrix.json\`: frozen provider, mission, and condition definitions.\n\n## Cross-provider benchmark\n\n- \`runs.csv\`: one isolated mission execution per row. \`strictPass\` means all reported phases passed. Selector tokens are separated from in-mission Jev tokens.\n- \`conditions.csv\`: condition-level strict completion, mean token, selection, and duration summaries.\n- \`matrix.json\`: frozen candidates, missions, and repetition count.\n\nEmpty secondary models indicate fixed-model conditions. Durations are wall-clock seconds. Token fields are raw token volume, not price-weighted cost.\n`);
+
+  const filesBeforeManifest = walk(PACKAGE)
+    .filter(file => !['manifest.json', 'SHA256SUMS'].includes(path.basename(file)))
+    .map(file => ({ path: path.relative(PACKAGE, file), bytes: fs.statSync(file).size, sha256: sha256(file) }))
+    .sort((a, b) => a.path.localeCompare(b.path));
+  const manifest = {
+    schemaVersion: 1,
+    publicationId: PUBLICATION_ID,
+    generatedAt: new Date().toISOString(),
+    cliBaseline: '1.11.0',
+    experiments: [
+      { id: 'optimization-pilot-v1', aggregateRuns: 70, missionFileAttempts: 210 },
+      { id: 'cross-provider-pilot-v1', isolatedMissionRuns: 120 },
+    ],
+    privacyExclusions: ['raw browser events and DOM', 'screenshots', 'HTML reports', 'environment files', 'credentials', 'absolute report paths', 'verbose logs'],
+    files: filesBeforeManifest,
+  };
+  fs.writeFileSync(path.join(PACKAGE, 'manifest.json'), `${JSON.stringify(manifest, null, 2)}\n`);
+  const checksumFiles = walk(PACKAGE).filter(file => path.basename(file) !== 'SHA256SUMS').sort();
+  fs.writeFileSync(path.join(PACKAGE, 'SHA256SUMS'), `${checksumFiles.map(file => `${sha256(file)}  ${path.relative(PACKAGE, file)}`).join('\n')}\n`);
+
+  fs.mkdirSync(OUTPUT, { recursive: true });
+  fs.copyFileSync(path.join(PACKAGE, 'REPORT.md'), path.join(OUTPUT, `${PUBLICATION_ID}-report.md`));
+  const archive = path.join(OUTPUT, `${PUBLICATION_ID}-data-package.tar.gz`);
+  const tar = spawnSync('tar', ['-czf', archive, '-C', OUTPUT, 'data-package'], { encoding: 'utf8' });
+  if (tar.status !== 0) throw new Error(tar.stderr || 'Could not create archive');
+  fs.writeFileSync(`${archive}.sha256`, `${sha256(archive)}  ${path.basename(archive)}\n`);
+  return { output: OUTPUT, package: PACKAGE, archive, report: path.join(OUTPUT, `${PUBLICATION_ID}-report.md`) };
+}
+
+if (process.argv[1] && path.resolve(process.argv[1]) === fileURLToPath(import.meta.url)) {
+  console.log(buildPublicationPackage());
+}

package/experiments/buildOptimizationDataPackage.js

@@ -0,0 +1,390 @@
+#!/usr/bin/env node
+
+import crypto from 'node:crypto';
+import fs from 'node:fs';
+import path from 'node:path';
+import { spawnSync } from 'node:child_process';
+import { fileURLToPath } from 'node:url';
+import { BENCHMARK_TARGETS } from './optimizationBenchmark.js';
+
+const HERE = path.dirname(fileURLToPath(import.meta.url));
+const DEFAULT_EXPERIMENT_ID = 'optimization-pilot-v1';
+const T95_DF4 = 2.776;
+const EXPECTED_RECORDS = {
+  openai: {
+    'missions/login.mission.js': 1,
+    'missions/removeTask.mission.js': 2,
+    'missions/fileTransfer.mission.js': 3,
+  },
+  gemini: {
+    'missions/profileNavigation.mission.js': 2,
+    'missions/projectsDashboard.mission.js': 2,
+    'missions/projectWorkspaceNavigation.mission.js': 2,
+  },
+};
+const CONTROL_CONDITION = {
+  openai: 'l1_terra_control',
+  gemini: 'l1_gemini38_control',
+};
+
+function mean(values) {
+  return values.reduce((sum, value) => sum + value, 0) / values.length;
+}
+
+function sampleSd(values) {
+  if (values.length < 2) return 0;
+  const average = mean(values);
+  return Math.sqrt(values.reduce((sum, value) => sum + (value - average) ** 2, 0) / (values.length - 1));
+}
+
+function round(value, digits = 2) {
+  return Number.isFinite(value) ? Number(value.toFixed(digits)) : null;
+}
+
+function csvEscape(value) {
+  const text = typeof value === 'object' && value !== null ? JSON.stringify(value) : String(value ?? '');
+  return /[",\n]/.test(text) ? `"${text.replaceAll('"', '""')}"` : text;
+}
+
+function writeCsv(file, columns, rows) {
+  fs.writeFileSync(file, `${columns.join(',')}\n${rows.map(row => columns.map(column => csvEscape(row[column])).join(',')).join('\n')}\n`);
+}
+
+function readReports(targetName, experimentId) {
+  const target = BENCHMARK_TARGETS[targetName];
+  const reportDir = path.join(target.repo, 'missions/mission_reports');
+  return fs.readdirSync(reportDir)
+    .filter(file => /^run_\d+\.json$/.test(file))
+    .map(file => ({ file, path: path.join(reportDir, file) }))
+    .map(entry => ({ ...entry, report: JSON.parse(fs.readFileSync(entry.path, 'utf8')) }))
+    .filter(entry => entry.report.experiment?.id === experimentId)
+    .map(entry => ({ ...entry, target: targetName }));
+}
+
+function stepMetrics(steps, report) {
+  const models = {};
+  for (const step of steps) {
+    const model = step.model || report.llm?.model || 'unknown';
+    models[model] ||= { turns: 0, tokens: 0, inputTokens: 0, outputTokens: 0 };
+    models[model].turns += 1;
+    models[model].tokens += Number(step.tokensUsed || 0);
+    models[model].inputTokens += Number(step.inputTokens || 0);
+    models[model].outputTokens += Number(step.outputTokens || 0);
+  }
+  return {
+    turns: steps.length,
+    tokens: steps.reduce((sum, step) => sum + Number(step.tokensUsed || 0), 0),
+    inputTokens: steps.reduce((sum, step) => sum + Number(step.inputTokens || 0), 0),
+    outputTokens: steps.reduce((sum, step) => sum + Number(step.outputTokens || 0), 0),
+    routedTurns: steps.filter(step => step.modelRouting?.applied).length,
+    forcedPrimaryRetries: steps.filter(step => step.modelRouting?.forcedPrimaryRetry).length,
+    requestFallbacks: steps.filter(step => step.modelRouting?.fallback).length,
+    models,
+  };
+}
+
+function normalizeRun(entry) {
+  const { report, target } = entry;
+  const expected = EXPECTED_RECORDS[target];
+  const steps = report.missions.flatMap(mission => mission.steps || []);
+  const step = stepMetrics(steps, report);
+  const missionOutcomes = Object.entries(expected).map(([file, expectedRecords]) => {
+    const records = report.missions.filter(mission => mission.file === file);
+    const recordSteps = records.flatMap(record => record.steps || []);
+    return {
+      experimentId: report.experiment.id,
+      target,
+      condition: report.experiment.condition,
+      block: report.experiment.block,
+      runId: report.runId,
+      file,
+      expectedRecords,
+      actualRecords: records.length,
+      passed: records.length === expectedRecords && records.every(record => record.status === 'passed'),
+      phasePassed: records.filter(record => record.status === 'passed').length,
+      phaseFailed: records.filter(record => record.status === 'failed').length,
+      turns: recordSteps.length,
+      tokens: recordSteps.reduce((sum, item) => sum + Number(item.tokensUsed || 0), 0),
+    };
+  });
+  const condition = BENCHMARK_TARGETS[target].conditions.find(item => item.id === report.experiment.condition);
+  return {
+    experimentId: report.experiment.id,
+    target,
+    condition: report.experiment.condition,
+    level: condition?.level,
+    block: report.experiment.block,
+    runId: report.runId,
+    provider: report.llm?.provider,
+    primaryModel: condition?.primary || report.llm?.model,
+    secondaryModel: condition?.secondary || '',
+    objective: report.optimization?.objective || condition?.objective,
+    mode: report.optimization?.mode || condition?.mode,
+    phasePassed: Number(report.summary?.passed || 0),
+    phaseFailed: Number(report.summary?.failed || 0),
+    phaseTotal: Number(report.summary?.totalMissions || 0),
+    missionFilesPassed: missionOutcomes.filter(outcome => outcome.passed).length,
+    missionFilesTotal: missionOutcomes.length,
+    durationSeconds: (new Date(report.endTime) - new Date(report.startTime)) / 1000,
+    ...step,
+    jevInputTokens: Number(report.jevShadow?.inputTokens || 0),
+    jevOutputTokens: Number(report.jevShadow?.outputTokens || 0),
+    jevEvaluations: Number(report.jevShadow?.evaluations || 0),
+    completionGates: Number(report.jevShadow?.triggeredCompletions || 0),
+    reportPath: entry.path,
+    reportFile: entry.file,
+    missionOutcomes,
+  };
+}
+
+function pairedDelta(conditionRuns, controlRuns, metric) {
+  const controlByBlock = Object.fromEntries(controlRuns.map(run => [run.block, run]));
+  const values = conditionRuns.map(run => {
+    const control = controlByBlock[run.block];
+    return control?.[metric] ? (run[metric] - control[metric]) / control[metric] * 100 : 0;
+  });
+  const average = mean(values);
+  const halfWidth = T95_DF4 * sampleSd(values) / Math.sqrt(values.length);
+  return { average, low: average - halfWidth, high: average + halfWidth };
+}
+
+function summarizeConditions(runs) {
+  const grouped = runs.reduce((groups, run) => {
+    (groups[`${run.target}:${run.condition}`] ||= []).push(run);
+    return groups;
+  }, {});
+  const controls = Object.fromEntries(Object.entries(CONTROL_CONDITION).map(([target, condition]) => [
+    target,
+    runs.filter(run => run.target === target && run.condition === condition),
+  ]));
+  return Object.values(grouped).map(conditionRuns => {
+    conditionRuns.sort((a, b) => a.block - b.block);
+    const first = conditionRuns[0];
+    const tokenDelta = pairedDelta(conditionRuns, controls[first.target], 'tokens');
+    const turnDelta = pairedDelta(conditionRuns, controls[first.target], 'turns');
+    const durationDelta = pairedDelta(conditionRuns, controls[first.target], 'durationSeconds');
+    return {
+      target: first.target,
+      condition: first.condition,
+      level: first.level,
+      primaryModel: first.primaryModel,
+      secondaryModel: first.secondaryModel,
+      objective: first.objective,
+      mode: first.mode,
+      n: conditionRuns.length,
+      missionFileAttempts: conditionRuns.reduce((sum, run) => sum + run.missionFilesTotal, 0),
+      missionFilePassed: conditionRuns.reduce((sum, run) => sum + run.missionFilesPassed, 0),
+      missionFilePassRate: conditionRuns.reduce((sum, run) => sum + run.missionFilesPassed, 0)
+        / conditionRuns.reduce((sum, run) => sum + run.missionFilesTotal, 0),
+      phasePassed: conditionRuns.reduce((sum, run) => sum + run.phasePassed, 0),
+      phaseTotal: conditionRuns.reduce((sum, run) => sum + run.phaseTotal, 0),
+      phasePassRate: conditionRuns.reduce((sum, run) => sum + run.phasePassed, 0)
+        / conditionRuns.reduce((sum, run) => sum + run.phaseTotal, 0),
+      meanTurns: mean(conditionRuns.map(run => run.turns)),
+      sdTurns: sampleSd(conditionRuns.map(run => run.turns)),
+      turnDeltaPct: turnDelta.average,
+      turnDeltaLow95: turnDelta.low,
+      turnDeltaHigh95: turnDelta.high,
+      meanTokens: mean(conditionRuns.map(run => run.tokens)),
+      sdTokens: sampleSd(conditionRuns.map(run => run.tokens)),
+      tokenDeltaPct: tokenDelta.average,
+      tokenDeltaLow95: tokenDelta.low,
+      tokenDeltaHigh95: tokenDelta.high,
+      meanDurationSeconds: mean(conditionRuns.map(run => run.durationSeconds)),
+      sdDurationSeconds: sampleSd(conditionRuns.map(run => run.durationSeconds)),
+      durationDeltaPct: durationDelta.average,
+      durationDeltaLow95: durationDelta.low,
+      durationDeltaHigh95: durationDelta.high,
+      meanRoutedTurns: mean(conditionRuns.map(run => run.routedTurns)),
+      meanForcedPrimaryRetries: mean(conditionRuns.map(run => run.forcedPrimaryRetries)),
+      meanCompletionGates: mean(conditionRuns.map(run => run.completionGates)),
+      meanJevTokens: mean(conditionRuns.map(run => run.jevInputTokens + run.jevOutputTokens)),
+    };
+  }).sort((a, b) => a.target.localeCompare(b.target) || a.level - b.level || a.condition.localeCompare(b.condition));
+}
+
+function markdownTable(rows) {
+  const header = '| Condition | Strict pass | Mean tokens | Δ tokens vs control (paired 95% CI) | Mean turns | Mean duration | Routes/run | Gates/run |';
+  const divider = '|---|---:|---:|---:|---:|---:|---:|---:|';
+  const body = rows.map(row => [
+    `| ${row.condition}`,
+    `${row.missionFilePassed}/${row.missionFileAttempts} (${(row.missionFilePassRate * 100).toFixed(0)}%)`,
+    Math.round(row.meanTokens).toLocaleString('en-US'),
+    `${row.tokenDeltaPct >= 0 ? '+' : ''}${row.tokenDeltaPct.toFixed(1)}% [${row.tokenDeltaLow95.toFixed(1)}, ${row.tokenDeltaHigh95.toFixed(1)}]`,
+    row.meanTurns.toFixed(1),
+    `${row.meanDurationSeconds.toFixed(0)}s`,
+    row.meanRoutedTurns.toFixed(1),
+    row.meanCompletionGates.toFixed(1),
+  ].join(' | ') + ' |').join('\n');
+  return `${header}\n${divider}\n${body}`;
+}
+
+function buildReport(experimentId, runs, summaries) {
+  const openai = summaries.filter(row => row.target === 'openai');
+  const gemini = summaries.filter(row => row.target === 'gemini');
+  const periods = runs.map(run => JSON.parse(fs.readFileSync(run.reportPath, 'utf8')));
+  const starts = periods.map(report => new Date(report.startTime)).filter(date => !Number.isNaN(date.valueOf()));
+  const ends = periods.map(report => new Date(report.endTime)).filter(date => !Number.isNaN(date.valueOf()));
+  return `# Testronaut Jev optimization pilot report
+
+Experiment: \`${experimentId}\`  
+Generated: ${new Date().toISOString()}  
+Benchmark period: ${new Date(Math.min(...starts)).toISOString()} to ${new Date(Math.max(...ends)).toISOString()}
+
+## Executive summary
+
+This blocked pilot completed **${runs.length} runs** across two repositories, seven conditions per provider, five repetitions per condition, and **${runs.length * 3} mission-file attempts**. Each routed condition used at most two execution models. Jev remained the guardrail/router and was not counted as an execution model.
+
+The clearest OpenAI result was guarded GPT-5.5: **15/15 strict mission-file passes** and **${Math.abs(openai.find(row => row.condition === 'l2_gpt55_guarded').tokenDeltaPct).toFixed(1)}% fewer model tokens** than the Terra control. Its paired 95% interval excluded zero, but it was materially slower. OpenAI routing coverage was too low—less than one routed turn per run—to distinguish secondary-model quality confidently.
+
+Gemini 3.8 Flash was already perfectly reliable on this workload. Guarded Gemini 3.5 Flash-Lite also achieved **15/15 strict passes** with a nominal **${Math.abs(gemini.find(row => row.condition === 'l2_gemini35lite_guarded').tokenDeltaPct).toFixed(1)}% token reduction**, but the paired interval included zero and duration increased. Gemini 3.8 → 3.7 was the strongest conservative routing policy: 15/15 strict passes, modest nominal token reduction, and nearly unchanged duration. Routing 3.8 → 3.5 Flash-Lite was less reliable and caused primary-model retries.
+
+## Methodology
+
+- Five randomized blocks per condition (\`n=5\`).
+- OpenAI repository: login, self-cleaning task creation/removal, and file transfer.
+- Gemini repository: profile navigation, projects dashboard, and project workspace navigation.
+- Level 1: single-model control without Jev.
+- Level 2: single model with live Jev guardrails and completion gating.
+- Level 3: current-generation/recent two-model routing with guardrails.
+- Level 4: current primary routed to older same-provider generations with guardrails.
+- Routing probability threshold: 0.40.
+- Strict mission-file success requires every expected phase for that file to exist and pass. This avoids treating a failed prerequisite that suppresses later phases as a smaller denominator.
+- Token, turn, and duration deltas are paired by randomized block against the provider's level-1 control. Intervals use a two-sided 95% Student-t interval with four degrees of freedom.
+
+## OpenAI results
+
+${markdownTable(openai)}
+
+### OpenAI interpretation
+
+- **Guarded GPT-5.5 was the pilot winner** for reliability and model-token volume: all 15 mission-file attempts passed.
+- GPT-5.5 used fewer model tokens and turns than Terra control, but mean duration increased substantially; it is a quality/cost candidate rather than a speed candidate.
+- Guarded Terra improved nominal tokens and strict completion, showing that completion gating contributed independently of model switching.
+- Terra → Luna was the best OpenAI routing condition by strict pass rate, but it routed only 0.4 turns per run and produced almost no model-token reduction.
+- Terra → GPT-5.4 and Terra → GPT-5.5 also routed fewer than one turn per run. Their secondary-model effects remain underpowered.
+- Guarded Luna showed high variance and incomplete mission-file execution in two blocks.
+
+## Gemini results
+
+${markdownTable(gemini)}
+
+### Gemini interpretation
+
+- The Gemini 3.8 control achieved 15/15 strict passes, leaving no reliability headroom.
+- Guarded Gemini 3.5 Flash-Lite retained 15/15 passes and had the largest nominal reduction among reliable Gemini conditions, at the cost of higher duration.
+- Gemini 3.8 → 3.7 retained perfect reliability and near-control duration; it is the strongest conservative Gemini routing candidate.
+- Gemini 3.8 → 3.6 retained reliability but offered little token or latency benefit.
+- Direct Gemini 3.6 and Gemini 3.8 → 3.5 Flash-Lite each fell to 11/15 strict passes. The latter also generated frequent primary-model retries.
+- No Gemini token delta had a paired 95% interval excluding zero at \`n=5\`; larger samples are required.
+
+## Cross-provider conclusions
+
+1. Keep the execution-model limit at two per run: one primary and one secondary. This preserved attribution and made fallback behavior auditable.
+2. Guardrails and completion gating produced more consistent savings than routing because routing coverage remained low.
+3. Model choice mattered more than routing sophistication on the OpenAI workload: guarded GPT-5.5 outperformed Terra-based policies.
+4. For Gemini, use guarded 3.5 Flash-Lite when cost is dominant and its slower runtime is acceptable; use 3.8 → 3.7 when preserving current-model reliability and latency is more important.
+5. Treat this as a pilot, not a universal model ranking. The mission subsets are intentionally small, provider pricing was not embedded, and application/model behavior may drift.
+
+## Limitations and next steps
+
+- Five repetitions provide useful variance estimates but limited statistical power.
+- Results apply to these six mission files and their test environments.
+- The package reports raw model-token volume, not price-weighted spend; pricing metadata should be versioned before asserting monetary savings.
+- Routed OpenAI conditions need either a larger sample or a routing policy that yields more eligible turns.
+- Repeat the winning and control conditions at a powered sample size derived from these observed variances.
+- Evaluate cross-provider routing first at mission boundaries to avoid transcript/tool-history incompatibilities.
+
+## Data package
+
+The accompanying package includes normalized run-, mission-, and condition-level CSV files; all 70 raw JSON reports; the frozen benchmark matrix; source state/CSV files where available; and SHA-256 checksums. Screenshots, HTML reports, environment files, credentials, and verbose logs are intentionally excluded.
+`;
+}
+
+function sha256(file) {
+  return crypto.createHash('sha256').update(fs.readFileSync(file)).digest('hex');
+}
+
+function walkFiles(root) {
+  return fs.readdirSync(root, { withFileTypes: true }).flatMap(entry => {
+    const full = path.join(root, entry.name);
+    return entry.isDirectory() ? walkFiles(full) : [full];
+  });
+}
+
+export function buildOptimizationDataPackage({ experimentId = DEFAULT_EXPERIMENT_ID } = {}) {
+  const root = path.resolve(HERE, 'results', experimentId);
+  const packageDir = path.join(root, 'data-package');
+  fs.rmSync(packageDir, { recursive: true, force: true });
+  fs.mkdirSync(path.join(packageDir, 'data/raw-reports/openai'), { recursive: true });
+  fs.mkdirSync(path.join(packageDir, 'data/raw-reports/gemini'), { recursive: true });
+  fs.mkdirSync(path.join(packageDir, 'data/source-state'), { recursive: true });
+
+  const entries = ['openai', 'gemini'].flatMap(target => readReports(target, experimentId));
+  if (entries.length !== 70) throw new Error(`Expected 70 reports for ${experimentId}; found ${entries.length}.`);
+  const runs = entries.map(normalizeRun).sort((a, b) => a.target.localeCompare(b.target) || a.block - b.block || a.condition.localeCompare(b.condition));
+  const summaries = summarizeConditions(runs);
+  const missionRows = runs.flatMap(run => run.missionOutcomes);
+
+  const runColumns = [
+    'experimentId', 'target', 'condition', 'level', 'block', 'runId', 'provider', 'primaryModel',
+    'secondaryModel', 'objective', 'mode', 'missionFilesPassed', 'missionFilesTotal', 'phasePassed',
+    'phaseFailed', 'phaseTotal', 'turns', 'tokens', 'inputTokens', 'outputTokens', 'durationSeconds',
+    'routedTurns', 'forcedPrimaryRetries', 'requestFallbacks', 'completionGates', 'jevEvaluations',
+    'jevInputTokens', 'jevOutputTokens', 'models', 'reportFile',
+  ];
+  const conditionColumns = Object.keys(summaries[0]);
+  const missionColumns = Object.keys(missionRows[0]);
+  writeCsv(path.join(packageDir, 'data/runs.csv'), runColumns, runs);
+  writeCsv(path.join(packageDir, 'data/conditions.csv'), conditionColumns, summaries.map(row => Object.fromEntries(Object.entries(row).map(([key, value]) => [key, typeof value === 'number' ? round(value, 6) : value]))));
+  writeCsv(path.join(packageDir, 'data/missions.csv'), missionColumns, missionRows);
+
+  for (const entry of entries) {
+    fs.copyFileSync(entry.path, path.join(packageDir, `data/raw-reports/${entry.target}/${entry.file}`));
+  }
+  for (const name of ['state-openai.json', 'state-gemini.json', 'runs-openai.csv', 'runs-gemini.csv']) {
+    const source = path.join(root, name);
+    if (fs.existsSync(source)) fs.copyFileSync(source, path.join(packageDir, `data/source-state/${name}`));
+  }
+
+  const matrix = Object.fromEntries(Object.entries(BENCHMARK_TARGETS).map(([target, value]) => [target, {
+    provider: value.provider,
+    repoName: path.basename(value.repo),
+    missions: value.missions,
+    conditions: value.conditions,
+  }]));
+  fs.writeFileSync(path.join(packageDir, 'data/matrix.json'), `${JSON.stringify(matrix, null, 2)}\n`);
+  const report = buildReport(experimentId, runs, summaries);
+  fs.writeFileSync(path.join(packageDir, 'REPORT.md'), report);
+  fs.writeFileSync(path.join(root, `${experimentId}-report.md`), report);
+  fs.writeFileSync(path.join(packageDir, 'README.md'), `# ${experimentId} data package\n\nStart with [REPORT.md](REPORT.md). Normalized datasets are under \`data/\`; raw Testronaut JSON reports are under \`data/raw-reports/\`. See \`manifest.json\` and \`SHA256SUMS\` for provenance and integrity. Screenshots, HTML, credentials, environment files, and verbose logs are excluded.\n`);
+
+  const dataFiles = walkFiles(packageDir).filter(file => !['manifest.json', 'SHA256SUMS'].includes(path.basename(file)));
+  const manifest = {
+    schemaVersion: 1,
+    experimentId,
+    generatedAt: new Date().toISOString(),
+    counts: { reports: entries.length, runs: runs.length, conditions: summaries.length, missionFileAttempts: missionRows.length },
+    exclusions: ['screenshots', 'HTML reports', 'environment files', 'credentials', 'verbose CLI logs'],
+    files: dataFiles.map(file => ({ path: path.relative(packageDir, file), bytes: fs.statSync(file).size, sha256: sha256(file) })),
+  };
+  fs.writeFileSync(path.join(packageDir, 'manifest.json'), `${JSON.stringify(manifest, null, 2)}\n`);
+  const checksumFiles = walkFiles(packageDir).filter(file => path.basename(file) !== 'SHA256SUMS');
+  fs.writeFileSync(path.join(packageDir, 'SHA256SUMS'), checksumFiles.map(file => `${sha256(file)}  ${path.relative(packageDir, file)}`).sort().join('\n') + '\n');
+
+  const archive = path.join(root, `${experimentId}-data-package.tar.gz`);
+  fs.rmSync(archive, { force: true });
+  const tar = spawnSync('tar', ['-czf', archive, '-C', root, 'data-package'], { encoding: 'utf8' });
+  if (tar.status !== 0) throw new Error(`Could not create archive: ${tar.stderr || tar.stdout}`);
+  fs.writeFileSync(`${archive}.sha256`, `${sha256(archive)}  ${path.basename(archive)}\n`);
+  return { root, packageDir, archive, manifest };
+}
+
+if (process.argv[1] && path.resolve(process.argv[1]) === fileURLToPath(import.meta.url)) {
+  const experimentId = process.argv.find(arg => arg.startsWith('--experiment-id='))?.split('=')[1] || DEFAULT_EXPERIMENT_ID;
+  const result = buildOptimizationDataPackage({ experimentId });
+  console.log(`Report and data package generated at ${result.packageDir}`);
+  console.log(`Archive: ${result.archive}`);
+}
+
+export const __dataPackageInternals = { mean, sampleSd, normalizeRun, summarizeConditions, pairedDelta };