testronaut 1.10.2 → 1.12.0

package/experiments/crossProviderBenchmark.js

@@ -0,0 +1,227 @@
+#!/usr/bin/env node
+
+import fs from 'node:fs';
+import path from 'node:path';
+import { spawnSync } from 'node:child_process';
+import { fileURLToPath } from 'node:url';
+import dotenv from 'dotenv';
+import { evaluateJevShadow, resolveJevShadowConfig } from '../core/jevShadow.js';
+import { __benchmarkInternals } from './optimizationBenchmark.js';
+
+const HERE = path.dirname(fileURLToPath(import.meta.url));
+const DEFAULT_CLI = path.resolve(HERE, '../bin/cli.js');
+const DEFAULT_REPO = '/home/shane/workspace/testronaut-examples';
+const MISSIONS = [
+  'missions/login.mission.js',
+  'missions/removeTask.mission.js',
+  'missions/fileTransfer.mission.js',
+];
+
+const MODELS = {
+  gpt55: { provider: 'openai', model: 'gpt-5.5', profile: 'Strongest OpenAI reliability and token result in the prior pilot; slower than Terra.' },
+  terra: { provider: 'openai', model: 'gpt-5.6-terra', profile: 'Current-generation balanced OpenAI reference.' },
+  gemini38: { provider: 'gemini', model: 'gemini-3.8-flash', profile: 'Reliable current-generation Gemini reference.' },
+  gemini35lite: { provider: 'gemini', model: 'gemini-3.5-flash-lite', profile: 'Low-cost Gemini candidate; prior guarded runs were reliable but slower.' },
+};
+
+export const CROSS_PROVIDER_CONDITIONS = [
+  { id: 'c1_gpt55', candidates: [MODELS.gpt55] },
+  { id: 'c2_terra', candidates: [MODELS.terra] },
+  { id: 'c3_gemini38', candidates: [MODELS.gemini38] },
+  { id: 'c4_gemini35lite', candidates: [MODELS.gemini35lite] },
+  { id: 'x1_gpt55_gemini38', candidates: [MODELS.gpt55, MODELS.gemini38] },
+  { id: 'x2_gpt55_gemini35lite', candidates: [MODELS.gpt55, MODELS.gemini35lite] },
+  { id: 'x3_terra_gemini38', candidates: [MODELS.terra, MODELS.gemini38] },
+  { id: 'x4_terra_gemini35lite', candidates: [MODELS.terra, MODELS.gemini35lite] },
+];
+
+export function buildCrossProviderSchedule({ repetitions = 5, seed = 'cross-provider-pilot-v1', repo = DEFAULT_REPO } = {}) {
+  const schedule = [];
+  for (let block = 1; block <= repetitions; block += 1) {
+    const conditions = __benchmarkInternals.shuffled(CROSS_PROVIDER_CONDITIONS, `${seed}:conditions:${block}`);
+    for (const condition of conditions) {
+      const missions = __benchmarkInternals.shuffled(MISSIONS, `${seed}:${condition.id}:${block}`);
+      for (const mission of missions) {
+        const unit = path.basename(mission, '.mission.js');
+        schedule.push({
+          key: `b${block}:${condition.id}:${unit}`,
+          block,
+          unit,
+          mission,
+          repo,
+          ...condition,
+        });
+      }
+    }
+  }
+  return schedule;
+}
+
+export function interpretSelection(evaluation, candidates, threshold = 0.55) {
+  const fallback = candidates[0];
+  if (candidates.length === 1) return { selected: fallback, selectedBy: 'fixed', score: 1, evaluation: null };
+  const answer = evaluation?.answers?.executionModel;
+  const index = String(answer?.choice || '').match(/^model_(\d+)$/)?.[1];
+  const candidate = index ? candidates[Number(index) - 1] : null;
+  const score = candidate ? Number(answer?.probabilities?.[`model_${index}`]) : Number.NaN;
+  if (evaluation?.status === 'ok' && candidate && Number.isFinite(score) && score >= threshold) {
+    return { selected: candidate, selectedBy: 'jev', score, evaluation };
+  }
+  return {
+    selected: fallback,
+    selectedBy: 'fallback',
+    score: Number.isFinite(score) ? score : null,
+    reason: evaluation?.status === 'ok' ? 'low_or_invalid_probability' : evaluation?.reason || evaluation?.error || 'no_evaluation',
+    evaluation,
+  };
+}
+
+export async function selectExecutionModel(run, { env = process.env, threshold = 0.55, fetchImpl } = {}) {
+  if (run.candidates.length === 1) return interpretSelection(null, run.candidates, threshold);
+  const missionText = fs.readFileSync(path.join(run.repo, run.mission), 'utf8').slice(0, 12000);
+  const criteria = Object.fromEntries(run.candidates.map((candidate, index) => [
+    `model_${index + 1}`,
+    `${candidate.provider}/${candidate.model}: ${candidate.profile}`,
+  ]));
+  const evaluation = await evaluateJevShadow({
+    task: 'Choose one execution model before a browser-testing mission starts.',
+    mission: missionText,
+    constraints: ['One model owns the entire mission transcript.', 'Do not assume cross-provider failover.', 'Prefer lower expected spend when reliability is comparable.'],
+  }, {
+    config: { ...resolveJevShadowConfig(env), enabled: true, strategiesEnabled: false },
+    questions: {
+      executionModel: {
+        type: 'choice',
+        instructions: 'Choose the least expensive candidate likely to complete this mission reliably.',
+        criteria,
+      },
+    },
+    ...(fetchImpl ? { fetchImpl } : {}),
+  });
+  return interpretSelection(evaluation, run.candidates, threshold);
+}
+
+export function commandForCrossProviderRun(run, selection, cliPath = DEFAULT_CLI) {
+  return {
+    command: process.execPath,
+    args: [cliPath, `--provider=${selection.selected.provider}`, `--model=${selection.selected.model}`, '--optimize=tokens', '--optimization-mode=live', run.mission],
+  };
+}
+
+function parseArgs(argv) {
+  const read = (name, fallback) => argv.find(arg => arg.startsWith(`--${name}=`))?.slice(name.length + 3) ?? fallback;
+  return {
+    dryRun: argv.includes('--dry-run'),
+    repetitions: Math.max(1, Number.parseInt(read('repetitions', '5'), 10) || 5),
+    seed: read('seed', 'cross-provider-pilot-v1'),
+    experimentId: read('experiment-id', 'cross-provider-pilot-v1'),
+    repo: path.resolve(read('repo', DEFAULT_REPO)),
+    cliPath: path.resolve(read('cli', DEFAULT_CLI)),
+    threshold: Math.min(1, Math.max(0, Number(read('selection-threshold', '0.55')) || 0.55)),
+  };
+}
+
+function writeJson(file, value) {
+  fs.mkdirSync(path.dirname(file), { recursive: true });
+  fs.writeFileSync(file, `${JSON.stringify(value, null, 2)}\n`);
+}
+
+function reportFiles(repo) {
+  const directory = path.join(repo, 'missions/mission_reports');
+  if (!fs.existsSync(directory)) return new Set();
+  return new Set(fs.readdirSync(directory).filter(file => /^run_\d+\.json$/.test(file)));
+}
+
+function findNewReport(repo, before) {
+  const directory = path.join(repo, 'missions/mission_reports');
+  if (!fs.existsSync(directory)) return null;
+  const file = fs.readdirSync(directory)
+    .filter(name => /^run_\d+\.json$/.test(name) && !before.has(name))
+    .map(name => ({ name, time: fs.statSync(path.join(directory, name)).mtimeMs }))
+    .sort((a, b) => b.time - a.time)[0]?.name;
+  return file ? path.join(directory, file) : null;
+}
+
+function summarize(reportPath) {
+  if (!reportPath) return null;
+  try {
+    const report = JSON.parse(fs.readFileSync(reportPath, 'utf8'));
+    const steps = report.missions?.flatMap(mission => mission.steps || []) || [];
+    return {
+      runId: report.runId,
+      passed: report.summary?.passed || 0,
+      failed: report.summary?.failed || 0,
+      turns: steps.length,
+      tokens: steps.reduce((sum, step) => sum + Number(step.tokensUsed || 0), 0),
+      jevInputTokens: Number(report.jevShadow?.inputTokens || 0),
+      jevOutputTokens: Number(report.jevShadow?.outputTokens || 0),
+    };
+  } catch { return null; }
+}
+
+function writeCsv(file, state) {
+  const columns = ['key', 'block', 'condition', 'mission', 'provider', 'model', 'selectedBy', 'score', 'status', 'passed', 'failed', 'turns', 'tokens', 'jevInputTokens', 'jevOutputTokens'];
+  const quote = value => /[",\n]/.test(String(value ?? '')) ? `"${String(value ?? '').replaceAll('"', '""')}"` : String(value ?? '');
+  const rows = Object.values(state.runs).map(run => [run.key, run.block, run.id, run.mission, run.selection?.selected?.provider, run.selection?.selected?.model, run.selection?.selectedBy, run.selection?.score, run.status, run.metrics?.passed, run.metrics?.failed, run.metrics?.turns, run.metrics?.tokens, run.metrics?.jevInputTokens, run.metrics?.jevOutputTokens].map(quote).join(','));
+  fs.writeFileSync(file, `${columns.join(',')}\n${rows.join('\n')}\n`);
+}
+
+export async function runCrossProviderBenchmark(options) {
+  dotenv.config({ path: path.join(options.repo, '.env'), override: false, quiet: true });
+  const schedule = buildCrossProviderSchedule(options);
+  if (options.dryRun) return { schedule, completed: 0, skipped: 0 };
+  const root = path.resolve(HERE, 'results', options.experimentId);
+  const stateFile = path.join(root, 'state.json');
+  let state;
+  try { state = JSON.parse(fs.readFileSync(stateFile, 'utf8')); } catch { state = { experimentId: options.experimentId, seed: options.seed, repetitions: options.repetitions, createdAt: new Date().toISOString(), runs: {} }; }
+  let completed = 0;
+  let skipped = 0;
+  for (const run of schedule) {
+    if (state.runs[run.key]?.status === 'completed') { skipped += 1; continue; }
+    let selection = state.runs[run.key]?.selection;
+    if (!selection) {
+      selection = await selectExecutionModel(run, { threshold: options.threshold });
+      state.runs[run.key] = { ...run, status: 'selected', selection };
+      writeJson(stateFile, state);
+    }
+    const invocation = commandForCrossProviderRun(run, selection, options.cliPath);
+    console.log(`\n[${run.key}] ${selection.selected.provider}/${selection.selected.model} (${selection.selectedBy})`);
+    const before = reportFiles(run.repo);
+    const logFile = path.join(root, 'logs', `${run.key.replaceAll(':', '_')}.log`);
+    fs.mkdirSync(path.dirname(logFile), { recursive: true });
+    const env = { ...process.env,
+      TESTRONAUT_EXPERIMENT_ID: options.experimentId,
+      TESTRONAUT_EXPERIMENT_CONDITION: run.id,
+      TESTRONAUT_EXPERIMENT_BLOCK: String(run.block),
+      TESTRONAUT_EXPERIMENT_UNIT: run.unit,
+      TESTRONAUT_EXPERIMENT_SELECTED_BY: selection.selectedBy,
+      TESTRONAUT_EXPERIMENT_SELECTION_SCORE: selection.score == null ? '' : String(selection.score),
+    };
+    delete env.TESTRONAUT_JEV_FAST_MODEL;
+    const startedAt = new Date().toISOString();
+    const result = spawnSync(invocation.command, invocation.args, { cwd: run.repo, env, encoding: 'utf8', stdio: ['inherit', 'pipe', 'pipe'], maxBuffer: 50 * 1024 * 1024 });
+    fs.writeFileSync(logFile, `${result.stdout || ''}${result.stderr || ''}`);
+    const reportPath = findNewReport(run.repo, before);
+    const status = reportPath ? 'completed' : 'no_report';
+    state.runs[run.key] = { ...run, status, selection, exitCode: result.status, signal: result.signal, startedAt, endedAt: new Date().toISOString(), reportPath, logFile, metrics: summarize(reportPath) };
+    writeJson(stateFile, state);
+    writeCsv(path.join(root, 'runs.csv'), state);
+    console.log(`[${run.key}] ${status}; exit=${result.status}; report=${reportPath || 'none'}`);
+    if (status === 'completed') completed += 1;
+  }
+  return { schedule, completed, skipped, stateFile };
+}
+
+function printSchedule(schedule) {
+  console.log(`Cross-provider benchmark schedule: ${schedule.length} isolated mission executions`);
+  for (const run of schedule) console.log(`${run.key} | ${run.candidates.map(item => `${item.provider}/${item.model}`).join(' vs ')} | ${run.mission}`);
+}
+
+if (process.argv[1] && path.resolve(process.argv[1]) === fileURLToPath(import.meta.url)) {
+  const options = parseArgs(process.argv.slice(2));
+  const result = await runCrossProviderBenchmark(options);
+  if (options.dryRun) printSchedule(result.schedule);
+  else console.log(`Benchmark progress saved to ${result.stateFile}. Completed ${result.completed}; resumed/skipped ${result.skipped}.`);
+}
+
+export const __crossProviderInternals = { parseArgs };

package/experiments/optimizationBenchmark.js

@@ -0,0 +1,339 @@
+#!/usr/bin/env node
+
+import fs from 'node:fs';
+import path from 'node:path';
+import { spawnSync } from 'node:child_process';
+import { fileURLToPath } from 'node:url';
+
+const HERE = path.dirname(fileURLToPath(import.meta.url));
+const DEFAULT_CLI = path.resolve(HERE, '../bin/cli.js');
+const DEFAULT_REPOS = {
+  openai: '/home/shane/workspace/testronaut-examples',
+  gemini: '/home/shane/workspace/e2e-testing-collegium',
+};
+
+export const BENCHMARK_TARGETS = {
+  openai: {
+    provider: 'openai',
+    repo: DEFAULT_REPOS.openai,
+    missions: [
+      'missions/login.mission.js',
+      'missions/removeTask.mission.js',
+      'missions/fileTransfer.mission.js',
+    ],
+    conditions: [
+      { id: 'l1_terra_control', level: 1, primary: 'gpt-5.6-terra', objective: 'off', mode: 'off' },
+      { id: 'l2_terra_guarded', level: 2, primary: 'gpt-5.6-terra', objective: 'tokens', mode: 'live' },
+      { id: 'l2_luna_guarded', level: 2, primary: 'gpt-5.6-luna', objective: 'tokens', mode: 'live' },
+      { id: 'l2_gpt55_guarded', level: 2, primary: 'gpt-5.5', objective: 'tokens', mode: 'live' },
+      { id: 'l3_terra_luna', level: 3, primary: 'gpt-5.6-terra', secondary: 'gpt-5.6-luna', objective: 'cost', mode: 'live' },
+      { id: 'l4_terra_gpt55', level: 4, primary: 'gpt-5.6-terra', secondary: 'gpt-5.5', objective: 'cost', mode: 'live' },
+      { id: 'l4_terra_gpt54', level: 4, primary: 'gpt-5.6-terra', secondary: 'gpt-5.4', objective: 'cost', mode: 'live' },
+    ],
+  },
+  gemini: {
+    provider: 'gemini',
+    repo: DEFAULT_REPOS.gemini,
+    missions: [
+      'missions/profileNavigation.mission.js',
+      'missions/projectsDashboard.mission.js',
+      'missions/projectWorkspaceNavigation.mission.js',
+    ],
+    conditions: [
+      { id: 'l1_gemini38_control', level: 1, primary: 'gemini-3.8-flash', objective: 'off', mode: 'off' },
+      { id: 'l2_gemini38_guarded', level: 2, primary: 'gemini-3.8-flash', objective: 'tokens', mode: 'live' },
+      { id: 'l2_gemini36_guarded', level: 2, primary: 'gemini-3.6-flash', objective: 'tokens', mode: 'live' },
+      { id: 'l2_gemini35lite_guarded', level: 2, primary: 'gemini-3.5-flash-lite', objective: 'tokens', mode: 'live' },
+      { id: 'l3_gemini38_gemini37', level: 3, primary: 'gemini-3.8-flash', secondary: 'gemini-3.7-flash', objective: 'cost', mode: 'live' },
+      { id: 'l4_gemini38_gemini36', level: 4, primary: 'gemini-3.8-flash', secondary: 'gemini-3.6-flash', objective: 'cost', mode: 'live' },
+      { id: 'l4_gemini38_gemini35lite', level: 4, primary: 'gemini-3.8-flash', secondary: 'gemini-3.5-flash-lite', objective: 'cost', mode: 'live' },
+    ],
+  },
+};
+
+function hashSeed(value) {
+  let hash = 2166136261;
+  for (const char of String(value)) {
+    hash ^= char.charCodeAt(0);
+    hash = Math.imul(hash, 16777619);
+  }
+  return hash >>> 0;
+}
+
+function seededRandom(seed) {
+  let value = seed || 1;
+  return () => {
+    value += 0x6D2B79F5;
+    let next = value;
+    next = Math.imul(next ^ (next >>> 15), next | 1);
+    next ^= next + Math.imul(next ^ (next >>> 7), next | 61);
+    return ((next ^ (next >>> 14)) >>> 0) / 4294967296;
+  };
+}
+
+function shuffled(values, seed) {
+  const result = [...values];
+  const random = seededRandom(hashSeed(seed));
+  for (let index = result.length - 1; index > 0; index -= 1) {
+    const swap = Math.floor(random() * (index + 1));
+    [result[index], result[swap]] = [result[swap], result[index]];
+  }
+  return result;
+}
+
+export function buildBenchmarkSchedule({ repetitions = 5, targets = ['openai', 'gemini'], seed = 'optimization-pilot-v1' } = {}) {
+  const schedule = [];
+  for (let block = 1; block <= repetitions; block += 1) {
+    for (const targetName of targets) {
+      const target = BENCHMARK_TARGETS[targetName];
+      if (!target) throw new Error(`Unknown benchmark target: ${targetName}`);
+      for (const condition of shuffled(target.conditions, `${seed}:${targetName}:${block}`)) {
+        schedule.push({
+          key: `${targetName}:b${block}:${condition.id}`,
+          target: targetName,
+          block,
+          provider: target.provider,
+          repo: target.repo,
+          missions: target.missions,
+          ...condition,
+        });
+      }
+    }
+  }
+  return schedule;
+}
+
+export function commandForRun(run, cliPath = DEFAULT_CLI) {
+  return {
+    command: process.execPath,
+    args: [
+      cliPath,
+      `--provider=${run.provider}`,
+      `--model=${run.primary}`,
+      `--optimize=${run.objective}`,
+      `--optimization-mode=${run.mode}`,
+      ...run.missions,
+    ],
+    env: {
+      TESTRONAUT_JEV_ROUTING_THRESHOLD: '0.4',
+      ...(run.secondary ? { TESTRONAUT_BETA_MODEL_ROUTING: '1' } : {}),
+      ...(run.secondary ? { TESTRONAUT_JEV_FAST_MODEL: run.secondary } : {}),
+    },
+  };
+}
+
+function parseArgs(argv) {
+  const read = (name, fallback) => argv.find(arg => arg.startsWith(`--${name}=`))?.slice(name.length + 3) ?? fallback;
+  const target = read('target', 'both');
+  return {
+    dryRun: argv.includes('--dry-run'),
+    repetitions: Math.max(1, Number.parseInt(read('repetitions', '5'), 10) || 5),
+    targets: target === 'both' ? ['openai', 'gemini'] : target.split(',').filter(Boolean),
+    seed: read('seed', 'optimization-pilot-v1'),
+    experimentId: read('experiment-id', 'optimization-pilot-v1'),
+    cliPath: path.resolve(read('cli', DEFAULT_CLI)),
+  };
+}
+
+function reportFiles(repo) {
+  const directory = path.join(repo, 'missions/mission_reports');
+  if (!fs.existsSync(directory)) return new Set();
+  return new Set(fs.readdirSync(directory).filter(file => /^run_\d+\.json$/.test(file)));
+}
+
+function findNewReport(repo, before) {
+  const directory = path.join(repo, 'missions/mission_reports');
+  if (!fs.existsSync(directory)) return null;
+  const candidates = fs.readdirSync(directory)
+    .filter(file => /^run_\d+\.json$/.test(file) && !before.has(file))
+    .map(file => ({ file, time: fs.statSync(path.join(directory, file)).mtimeMs }))
+    .sort((a, b) => b.time - a.time);
+  return candidates[0] ? path.join(directory, candidates[0].file) : null;
+}
+
+function existingExperimentReports(repo, experimentId) {
+  const directory = path.join(repo, 'missions/mission_reports');
+  const reports = new Map();
+  if (!fs.existsSync(directory)) return reports;
+  for (const file of fs.readdirSync(directory).filter(name => /^run_\d+\.json$/.test(name))) {
+    const reportPath = path.join(directory, file);
+    try {
+      const report = JSON.parse(fs.readFileSync(reportPath, 'utf8'));
+      if (report.experiment?.id !== experimentId) continue;
+      reports.set(`${report.experiment.condition}:${report.experiment.block}`, reportPath);
+    } catch { /* ignore incomplete or unrelated reports */ }
+  }
+  return reports;
+}
+
+function loadState(file, metadata) {
+  try { return JSON.parse(fs.readFileSync(file, 'utf8')); } catch { return { ...metadata, runs: {} }; }
+}
+
+function writeJson(file, value) {
+  fs.mkdirSync(path.dirname(file), { recursive: true });
+  fs.writeFileSync(file, `${JSON.stringify(value, null, 2)}\n`);
+}
+
+function writeSummaryCsv(file, state) {
+  const columns = [
+    'key', 'target', 'block', 'level', 'condition', 'primary', 'secondary', 'objective',
+    'status', 'runId', 'passed', 'failed', 'turns', 'tokens', 'routedTurns',
+    'forcedPrimaryRetries', 'completionGates', 'jevInputTokens', 'jevOutputTokens',
+  ];
+  const escape = value => {
+    const text = String(value ?? '');
+    return /[",\n]/.test(text) ? `"${text.replaceAll('"', '""')}"` : text;
+  };
+  const rows = Object.entries(state.runs).map(([key, run]) => {
+    const metrics = run.metrics || {};
+    return [
+      key, run.target, run.block, run.level, run.id, run.primary, run.secondary, run.objective,
+      run.status, metrics.runId, metrics.passed, metrics.failed, metrics.turns, metrics.tokens,
+      metrics.routedTurns, metrics.forcedPrimaryRetries, metrics.completionGates,
+      metrics.jevInputTokens, metrics.jevOutputTokens,
+    ].map(escape).join(',');
+  });
+  fs.writeFileSync(file, `${columns.join(',')}\n${rows.join('\n')}\n`);
+}
+
+function summarizeReport(reportPath) {
+  if (!reportPath) return null;
+  try {
+    const report = JSON.parse(fs.readFileSync(reportPath, 'utf8'));
+    const steps = report.missions?.flatMap(mission => mission.steps || []) || [];
+    const models = {};
+    for (const step of steps) {
+      const model = step.model || report.llm?.model || 'unknown';
+      models[model] ||= { turns: 0, tokens: 0 };
+      models[model].turns += 1;
+      models[model].tokens += Number(step.tokensUsed || 0);
+    }
+    return {
+      runId: report.runId,
+      passed: report.summary?.passed || 0,
+      failed: report.summary?.failed || 0,
+      turns: steps.length,
+      tokens: steps.reduce((sum, step) => sum + Number(step.tokensUsed || 0), 0),
+      routedTurns: steps.filter(step => step.modelRouting?.applied).length,
+      forcedPrimaryRetries: steps.filter(step => step.modelRouting?.forcedPrimaryRetry).length,
+      jevInputTokens: Number(report.jevShadow?.inputTokens || 0),
+      jevOutputTokens: Number(report.jevShadow?.outputTokens || 0),
+      completionGates: Number(report.jevShadow?.triggeredCompletions || 0),
+      models,
+    };
+  } catch {
+    return null;
+  }
+}
+
+export function runBenchmark(options) {
+  const schedule = buildBenchmarkSchedule(options);
+  if (options.dryRun) return { schedule, completed: 0, skipped: 0 };
+
+  const stateRoot = path.resolve(HERE, 'results', options.experimentId);
+  const legacyStateFile = path.join(stateRoot, 'state.json');
+  const legacyState = loadState(legacyStateFile, { runs: {} });
+  const states = {};
+  const stateFiles = {};
+  for (const target of options.targets) {
+    const stateFile = path.join(stateRoot, `state-${target}.json`);
+    const legacyRuns = Object.fromEntries(
+      Object.entries(legacyState.runs || {}).filter(([, run]) => run.target === target),
+    );
+    const state = loadState(stateFile, {
+      experimentId: options.experimentId,
+      seed: options.seed,
+      repetitions: options.repetitions,
+      targets: [target],
+      createdAt: new Date().toISOString(),
+      runs: legacyRuns,
+    });
+    const targetSchedule = schedule.filter(run => run.target === target);
+    const existing = existingExperimentReports(targetSchedule[0]?.repo, options.experimentId);
+    for (const run of targetSchedule) {
+      if (state.runs[run.key]?.status === 'completed') continue;
+      const reportPath = existing.get(`${run.id}:${run.block}`);
+      if (!reportPath) continue;
+      state.runs[run.key] = {
+        ...run,
+        status: 'completed',
+        exitCode: null,
+        recoveredFromReport: true,
+        reportPath,
+        metrics: summarizeReport(reportPath),
+      };
+    }
+    states[target] = state;
+    stateFiles[target] = stateFile;
+    writeJson(stateFile, state);
+    writeSummaryCsv(path.join(stateRoot, `runs-${target}.csv`), state);
+  }
+  let completed = 0;
+  let skipped = 0;
+
+  for (const run of schedule) {
+    const state = states[run.target];
+    const stateFile = stateFiles[run.target];
+    if (state.runs[run.key]?.status === 'completed') {
+      skipped += 1;
+      continue;
+    }
+    const invocation = commandForRun(run, options.cliPath);
+    console.log(`\n[${run.key}] ${run.primary}${run.secondary ? ` -> ${run.secondary}` : ''} (${run.objective}/${run.mode})`);
+    const before = reportFiles(run.repo);
+    const logFile = path.join(stateRoot, 'logs', `${run.key.replaceAll(':', '_')}.log`);
+    fs.mkdirSync(path.dirname(logFile), { recursive: true });
+    const startedAt = new Date().toISOString();
+    const env = { ...process.env, ...invocation.env };
+    if (!run.secondary) delete env.TESTRONAUT_JEV_FAST_MODEL;
+    Object.assign(env, {
+      TESTRONAUT_EXPERIMENT_ID: options.experimentId,
+      TESTRONAUT_EXPERIMENT_CONDITION: run.id,
+      TESTRONAUT_EXPERIMENT_BLOCK: String(run.block),
+    });
+    const result = spawnSync(invocation.command, invocation.args, {
+      cwd: run.repo,
+      env,
+      encoding: 'utf8',
+      stdio: ['inherit', 'pipe', 'pipe'],
+      maxBuffer: 50 * 1024 * 1024,
+    });
+    fs.writeFileSync(logFile, `${result.stdout || ''}${result.stderr || ''}`);
+    const reportPath = findNewReport(run.repo, before);
+    const status = reportPath ? 'completed' : 'no_report';
+    state.runs[run.key] = {
+      ...run,
+      status,
+      exitCode: result.status,
+      signal: result.signal,
+      startedAt,
+      endedAt: new Date().toISOString(),
+      reportPath,
+      logFile,
+      metrics: summarizeReport(reportPath),
+    };
+    writeJson(stateFile, state);
+    writeSummaryCsv(path.join(stateRoot, `runs-${run.target}.csv`), state);
+    console.log(`[${run.key}] ${status}; exit=${result.status}; report=${reportPath || 'none'}`);
+    if (status === 'completed') completed += 1;
+  }
+  return { schedule, completed, skipped, stateFiles };
+}
+
+function printSchedule(schedule) {
+  console.log(`Optimization benchmark schedule: ${schedule.length} runs`);
+  for (const run of schedule) {
+    const secondary = run.secondary ? ` -> ${run.secondary}` : '';
+    console.log(`${run.key} | L${run.level} | ${run.primary}${secondary} | ${run.objective}/${run.mode} | ${run.missions.length} missions`);
+  }
+}
+
+if (process.argv[1] && path.resolve(process.argv[1]) === fileURLToPath(import.meta.url)) {
+  const options = parseArgs(process.argv.slice(2));
+  const result = runBenchmark(options);
+  if (options.dryRun) printSchedule(result.schedule);
+  else console.log(`Benchmark progress saved to ${Object.values(result.stateFiles).join(', ')}. Completed ${result.completed}; resumed/skipped ${result.skipped}.`);
+}
+
+export const __benchmarkInternals = { hashSeed, seededRandom, shuffled, parseArgs, summarizeReport };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "testronaut",
-  "version": "1.10.2",
+  "version": "1.12.0",
   "main": "./runner/testronaut.js",
   "bin": {
     "testronaut": "bin/cli.js"
@@ -12,6 +12,10 @@
   },
   "type": "module",
   "scripts": {
+    "benchmark:optimization": "node ./experiments/optimizationBenchmark.js",
+    "benchmark:cross-provider": "node ./experiments/crossProviderBenchmark.js",
+    "benchmark:package": "node ./experiments/buildOptimizationDataPackage.js",
+    "benchmark:publish-package": "node ./experiments/buildExperimentPublicationPackage.js",
     "test": "vitest run",
     "test:watch": "vitest",
     "start": "node ./bin/cli.js"

package/tests/binTests/cli.helpers.test.js

@@ -108,6 +108,7 @@ describe('cli helpers', () => {
     }, '/tmp/project');
     expect(effective.outputDir).toEqual({ value: '/tmp/project/artifacts', source: 'config' });
     expect(effective.authenticated).toBe(true);
+    expect(effective.guardrails).toMatchObject({ enabled: false, failOpen: true, credentialConfigured: false });
     expect(JSON.stringify(effective)).not.toContain('secret');
   });
 
@@ -128,6 +129,21 @@ describe('cli helpers', () => {
     expect(res.invalid).toBe(false);
   });
 
+  it('parses optimization objective and execution mode flags', () => {
+    expect(__test__.parseOptimizationArgs([
+      '--optimize=balanced', '--optimization-mode', 'live', '--beta-model-routing', 'login.mission.js',
+    ])).toMatchObject({
+      objective: 'balanced', mode: 'live', betaModelRouting: true, args: ['login.mission.js'], invalid: false,
+    });
+    expect(__test__.parseOptimizationArgs(['--optimize=unknown'])).toMatchObject({ invalid: true });
+    expect(__test__.parseOptimizationArgs(['--guardrails', 'login.mission.js'])).toMatchObject({
+      objective: 'tokens', mode: 'live', guardrailsMode: 'live', args: ['login.mission.js'], invalid: false,
+    });
+    expect(__test__.parseOptimizationArgs(['--guardrails=shadow', 'login.mission.js'])).toMatchObject({
+      objective: 'tokens', mode: 'shadow', args: ['login.mission.js'], invalid: false,
+    });
+  });
+
   it('parses vercel bypass value from next arg', () => {
     const { parseVercelBypassArgs } = __test__;
     const res = parseVercelBypassArgs(['--vercel_bypass', 'abc', 'login']);

package/tests/binTests/cli.integration.test.js

@@ -62,6 +62,20 @@ describe('CLI inspection commands', () => {
     expect(result.stdout).not.toContain('private');
   });
 
+  it('reports opt-in guardrail readiness without exposing the Jev key', () => {
+    const cwd = makeProject();
+    fs.writeFileSync(path.join(cwd, 'testronaut-config.json'), JSON.stringify({
+      provider: 'openai', model: 'gpt-5.6', guardrails: { enabled: true, completionProbability: 0.85 },
+    }));
+    const result = spawnSync(process.execPath, [cliPath, 'config', '--json'], {
+      cwd, encoding: 'utf8', env: { ...process.env, TESTRONAUT_JEV_API_KEY: 'private-jev-key' },
+    });
+    expect(result.status).toBe(0);
+    const output = JSON.parse(result.stdout);
+    expect(output.guardrails).toMatchObject({ enabled: true, mode: 'live', completionProbability: 0.85, credentialConfigured: true, credentialSource: 'TESTRONAUT_JEV_API_KEY' });
+    expect(result.stdout).not.toContain('private-jev-key');
+  });
+
   it('suggests a close mission name and exits nonzero during dry-run', () => {
     const result = run(makeProject(), ['logn.mission.js', '--dry-run', '--json']);
     expect(result.status).toBe(1);

package/tests/binTests/terminalUi.test.js

@@ -0,0 +1,34 @@
+import { describe, expect, it } from 'vitest';
+import { createTerminalUi, shouldUseColor } from '../../bin/terminalUi.js';
+
+function capture() {
+  let output = '';
+  return {
+    stream: { isTTY: false, write(value) { output += value; } },
+    read: () => output,
+  };
+}
+
+describe('terminal UI', () => {
+  it('renders a readable color-free mission lifecycle', () => {
+    const target = capture();
+    const ui = createTerminalUi({ stream: target.stream, color: false });
+
+    ui.banner({ version: '1.2.3', provider: 'openai', model: 'gpt-test', session: 'isolated' });
+    ui.phase('launch', 'login.mission.js');
+    ui.mission(true, 'login.mission.js', ['smoke']);
+    ui.summary({ totalMissions: 1, passed: 1, failed: 0, reportPath: 'reports/run.html' });
+
+    expect(target.read()).toContain('🧑‍🚀 TESTRONAUT  /  MISSION RUNNER');
+    expect(target.read()).toContain('LAUNCH    login.mission.js');
+    expect(target.read()).toContain('#smoke');
+    expect(target.read()).toContain('MISSION COMPLETE');
+    expect(target.read()).not.toContain('\u001b[');
+  });
+
+  it('honors terminal capability and NO_COLOR', () => {
+    expect(shouldUseColor({ isTTY: true }, {})).toBe(true);
+    expect(shouldUseColor({ isTTY: true }, { NO_COLOR: '' })).toBe(false);
+    expect(shouldUseColor({ isTTY: false }, {})).toBe(false);
+  });
+});