testronaut 1.11.0 → 1.12.1

package/core/jevUsage.js

@@ -0,0 +1,62 @@
+export const DEFAULT_JEV_INPUT_USD_PER_MILLION = 0.042;
+
+function nonNegativeNumber(value, fallback = 0) {
+  const number = Number(value);
+  return Number.isFinite(number) && number >= 0 ? number : fallback;
+}
+
+export function resolveJevInputPrice(env = process.env) {
+  return nonNegativeNumber(
+    env.TESTRONAUT_JEV_INPUT_USD_PER_MILLION,
+    DEFAULT_JEV_INPUT_USD_PER_MILLION,
+  );
+}
+
+export function summarizeJevUsage(steps = [], {
+  inputUsdPerMillion = resolveJevInputPrice(),
+} = {}) {
+  const entries = (Array.isArray(steps) ? steps : [])
+    .map(step => step?.jevShadow)
+    .filter(Boolean);
+  const completed = entries.filter(entry => entry.status === 'ok');
+  const inputTokens = completed.reduce(
+    (sum, entry) => sum + nonNegativeNumber(entry.usage?.input_tokens),
+    0,
+  );
+  const outputTokens = completed.reduce(
+    (sum, entry) => sum + nonNegativeNumber(entry.usage?.output_tokens),
+    0,
+  );
+  const strategyEntries = completed.map(entry => entry.strategies).filter(Boolean);
+  const countChoices = (values) => values.reduce((counts, value) => {
+    if (value) counts[value] = (counts[value] || 0) + 1;
+    return counts;
+  }, {});
+  const regionDecisions = strategyEntries.flatMap(entry => entry.domRelevance || []);
+
+  return {
+    evaluations: entries.length,
+    completed: completed.length,
+    errors: entries.filter(entry => entry.status === 'error').length,
+    skipped: entries.filter(entry => entry.status === 'skipped').length,
+    inputTokens,
+    outputTokens,
+    latencyMs: completed.reduce((sum, entry) => sum + nonNegativeNumber(entry.latencyMs), 0),
+    completionCandidates: completed.filter(entry => entry.completionGate?.candidate).length,
+    triggeredCompletions: (Array.isArray(steps) ? steps : [])
+      .filter(step => step?.jevGate?.triggered).length,
+    inputUsdPerMillion,
+    estimatedInputCostUsd: inputTokens * inputUsdPerMillion / 1_000_000,
+    strategies: strategyEntries.length ? {
+      evaluations: strategyEntries.length,
+      modelRoutes: countChoices(strategyEntries.map(entry => entry.modelRoute?.choice)),
+      actionChoices: countChoices(strategyEntries.map(entry => entry.actionChoice?.choice)),
+      recoveryRoutes: countChoices(strategyEntries.map(entry => entry.recovery?.choice)),
+      missionStages: countChoices(strategyEntries.map(entry => entry.missionStage?.choice)),
+      verifiedPostconditions: strategyEntries.filter(entry => Number(entry.verification?.noul) >= 0.8).length,
+      satisfiedPrerequisites: strategyEntries.filter(entry => Number(entry.prerequisite?.noul) >= 0.8).length,
+      regionsEvaluated: regionDecisions.length,
+      relevantRegions: regionDecisions.filter(region => Number(region.probability) >= 0.5).length,
+    } : null,
+  };
+}

package/core/missionDiscovery.js

@@ -6,6 +6,54 @@ import { loadConfig } from './config.js';
 const DEFAULT_ROOT = 'missions';
 const DEFAULT_INCLUDE = ['**/*.mission.js', '**/*.mission.ts'];
 
+function orderMissionFiles(files, order, root) {
+  const alphabetical = [...files].sort();
+  if (!Array.isArray(order) || order.length === 0) {
+    return { files: alphabetical, warnings: [] };
+  }
+
+  const remaining = new Set(alphabetical);
+  const ordered = [];
+  const warnings = [];
+
+  for (let index = 0; index < order.length; index += 1) {
+    const rawPattern = order[index];
+    const pattern = String(rawPattern ?? '').trim();
+    if (!pattern) continue;
+
+    if (pattern === '*') {
+      const reserved = new Set(order.slice(index + 1).flatMap(laterPattern => {
+        const later = String(laterPattern ?? '').trim();
+        if (!later || later === '*') return [];
+        return fg.sync(later, { cwd: root, onlyFiles: true });
+      }));
+      for (const file of alphabetical) {
+        if (!reserved.has(file) && remaining.delete(file)) ordered.push(file);
+      }
+      continue;
+    }
+
+    const matches = fg.sync(pattern, { cwd: root, onlyFiles: true })
+      .filter(file => remaining.has(file))
+      .sort();
+    for (const file of matches) {
+      remaining.delete(file);
+      ordered.push(file);
+    }
+
+    if (!fg.isDynamicPattern(pattern) && !alphabetical.includes(pattern)) {
+      warnings.push(`Configured mission order entry was not discovered: ${pattern}`);
+    }
+  }
+
+  // A partial order only moves the named groups; all other missions retain
+  // deterministic legacy ordering after the configured entries.
+  for (const file of alphabetical) {
+    if (remaining.delete(file)) ordered.push(file);
+  }
+  return { files: ordered, warnings };
+}
+
 /**
  * Discover mission files based on testronaut-config.json.
  * - When no missions block exists, preserve legacy behavior:
@@ -14,7 +62,7 @@ const DEFAULT_INCLUDE = ['**/*.mission.js', '**/*.mission.ts'];
  *
  * @param {object} opts
  * @param {string} [opts.cwd=process.cwd()] - project root for config + resolution
- * @returns {Promise<{ root:string, files:string[] }>} root is absolute; files are relative to root
+ * @returns {Promise<{ root:string, files:string[], warnings:string[] }>} root is absolute; files are relative to root
  */
 export async function discoverMissionFiles({ cwd = process.cwd() } = {}) {
   const defaultRoot = path.resolve(cwd, DEFAULT_ROOT);
@@ -24,17 +72,17 @@ export async function discoverMissionFiles({ cwd = process.cwd() } = {}) {
   // Legacy behavior remains non-recursive, now with first-class TypeScript support.
   if (!missionsCfg) {
     if (!fs.existsSync(defaultRoot)) {
-      return { root: defaultRoot, files: [] };
+      return { root: defaultRoot, files: [], warnings: [] };
     }
     const files = fs.readdirSync(defaultRoot)
       .filter(f => f.endsWith('.mission.js') || f.endsWith('.mission.ts'))
       .sort();
-    return { root: defaultRoot, files };
+    return { root: defaultRoot, files, warnings: [] };
   }
 
   const root = path.resolve(cwd, missionsCfg.root || DEFAULT_ROOT);
   if (!fs.existsSync(root)) {
-    return { root, files: [] };
+    return { root, files: [], warnings: [] };
   }
 
   const include = Array.isArray(missionsCfg.include) && missionsCfg.include.length
@@ -49,7 +97,9 @@ export async function discoverMissionFiles({ cwd = process.cwd() } = {}) {
   });
 
   const unique = Array.from(new Set(matches));
-  unique.sort();
+  const ordered = orderMissionFiles(unique, missionsCfg.order, root);
 
-  return { root, files: unique };
+  return { root, ...ordered };
 }
+
+export const __test__ = { orderMissionFiles };

package/core/optimization.js

@@ -0,0 +1,98 @@
+export const OPTIMIZATION_OBJECTIVES = ['off', 'cost', 'tokens', 'balanced', 'speed'];
+export const OPTIMIZATION_MODES = ['off', 'shadow', 'live'];
+
+const OBJECTIVE_ALIASES = {
+  neither: 'off',
+  'lower-spend': 'cost',
+  spend: 'cost',
+  'fewer-tokens': 'tokens',
+  both: 'balanced',
+  latency: 'speed',
+};
+
+function normalize(value, allowed, aliases = {}) {
+  const normalized = String(value ?? '').trim().toLowerCase();
+  const resolved = aliases[normalized] || normalized;
+  return allowed.includes(resolved) ? resolved : null;
+}
+
+function truthy(value) {
+  return /^(1|true|yes|on)$/i.test(String(value ?? '').trim());
+}
+
+export function resolveOptimizationPolicy({ env = process.env, config = {} } = {}) {
+  const configValue = config.optimization || {};
+  const guardrailsValue = config.guardrails || {};
+  const guardrailsConfigured = guardrailsValue.enabled !== undefined || guardrailsValue.mode !== undefined;
+  const guardrailsEnabled = guardrailsValue.enabled !== false && guardrailsValue.mode !== 'off';
+  const rawObjective = env.TESTRONAUT_OPTIMIZE
+    ?? configValue.objective
+    ?? (guardrailsConfigured ? (guardrailsEnabled ? 'tokens' : 'off') : undefined);
+  const rawMode = env.TESTRONAUT_OPTIMIZATION_MODE
+    ?? configValue.mode
+    ?? (guardrailsConfigured ? (guardrailsEnabled ? (guardrailsValue.mode || 'live') : 'off') : undefined);
+  const objective = normalize(rawObjective, OPTIMIZATION_OBJECTIVES, OBJECTIVE_ALIASES);
+  const mode = normalize(rawMode, OPTIMIZATION_MODES);
+  const configured = rawObjective !== undefined || rawMode !== undefined;
+  const effectiveObjective = objective ?? (configured ? 'off' : null);
+  const effectiveMode = mode ?? (configured && effectiveObjective !== 'off' ? 'shadow' : 'off');
+  const routingRequested = ['cost', 'balanced', 'speed'].includes(effectiveObjective);
+  const betaModelRouting = truthy(
+    env.TESTRONAUT_BETA_MODEL_ROUTING ?? configValue.beta?.modelRouting,
+  );
+  const threshold = Number(
+    env.TESTRONAUT_JEV_ROUTING_THRESHOLD
+      ?? configValue.guardrails?.minimumRoutingProbability,
+  );
+
+  return {
+    configured,
+    objective: effectiveObjective,
+    mode: effectiveObjective === 'off' ? 'off' : effectiveMode,
+    routingRequested,
+    betaModelRouting,
+    routing: routingRequested && betaModelRouting,
+    tokenReduction: ['tokens', 'balanced'].includes(effectiveObjective),
+    completionGate: effectiveMode === 'live' && effectiveObjective !== 'off',
+    routingProbability: Number.isFinite(threshold) && threshold >= 0 && threshold <= 1
+      ? threshold
+      : 0.4,
+    completionProbability: (() => {
+      const value = Number(
+        env.TESTRONAUT_JEV_GATE_THRESHOLD
+          ?? guardrailsValue.completionProbability
+          ?? configValue.guardrails?.completionProbability,
+      );
+      return Number.isFinite(value) && value >= 0 && value <= 1 ? value : 0.8;
+    })(),
+    valid: (!configured || Boolean(objective || rawObjective === undefined))
+      && (rawMode === undefined || Boolean(mode)),
+  };
+}
+
+export function applyOptimizationPolicy(policy, env = process.env, { force = false } = {}) {
+  if (!policy?.configured) return env;
+  env.TESTRONAUT_OPTIMIZATION_OBJECTIVE_EFFECTIVE = policy.objective;
+  env.TESTRONAUT_OPTIMIZATION_MODE_EFFECTIVE = policy.mode;
+  if (force || env.TESTRONAUT_JEV_ROUTING_THRESHOLD === undefined) {
+    env.TESTRONAUT_JEV_ROUTING_THRESHOLD = String(policy.routingProbability);
+  }
+  if (force || env.TESTRONAUT_JEV_GATE_THRESHOLD === undefined) {
+    env.TESTRONAUT_JEV_GATE_THRESHOLD = String(policy.completionProbability);
+  }
+  if (force || env.TESTRONAUT_JEV_ROUTING === undefined) {
+    env.TESTRONAUT_JEV_ROUTING = policy.routing ? policy.mode : 'off';
+  }
+  if (force || env.TESTRONAUT_JEV_GATE === undefined) {
+    env.TESTRONAUT_JEV_GATE = policy.completionGate ? '1' : '0';
+  }
+  if (force || env.TESTRONAUT_JEV_STRATEGIES === undefined) {
+    env.TESTRONAUT_JEV_STRATEGIES = policy.routing ? '1' : '0';
+  }
+  if (force || env.TESTRONAUT_JEV_SHADOW === undefined) {
+    env.TESTRONAUT_JEV_SHADOW = policy.mode === 'off' ? '0' : '1';
+  }
+  return env;
+}
+
+export const __optimizationInternals = { normalize, OBJECTIVE_ALIASES };

package/core/turnLoop.js

@@ -49,7 +49,10 @@ import {
 import { resolveProviderModel } from '../llm/modelResolver.js';
 import { getLLM } from '../llm/llmFactory.js';
 import { summarizeTurnIntentFromMessage } from './turnIntent.js';
-import { maskPreview, redactArgs } from './redaction.js';
+import { maskPreview, redactArgs, redactPasswordInText } from './redaction.js';
+import { buildJevShadowState, buildJevStrategyQuestions, evaluateJevCompletionCandidate, evaluateJevShadow, extractJevStrategyResults, resolveJevShadowConfig, sanitizeJevShadowEvents } from './jevShadow.js';
+import { buildBrowserEvidence, collectBrowserControlState, mergeBrowserControlState } from './browserEvidence.js';
+import { chooseJevTurnModel, resolveJevRoutingConfig } from './jevRouting.js';
 import { 
   sanitizeHeavyToolHistory, 
   pruneConversationContext,
@@ -169,6 +172,20 @@ const FIRE_AND_FORGET_TOOLS = new Set([
   'record_mission_telemetry',
 ]);
 
+function currentBrowserUrl(browser) {
+  try { return browser?.page?.url?.() || null; } catch { return null; }
+}
+
+async function captureBrowserEvidence(browser, html, previous) {
+  const evidence = buildBrowserEvidence(html, {
+    url: currentBrowserUrl(browser),
+    previous,
+    redactText: redactPasswordInText,
+  });
+  const states = await collectBrowserControlState(browser);
+  return mergeBrowserControlState(evidence, states);
+}
+
 /**
  * Utility: format byte counts into human-readable strings.
  * Used when reporting upload/download events in mission logs.
@@ -221,6 +238,11 @@ const pushDOMAssistant = async (browser, messages, agentMemory, { skipIfLastTool
     agentMemory
   );
   await tokenEstimate(MODEL_ID, domHtml);
+  agentMemory.browserEvidence = await captureBrowserEvidence(
+    browser,
+    domHtml,
+    agentMemory.browserEvidence,
+  );
 
   // Push the corresponding tool response back into the conversation
   messages.push({
@@ -269,6 +291,11 @@ export const turnLoop = async (
     dataTypes: ['document', 'file', 'item', 'row'],
   };
   const stepsArchive = ctx.stepsArchive || steps;
+  const jevShadowConfig = resolveJevShadowConfig();
+  const jevRoutingConfig = resolveJevRoutingConfig(process.env, {
+    provider: PROVIDER_ID,
+    primaryModel: MODEL_ID,
+  });
   const retryLimitClamped = Math.min(10, Math.max(1, Number.isFinite(retryLimitRaw) ? retryLimitRaw : DEFAULT_TURN_RETRY_LIMIT)); // retries allowed (excludes initial)
   const maxAttempts = retryLimitClamped + 1; // includes initial attempt
   ctx.groundControl = groundControl;
@@ -281,11 +308,109 @@ export const turnLoop = async (
   ensureDocProgress(agentMemory, resourceGuardCfg);
   let turnRetries = 0;
   let stepSeq = ctx._stepSeq || 0;
+  let nextJevStrategy = null;
+  let forcePrimaryRetry = false;
+
+  const recordJevShadow = async (step, msg, phase, proposedTools = []) => {
+    if (!jevShadowConfig.enabled) return;
+    nextJevStrategy = null;
+    // Keep status lines and purpose-built telemetry, but exclude raw tool payload
+    // lines (which may contain DOM, page text, or other third-party data).
+    const safeEvents = sanitizeJevShadowEvents(step.events, redactPasswordInText);
+    agentMemory.browserEvidence = await captureBrowserEvidence(
+      browser,
+      null,
+      agentMemory.browserEvidence,
+    );
+    const state = buildJevShadowState({
+      mission: redactPasswordInText(ctx.goal || missionName),
+      turn: step.turn,
+      maxTurns,
+      phase,
+      assistantSummary: redactPasswordInText(step.summary || ''),
+      assistantContent: redactPasswordInText(msg?.content || ''),
+      proposedTools,
+      events: safeEvents,
+      groundControl,
+      browserEvidence: agentMemory.browserEvidence,
+    });
+    const strategyCatalog = jevShadowConfig.strategiesEnabled
+      ? buildJevStrategyQuestions(agentMemory.browserEvidence)
+      : null;
+    step.jevShadow = await evaluateJevShadow(state, {
+      config: jevShadowConfig,
+      ...(strategyCatalog ? { questions: strategyCatalog.questions } : {}),
+    });
+    if (step.jevShadow.status === 'ok' && strategyCatalog) {
+      step.jevShadow.strategies = extractJevStrategyResults(
+        step.jevShadow.answers,
+        strategyCatalog,
+      );
+      nextJevStrategy = step.jevShadow.strategies;
+    }
+    step.jevShadow.completionGate = evaluateJevCompletionCandidate(step.jevShadow, {
+      threshold: jevShadowConfig.gateThreshold,
+      browserEvidence: agentMemory.browserEvidence,
+    });
+    const status = step.jevShadow?.status || 'unknown';
+    const latency = Number.isFinite(step.jevShadow?.latencyMs)
+      ? ` in ${step.jevShadow.latencyMs}ms`
+      : '';
+    console.log(`🔬 Jev shadow: ${status}${latency}`);
+  };
+
+  const finalizeWithJevGate = async (step) => {
+    if (!jevShadowConfig.gateEnabled || !step.jevShadow?.completionGate?.candidate) return null;
+
+    const prog = ensureDocProgress(agentMemory, resourceGuardCfg);
+    if (prog?.items.length && prog.downloaded.size < prog.items.length) {
+      step.jevGate = { triggered: false, reason: 'resource_guard_incomplete' };
+      return null;
+    }
+
+    const existingScreenshot = Boolean(
+      step.screenshotPath
+        || steps.some(existing => existing?.screenshotPath)
+        || stepsArchive.some(existing => existing?.screenshotPath),
+    );
+    if (process.env.TESTRONAUT_SCREENSHOTS !== '0' && !existingScreenshot) {
+      try {
+        const screenshotResult = await CHROME_TOOL_MAP.screenshot(
+          browser,
+          { name: `jev-completion-turn-${step.turn + 1}` },
+          agentMemory,
+        );
+        const match = String(screenshotResult).match(/screenshot.*?saved at: (.+\.png)/i);
+        if (match?.[1]) step.screenshotPath = match[1];
+        step.events.push('🖼️ Jev completion gate captured final screenshot');
+      } catch (error) {
+        step.jevGate = { triggered: false, reason: 'screenshot_failed', error: error?.message || String(error) };
+        step.events.push(`⚠️ Jev completion gate screenshot failed: ${error?.message || error}`);
+        return null;
+      }
+    }
+
+    const gate = step.jevShadow.completionGate;
+    recordGroundTelemetry(groundControl, {
+      kind: 'assertion',
+      text: `Jev completion gate satisfied at ${Math.round(gate.threshold * 100)}% threshold.`,
+      status: 'passed',
+    }, { turn: step.turn, source: 'jev_completion_gate', signals: gate.signals });
+
+    step.jevGate = { triggered: true, ...gate };
+    step.events.push(`🔬 Jev completion gate triggered at ${Math.round(gate.threshold * 100)}% threshold`);
+    step.result = '✅ Mission Success (Jev gate)';
+    recordStep(step);
+    const finalMessage = 'SUCCESS: Jev completion gate found sufficient browser evidence.';
+    return { success: true, finalMessage, steps: stepsArchive, completedBy: 'jev_gate' };
+  };
   
   // Centralized recorder: push to in-memory buffer AND fire optional callback for streaming
   // Idempotent recorder – prevents accidental double-push
   const recordStep = (step) => {
     if (!step || step.__recorded) return;
+    step.endTime = Date.now();
+    step.durationMs = Math.max(0, step.endTime - (step.startTime ?? step.endTime));
     step._seq = stepSeq++;
     step.__recorded = true;
     try { steps.push(step); } catch {}
@@ -304,12 +429,27 @@ export const turnLoop = async (
     const attempt = turnRetries + 1;
     const step = {
       turn,
+      startTime: Date.now(),
       retryAttempt: attempt,
       retryLimit: retryLimitClamped, // number of retries allowed (excludes initial)
       events: [],
       result: '🟡 In Progress',
       missionName,
     };
+    const modelRouting = chooseJevTurnModel(nextJevStrategy, jevRoutingConfig, {
+      provider: PROVIDER_ID,
+      primaryModel: MODEL_ID,
+    });
+    if (forcePrimaryRetry) {
+      modelRouting.selectedModel = MODEL_ID;
+      modelRouting.applied = false;
+      modelRouting.reason = 'routed_retry_primary';
+      modelRouting.forcedPrimaryRetry = true;
+      forcePrimaryRetry = false;
+    }
+    step.modelRouting = modelRouting;
+    step.provider = PROVIDER_ID;
+    step.model = modelRouting.selectedModel;
     if (attempt > 1) {
       const retryNumber = attempt - 1;
       step.events.push(`🔁 Re-attempt ${retryNumber}/${retryLimitClamped} for turn`);
@@ -372,13 +512,14 @@ export const turnLoop = async (
       // ─────────────────────────────────────────────
       // STEP 2: Request next reasoning turn from model
       // ─────────────────────────────────────────────
-      const contextCheck = await warnIfContextNearLimit(MODEL_ID, { messages, tools: activeToolsSchema });
+      const turnModel = modelRouting.selectedModel;
+      const contextCheck = await warnIfContextNearLimit(turnModel, { messages, tools: activeToolsSchema });
       const projectedTokens = contextCheck.estimatedTokens
-        ?? await tokenEstimate(MODEL_ID, { messages, tools: activeToolsSchema });
+        ?? await tokenEstimate(turnModel, { messages, tools: activeToolsSchema });
       ({ totalTokensUsed, turnTimestamps, shouldBackoff } = await tokenUseCoolOff(
         totalTokensUsed,
         turnTimestamps,
-        MODEL_ID,
+        turnModel,
         projectedTokens
       ));
       if (shouldBackoff) {
@@ -386,13 +527,21 @@ export const turnLoop = async (
         turn -= 1;
         continue;
       }
-      const { message, usage, headers } = await llm.chat({
-        model: MODEL_ID,
-        messages,
-        tools: activeToolsSchema,
-      });
+      let modelResponse;
+      try {
+        modelResponse = await llm.chat({ model: turnModel, messages, tools: activeToolsSchema });
+      } catch (routingError) {
+        if (!modelRouting.applied) throw routingError;
+        modelRouting.fallback = true;
+        modelRouting.fallbackReason = routingError?.message || String(routingError);
+        modelRouting.selectedModel = MODEL_ID;
+        step.model = MODEL_ID;
+        step.events.push(`↩️ Jev route fallback to ${MODEL_ID}`);
+        modelResponse = await llm.chat({ model: MODEL_ID, messages, tools: activeToolsSchema });
+      }
+      const { message, usage, headers } = modelResponse;
       response = { message, usage, headers };
-      updateLimitsFromHeaders(MODEL_ID, headers);
+      updateLimitsFromHeaders(modelRouting.selectedModel, headers);
 
     } catch (err) {
       // ─────────────────────────────────────────────
@@ -460,6 +609,7 @@ export const turnLoop = async (
     if (msg.tool_calls?.length) {
       console.log('Processing tool calls...');
       const toolResponses = [];
+      const proposedTools = [];
       let hadToolIssues = false;
       let domRefreshSource = null;
 
@@ -467,6 +617,7 @@ export const turnLoop = async (
         const fnName = call.function.name;
         const args = JSON.parse(call.function.arguments || '{}');
         const safeArgs = redactArgs(fnName, args);
+        proposedTools.push({ name: fnName, arguments: safeArgs });
         console.log(`[model] → ${fnName}`, safeArgs);
         step.events.push(`[model] → ${fnName} ${JSON.stringify(safeArgs)}`);
 
@@ -761,6 +912,14 @@ export const turnLoop = async (
           }
         }
 
+        if (!errorMessage && fnName === 'get_dom') {
+          agentMemory.browserEvidence = await captureBrowserEvidence(
+            browser,
+            result,
+            agentMemory.browserEvidence,
+          );
+        }
+
         // Decide what to send back to the LLM for this tool.
         // - For "fire-and-forget" tools, send a tiny stub (OK/error) to avoid
         //   bloating context with large payloads (file JSON, etc.).
@@ -833,11 +992,17 @@ export const turnLoop = async (
           step.events.push(`[auto] → DOM size after ${domRefreshSource}: ${domRefresh.domLength} chars`);
         }
       }
+      await recordJevShadow(step, msg, hadToolIssues ? 'after_action_error' : 'after_action', proposedTools);
+      if (!hadToolIssues) {
+        const gatedResult = await finalizeWithJevGate(step);
+        if (gatedResult) return gatedResult;
+      }
       if (hadToolIssues && turnRetries < retryLimitClamped) {
         step.result = '⏳ Retrying turn';
         const retryNumber = attempt - 1;
         step.events.push(`🔁 Re-attempt ${retryNumber}/${retryLimitClamped} after tool issues`);
         recordStep(step);
+        forcePrimaryRetry = modelRouting.applied;
         turnRetries += 1;
         const delay = Math.min(TURN_RETRY_BASE_DELAY_MS * 2 ** (turnRetries - 1), 2000);
         await wait(delay);
@@ -875,6 +1040,7 @@ export const turnLoop = async (
         turnRetries = 0;
         continue;
       }
+      await recordJevShadow(step, msg, 'final_response');
       step.events.push(finalResponse.finalMessage);
       step.result = finalResponse.success ? '✅ Mission Success' : '❌ Mission Failure';
       recordStep(step);
@@ -891,13 +1057,68 @@ export const turnLoop = async (
     });
     console.log(`[auto] → Injected DOM for next reasoning step`);
     step.events.push(`[auto] → Injected DOM for next reasoning step`);
+    await recordJevShadow(step, msg, 'after_observation');
     // This is a meaningful turn even without tool calls — record it.
     step.result = step.result || '🟡 In Progress';
     recordStep(step);
     turnRetries = 0;
   }
 
-  return { success: false, steps: stepsArchive };
+  // The configured budget covers working turns. Once those are consumed, make
+  // one final tool-free request so every mission ends with an explicit verdict.
+  const closureStep = {
+    turn: maxTurns,
+    startTime: Date.now(),
+    isClosureTurn: true,
+    retryAttempt: 1,
+    retryLimit: 0,
+    events: ['⏱️ Working-turn budget exhausted; requesting final mission verdict.'],
+    result: '🟡 Awaiting Closure',
+    missionName,
+    provider: PROVIDER_ID,
+    model: MODEL_ID,
+  };
+  console.log(`\n🏁 Turn budget exhausted after ${maxTurns} working turn(s). Requesting closure.`);
+  messages.push({
+    role: 'user',
+    content: 'The working-turn budget is exhausted. Do not call tools or continue the task. Based only on the evidence already collected, respond with exactly SUCCESS: followed by a short reason if the goal was achieved, otherwise FAILURE: followed by a short reason.',
+  });
+
+  try {
+    const closureResponse = await llm.chat({ model: MODEL_ID, messages, tools: [] });
+    const msg = closureResponse?.message ?? { role: 'assistant', content: '' };
+    const usage = closureResponse?.usage;
+    if (usage) {
+      closureStep.tokensUsed = usage.total_tokens || 0;
+      closureStep.inputTokens = usage.input_tokens ?? usage.prompt_tokens ?? usage.providerRaw?.input_tokens ?? usage.providerRaw?.prompt_tokens;
+      closureStep.outputTokens = usage.output_tokens ?? usage.completion_tokens ?? usage.providerRaw?.output_tokens ?? usage.providerRaw?.completion_tokens;
+    }
+    closureStep.summary = summarizeTurnIntentFromMessage(msg, { emoji: true });
+    const finalResponse = finalResponseHandler(msg);
+    if (finalResponse) {
+      closureStep.events.push(finalResponse.finalMessage);
+      closureStep.result = finalResponse.success ? '✅ Mission Success' : '❌ Mission Failure';
+      recordStep(closureStep);
+      return {
+        success: finalResponse.success,
+        finalMessage: finalResponse.finalMessage,
+        steps: stepsArchive,
+        completedBy: 'budget_closure',
+      };
+    }
+    closureStep.events.push('FAILURE: The closure response did not provide an explicit SUCCESS or FAILURE verdict.');
+  } catch (error) {
+    closureStep.events.push(`FAILURE: Unable to obtain the closure verdict: ${error?.message || error}`);
+  }
+
+  closureStep.result = '❌ Mission Failure';
+  recordStep(closureStep);
+  return {
+    success: false,
+    finalMessage: closureStep.events.at(-1),
+    steps: stepsArchive,
+    completedBy: 'budget_closure',
+  };
 };
 
 // Expose small helper bundle for unit tests (no production use).

package/experiments/CROSS_PROVIDER.md

@@ -0,0 +1,19 @@
+# Cross-provider benchmark
+
+This benchmark runs three example-repository missions across four fixed-model controls and four two-model OpenAI/Gemini portfolios. Jev selects one model at the mission boundary for portfolio conditions; that model owns the complete mission transcript.
+
+The default design is 8 conditions × 5 randomized blocks × 3 missions, or 120 isolated mission executions. Failed missions are not retried with the other provider.
+
+Preview the complete deterministic schedule without making API calls:
+
+```sh
+npm run benchmark:cross-provider -- --dry-run
+```
+
+Run or resume it:
+
+```sh
+npm run benchmark:cross-provider
+```
+
+Keys are loaded from `/home/shane/workspace/testronaut-examples/.env`. Progress, selections, logs, report paths, and a summary CSV are written under `experiments/results/cross-provider-pilot-v1/`. Re-running the command resumes completed work. Useful overrides include `--repetitions=5`, `--seed=...`, `--repo=...`, `--experiment-id=...`, and `--selection-threshold=0.55`.