testronaut 1.11.0 → 1.12.0

package/core/optimization.js

@@ -0,0 +1,98 @@
+export const OPTIMIZATION_OBJECTIVES = ['off', 'cost', 'tokens', 'balanced', 'speed'];
+export const OPTIMIZATION_MODES = ['off', 'shadow', 'live'];
+
+const OBJECTIVE_ALIASES = {
+  neither: 'off',
+  'lower-spend': 'cost',
+  spend: 'cost',
+  'fewer-tokens': 'tokens',
+  both: 'balanced',
+  latency: 'speed',
+};
+
+function normalize(value, allowed, aliases = {}) {
+  const normalized = String(value ?? '').trim().toLowerCase();
+  const resolved = aliases[normalized] || normalized;
+  return allowed.includes(resolved) ? resolved : null;
+}
+
+function truthy(value) {
+  return /^(1|true|yes|on)$/i.test(String(value ?? '').trim());
+}
+
+export function resolveOptimizationPolicy({ env = process.env, config = {} } = {}) {
+  const configValue = config.optimization || {};
+  const guardrailsValue = config.guardrails || {};
+  const guardrailsConfigured = guardrailsValue.enabled !== undefined || guardrailsValue.mode !== undefined;
+  const guardrailsEnabled = guardrailsValue.enabled !== false && guardrailsValue.mode !== 'off';
+  const rawObjective = env.TESTRONAUT_OPTIMIZE
+    ?? configValue.objective
+    ?? (guardrailsConfigured ? (guardrailsEnabled ? 'tokens' : 'off') : undefined);
+  const rawMode = env.TESTRONAUT_OPTIMIZATION_MODE
+    ?? configValue.mode
+    ?? (guardrailsConfigured ? (guardrailsEnabled ? (guardrailsValue.mode || 'live') : 'off') : undefined);
+  const objective = normalize(rawObjective, OPTIMIZATION_OBJECTIVES, OBJECTIVE_ALIASES);
+  const mode = normalize(rawMode, OPTIMIZATION_MODES);
+  const configured = rawObjective !== undefined || rawMode !== undefined;
+  const effectiveObjective = objective ?? (configured ? 'off' : null);
+  const effectiveMode = mode ?? (configured && effectiveObjective !== 'off' ? 'shadow' : 'off');
+  const routingRequested = ['cost', 'balanced', 'speed'].includes(effectiveObjective);
+  const betaModelRouting = truthy(
+    env.TESTRONAUT_BETA_MODEL_ROUTING ?? configValue.beta?.modelRouting,
+  );
+  const threshold = Number(
+    env.TESTRONAUT_JEV_ROUTING_THRESHOLD
+      ?? configValue.guardrails?.minimumRoutingProbability,
+  );
+
+  return {
+    configured,
+    objective: effectiveObjective,
+    mode: effectiveObjective === 'off' ? 'off' : effectiveMode,
+    routingRequested,
+    betaModelRouting,
+    routing: routingRequested && betaModelRouting,
+    tokenReduction: ['tokens', 'balanced'].includes(effectiveObjective),
+    completionGate: effectiveMode === 'live' && effectiveObjective !== 'off',
+    routingProbability: Number.isFinite(threshold) && threshold >= 0 && threshold <= 1
+      ? threshold
+      : 0.4,
+    completionProbability: (() => {
+      const value = Number(
+        env.TESTRONAUT_JEV_GATE_THRESHOLD
+          ?? guardrailsValue.completionProbability
+          ?? configValue.guardrails?.completionProbability,
+      );
+      return Number.isFinite(value) && value >= 0 && value <= 1 ? value : 0.8;
+    })(),
+    valid: (!configured || Boolean(objective || rawObjective === undefined))
+      && (rawMode === undefined || Boolean(mode)),
+  };
+}
+
+export function applyOptimizationPolicy(policy, env = process.env, { force = false } = {}) {
+  if (!policy?.configured) return env;
+  env.TESTRONAUT_OPTIMIZATION_OBJECTIVE_EFFECTIVE = policy.objective;
+  env.TESTRONAUT_OPTIMIZATION_MODE_EFFECTIVE = policy.mode;
+  if (force || env.TESTRONAUT_JEV_ROUTING_THRESHOLD === undefined) {
+    env.TESTRONAUT_JEV_ROUTING_THRESHOLD = String(policy.routingProbability);
+  }
+  if (force || env.TESTRONAUT_JEV_GATE_THRESHOLD === undefined) {
+    env.TESTRONAUT_JEV_GATE_THRESHOLD = String(policy.completionProbability);
+  }
+  if (force || env.TESTRONAUT_JEV_ROUTING === undefined) {
+    env.TESTRONAUT_JEV_ROUTING = policy.routing ? policy.mode : 'off';
+  }
+  if (force || env.TESTRONAUT_JEV_GATE === undefined) {
+    env.TESTRONAUT_JEV_GATE = policy.completionGate ? '1' : '0';
+  }
+  if (force || env.TESTRONAUT_JEV_STRATEGIES === undefined) {
+    env.TESTRONAUT_JEV_STRATEGIES = policy.routing ? '1' : '0';
+  }
+  if (force || env.TESTRONAUT_JEV_SHADOW === undefined) {
+    env.TESTRONAUT_JEV_SHADOW = policy.mode === 'off' ? '0' : '1';
+  }
+  return env;
+}
+
+export const __optimizationInternals = { normalize, OBJECTIVE_ALIASES };

package/core/turnLoop.js

@@ -49,7 +49,10 @@ import {
 import { resolveProviderModel } from '../llm/modelResolver.js';
 import { getLLM } from '../llm/llmFactory.js';
 import { summarizeTurnIntentFromMessage } from './turnIntent.js';
-import { maskPreview, redactArgs } from './redaction.js';
+import { maskPreview, redactArgs, redactPasswordInText } from './redaction.js';
+import { buildJevShadowState, buildJevStrategyQuestions, evaluateJevCompletionCandidate, evaluateJevShadow, extractJevStrategyResults, resolveJevShadowConfig, sanitizeJevShadowEvents } from './jevShadow.js';
+import { buildBrowserEvidence, collectBrowserControlState, mergeBrowserControlState } from './browserEvidence.js';
+import { chooseJevTurnModel, resolveJevRoutingConfig } from './jevRouting.js';
 import { 
   sanitizeHeavyToolHistory, 
   pruneConversationContext,
@@ -169,6 +172,20 @@ const FIRE_AND_FORGET_TOOLS = new Set([
   'record_mission_telemetry',
 ]);
 
+function currentBrowserUrl(browser) {
+  try { return browser?.page?.url?.() || null; } catch { return null; }
+}
+
+async function captureBrowserEvidence(browser, html, previous) {
+  const evidence = buildBrowserEvidence(html, {
+    url: currentBrowserUrl(browser),
+    previous,
+    redactText: redactPasswordInText,
+  });
+  const states = await collectBrowserControlState(browser);
+  return mergeBrowserControlState(evidence, states);
+}
+
 /**
  * Utility: format byte counts into human-readable strings.
  * Used when reporting upload/download events in mission logs.
@@ -221,6 +238,11 @@ const pushDOMAssistant = async (browser, messages, agentMemory, { skipIfLastTool
     agentMemory
   );
   await tokenEstimate(MODEL_ID, domHtml);
+  agentMemory.browserEvidence = await captureBrowserEvidence(
+    browser,
+    domHtml,
+    agentMemory.browserEvidence,
+  );
 
   // Push the corresponding tool response back into the conversation
   messages.push({
@@ -269,6 +291,11 @@ export const turnLoop = async (
     dataTypes: ['document', 'file', 'item', 'row'],
   };
   const stepsArchive = ctx.stepsArchive || steps;
+  const jevShadowConfig = resolveJevShadowConfig();
+  const jevRoutingConfig = resolveJevRoutingConfig(process.env, {
+    provider: PROVIDER_ID,
+    primaryModel: MODEL_ID,
+  });
   const retryLimitClamped = Math.min(10, Math.max(1, Number.isFinite(retryLimitRaw) ? retryLimitRaw : DEFAULT_TURN_RETRY_LIMIT)); // retries allowed (excludes initial)
   const maxAttempts = retryLimitClamped + 1; // includes initial attempt
   ctx.groundControl = groundControl;
@@ -281,6 +308,102 @@ export const turnLoop = async (
   ensureDocProgress(agentMemory, resourceGuardCfg);
   let turnRetries = 0;
   let stepSeq = ctx._stepSeq || 0;
+  let nextJevStrategy = null;
+  let forcePrimaryRetry = false;
+
+  const recordJevShadow = async (step, msg, phase, proposedTools = []) => {
+    if (!jevShadowConfig.enabled) return;
+    nextJevStrategy = null;
+    // Keep status lines and purpose-built telemetry, but exclude raw tool payload
+    // lines (which may contain DOM, page text, or other third-party data).
+    const safeEvents = sanitizeJevShadowEvents(step.events, redactPasswordInText);
+    agentMemory.browserEvidence = await captureBrowserEvidence(
+      browser,
+      null,
+      agentMemory.browserEvidence,
+    );
+    const state = buildJevShadowState({
+      mission: redactPasswordInText(ctx.goal || missionName),
+      turn: step.turn,
+      maxTurns,
+      phase,
+      assistantSummary: redactPasswordInText(step.summary || ''),
+      assistantContent: redactPasswordInText(msg?.content || ''),
+      proposedTools,
+      events: safeEvents,
+      groundControl,
+      browserEvidence: agentMemory.browserEvidence,
+    });
+    const strategyCatalog = jevShadowConfig.strategiesEnabled
+      ? buildJevStrategyQuestions(agentMemory.browserEvidence)
+      : null;
+    step.jevShadow = await evaluateJevShadow(state, {
+      config: jevShadowConfig,
+      ...(strategyCatalog ? { questions: strategyCatalog.questions } : {}),
+    });
+    if (step.jevShadow.status === 'ok' && strategyCatalog) {
+      step.jevShadow.strategies = extractJevStrategyResults(
+        step.jevShadow.answers,
+        strategyCatalog,
+      );
+      nextJevStrategy = step.jevShadow.strategies;
+    }
+    step.jevShadow.completionGate = evaluateJevCompletionCandidate(step.jevShadow, {
+      threshold: jevShadowConfig.gateThreshold,
+      browserEvidence: agentMemory.browserEvidence,
+    });
+    const status = step.jevShadow?.status || 'unknown';
+    const latency = Number.isFinite(step.jevShadow?.latencyMs)
+      ? ` in ${step.jevShadow.latencyMs}ms`
+      : '';
+    console.log(`🔬 Jev shadow: ${status}${latency}`);
+  };
+
+  const finalizeWithJevGate = async (step) => {
+    if (!jevShadowConfig.gateEnabled || !step.jevShadow?.completionGate?.candidate) return null;
+
+    const prog = ensureDocProgress(agentMemory, resourceGuardCfg);
+    if (prog?.items.length && prog.downloaded.size < prog.items.length) {
+      step.jevGate = { triggered: false, reason: 'resource_guard_incomplete' };
+      return null;
+    }
+
+    const existingScreenshot = Boolean(
+      step.screenshotPath
+        || steps.some(existing => existing?.screenshotPath)
+        || stepsArchive.some(existing => existing?.screenshotPath),
+    );
+    if (process.env.TESTRONAUT_SCREENSHOTS !== '0' && !existingScreenshot) {
+      try {
+        const screenshotResult = await CHROME_TOOL_MAP.screenshot(
+          browser,
+          { name: `jev-completion-turn-${step.turn + 1}` },
+          agentMemory,
+        );
+        const match = String(screenshotResult).match(/screenshot.*?saved at: (.+\.png)/i);
+        if (match?.[1]) step.screenshotPath = match[1];
+        step.events.push('🖼️ Jev completion gate captured final screenshot');
+      } catch (error) {
+        step.jevGate = { triggered: false, reason: 'screenshot_failed', error: error?.message || String(error) };
+        step.events.push(`⚠️ Jev completion gate screenshot failed: ${error?.message || error}`);
+        return null;
+      }
+    }
+
+    const gate = step.jevShadow.completionGate;
+    recordGroundTelemetry(groundControl, {
+      kind: 'assertion',
+      text: `Jev completion gate satisfied at ${Math.round(gate.threshold * 100)}% threshold.`,
+      status: 'passed',
+    }, { turn: step.turn, source: 'jev_completion_gate', signals: gate.signals });
+
+    step.jevGate = { triggered: true, ...gate };
+    step.events.push(`🔬 Jev completion gate triggered at ${Math.round(gate.threshold * 100)}% threshold`);
+    step.result = '✅ Mission Success (Jev gate)';
+    recordStep(step);
+    const finalMessage = 'SUCCESS: Jev completion gate found sufficient browser evidence.';
+    return { success: true, finalMessage, steps: stepsArchive, completedBy: 'jev_gate' };
+  };
   
   // Centralized recorder: push to in-memory buffer AND fire optional callback for streaming
   // Idempotent recorder – prevents accidental double-push
@@ -310,6 +433,20 @@ export const turnLoop = async (
       result: '🟡 In Progress',
       missionName,
     };
+    const modelRouting = chooseJevTurnModel(nextJevStrategy, jevRoutingConfig, {
+      provider: PROVIDER_ID,
+      primaryModel: MODEL_ID,
+    });
+    if (forcePrimaryRetry) {
+      modelRouting.selectedModel = MODEL_ID;
+      modelRouting.applied = false;
+      modelRouting.reason = 'routed_retry_primary';
+      modelRouting.forcedPrimaryRetry = true;
+      forcePrimaryRetry = false;
+    }
+    step.modelRouting = modelRouting;
+    step.provider = PROVIDER_ID;
+    step.model = modelRouting.selectedModel;
     if (attempt > 1) {
       const retryNumber = attempt - 1;
       step.events.push(`🔁 Re-attempt ${retryNumber}/${retryLimitClamped} for turn`);
@@ -372,13 +509,14 @@ export const turnLoop = async (
       // ─────────────────────────────────────────────
       // STEP 2: Request next reasoning turn from model
       // ─────────────────────────────────────────────
-      const contextCheck = await warnIfContextNearLimit(MODEL_ID, { messages, tools: activeToolsSchema });
+      const turnModel = modelRouting.selectedModel;
+      const contextCheck = await warnIfContextNearLimit(turnModel, { messages, tools: activeToolsSchema });
       const projectedTokens = contextCheck.estimatedTokens
-        ?? await tokenEstimate(MODEL_ID, { messages, tools: activeToolsSchema });
+        ?? await tokenEstimate(turnModel, { messages, tools: activeToolsSchema });
       ({ totalTokensUsed, turnTimestamps, shouldBackoff } = await tokenUseCoolOff(
         totalTokensUsed,
         turnTimestamps,
-        MODEL_ID,
+        turnModel,
         projectedTokens
       ));
       if (shouldBackoff) {
@@ -386,13 +524,21 @@ export const turnLoop = async (
         turn -= 1;
         continue;
       }
-      const { message, usage, headers } = await llm.chat({
-        model: MODEL_ID,
-        messages,
-        tools: activeToolsSchema,
-      });
+      let modelResponse;
+      try {
+        modelResponse = await llm.chat({ model: turnModel, messages, tools: activeToolsSchema });
+      } catch (routingError) {
+        if (!modelRouting.applied) throw routingError;
+        modelRouting.fallback = true;
+        modelRouting.fallbackReason = routingError?.message || String(routingError);
+        modelRouting.selectedModel = MODEL_ID;
+        step.model = MODEL_ID;
+        step.events.push(`↩️ Jev route fallback to ${MODEL_ID}`);
+        modelResponse = await llm.chat({ model: MODEL_ID, messages, tools: activeToolsSchema });
+      }
+      const { message, usage, headers } = modelResponse;
       response = { message, usage, headers };
-      updateLimitsFromHeaders(MODEL_ID, headers);
+      updateLimitsFromHeaders(modelRouting.selectedModel, headers);
 
     } catch (err) {
       // ─────────────────────────────────────────────
@@ -460,6 +606,7 @@ export const turnLoop = async (
     if (msg.tool_calls?.length) {
       console.log('Processing tool calls...');
       const toolResponses = [];
+      const proposedTools = [];
       let hadToolIssues = false;
       let domRefreshSource = null;
 
@@ -467,6 +614,7 @@ export const turnLoop = async (
         const fnName = call.function.name;
         const args = JSON.parse(call.function.arguments || '{}');
         const safeArgs = redactArgs(fnName, args);
+        proposedTools.push({ name: fnName, arguments: safeArgs });
         console.log(`[model] → ${fnName}`, safeArgs);
         step.events.push(`[model] → ${fnName} ${JSON.stringify(safeArgs)}`);
 
@@ -761,6 +909,14 @@ export const turnLoop = async (
           }
         }
 
+        if (!errorMessage && fnName === 'get_dom') {
+          agentMemory.browserEvidence = await captureBrowserEvidence(
+            browser,
+            result,
+            agentMemory.browserEvidence,
+          );
+        }
+
         // Decide what to send back to the LLM for this tool.
         // - For "fire-and-forget" tools, send a tiny stub (OK/error) to avoid
         //   bloating context with large payloads (file JSON, etc.).
@@ -833,11 +989,17 @@ export const turnLoop = async (
           step.events.push(`[auto] → DOM size after ${domRefreshSource}: ${domRefresh.domLength} chars`);
         }
       }
+      await recordJevShadow(step, msg, hadToolIssues ? 'after_action_error' : 'after_action', proposedTools);
+      if (!hadToolIssues) {
+        const gatedResult = await finalizeWithJevGate(step);
+        if (gatedResult) return gatedResult;
+      }
       if (hadToolIssues && turnRetries < retryLimitClamped) {
         step.result = '⏳ Retrying turn';
         const retryNumber = attempt - 1;
         step.events.push(`🔁 Re-attempt ${retryNumber}/${retryLimitClamped} after tool issues`);
         recordStep(step);
+        forcePrimaryRetry = modelRouting.applied;
         turnRetries += 1;
         const delay = Math.min(TURN_RETRY_BASE_DELAY_MS * 2 ** (turnRetries - 1), 2000);
         await wait(delay);
@@ -875,6 +1037,7 @@ export const turnLoop = async (
         turnRetries = 0;
         continue;
       }
+      await recordJevShadow(step, msg, 'final_response');
       step.events.push(finalResponse.finalMessage);
       step.result = finalResponse.success ? '✅ Mission Success' : '❌ Mission Failure';
       recordStep(step);
@@ -891,6 +1054,7 @@ export const turnLoop = async (
     });
     console.log(`[auto] → Injected DOM for next reasoning step`);
     step.events.push(`[auto] → Injected DOM for next reasoning step`);
+    await recordJevShadow(step, msg, 'after_observation');
     // This is a meaningful turn even without tool calls — record it.
     step.result = step.result || '🟡 In Progress';
     recordStep(step);

package/experiments/CROSS_PROVIDER.md

@@ -0,0 +1,19 @@
+# Cross-provider benchmark
+
+This benchmark runs three example-repository missions across four fixed-model controls and four two-model OpenAI/Gemini portfolios. Jev selects one model at the mission boundary for portfolio conditions; that model owns the complete mission transcript.
+
+The default design is 8 conditions × 5 randomized blocks × 3 missions, or 120 isolated mission executions. Failed missions are not retried with the other provider.
+
+Preview the complete deterministic schedule without making API calls:
+
+```sh
+npm run benchmark:cross-provider -- --dry-run
+```
+
+Run or resume it:
+
+```sh
+npm run benchmark:cross-provider
+```
+
+Keys are loaded from `/home/shane/workspace/testronaut-examples/.env`. Progress, selections, logs, report paths, and a summary CSV are written under `experiments/results/cross-provider-pilot-v1/`. Re-running the command resumes completed work. Useful overrides include `--repetitions=5`, `--seed=...`, `--repo=...`, `--experiment-id=...`, and `--selection-threshold=0.55`.

package/experiments/README.md

@@ -0,0 +1,62 @@
+# Optimization benchmark pilot
+
+This benchmark compares seven optimization conditions across the OpenAI-focused
+Testronaut Examples repository and the Gemini-focused Collegium repository. It
+uses five randomized blocks by default: 35 runs per repository and 70 total.
+
+The fixed mission subsets are:
+
+- OpenAI: login, self-cleaning task creation/removal, and file transfer.
+- Gemini: profile navigation, projects dashboard, and project workspace navigation.
+
+Preview the complete deterministic schedule without making model calls:
+
+```bash
+npm run benchmark:optimization -- --dry-run
+```
+
+Run or resume both repositories:
+
+```bash
+npm run benchmark:optimization
+```
+
+Run only one target:
+
+```bash
+npm run benchmark:optimization -- --target=openai
+npm run benchmark:optimization -- --target=gemini
+```
+
+Useful controls:
+
+```bash
+npm run benchmark:optimization -- --repetitions=1 --experiment-id=optimization-smoke
+npm run benchmark:optimization -- --seed=my-fixed-seed
+```
+
+Progress is saved after every condition under target-specific
+`experiments/results/<experiment-id>/state-openai.json` and
+`state-gemini.json` files, so both loops may run concurrently without clobbering
+state. Reusing the same experiment ID skips completed conditions. The matching
+`runs-openai.csv` and `runs-gemini.csv` files contain one row per completed run,
+and captured CLI output is stored under the corresponding `logs` directory.
+Each generated Testronaut JSON report also records its experiment ID, condition,
+and block. If state is lost, the runner recovers completed conditions from those
+report fields before resuming.
+
+The runner invokes this checkout's `bin/cli.js` directly and fixes the routing
+probability at 0.40. It does not alter either target repository's configuration.
+Model/API credentials must already be available in the environment or target
+repository `.env` files.
+
+After all 70 reports are present, generate the findings report and portable data
+package with:
+
+```bash
+npm run benchmark:package
+```
+
+This writes a Markdown report, normalized CSV datasets, the raw JSON reports,
+manifest, and checksums under `experiments/results/optimization-pilot-v1/`, plus
+a compressed `optimization-pilot-v1-data-package.tar.gz` archive.