testronaut 1.10.2 → 1.12.0

package/tests/coreTests/browserEvidence.test.js

@@ -0,0 +1,70 @@
+import { describe, expect, it } from 'vitest';
+import { buildBrowserEvidence, mergeBrowserControlState } from '../../core/browserEvidence.js';
+
+describe('browser evidence', () => {
+  it('extracts bounded structure without field values or URL secrets', () => {
+    const evidence = buildBrowserEvidence(`
+      <html><head><title>Mission Dashboard</title></head><body>
+        <main><h1>Mission Tasks</h1><form><label for="access">Access code</label>
+          <input id="access" name="access" type="password" value="super-secret" />
+          <button>Sign in</button>
+        </form></main>
+        <a href="https://example.test/tasks?token=secret#private">Tasks</a>
+        <p>private body content</p>
+      </body></html>
+    `, { url: 'https://example.test/dashboard?token=secret#private' });
+
+    expect(evidence).toMatchObject({
+      url: 'https://example.test/dashboard',
+      title: 'Mission Dashboard',
+      headings: ['Mission Tasks'],
+      counts: { headings: 1, forms: 1, buttons: 1, links: 1, inputs: 1 },
+    });
+    expect(JSON.stringify(evidence)).not.toContain('super-secret');
+    expect(JSON.stringify(evidence)).not.toContain('private body content');
+    expect(JSON.stringify(evidence)).not.toContain('token=secret');
+    expect(evidence.regions).toEqual(expect.arrayContaining([
+      expect.objectContaining({ kind: 'main', headings: ['Mission Tasks'] }),
+      expect.objectContaining({ kind: 'form', controls: ['Access code', 'Sign in'] }),
+    ]));
+  });
+
+  it('tracks route changes and clears stale page structure', () => {
+    const previous = buildBrowserEvidence('<title>Login</title><h1>Sign in</h1>', {
+      url: 'https://example.test/login',
+    });
+    const evidence = buildBrowserEvidence(null, {
+      url: 'https://example.test/dashboard',
+      previous,
+    });
+
+    expect(evidence).toEqual({
+      url: 'https://example.test/dashboard',
+      previousUrl: 'https://example.test/login',
+      routeChanged: true,
+    });
+  });
+
+  it('merges only boolean live state without exposing field contents', () => {
+    const evidence = { controls: [{ tag: 'input', id: 'password' }] };
+    const merged = mergeBrowserControlState(evidence, [{
+      hasValue: true,
+      checked: false,
+      disabled: false,
+      readOnly: true,
+      visible: true,
+      value: 'must-not-copy',
+    }]);
+
+    expect(merged.controls[0]).toEqual({
+      tag: 'input',
+      id: 'password',
+      hasValue: true,
+      checked: false,
+      disabled: false,
+      readOnly: true,
+      visible: true,
+    });
+    expect(JSON.stringify(merged)).not.toContain('must-not-copy');
+  });
+});

package/tests/coreTests/jevRouting.test.js

@@ -0,0 +1,57 @@
+import { describe, expect, it } from 'vitest';
+import { chooseJevTurnModel, resolveJevRoutingConfig } from '../../core/jevRouting.js';
+
+const safeFastRoute = {
+  modelRoute: { choice: 'fast_model', confidence: 0.22, probabilities: { fast_model: 0.52 } },
+  missionStage: { choice: 'interact' },
+  recovery: { choice: 'continue' },
+};
+
+describe('Jev dynamic routing', () => {
+  it('defaults OpenAI experiments to Luna without enabling routing', () => {
+    expect(resolveJevRoutingConfig({}, {
+      provider: 'openai', primaryModel: 'gpt-5.6-terra',
+    })).toMatchObject({ mode: 'off', fastModel: 'gpt-5.6-luna', confidenceThreshold: 0.4 });
+  });
+
+  it.each([
+    ['gemini', 'gemini-3.8-flash', 'gemini-3.5-flash-lite'],
+    ['anthropic', 'claude-sonnet-5', 'claude-haiku-4-5'],
+  ])('keeps the default fast model within the %s family', (provider, primaryModel, fastModel) => {
+    expect(resolveJevRoutingConfig({}, { provider, primaryModel })).toMatchObject({ fastModel });
+  });
+
+  it('records an eligible route without applying it in shadow mode', () => {
+    const result = chooseJevTurnModel(safeFastRoute, {
+      mode: 'shadow', fastModel: 'gpt-5.6-luna', confidenceThreshold: 0.4,
+    }, { provider: 'openai', primaryModel: 'gpt-5.6-terra' });
+    expect(result).toMatchObject({ eligible: true, applied: false, selectedModel: 'gpt-5.6-terra', reason: 'shadow_only' });
+  });
+
+  it('applies a high-confidence fast route for one live turn', () => {
+    const result = chooseJevTurnModel(safeFastRoute, {
+      mode: 'live', fastModel: 'gpt-5.6-luna', confidenceThreshold: 0.4,
+    }, { provider: 'openai', primaryModel: 'gpt-5.6-terra' });
+    expect(result).toMatchObject({ eligible: true, applied: true, selectedModel: 'gpt-5.6-luna', reason: 'jev_fast_model', score: 0.52, confidence: 0.22 });
+  });
+
+  it.each([
+    [{ ...safeFastRoute, missionStage: { choice: 'authenticate' } }, 'stage_authenticate'],
+    [{ ...safeFastRoute, recovery: { choice: 'reobserve' } }, 'recovery_reobserve'],
+    [{ ...safeFastRoute, modelRoute: { choice: 'fast_model', confidence: 0.5, probabilities: { fast_model: 0.39 } } }, 'low_probability'],
+  ])('keeps sensitive or uncertain turns on the primary model', (strategy, reason) => {
+    expect(chooseJevTurnModel(strategy, {
+      mode: 'live', fastModel: 'gpt-5.6-luna', confidenceThreshold: 0.4,
+    }, { provider: 'openai', primaryModel: 'gpt-5.6-terra' })).toMatchObject({ applied: false, reason });
+  });
+
+  it('rejects a configured model from another provider family', () => {
+    expect(chooseJevTurnModel(safeFastRoute, {
+      mode: 'live', fastModel: 'gpt-5.6-luna', confidenceThreshold: 0.4,
+    }, { provider: 'gemini', primaryModel: 'gemini-3.8-flash' })).toMatchObject({
+      applied: false,
+      reason: 'fast_model_provider_mismatch',
+      selectedModel: 'gemini-3.8-flash',
+    });
+  });
+});

package/tests/coreTests/jevShadow.test.js

@@ -0,0 +1,158 @@
+import { describe, expect, it, vi } from 'vitest';
+import {
+  buildJevShadowState,
+  buildJevStrategyQuestions,
+  evaluateJevCompletionCandidate,
+  evaluateJevShadow,
+  resolveJevShadowConfig,
+  sanitizeJevShadowEvents,
+  extractJevStrategyResults,
+} from '../../core/jevShadow.js';
+
+describe('Jev shadow evaluation', () => {
+  it('is disabled by default and resolves supported environment overrides', () => {
+    expect(resolveJevShadowConfig({}).enabled).toBe(false);
+    expect(resolveJevShadowConfig({
+      TESTRONAUT_JEV_GATE: 'true',
+      TYPESAFE_API_KEY: 'secret',
+      TESTRONAUT_JEV_MODEL: 'jev-test',
+      TESTRONAUT_JEV_TIMEOUT_MS: '1234',
+      TESTRONAUT_JEV_GATE_THRESHOLD: '0.85',
+    })).toMatchObject({
+      enabled: true,
+      gateEnabled: true,
+      gateThreshold: 0.85,
+      apiKey: 'secret',
+      model: 'jev-test',
+      timeoutMs: 1234,
+    });
+  });
+
+  it('requires all completion signals and browser evidence to clear the gate', () => {
+    const result = {
+      status: 'ok',
+      answers: {
+        missionComplete: { noul: 0.82 },
+        progressState: { probabilities: { complete: 0.91 } },
+        nextCompute: { probabilities: { finish: 0.88 } },
+      },
+    };
+
+    expect(evaluateJevCompletionCandidate(result, {
+      threshold: 0.8,
+      browserEvidence: { headings: ['Dashboard'] },
+    }).candidate).toBe(true);
+    expect(evaluateJevCompletionCandidate(result, {
+      threshold: 0.9,
+      browserEvidence: { headings: ['Dashboard'] },
+    }).candidate).toBe(false);
+    expect(evaluateJevCompletionCandidate(result, {
+      threshold: 0.8,
+      browserEvidence: null,
+    }).candidate).toBe(false);
+  });
+
+  it('builds and maps the optional strategy questions', () => {
+    const evidence = {
+      regions: [{ id: 'region_1', kind: 'main', headings: ['Dashboard'] }],
+      controls: [{ tag: 'button', id: 'save', label: 'Save' }],
+    };
+    const catalog = buildJevStrategyQuestions(evidence);
+
+    expect(catalog.questions).toMatchObject({
+      modelRoute: { type: 'choice' },
+      postconditionSatisfied: { type: 'noul' },
+      recoveryRoute: { type: 'choice' },
+      missionStage: { type: 'choice' },
+      prerequisiteSatisfied: { type: 'noul' },
+      regionRelevant_region_1: { type: 'noul' },
+      actionChoice: { type: 'choice' },
+    });
+
+    const result = extractJevStrategyResults({
+      regionRelevant_region_1: { type: 'noul', noul: 0.9 },
+      actionChoice: { type: 'choice', choice: 'action_1', confidence: 0.8 },
+      modelRoute: { type: 'choice', choice: 'deterministic' },
+    }, catalog);
+    expect(result.domRelevance[0].probability).toBe(0.9);
+    expect(result.actionChoice.selectedControl).toEqual(evidence.controls[0]);
+    expect(result.modelRoute.choice).toBe('deterministic');
+  });
+
+  it('builds a bounded state for the decision request', () => {
+    const state = buildJevShadowState({
+      mission: 'visit the dashboard',
+      turn: 1,
+      maxTurns: 10,
+      phase: 'after_action',
+      assistantContent: 'x'.repeat(3000),
+      proposedTools: [{ name: 'click_text', arguments: { text: 'Continue' } }],
+      events: Array.from({ length: 20 }, (_, index) => `event ${index}`),
+    });
+
+    expect(state.turn).toEqual({ current: 2, maximum: 10 });
+    expect(state.assistant.content).toContain('[truncated');
+    expect(state.recentEvents).toHaveLength(12);
+  });
+
+  it('excludes raw tool payloads while retaining tool status telemetry', () => {
+    expect(sanitizeJevShadowEvents([
+      '[tool ] ← get_dom result: OK',
+      '[tool ] ← <html>private page text</html>',
+      'Navigation completed',
+    ])).toEqual([
+      '[tool ] ← get_dom result: OK',
+      'Navigation completed',
+    ]);
+  });
+
+  it('returns API answers and usage without exposing the credential', async () => {
+    const fetchImpl = vi.fn(async (url, options) => ({
+      ok: true,
+      json: async () => ({
+        model: 'jev-resolved',
+        answers: { missionComplete: { type: 'noul', noul: 0.8 } },
+        usage: { input_tokens: 100, output_tokens: 3 },
+      }),
+    }));
+    const times = [100, 125];
+
+    const result = await evaluateJevShadow({ mission: 'test' }, {
+      config: {
+        enabled: true,
+        apiKey: 'top-secret',
+        apiBase: 'https://jev.example',
+        model: 'jev-latest',
+        timeoutMs: 1000,
+      },
+      fetchImpl,
+      now: () => times.shift(),
+    });
+
+    expect(result).toMatchObject({ status: 'ok', model: 'jev-resolved', latencyMs: 25 });
+    const [, request] = fetchImpl.mock.calls[0];
+    expect(request.headers.authorization).toBe('Bearer top-secret');
+    expect(JSON.stringify(result)).not.toContain('top-secret');
+  });
+
+  it('turns API failures into shadow telemetry', async () => {
+    const result = await evaluateJevShadow({}, {
+      config: {
+        enabled: true,
+        apiKey: 'secret',
+        apiBase: 'https://jev.example',
+        model: 'jev-latest',
+        timeoutMs: 1000,
+      },
+      fetchImpl: async () => ({ ok: false, status: 429 }),
+    });
+
+    expect(result).toMatchObject({ status: 'error', error: 'http_429' });
+  });
+
+  it('skips safely when enabled without a key', async () => {
+    await expect(evaluateJevShadow({}, {
+      config: { enabled: true, apiKey: '' },
+    })).resolves.toEqual({ status: 'skipped', reason: 'missing_api_key' });
+  });
+});

package/tests/coreTests/jevSuiteSelection.test.js

@@ -0,0 +1,24 @@
+import { describe, expect, it } from 'vitest';
+import { buildJevSuiteSelectionRequest, interpretJevSuiteSelection } from '../../core/jevSuiteSelection.js';
+
+describe('Jev suite selection', () => {
+  it('creates one bounded decision per mission and applies a threshold', () => {
+    const request = buildJevSuiteSelectionRequest({
+      changedFiles: ['src/auth/login.js'],
+      missions: [
+        { file: 'login.mission.js', tags: ['auth'] },
+        { file: 'billing.mission.js', tags: ['billing'] },
+      ],
+    });
+    expect(Object.keys(request.questions)).toEqual(['select_mission_1', 'select_mission_2']);
+    expect(request.state.changedFiles).toEqual(['src/auth/login.js']);
+
+    expect(interpretJevSuiteSelection({
+      select_mission_1: { noul: 0.95 },
+      select_mission_2: { noul: 0.2 },
+    }, request.catalog, 0.7)).toEqual([
+      { file: 'login.mission.js', tags: ['auth'], probability: 0.95, selected: true },
+      { file: 'billing.mission.js', tags: ['billing'], probability: 0.2, selected: false },
+    ]);
+  });
+});

package/tests/coreTests/jevUsage.test.js

@@ -0,0 +1,58 @@
+import { describe, expect, it } from 'vitest';
+import { resolveJevInputPrice, summarizeJevUsage } from '../../core/jevUsage.js';
+
+describe('Jev usage summaries', () => {
+  it('aggregates usage, outcomes, gates, latency, and estimated spend', () => {
+    const summary = summarizeJevUsage([
+      {
+        jevShadow: {
+          status: 'ok',
+          usage: { input_tokens: 1000, output_tokens: 25 },
+          latencyMs: 120,
+          completionGate: { candidate: true },
+          strategies: {
+            modelRoute: { choice: 'deterministic' },
+            actionChoice: { choice: 'action_1' },
+            recovery: { choice: 'continue' },
+            missionStage: { choice: 'interact' },
+            verification: { noul: 0.9 },
+            prerequisite: { noul: 0.85 },
+            domRelevance: [{ probability: 0.8 }, { probability: 0.2 }],
+          },
+        },
+        jevGate: { triggered: true },
+      },
+      { jevShadow: { status: 'error' } },
+      { jevShadow: { status: 'skipped' } },
+    ], { inputUsdPerMillion: 1 });
+
+    expect(summary).toEqual({
+      evaluations: 3,
+      completed: 1,
+      errors: 1,
+      skipped: 1,
+      inputTokens: 1000,
+      outputTokens: 25,
+      latencyMs: 120,
+      completionCandidates: 1,
+      triggeredCompletions: 1,
+      inputUsdPerMillion: 1,
+      estimatedInputCostUsd: 0.001,
+      strategies: {
+        evaluations: 1,
+        modelRoutes: { deterministic: 1 },
+        actionChoices: { action_1: 1 },
+        recoveryRoutes: { continue: 1 },
+        missionStages: { interact: 1 },
+        verifiedPostconditions: 1,
+        satisfiedPrerequisites: 1,
+        regionsEvaluated: 2,
+        relevantRegions: 1,
+      },
+    });
+  });
+
+  it('supports a configured input price', () => {
+    expect(resolveJevInputPrice({ TESTRONAUT_JEV_INPUT_USD_PER_MILLION: '0.05' })).toBe(0.05);
+  });
+});

package/tests/coreTests/optimization.test.js

@@ -0,0 +1,80 @@
+import { describe, expect, it } from 'vitest';
+import { applyOptimizationPolicy, resolveOptimizationPolicy } from '../../core/optimization.js';
+
+describe('optimization policy', () => {
+  it.each([
+    ['off', false, false],
+    ['cost', false, false],
+    ['tokens', false, true],
+    ['balanced', false, true],
+    ['speed', false, false],
+  ])('maps %s to implemented capabilities', (objective, routing, tokenReduction) => {
+    expect(resolveOptimizationPolicy({
+      env: {}, config: { optimization: { objective, mode: 'live' } },
+    })).toMatchObject({ objective, mode: objective === 'off' ? 'off' : 'live', routing, tokenReduction });
+  });
+
+  it('accepts user-facing aliases and defaults configured policies to shadow', () => {
+    expect(resolveOptimizationPolicy({
+      env: { TESTRONAUT_OPTIMIZE: 'both' }, config: {},
+    })).toMatchObject({ objective: 'balanced', mode: 'shadow' });
+  });
+
+  it('gives environment values precedence over config', () => {
+    expect(resolveOptimizationPolicy({
+      env: { TESTRONAUT_OPTIMIZE: 'cost', TESTRONAUT_OPTIMIZATION_MODE: 'live' },
+      config: { optimization: { objective: 'tokens', mode: 'shadow' } },
+    })).toMatchObject({ objective: 'cost', mode: 'live', routingRequested: true, routing: false });
+  });
+
+  it('requires an explicit beta opt-in before model routing can activate', () => {
+    expect(resolveOptimizationPolicy({
+      env: { TESTRONAUT_BETA_MODEL_ROUTING: '1' },
+      config: { optimization: { objective: 'balanced', mode: 'live' } },
+    })).toMatchObject({ routingRequested: true, betaModelRouting: true, routing: true });
+  });
+
+  it('supports stable guardrails without strategy or model routing calls', () => {
+    const env = {};
+    applyOptimizationPolicy(resolveOptimizationPolicy({
+      env: {}, config: { optimization: { objective: 'tokens', mode: 'live', guardrails: { completionProbability: 0.9 } } },
+    }), env);
+    expect(env).toMatchObject({
+      TESTRONAUT_JEV_ROUTING: 'off',
+      TESTRONAUT_JEV_STRATEGIES: '0',
+      TESTRONAUT_JEV_GATE: '1',
+      TESTRONAUT_JEV_GATE_THRESHOLD: '0.9',
+    });
+  });
+
+  it('accepts the top-level opt-in guardrails configuration', () => {
+    expect(resolveOptimizationPolicy({
+      env: {}, config: { guardrails: { enabled: true, mode: 'live', completionProbability: 0.85 } },
+    })).toMatchObject({ objective: 'tokens', mode: 'live', routing: false, completionProbability: 0.85 });
+    expect(resolveOptimizationPolicy({
+      env: {}, config: { guardrails: { enabled: false } },
+    })).toMatchObject({ objective: 'off', mode: 'off' });
+  });
+
+  it('translates policy to legacy Jev controls without replacing explicit overrides', () => {
+    const env = { TESTRONAUT_JEV_GATE: '0' };
+    applyOptimizationPolicy(resolveOptimizationPolicy({
+      env: { TESTRONAUT_BETA_MODEL_ROUTING: '1' }, config: { optimization: { objective: 'balanced', mode: 'live' } },
+    }), env);
+    expect(env).toMatchObject({
+      TESTRONAUT_JEV_ROUTING: 'live',
+      TESTRONAUT_JEV_GATE: '0',
+      TESTRONAUT_JEV_STRATEGIES: '1',
+      TESTRONAUT_JEV_SHADOW: '1',
+      TESTRONAUT_JEV_ROUTING_THRESHOLD: '0.4',
+    });
+  });
+
+  it('allows explicit CLI policy to replace lower-level overrides', () => {
+    const env = { TESTRONAUT_JEV_GATE: '1', TESTRONAUT_JEV_ROUTING: 'live' };
+    applyOptimizationPolicy(resolveOptimizationPolicy({
+      env: { TESTRONAUT_OPTIMIZE: 'off' }, config: {},
+    }), env, { force: true });
+    expect(env).toMatchObject({ TESTRONAUT_JEV_GATE: '0', TESTRONAUT_JEV_ROUTING: 'off' });
+  });
+});