testronaut 1.11.0 → 1.12.1

package/tests/coreTests/jevShadow.test.js

@@ -0,0 +1,158 @@
+import { describe, expect, it, vi } from 'vitest';
+import {
+  buildJevShadowState,
+  buildJevStrategyQuestions,
+  evaluateJevCompletionCandidate,
+  evaluateJevShadow,
+  resolveJevShadowConfig,
+  sanitizeJevShadowEvents,
+  extractJevStrategyResults,
+} from '../../core/jevShadow.js';
+
+describe('Jev shadow evaluation', () => {
+  it('is disabled by default and resolves supported environment overrides', () => {
+    expect(resolveJevShadowConfig({}).enabled).toBe(false);
+    expect(resolveJevShadowConfig({
+      TESTRONAUT_JEV_GATE: 'true',
+      TYPESAFE_API_KEY: 'secret',
+      TESTRONAUT_JEV_MODEL: 'jev-test',
+      TESTRONAUT_JEV_TIMEOUT_MS: '1234',
+      TESTRONAUT_JEV_GATE_THRESHOLD: '0.85',
+    })).toMatchObject({
+      enabled: true,
+      gateEnabled: true,
+      gateThreshold: 0.85,
+      apiKey: 'secret',
+      model: 'jev-test',
+      timeoutMs: 1234,
+    });
+  });
+
+  it('requires all completion signals and browser evidence to clear the gate', () => {
+    const result = {
+      status: 'ok',
+      answers: {
+        missionComplete: { noul: 0.82 },
+        progressState: { probabilities: { complete: 0.91 } },
+        nextCompute: { probabilities: { finish: 0.88 } },
+      },
+    };
+
+    expect(evaluateJevCompletionCandidate(result, {
+      threshold: 0.8,
+      browserEvidence: { headings: ['Dashboard'] },
+    }).candidate).toBe(true);
+    expect(evaluateJevCompletionCandidate(result, {
+      threshold: 0.9,
+      browserEvidence: { headings: ['Dashboard'] },
+    }).candidate).toBe(false);
+    expect(evaluateJevCompletionCandidate(result, {
+      threshold: 0.8,
+      browserEvidence: null,
+    }).candidate).toBe(false);
+  });
+
+  it('builds and maps the optional strategy questions', () => {
+    const evidence = {
+      regions: [{ id: 'region_1', kind: 'main', headings: ['Dashboard'] }],
+      controls: [{ tag: 'button', id: 'save', label: 'Save' }],
+    };
+    const catalog = buildJevStrategyQuestions(evidence);
+
+    expect(catalog.questions).toMatchObject({
+      modelRoute: { type: 'choice' },
+      postconditionSatisfied: { type: 'noul' },
+      recoveryRoute: { type: 'choice' },
+      missionStage: { type: 'choice' },
+      prerequisiteSatisfied: { type: 'noul' },
+      regionRelevant_region_1: { type: 'noul' },
+      actionChoice: { type: 'choice' },
+    });
+
+    const result = extractJevStrategyResults({
+      regionRelevant_region_1: { type: 'noul', noul: 0.9 },
+      actionChoice: { type: 'choice', choice: 'action_1', confidence: 0.8 },
+      modelRoute: { type: 'choice', choice: 'deterministic' },
+    }, catalog);
+    expect(result.domRelevance[0].probability).toBe(0.9);
+    expect(result.actionChoice.selectedControl).toEqual(evidence.controls[0]);
+    expect(result.modelRoute.choice).toBe('deterministic');
+  });
+
+  it('builds a bounded state for the decision request', () => {
+    const state = buildJevShadowState({
+      mission: 'visit the dashboard',
+      turn: 1,
+      maxTurns: 10,
+      phase: 'after_action',
+      assistantContent: 'x'.repeat(3000),
+      proposedTools: [{ name: 'click_text', arguments: { text: 'Continue' } }],
+      events: Array.from({ length: 20 }, (_, index) => `event ${index}`),
+    });
+
+    expect(state.turn).toEqual({ current: 2, maximum: 10 });
+    expect(state.assistant.content).toContain('[truncated');
+    expect(state.recentEvents).toHaveLength(12);
+  });
+
+  it('excludes raw tool payloads while retaining tool status telemetry', () => {
+    expect(sanitizeJevShadowEvents([
+      '[tool ] ← get_dom result: OK',
+      '[tool ] ← <html>private page text</html>',
+      'Navigation completed',
+    ])).toEqual([
+      '[tool ] ← get_dom result: OK',
+      'Navigation completed',
+    ]);
+  });
+
+  it('returns API answers and usage without exposing the credential', async () => {
+    const fetchImpl = vi.fn(async (url, options) => ({
+      ok: true,
+      json: async () => ({
+        model: 'jev-resolved',
+        answers: { missionComplete: { type: 'noul', noul: 0.8 } },
+        usage: { input_tokens: 100, output_tokens: 3 },
+      }),
+    }));
+    const times = [100, 125];
+
+    const result = await evaluateJevShadow({ mission: 'test' }, {
+      config: {
+        enabled: true,
+        apiKey: 'top-secret',
+        apiBase: 'https://jev.example',
+        model: 'jev-latest',
+        timeoutMs: 1000,
+      },
+      fetchImpl,
+      now: () => times.shift(),
+    });
+
+    expect(result).toMatchObject({ status: 'ok', model: 'jev-resolved', latencyMs: 25 });
+    const [, request] = fetchImpl.mock.calls[0];
+    expect(request.headers.authorization).toBe('Bearer top-secret');
+    expect(JSON.stringify(result)).not.toContain('top-secret');
+  });
+
+  it('turns API failures into shadow telemetry', async () => {
+    const result = await evaluateJevShadow({}, {
+      config: {
+        enabled: true,
+        apiKey: 'secret',
+        apiBase: 'https://jev.example',
+        model: 'jev-latest',
+        timeoutMs: 1000,
+      },
+      fetchImpl: async () => ({ ok: false, status: 429 }),
+    });
+
+    expect(result).toMatchObject({ status: 'error', error: 'http_429' });
+  });
+
+  it('skips safely when enabled without a key', async () => {
+    await expect(evaluateJevShadow({}, {
+      config: { enabled: true, apiKey: '' },
+    })).resolves.toEqual({ status: 'skipped', reason: 'missing_api_key' });
+  });
+});

package/tests/coreTests/jevSuiteSelection.test.js

@@ -0,0 +1,24 @@
+import { describe, expect, it } from 'vitest';
+import { buildJevSuiteSelectionRequest, interpretJevSuiteSelection } from '../../core/jevSuiteSelection.js';
+
+describe('Jev suite selection', () => {
+  it('creates one bounded decision per mission and applies a threshold', () => {
+    const request = buildJevSuiteSelectionRequest({
+      changedFiles: ['src/auth/login.js'],
+      missions: [
+        { file: 'login.mission.js', tags: ['auth'] },
+        { file: 'billing.mission.js', tags: ['billing'] },
+      ],
+    });
+    expect(Object.keys(request.questions)).toEqual(['select_mission_1', 'select_mission_2']);
+    expect(request.state.changedFiles).toEqual(['src/auth/login.js']);
+
+    expect(interpretJevSuiteSelection({
+      select_mission_1: { noul: 0.95 },
+      select_mission_2: { noul: 0.2 },
+    }, request.catalog, 0.7)).toEqual([
+      { file: 'login.mission.js', tags: ['auth'], probability: 0.95, selected: true },
+      { file: 'billing.mission.js', tags: ['billing'], probability: 0.2, selected: false },
+    ]);
+  });
+});

package/tests/coreTests/jevUsage.test.js

@@ -0,0 +1,58 @@
+import { describe, expect, it } from 'vitest';
+import { resolveJevInputPrice, summarizeJevUsage } from '../../core/jevUsage.js';
+
+describe('Jev usage summaries', () => {
+  it('aggregates usage, outcomes, gates, latency, and estimated spend', () => {
+    const summary = summarizeJevUsage([
+      {
+        jevShadow: {
+          status: 'ok',
+          usage: { input_tokens: 1000, output_tokens: 25 },
+          latencyMs: 120,
+          completionGate: { candidate: true },
+          strategies: {
+            modelRoute: { choice: 'deterministic' },
+            actionChoice: { choice: 'action_1' },
+            recovery: { choice: 'continue' },
+            missionStage: { choice: 'interact' },
+            verification: { noul: 0.9 },
+            prerequisite: { noul: 0.85 },
+            domRelevance: [{ probability: 0.8 }, { probability: 0.2 }],
+          },
+        },
+        jevGate: { triggered: true },
+      },
+      { jevShadow: { status: 'error' } },
+      { jevShadow: { status: 'skipped' } },
+    ], { inputUsdPerMillion: 1 });
+
+    expect(summary).toEqual({
+      evaluations: 3,
+      completed: 1,
+      errors: 1,
+      skipped: 1,
+      inputTokens: 1000,
+      outputTokens: 25,
+      latencyMs: 120,
+      completionCandidates: 1,
+      triggeredCompletions: 1,
+      inputUsdPerMillion: 1,
+      estimatedInputCostUsd: 0.001,
+      strategies: {
+        evaluations: 1,
+        modelRoutes: { deterministic: 1 },
+        actionChoices: { action_1: 1 },
+        recoveryRoutes: { continue: 1 },
+        missionStages: { interact: 1 },
+        verifiedPostconditions: 1,
+        satisfiedPrerequisites: 1,
+        regionsEvaluated: 2,
+        relevantRegions: 1,
+      },
+    });
+  });
+
+  it('supports a configured input price', () => {
+    expect(resolveJevInputPrice({ TESTRONAUT_JEV_INPUT_USD_PER_MILLION: '0.05' })).toBe(0.05);
+  });
+});

package/tests/coreTests/missionDiscovery.test.js

@@ -78,4 +78,48 @@ describe('discoverMissionFiles', () => {
     const res = await discoverMissionFiles({ cwd: tmp });
     expect(res.files).toEqual(['keep.mission.js']);
   });
+
+  it('orders configured patterns and places remaining missions at the wildcard', async () => {
+    const tmp = makeTempDir();
+    temps.push(tmp);
+    const missionsDir = path.join(tmp, 'missions');
+    fs.mkdirSync(path.join(missionsDir, 'account'), { recursive: true });
+    for (const file of ['login.mission.js', 'logout.mission.js', 'search.mission.js']) {
+      fs.writeFileSync(path.join(missionsDir, file), '// ok');
+    }
+    fs.writeFileSync(path.join(missionsDir, 'account', 'profile.mission.js'), '// ok');
+
+    writeJson(path.join(tmp, 'testronaut-config.json'), {
+      missions: {
+        include: ['**/*.mission.js'],
+        order: ['login.mission.js', 'account/**/*.mission.js', '*', 'logout.mission.js'],
+      },
+    });
+
+    const res = await discoverMissionFiles({ cwd: tmp });
+    expect(res.files).toEqual([
+      'login.mission.js',
+      'account/profile.mission.js',
+      'search.mission.js',
+      'logout.mission.js',
+    ]);
+  });
+
+  it('warns about missing exact order entries while ignoring unmatched globs', async () => {
+    const tmp = makeTempDir();
+    temps.push(tmp);
+    fs.mkdirSync(path.join(tmp, 'missions'));
+    fs.writeFileSync(path.join(tmp, 'missions', 'actual.mission.js'), '// ok');
+    writeJson(path.join(tmp, 'testronaut-config.json'), {
+      missions: {
+        order: ['missing.mission.js', 'optional/**/*.mission.js', '*'],
+      },
+    });
+
+    const res = await discoverMissionFiles({ cwd: tmp });
+    expect(res.files).toEqual(['actual.mission.js']);
+    expect(res.warnings).toEqual([
+      'Configured mission order entry was not discovered: missing.mission.js',
+    ]);
+  });
 });

package/tests/coreTests/optimization.test.js

@@ -0,0 +1,80 @@
+import { describe, expect, it } from 'vitest';
+import { applyOptimizationPolicy, resolveOptimizationPolicy } from '../../core/optimization.js';
+
+describe('optimization policy', () => {
+  it.each([
+    ['off', false, false],
+    ['cost', false, false],
+    ['tokens', false, true],
+    ['balanced', false, true],
+    ['speed', false, false],
+  ])('maps %s to implemented capabilities', (objective, routing, tokenReduction) => {
+    expect(resolveOptimizationPolicy({
+      env: {}, config: { optimization: { objective, mode: 'live' } },
+    })).toMatchObject({ objective, mode: objective === 'off' ? 'off' : 'live', routing, tokenReduction });
+  });
+
+  it('accepts user-facing aliases and defaults configured policies to shadow', () => {
+    expect(resolveOptimizationPolicy({
+      env: { TESTRONAUT_OPTIMIZE: 'both' }, config: {},
+    })).toMatchObject({ objective: 'balanced', mode: 'shadow' });
+  });
+
+  it('gives environment values precedence over config', () => {
+    expect(resolveOptimizationPolicy({
+      env: { TESTRONAUT_OPTIMIZE: 'cost', TESTRONAUT_OPTIMIZATION_MODE: 'live' },
+      config: { optimization: { objective: 'tokens', mode: 'shadow' } },
+    })).toMatchObject({ objective: 'cost', mode: 'live', routingRequested: true, routing: false });
+  });
+
+  it('requires an explicit beta opt-in before model routing can activate', () => {
+    expect(resolveOptimizationPolicy({
+      env: { TESTRONAUT_BETA_MODEL_ROUTING: '1' },
+      config: { optimization: { objective: 'balanced', mode: 'live' } },
+    })).toMatchObject({ routingRequested: true, betaModelRouting: true, routing: true });
+  });
+
+  it('supports stable guardrails without strategy or model routing calls', () => {
+    const env = {};
+    applyOptimizationPolicy(resolveOptimizationPolicy({
+      env: {}, config: { optimization: { objective: 'tokens', mode: 'live', guardrails: { completionProbability: 0.9 } } },
+    }), env);
+    expect(env).toMatchObject({
+      TESTRONAUT_JEV_ROUTING: 'off',
+      TESTRONAUT_JEV_STRATEGIES: '0',
+      TESTRONAUT_JEV_GATE: '1',
+      TESTRONAUT_JEV_GATE_THRESHOLD: '0.9',
+    });
+  });
+
+  it('accepts the top-level opt-in guardrails configuration', () => {
+    expect(resolveOptimizationPolicy({
+      env: {}, config: { guardrails: { enabled: true, mode: 'live', completionProbability: 0.85 } },
+    })).toMatchObject({ objective: 'tokens', mode: 'live', routing: false, completionProbability: 0.85 });
+    expect(resolveOptimizationPolicy({
+      env: {}, config: { guardrails: { enabled: false } },
+    })).toMatchObject({ objective: 'off', mode: 'off' });
+  });
+
+  it('translates policy to legacy Jev controls without replacing explicit overrides', () => {
+    const env = { TESTRONAUT_JEV_GATE: '0' };
+    applyOptimizationPolicy(resolveOptimizationPolicy({
+      env: { TESTRONAUT_BETA_MODEL_ROUTING: '1' }, config: { optimization: { objective: 'balanced', mode: 'live' } },
+    }), env);
+    expect(env).toMatchObject({
+      TESTRONAUT_JEV_ROUTING: 'live',
+      TESTRONAUT_JEV_GATE: '0',
+      TESTRONAUT_JEV_STRATEGIES: '1',
+      TESTRONAUT_JEV_SHADOW: '1',
+      TESTRONAUT_JEV_ROUTING_THRESHOLD: '0.4',
+    });
+  });
+
+  it('allows explicit CLI policy to replace lower-level overrides', () => {
+    const env = { TESTRONAUT_JEV_GATE: '1', TESTRONAUT_JEV_ROUTING: 'live' };
+    applyOptimizationPolicy(resolveOptimizationPolicy({
+      env: { TESTRONAUT_OPTIMIZE: 'off' }, config: {},
+    }), env, { force: true });
+    expect(env).toMatchObject({ TESTRONAUT_JEV_GATE: '0', TESTRONAUT_JEV_ROUTING: 'off' });
+  });
+});