testronaut 1.2.5 → 1.3.1

package/runner/testronaut.js

@@ -25,7 +25,7 @@
 
 import { runAgent } from '../core/agent.js';
 import { redactPasswordInText } from '../core/redaction.js';
-import { loadConfig, enforceTurnBudget } from '../core/config.js';
+import { loadConfig, enforceTurnBudget, getRetryLimit } from '../core/config.js';
 
 /**
  * Run a mission flow.
@@ -38,9 +38,14 @@ export async function runMissions({ preMission, mission, postMission }, missionN
   const cfg = await loadConfig();
   const { effectiveMax, limits, notes } = enforceTurnBudget(cfg);
   const maxTurns = effectiveMax;
+  const retryInfo = getRetryLimit(cfg);
+  const retryLimit = retryInfo.value;
   if (notes.length) {
     console.warn(notes.join('\n'));
   }
+  if (retryInfo.clamped) {
+    console.warn(`⚠️ Retry limit clamped to ${retryLimit} (allowed range 1-10).`);
+  }
 
   // 2) Normalize submissions
   const normalizeToArray = (x) => (Array.isArray(x) ? x : x ? [x] : []);
@@ -123,7 +128,7 @@ export async function runMissions({ preMission, mission, postMission }, missionN
   );
 
   // 3) Execute
-  const success = await runAgent(goals, missionName, maxTurns);
+  const success = await runAgent(goals, missionName, maxTurns, retryLimit);
   if (!success) {
     console.log(`❌ Aborting after failed goal.`);
     return;

package/tests/binTests/cli.helpers.test.js

@@ -0,0 +1,36 @@
+import { describe, it, expect } from 'vitest';
+import path from 'path';
+import fs from 'fs';
+
+// Import helper exports from cli.js
+import { __test__ } from '../../bin/cli.js';
+
+describe('cli helpers', () => {
+  it('guessMimeType returns expected types', () => {
+    const { guessMimeType } = __test__;
+    expect(guessMimeType('file.html')).toContain('text/html');
+    expect(guessMimeType('file.json')).toContain('application/json');
+    expect(guessMimeType('file.unknown')).toBe('application/octet-stream');
+  });
+
+  it('safeJoin prevents traversal', () => {
+    const { safeJoin } = __test__;
+    const root = '/tmp/root';
+    expect(safeJoin(root, '/sub/ok.txt')).toBe(path.resolve('/tmp/root/sub/ok.txt'));
+    expect(safeJoin(root, '/../etc/passwd')).toBeNull();
+  });
+
+  it('findLatestReportPair finds latest html/json', () => {
+    const { findLatestReportPair } = __test__;
+    const tmp = fs.mkdtempSync(path.join(process.cwd(), 'testronaut-report-'));
+    const r1 = path.join(tmp, 'run_100.html');
+    const r2 = path.join(tmp, 'run_200.html');
+    fs.writeFileSync(r1, 'a');
+    fs.writeFileSync(r2, 'b');
+    fs.writeFileSync(path.join(tmp, 'run_200.json'), '{}');
+
+    const latest = findLatestReportPair(tmp);
+    expect(latest?.htmlFile).toBe('run_200.html');
+    expect(latest?.jsonFile).toBe('run_200.json');
+  });
+});

package/tests/coreTests/config.test.js

@@ -8,6 +8,7 @@ import { readFile } from 'node:fs/promises';
 import {
   loadConfig,
   getMaxTurns,
+  getRetryLimit,
   resolveTurnLimits,
   enforceTurnBudget,
 } from '../../core/config.js';
@@ -64,6 +65,46 @@ describe('core/config', () => {
     });
   });
 
+  describe('getRetryLimit', () => {
+    const OLD_ENV = { ...process.env };
+    beforeEach(() => {
+      process.env = { ...OLD_ENV };
+      delete process.env.TESTRONAUT_RETRY_LIMIT;
+    });
+    afterEach(() => {
+      process.env = { ...OLD_ENV };
+    });
+
+    it('returns default when missing', () => {
+      const res = getRetryLimit({});
+      expect(res.value).toBe(2);
+      expect(res.source).toBe('default');
+      expect(res.clamped).toBe(false);
+    });
+
+    it('uses config value and clamps to range', () => {
+      expect(getRetryLimit({ retryLimit: 5 }).value).toBe(5);
+      const high = getRetryLimit({ retryLimit: 99 });
+      expect(high.value).toBe(10);
+      expect(high.clamped).toBe(true);
+      const low = getRetryLimit({ retryLimit: 0 });
+      expect(low.value).toBe(1);
+      expect(low.clamped).toBe(true);
+    });
+
+    it('prefers env over config and clamps', () => {
+      process.env.TESTRONAUT_RETRY_LIMIT = '8';
+      const res = getRetryLimit({ retryLimit: 3 });
+      expect(res.value).toBe(8);
+      expect(res.source).toBe('env');
+
+      process.env.TESTRONAUT_RETRY_LIMIT = '0';
+      const clampedLow = getRetryLimit({ retryLimit: 3 });
+      expect(clampedLow.value).toBe(1);
+      expect(clampedLow.clamped).toBe(true);
+    });
+  });
+
   describe('enforceTurnBudget (lenient)', () => {
     it('clamps > hardMaxTurns', () => {
       const cfg = { maxTurns: 1000 };

package/tests/coreTests/turnLoop.test.js

@@ -128,7 +128,7 @@ describe('turnLoop', () => {
     const messages = baseMessages();
     const result = await turnLoop(browser, messages, 1, 0, 0, {}, { steps: [], missionName: 'demo' });
 
-    expect(result).toBeTruthy();
+    expect(result.success).toBe(true);
     expect(result.steps?.length).toBe(1);
     const step = result.steps[0];
     expect(step.tokensUsed).toBe(42);
@@ -163,7 +163,7 @@ describe('turnLoop', () => {
     const messages = baseMessages();
     const res = await turnLoop(browser, messages, 2, 0, 0, {}, { steps: [], missionName: 'demo' });
 
-    expect(res).toBeTruthy();
+    expect(res.success).toBe(true);
     expect(res.steps.length).toBe(2);
 
     const step1 = res.steps[0];
@@ -174,4 +174,56 @@ describe('turnLoop', () => {
     const step2 = res.steps[1];
     expect(step2.result).toMatch(/Success/);
   });
+
+  it('retries a turn on tool error before marking issues', async () => {
+    // First turn: tool call → click_text will throw
+    shared.chatMock.mockResolvedValueOnce({
+      message: {
+        role: 'assistant',
+        content: '',
+        tool_calls: [
+          {
+            id: 'tool_retry',
+            type: 'function',
+            function: { name: 'click_text', arguments: JSON.stringify({ text: 'Hello' }) },
+          },
+        ],
+      },
+      usage: { total_tokens: 7 },
+    });
+    // Retry turn: same tool call, click_text succeeds
+    shared.chatMock.mockResolvedValueOnce({
+      message: {
+        role: 'assistant',
+        content: '',
+        tool_calls: [
+          {
+            id: 'tool_retry_2',
+            type: 'function',
+            function: { name: 'click_text', arguments: JSON.stringify({ text: 'Hello' }) },
+          },
+        ],
+      },
+      usage: { total_tokens: 6 },
+    });
+    // Final turn: mission completes
+    shared.chatMock.mockResolvedValueOnce({
+      message: { role: 'assistant', content: 'FINAL: done' },
+      usage: { total_tokens: 5 },
+    });
+
+    // click_text fails once, then succeeds
+    shared.chromeToolSpies.click_text
+      .mockImplementationOnce(async () => { throw new Error('no locator'); })
+      .mockImplementationOnce(async () => JSON.stringify({ ok: true, clicked: 'Hello' }));
+
+    const messages = baseMessages();
+    const res = await turnLoop(browser, messages, 3, 0, 0, {}, { steps: [], missionName: 'demo' });
+
+    expect(res.success).toBe(true);
+    expect(res.steps.length).toBe(3);
+    expect(res.steps[0].result).toBe('⏳ Retrying turn');
+    expect(res.steps[1].result).toBe('✅ Passed');
+    expect(res.steps[2].result).toMatch(/Success/);
+  });
 });

package/tests/llmTests/geminiProvider.test.js

@@ -77,6 +77,35 @@ describe('GeminiProvider', () => {
     expect(shared.getModelOpts.tools[0].functionDeclarations.map(d => d.name)).toEqual(['foo', 'bar']);
   });
 
+  it('strips additionalProperties from tool schemas (unsupported by Gemini)', async () => {
+    const prov = new GeminiProvider({ apiKey: 'gk-123' });
+    const tools = [
+      {
+        type: 'function',
+        function: {
+          name: 'set_ground_control_state',
+          parameters: {
+            type: 'object',
+            properties: {
+              app: {
+                type: 'object',
+                additionalProperties: false,
+                properties: { baseUrl: { type: 'string' } },
+              },
+            },
+            additionalProperties: false,
+          },
+        },
+      },
+    ];
+
+    await prov.chat({ model: 'gemini-2.5-flash', messages: [{ role: 'user', content: 'go' }], tools });
+
+    const decl = shared.getModelOpts.tools[0].functionDeclarations[0];
+    expect(decl.parameters.additionalProperties).toBeUndefined();
+    expect(decl.parameters.properties.app.additionalProperties).toBeUndefined();
+  });
+
   it('converts OpenAI-like messages → Gemini contents (system prefix + tool calls + tool results)', async () => {
     const prov = new GeminiProvider({ apiKey: 'gk-xyz' });
 

package/tests/runnerTests/testronaut.test.js

@@ -9,10 +9,11 @@ vi.mock('../../core/redaction.js', () => ({
 vi.mock('../../core/config.js', () => ({
   loadConfig: vi.fn(),
   enforceTurnBudget: vi.fn(),
+  getRetryLimit: vi.fn(),
 }));
 
 import { runAgent } from '../../core/agent.js';
-import { loadConfig, enforceTurnBudget } from '../../core/config.js';
+import { loadConfig, enforceTurnBudget, getRetryLimit } from '../../core/config.js';
 
 // Adjust the import path if your file lives elsewhere
 import { runMissions } from '../../runner/testronaut.js';
@@ -20,6 +21,8 @@ import { runMissions } from '../../runner/testronaut.js';
 describe('cli/testronaut.runMissions (with enforceTurnBudget)', () => {
   beforeEach(() => {
     vi.clearAllMocks();
+    // Default retry limit unless overridden in a test
+    getRetryLimit.mockReturnValue({ value: 2, source: 'default', clamped: false });
   });
 
   it('passes effectiveMax to runAgent and logs any notes', async () => {
@@ -30,6 +33,7 @@ describe('cli/testronaut.runMissions (with enforceTurnBudget)', () => {
       notes: ['⚠️ maxTurns (999) exceeds hardMaxTurns (200). Clamping to 200.'],
       strict: false,
     });
+    getRetryLimit.mockReturnValue({ value: 2, source: 'default', clamped: false });
 
     const warn = vi.spyOn(console, 'warn').mockImplementation(() => {});
     const log = vi.spyOn(console, 'log').mockImplementation(() => {});
@@ -46,7 +50,7 @@ describe('cli/testronaut.runMissions (with enforceTurnBudget)', () => {
       expect.stringContaining('Clamping to 200')
     );
     expect(runAgent).toHaveBeenCalledWith(
-      expect.any(Array), 'Budgeted Run', 200
+      expect.any(Array), 'Budgeted Run', 200, 2
     );
 
     warn.mockRestore();
@@ -61,6 +65,7 @@ describe('cli/testronaut.runMissions (with enforceTurnBudget)', () => {
       notes: [],
       strict: false,
     });
+    getRetryLimit.mockReturnValue({ value: 3, source: 'default', clamped: false });
 
     const warn = vi.spyOn(console, 'warn').mockImplementation(() => {});
     const log = vi.spyOn(console, 'log').mockImplementation(() => {});
@@ -72,7 +77,7 @@ describe('cli/testronaut.runMissions (with enforceTurnBudget)', () => {
     await runMissions({ mission: 'No warnings' }, 'Clean');
 
     expect(warn).not.toHaveBeenCalled();
-    expect(runAgent).toHaveBeenCalledWith(expect.any(Array), 'Clean', 20);
+    expect(runAgent).toHaveBeenCalledWith(expect.any(Array), 'Clean', 20, 3);
 
     warn.mockRestore();
     log.mockRestore();
@@ -138,8 +143,8 @@ describe('cli/testronaut.runMissions (with enforceTurnBudget)', () => {
     // Mission name is the passed-in missionName for main
     expect(goals[1].submissionName).toMatch(/^My Mission/);
 
-    // Effective max turns passed through
-    expect(runAgent).toHaveBeenCalledWith(expect.any(Array), 'My Mission', 15);
+    // Effective max turns and retry limit passed through
+    expect(runAgent).toHaveBeenCalledWith(expect.any(Array), 'My Mission', 15, 2);
 
     log.mockRestore();
   });