testronaut 1.3.6 → 1.4.0

package/core/turnLoop.js

@@ -107,6 +107,28 @@ function extractDocIdFromUrl(url = '') {
   return m ? m[1] : '';
 }
 
+function isMfaLikeFill(fnName, args = {}) {
+  if (fnName !== 'fill') return false;
+  const haystack = [
+    args.selector,
+    args.label,
+    args.placeholder,
+    args.name,
+    args.testId,
+    args.role,
+  ].map(v => String(v || '').toLowerCase()).join(' ');
+
+  return /\b(mfa|totp|otp|verification|2fa|code)\b/.test(haystack);
+}
+
+function getFillText(args = {}) {
+  return String(args.text ?? args.value ?? args.input ?? args.keys ?? '');
+}
+
+function safeListLabel(list = []) {
+  return Array.isArray(list) && list.length ? list.join(', ') : '(none)';
+}
+
 // Rolling token counters used for self-throttling
 let totalTokensUsed = 0;
 let turnTimestamps = [];
@@ -428,6 +450,97 @@ export const turnLoop = async (
           step.events.push(errorMessage
             ? `👤 Human-in-the-loop input ${step.humanInput.status}: ${errorMessage.replace(/^ERROR:\s*/, '')}`
             : '👤 Human-in-the-loop input provided.');
+
+          if (!errorMessage) {
+            try {
+              const parsed = JSON.parse(result);
+              agentMemory.lastVerificationInput = {
+                source: 'human_input',
+                value: parsed.value,
+                codeType: parsed.codeType || args.codeType || 'verification_code',
+              };
+            } catch {
+              // ignore malformed tool result
+            }
+          }
+        }
+
+        if (fnName === 'get_mfa_code') {
+          step.mfa = step.mfa || {};
+          step.mfa.requested = true;
+          try {
+            const parsed = JSON.parse(result);
+            step.mfa.nickname = parsed.nickname || args.nickname || null;
+            step.mfa.status = parsed.ok ? 'provided' : parsed.code || 'unavailable';
+            step.mfa.availableNicknames = parsed.availableNicknames || [];
+            step.mfa.responseKeys = parsed.responseKeys || [];
+            step.mfa.listStatus = parsed.mfaListStatus || null;
+            agentMemory.lastMfaLookup = parsed.ok
+              ? {
+                  ok: true,
+                  nickname: parsed.nickname || args.nickname || null,
+                  value: parsed.value,
+                  secondsRemaining: parsed.mfaCode?.secondsRemaining,
+                  resolvedFromList: !!parsed.resolvedFromList,
+                  requestedNickname: parsed.requestedNickname,
+                  availableNicknames: parsed.availableNicknames || [],
+                }
+              : {
+                  ok: false,
+                  nickname: parsed.nickname || args.nickname || null,
+                  code: parsed.code,
+                  error: parsed.error,
+                  availableNicknames: parsed.availableNicknames || [],
+                  responseKeys: parsed.responseKeys || [],
+                  mfaListStatus: parsed.mfaListStatus || null,
+                };
+
+            step.events.push(
+              parsed.ok
+                ? `🔐 MFA code retrieved for "${step.mfa.nickname || 'configured MFA'}".`
+                : `🔐 MFA code unavailable: ${parsed.error || parsed.code || 'unknown error'}`
+            );
+            if (parsed.ok) {
+              const detailLine = [
+                `🔐 MFA source: API`,
+                `nickname="${step.mfa.nickname || 'configured MFA'}"`,
+                parsed.resolvedFromList ? `resolvedFromList=true` : null,
+                Number.isFinite(parsed.mfaCode?.secondsRemaining)
+                  ? `secondsRemaining=${parsed.mfaCode.secondsRemaining}`
+                  : null,
+              ].filter(Boolean).join(' ');
+              console.log(detailLine);
+              step.events.push(detailLine);
+            } else {
+              const reasonLine = `🔐 MFA unavailable reason: ${parsed.code || 'unknown'} - ${parsed.error || 'unknown error'}`;
+              console.log(reasonLine);
+              step.events.push(reasonLine);
+
+              if (Array.isArray(parsed.availableNicknames)) {
+                const listLine = `🔐 MFA list endpoint nicknames: ${safeListLabel(parsed.availableNicknames)}`;
+                console.log(listLine);
+                step.events.push(listLine);
+              }
+
+              if (parsed.mfaListStatus && parsed.mfaListStatus !== 'available') {
+                const listStatusLine = `🔐 MFA list endpoint status: ${JSON.stringify(parsed.mfaListStatus)}`;
+                console.log(listStatusLine);
+                step.events.push(listStatusLine);
+              } else if (parsed.mfaListStatus === 'available') {
+                const listStatusLine = '🔐 MFA list endpoint status: available';
+                console.log(listStatusLine);
+                step.events.push(listStatusLine);
+              }
+
+              if (Array.isArray(parsed.responseKeys) && parsed.responseKeys.length) {
+                const keysLine = `🔐 MFA API response keys: ${parsed.responseKeys.join(', ')}`;
+                console.log(keysLine);
+                step.events.push(keysLine);
+              }
+            }
+          } catch {
+            step.mfa.status = errorMessage ? 'error' : 'unknown';
+          }
         }
 
         // Capture and log any file upload/download events (for reports)
@@ -460,7 +573,36 @@ export const turnLoop = async (
           // non-JSON results ignored
         }
 
-        console.log(`[tool ] ← ${fnName} result:`, errorMessage ? '❌ Failed' : '✅ Success');
+        let toolStatusLabel = errorMessage ? '❌ Failed' : '✅ Success';
+        if (fnName === 'get_mfa_code' && !errorMessage) {
+          try {
+            const parsed = JSON.parse(result);
+            toolStatusLabel = parsed.ok
+              ? '✅ Code retrieved'
+              : `⚠️ Unavailable${parsed.code ? ` (${parsed.code})` : ''}`;
+          } catch {
+            toolStatusLabel = '⚠️ Unavailable';
+          }
+        }
+
+        console.log(`[tool ] ← ${fnName} result:`, toolStatusLabel);
+
+        if (isMfaLikeFill(fnName, args)) {
+          const fillText = getFillText(args);
+          let sourceLine;
+          if (agentMemory.lastMfaLookup?.ok && fillText === agentMemory.lastMfaLookup.value) {
+            sourceLine = `🔐 MFA fill source: get_mfa_code nickname="${agentMemory.lastMfaLookup.nickname || 'configured MFA'}"`;
+          } else if (agentMemory.lastVerificationInput?.value && fillText === agentMemory.lastVerificationInput.value) {
+            sourceLine = `🔐 MFA fill source: request_human_input codeType="${agentMemory.lastVerificationInput.codeType}"`;
+          } else if (agentMemory.lastMfaLookup && !agentMemory.lastMfaLookup.ok) {
+            sourceLine = `⚠️ MFA fill source: not from get_mfa_code. Last MFA lookup failed with ${agentMemory.lastMfaLookup.code || 'unknown'}: ${agentMemory.lastMfaLookup.error || 'unknown error'}`;
+          } else {
+            sourceLine = '⚠️ MFA fill source: not from a recorded get_mfa_code or request_human_input result.';
+          }
+          console.log(sourceLine);
+          step.events.push(sourceLine);
+        }
+
         let resultForLog = result;
         if (fnName === 'request_human_input') {
           try {
@@ -474,8 +616,26 @@ export const turnLoop = async (
             resultForLog = errorMessage || 'Human input received.';
           }
         }
+        if (fnName === 'get_mfa_code') {
+          try {
+            const parsed = JSON.parse(result);
+            resultForLog = JSON.stringify({
+              ...parsed,
+              value: maskPreview(parsed.value),
+              redactedValue: maskPreview(parsed.value),
+              mfaCode: parsed.mfaCode
+                ? {
+                    ...parsed.mfaCode,
+                    code: maskPreview(parsed.mfaCode.code),
+                  }
+                : parsed.mfaCode,
+            });
+          } catch {
+            resultForLog = errorMessage || 'MFA code lookup completed.';
+          }
+        }
         const truncated = resultForLog.length > 1000 ? resultForLog.slice(0, 1000) + '…' : resultForLog;
-        step.events.push(`[tool ] ← ${fnName} result: ${errorMessage ? '❌ Failed' : '✅ Success'}`);
+        step.events.push(`[tool ] ← ${fnName} result: ${toolStatusLabel}`);
         step.events.push(`[tool ] ← ${truncated}`);
 
         // Screenshot detection (for report metadata only)

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "testronaut",
-  "version": "1.3.6",
+  "version": "1.4.0",
   "main": "./runner/testronaut.js",
   "bin": {
     "testronaut": "bin/cli.js"
@@ -44,6 +44,7 @@
   ],
   "devDependencies": {
     "@types/node": "^24.9.1",
-    "vitest": "^4.0.1"
+    "periapsis": "^2.0.1",
+    "vitest": "^4.1.9"
   }
 }

package/policy/vulnerability-exceptions.json

@@ -0,0 +1,36 @@
+[
+  {
+    "id": "VEX-2026-001",
+    "type": "alert",
+    "repo": "mission-testronaut/testronaut-cli",
+    "alertNumber": 16,
+    "appliesToSeverities": [
+      "critical"
+    ],
+    "reason": "Upstream fix not yet available. Reviewing impact and testing patched version. No production exposure — dev dependency only.",
+    "acceptedUntil": "2026-06-30T00:00:00.000Z",
+    "approvedBy": [
+      "Shane Fast"
+    ],
+    "createdAt": "2026-06-17T17:14:47.945Z",
+    "evidenceRef": "https://github.com/mission-testronaut/testronaut-cli/security/dependabot/16"
+  },
+  {
+    "id": "VEX-2026-002",
+    "type": "package",
+    "package": "vitest",
+    "ecosystem": "npm",
+    "repo": "mission-testronaut/testronaut-cli",
+    "alertNumber": 16,
+    "appliesToSeverities": [
+      "critical"
+    ],
+    "reason": "need extra time to address",
+    "acceptedUntil": "2026-07-04T00:00:00.000Z",
+    "approvedBy": [
+      "Shane Fast"
+    ],
+    "createdAt": "2026-07-03T01:28:55.590Z",
+    "evidenceRef": "https://github.com/mission-testronaut/testronaut-cli/security/dependabot/16"
+  }
+]

package/policy/vulnerability-notifications.json

@@ -0,0 +1 @@
+{}

package/policy/vulnerability-owners.json

@@ -0,0 +1 @@
+{}

package/policy/vulnerability-policy.json

@@ -0,0 +1,33 @@
+{
+  "version": 1,
+  "rollout": {
+    "enabled": true,
+    "gracePeriodUntil": null,
+    "mode": "observe"
+  },
+  "slaDays": {
+    "critical": 3,
+    "high": 30,
+    "medium": 60,
+    "low": 100
+  },
+  "warningThresholdDays": {
+    "critical": 1,
+    "high": 7,
+    "medium": 14,
+    "low": 30
+  },
+  "prCheck": {
+    "enabled": true,
+    "failOnBreach": true,
+    "advisoryOnly": false,
+    "minimumSeverity": "high"
+  },
+  "exceptions": {
+    "requireApproval": true,
+    "maxDurationDays": 90,
+    "allowPackageScope": true,
+    "allowRepoWideScope": true,
+    "allowOrgWideScope": false
+  }
+}

package/tests/binTests/cli.helpers.test.js

@@ -115,4 +115,59 @@ describe('cli helpers', () => {
     expect(createVercelBypassHeader('abc')).toEqual({ 'x-vercel-protection-bypass': 'abc' });
     expect(createVercelBypassHeader('')).toEqual({});
   });
+
+  it('parses run options from -o and removes them from args', () => {
+    const { parseRunOptionsArgs } = __test__;
+    const res = parseRunOptionsArgs(['-o', 'mfa=github-test-mfa', 'login.mission.js']);
+
+    expect(res.options).toEqual({ mfa: 'github-test-mfa' });
+    expect(res.args).toEqual(['login.mission.js']);
+    expect(res.invalid).toBe(false);
+  });
+
+  it('parses comma-separated inline run options', () => {
+    const { parseRunOptionsArgs } = __test__;
+    const res = parseRunOptionsArgs(['--options=mfa=github-test-mfa,foo=bar', 'login']);
+
+    expect(res.options).toEqual({ mfa: 'github-test-mfa', foo: 'bar' });
+    expect(res.args).toEqual(['login']);
+    expect(res.invalid).toBe(false);
+  });
+
+  it('flags invalid run options', () => {
+    const { parseRunOptionsArgs } = __test__;
+    const res = parseRunOptionsArgs(['--options', 'mfa', 'login']);
+
+    expect(res.options).toEqual({});
+    expect(res.args).toEqual(['login']);
+    expect(res.invalid).toBe(true);
+  });
+
+  describe('detectCliName', () => {
+    const { detectCliName } = __test__;
+
+    it('returns "npx testronaut" when npm_command is exec (npx invocation)', () => {
+      expect(detectCliName('exec', '/some/path/cli.js')).toBe('npx testronaut');
+    });
+
+    it('returns "testronaut" when argv[1] basename is testronaut (global install)', () => {
+      expect(detectCliName(undefined, '/usr/local/bin/testronaut')).toBe('testronaut');
+    });
+
+    it('returns "npx testronaut" when argv[1] is cli.js (direct node invocation)', () => {
+      expect(detectCliName(undefined, '/home/user/testronaut-cli/bin/cli.js')).toBe('npx testronaut');
+    });
+
+    it('npm_command=exec takes precedence over a global-bin argv[1]', () => {
+      expect(detectCliName('exec', '/usr/local/bin/testronaut')).toBe('npx testronaut');
+    });
+
+    it('returns "npx testronaut" when argv[1] is undefined', () => {
+      expect(detectCliName(undefined, undefined)).toBe('npx testronaut');
+    });
+
+    it('returns "npx testronaut" when argv[1] is empty string', () => {
+      expect(detectCliName(undefined, '')).toBe('npx testronaut');
+    });
+  });
 });

package/tests/coreTests/config.test.js

@@ -1,10 +1,11 @@
 import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest';
 
+const { readFile } = vi.hoisted(() => ({ readFile: vi.fn() }));
+
 vi.mock('node:fs/promises', () => ({
-  readFile: vi.fn(),
+  readFile,
 }));
 
-import { readFile } from 'node:fs/promises';
 import {
   loadConfig,
   getMaxTurns,

package/tests/coreTests/turnLoop.test.js

@@ -12,6 +12,7 @@ const { shared } = vi.hoisted(() => {
 
       // Browser tool spies available to the chromeBrowser mock
       chromeToolSpies: {
+        fill: vi.fn(async () => 'filled'),
         click_text: vi.fn(async () => JSON.stringify({ ok: true, clicked: 'Hello' })),
         get_dom: vi.fn(async () => '<html>dom</html>'),
         screenshot: vi.fn(async () =>
@@ -23,6 +24,15 @@ const { shared } = vi.hoisted(() => {
           })
         ),
         list_local_files: vi.fn(async () => JSON.stringify({ files: ['a.pdf', 'b.pdf'] })),
+        get_mfa_code: vi.fn(async () => JSON.stringify({
+          ok: false,
+          code: 'invalid_response',
+          error: 'MFA API response did not include a recognized code field. Response keys: nickname, message.',
+          nickname: 'rudy-poo',
+          availableNicknames: ['rudy-poo', 'github-prod'],
+          mfaListStatus: 'available',
+          responseKeys: ['nickname', 'message'],
+        })),
       },
     },
   };
@@ -62,9 +72,11 @@ vi.mock('../../tools/tokenControl.js', () => ({
 vi.mock('../../tools/toolSchema.js', () => ({
   default: [
     { type: 'function', function: { name: 'click_text', description: 'click by text', parameters: {} } },
+    { type: 'function', function: { name: 'fill', description: 'fill input', parameters: {} } },
     { type: 'function', function: { name: 'get_dom', description: 'get DOM html', parameters: {} } },
     { type: 'function', function: { name: 'screenshot', description: 'take a screenshot', parameters: {} } },
     { type: 'function', function: { name: 'list_local_files', description: 'list files', parameters: {} } },
+    { type: 'function', function: { name: 'get_mfa_code', description: 'get mfa', parameters: {} } },
   ],
 }));
 
@@ -115,9 +127,11 @@ describe('turnLoop', () => {
   beforeEach(() => {
     browser = {};
     // reset spies and chat mock
+    shared.chromeToolSpies.fill.mockClear();
     shared.chromeToolSpies.click_text.mockClear();
     shared.chromeToolSpies.get_dom.mockClear();
     shared.chromeToolSpies.screenshot.mockClear();
+    shared.chromeToolSpies.get_mfa_code.mockClear();
     if (shared.chatMock) shared.chatMock.mockReset();
   });
 
@@ -230,6 +244,106 @@ describe('turnLoop', () => {
     expect(res.steps[2].result).toMatch(/Success/);
   });
 
+  it('logs unavailable MFA lookups without marking the tool as a success', async () => {
+    shared.chatMock.mockResolvedValueOnce({
+      message: {
+        role: 'assistant',
+        content: '',
+        tool_calls: [
+          {
+            id: 'tool_mfa',
+            type: 'function',
+            function: { name: 'get_mfa_code', arguments: JSON.stringify({ nickname: 'rudy-poo' }) },
+          },
+        ],
+      },
+      usage: { total_tokens: 7 },
+    });
+
+    shared.chatMock.mockResolvedValueOnce({
+      message: { role: 'assistant', content: 'FINAL: graceful failure noted' },
+      usage: { total_tokens: 5 },
+    });
+
+    const messages = baseMessages();
+    const res = await turnLoop(browser, messages, 2, 0, 0, {}, { steps: [], missionName: 'demo' });
+
+    expect(res.success).toBe(true);
+    expect(shared.chromeToolSpies.get_mfa_code).toHaveBeenCalledWith(
+      browser,
+      { nickname: 'rudy-poo' },
+      expect.any(Object)
+    );
+    expect(res.steps[0].mfa).toMatchObject({
+      requested: true,
+      nickname: 'rudy-poo',
+      status: 'invalid_response',
+      availableNicknames: ['rudy-poo', 'github-prod'],
+      responseKeys: ['nickname', 'message'],
+      listStatus: 'available',
+    });
+    expect(res.steps[0].events).toContain(
+      '[tool ] ← get_mfa_code result: ⚠️ Unavailable (invalid_response)'
+    );
+    expect(res.steps[0].events.join('\n')).toContain('Response keys: nickname, message');
+    expect(res.steps[0].events).toContain('🔐 MFA list endpoint nicknames: rudy-poo, github-prod');
+    expect(res.steps[0].events).toContain('🔐 MFA list endpoint status: available');
+    expect(res.steps[0].events).toContain('🔐 MFA API response keys: nickname, message');
+    expect(res.steps[0].events.join('\n')).not.toContain(
+      '[tool ] ← get_mfa_code result: ✅ Success'
+    );
+  });
+
+  it('logs when an MFA field is filled with a value not sourced from MFA tools', async () => {
+    shared.chatMock.mockResolvedValueOnce({
+      message: {
+        role: 'assistant',
+        content: '',
+        tool_calls: [
+          {
+            id: 'tool_mfa',
+            type: 'function',
+            function: { name: 'get_mfa_code', arguments: JSON.stringify({ nickname: 'rudy-poo' }) },
+          },
+        ],
+      },
+      usage: { total_tokens: 7 },
+    });
+
+    shared.chatMock.mockResolvedValueOnce({
+      message: {
+        role: 'assistant',
+        content: '',
+        tool_calls: [
+          {
+            id: 'tool_fill_mfa',
+            type: 'function',
+            function: { name: 'fill', arguments: JSON.stringify({ selector: '#mfa-code', text: '123456' }) },
+          },
+        ],
+      },
+      usage: { total_tokens: 6 },
+    });
+
+    shared.chatMock.mockResolvedValueOnce({
+      message: { role: 'assistant', content: 'FINAL: graceful failure noted' },
+      usage: { total_tokens: 5 },
+    });
+
+    const messages = baseMessages();
+    const res = await turnLoop(browser, messages, 3, 0, 0, {}, { steps: [], missionName: 'demo' });
+
+    expect(res.success).toBe(true);
+    expect(shared.chromeToolSpies.fill).toHaveBeenCalledWith(
+      browser,
+      { selector: '#mfa-code', text: '123456' },
+      expect.any(Object)
+    );
+    expect(res.steps[1].events.join('\n')).toContain(
+      '⚠️ MFA fill source: not from get_mfa_code. Last MFA lookup failed with invalid_response'
+    );
+  });
+
   describe('__docProgressInternals helpers', () => {
     const { parseDocListFromDom, extractDocIdFromUrl, ensureDocProgress } = __docProgressInternals;