testronaut 1.2.3 → 1.2.5

package/README.md

@@ -127,7 +127,7 @@ Current supported providers:
 
 | Provider | Example Models |
 |-----------|----------------|
-| **OpenAI** | gpt-4o, gpt-4.1, o3, gpt-5, etc. |
+| **OpenAI** | gpt-4o, gpt-4.1, o3, gpt-5, gpt-5.1, etc. |
 | **Google Gemini** | gemini-2.5-pro, gemini-2.5-flash, gemini-2.5-flash-8b |
 
 More providers coming soon (Anthropic, Mistral, etc.).

package/bin/cli.js

@@ -51,7 +51,7 @@ import { ensureBrowsers } from '../tools/playwrightSetup.js';
 // Keep PW browsers inside the project to avoid global cache skew
 process.env.PLAYWRIGHT_BROWSERS_PATH = process.env.PLAYWRIGHT_BROWSERS_PATH || '0';
 
-
+const TMP_DIR = path.resolve('./missions/tmp');
 const __dirname = path.dirname(fileURLToPath(import.meta.url));
 
 // const apiBase = `http://localhost:3002` //
@@ -60,6 +60,52 @@ const apiBase = 'http://api.testronaut.app'; // Replace with your actual API bas
 
 const args = process.argv.slice(2);
 
+function readJsonlSteps(p) {
+  if (!p || !fs.existsSync(p)) return null;
+  const lines = fs.readFileSync(p, 'utf8').split(/\r?\n/).filter(Boolean);
+  const parsed = [];
+  for (const line of lines) {
+    try { parsed.push(JSON.parse(line)); } catch { /* ignore bad lines */ }
+  }
+  return parsed;
+}
+
+/**
+ * Merge duplicate turns without losing information.
+ * Strategy:
+ *  - Keep the latest **non-empty** version for a given turn if both exist
+ *  - If both are non-empty, keep the later one (last write wins)
+ *  - Preserve original order by line index as a tiebreaker
+ */
+function mergeDuplicateTurns(steps) {
+  const out = [];
+  const byTurn = new Map(); // turn -> index in out
+  for (const s of steps) {
+    const turn = Number.isFinite(s.turn) ? s.turn : out.length;
+    const hasEvents = Array.isArray(s.events) && s.events.length > 0;
+    if (!byTurn.has(turn)) {
+      out.push(s);
+      byTurn.set(turn, out.length - 1);
+    } else {
+      const idx = byTurn.get(turn);
+      const existing = out[idx];
+      const existingHasEvents = Array.isArray(existing.events) && existing.events.length > 0;
+      // Prefer the one with events; if both have events, prefer the newer (s)
+      if (!existingHasEvents && hasEvents) {
+        out[idx] = s;
+      } else if (existingHasEvents && hasEvents) {
+        out[idx] = s;
+      } else {
+        // both empty or both sparse — keep latest
+        out[idx] = s;
+      }
+    }
+  }
+  // stable sort by `turn` ascending, then by original order via array index already preserved
+  out.sort((a, b) => (a.turn ?? 0) - (b.turn ?? 0));
+  return out;
+}
+
 // Look for --model=<id> or --model <id>
 let modelOverride;
 const modelFlagIndex = args.findIndex(a => a === '--model' || a.startsWith('--model='));
@@ -236,11 +282,22 @@ const endTime = new Date();
 
 const flatMissions = allResults.flatMap(entry => {
   const result = entry.result;
-  const missions = Array.isArray(result) ? result : [result]; // normalize
-  return missions.map(m => ({
-    ...m,
-    file: entry.file
-  }));
+  const missions = Array.isArray(result) ? result : [result];
+
+  return missions.map(m => {
+    let steps = m.steps;
+    if (m.stepFile && fs.existsSync(m.stepFile)) {
+      const fromJsonl = readJsonlSteps(m.stepFile);
+      if (fromJsonl && fromJsonl.length) {
+        steps = mergeDuplicateTurns(fromJsonl);
+      }
+    }
+    return {
+      ...m,
+      steps,
+      file: entry.file,
+    };
+  });
 });
 
 // Read provider/model from config (allow env override for model)
@@ -753,16 +810,22 @@ async function serveLatestReport() {
   await new Promise((resolve, reject) => {
     server.once('error', reject);
     server.listen(0, '127.0.0.1', resolve);
-
-    process.on('SIGINT', () => {
-      console.log('\n🛑 Shutting down…');
-      server.close(() => {
-        resolve();
-        process.exit(0);
-      });
-    });
   });
 
+  // Move signal handling to after server is listening
+  const shutdown = () => {
+    console.log('\n🛑 Shutting down…');
+    try {
+      server.close(); // best-effort; don’t wait for it
+    } catch (_) {
+      // ignore
+    }
+    process.exit(0);
+  };
+
+  process.once('SIGINT', shutdown);
+  process.once('SIGTERM', shutdown);
+
   const address = server.address();
   const port = typeof address === 'object' && address ? address.port : 0;
   const baseUrl = `http://127.0.0.1:${port}`;
@@ -775,3 +838,14 @@ async function serveLatestReport() {
   // Best-effort auto-open
   await openInBrowser(reportUrl).catch(() => {});
 }
+
+try {
+  if (!process.env.TN_KEEP_TMP && fs.existsSync(TMP_DIR)) {
+    fs.rmSync(TMP_DIR, { recursive: true, force: true });
+    console.log('🧹 Cleaned up temporary files.');
+  } else {
+    console.log('⚠️ Skipped tmp cleanup (TN_KEEP_TMP set).');
+  }
+} catch (err) {
+  console.warn(`⚠️ Could not remove tmp folder: ${err.message}`);
+}

package/bin/init.js

@@ -94,9 +94,10 @@ export async function initializeTestronautProject() {
           { title: 'o3 (reasoning w/ native tool use)', value: 'o3' },
           { title: 'o4-mini (reasoning, cost-effective)', value: 'o4-mini' },
           // GPT-5 family (availability varies)
-          { title: 'GPT-5 (latest reasoning model)', value: 'gpt-5' },
+          { title: 'GPT-5 (previous reasoning model)', value: 'gpt-5' },
           { title: 'GPT-5 mini (faster)', value: 'gpt-5-mini' },
           { title: 'GPT-5 nano (lightweight)', value: 'gpt-5-nano' },
+          { title: 'GPT-5.1 (latest reasoning model)', value: 'gpt-5.1' },
         ],
         // Keep prior selection highlighted if present; fallback to 4.1-mini.
         initial: pickInitialIndex(models, config.model, 'gpt-4.1-mini')

package/bin/initHelpers.js

@@ -92,6 +92,7 @@ export function openAIModels() {
     'gpt-5',
     'gpt-5-mini',
     'gpt-5-nano',
+    'gpt-5.1',
   ];
 }
 

package/core/agent.js

@@ -25,6 +25,8 @@
 import 'dotenv/config';
 import { ChromeBrowser } from '../tools/chromeBrowser.js';
 import { turnLoop } from './turnLoop.js';
+import fs from 'fs';
+import path from 'path';
 
 /**
  * Execute goals with a browser agent.
@@ -41,9 +43,18 @@ export async function runAgent(goals, missionName, maxTurns = 20) {
 
   try {
     const missionResults = [];
+    const tmpDir = path.resolve(process.cwd(), 'missions/tmp');
+    fs.mkdirSync(tmpDir, { recursive: true });
 
     for (const goal of goals) {
       const steps = [];
+      // Unique-ish JSONL file for this mission’s steps
+      const stepFile = path.join(
+        tmpDir,
+        `${missionName.replace(/[^\w.-]+/g, '_')}_${Date.now()}_steps.jsonl`
+      );
+      // Start clean
+      fs.writeFileSync(stepFile, '');
 
       // Ensure user message is a string (functions/objects → toString fallback)
       const userContent =
@@ -78,16 +89,44 @@ export async function runAgent(goals, missionName, maxTurns = 20) {
         0,    // currentTurn
         0,    // retryCount
         null, // currentStep
-        { steps, missionName }
+        {
+          steps,
+          missionName,
+          onStep: (s) => {
+            // Append each step as a JSON line
+            try {
+              fs.appendFileSync(stepFile, JSON.stringify(s) + '\n');
+              // Keep memory in check: retain only the last ~20 steps in RAM
+              if (steps.length > 20) steps.splice(0, steps.length - 20);
+            } catch {} // best-effort; don’t crash the agent
+          }
+        }
       );
 
+      function dedupeSteps(steps) {
+        const seen = new Set();
+        const out = [];
+        for (const s of steps) {
+          const first = s?.events?.[0] || '';
+          const key = `${s.turn}::${s.summary}::${first}`;
+          if (!seen.has(key)) {
+            seen.add(key);
+            out.push(s);
+          }
+        }
+        return out;
+      }
+
+      const compact = dedupeSteps(steps);
+
       // Snapshot steps (avoid retaining references to mutable arrays)
       missionResults.push({
         missionName,
         submissionType: goal.submissionType || goal.label,
         submissionName: goal.submissionName || null,
         status: result?.success ? 'passed' : 'failed',
-        steps: JSON.parse(JSON.stringify(steps)), // deep-clone
+        steps: JSON.parse(JSON.stringify(compact)), // last 20 (by memory design)
+        stepFile, // full history is in this JSONL (one step per line)
         startTime: Date.now() - 1, // TODO: wire actual timings if needed
         endTime: Date.now(),
       });

package/core/turnLoop.js

@@ -60,7 +60,7 @@ const llm = getLLM(PROVIDER_ID);
 let totalTokensUsed = 0;
 let turnTimestamps = [];
 let shouldBackoff;
-let steps = [];
+// let steps = [];
 
 /**
  * Utility: format byte counts into human-readable strings.
@@ -148,6 +148,15 @@ export const turnLoop = async (
   const { steps = [], missionName } = ctx;
   let agentMemory = { lastMenuExpanded: false };
   
+  // Centralized recorder: push to in-memory buffer AND fire optional callback for streaming
+  // Idempotent recorder – prevents accidental double-push
+  const recordStep = (step) => {
+    if (!step || step.__recorded) return;
+    step.__recorded = true;
+    try { steps.push(step); } catch {}
+    try { ctx?.onStep?.(step); } catch {}
+  };
+  
   // ─────────────────────────────────────────────
   // STEP 1: Begin reasoning cycle
   // ─────────────────────────────────────────────
@@ -163,14 +172,20 @@ export const turnLoop = async (
       // Adaptive cooldown if nearing provider rate limit
       ({ totalTokensUsed, turnTimestamps, shouldBackoff } =
         await tokenUseCoolOff(totalTokensUsed, turnTimestamps, MODEL_ID));
-      if (shouldBackoff) return await turnLoop(browser, messages, maxTurns, turn, currentStep);
+      if (shouldBackoff) {
+        // step.result = '🟡 In Progress';
+        recordStep(step);         // ✅ write the partial step before sleeping
+        turn -= 1;                // retry same turn index
+        continue;
+        // return await turnLoop(browser, messages, maxTurns, turn, currentStep);
+      }
 
       // Verify message structure integrity before requesting next model step
       if (!validateAndInsertMissingToolResponses(messages, { insertPlaceholders: true })) {
         console.error('❌ Tool call structure invalid and no placeholders inserted.');
         step.events.push('❌ Tool call structure invalid and no placeholders inserted.');
         step.result = '❌ Failure';
-        steps.push(step);
+        recordStep(step);
         return steps;
       }
 
@@ -190,7 +205,7 @@ export const turnLoop = async (
         console.dir(unrespondedCalls, { depth: 5 });
         step.events.push('🛑 Detected assistant tool calls without matching tool responses');
         step.result = '❌ Failure';
-        steps.push(step);
+        recordStep(step);
         return steps;
       }
 
@@ -212,21 +227,28 @@ export const turnLoop = async (
         try { updateLimitsFromHeaders(MODEL_ID, err.headers || err.response?.headers || {}); } catch {}
         const delay = Math.min(60000, 2 ** retryCount * 2000);
         console.warn(`⚠️ Rate limited. Retrying in ${delay / 1000}s... (retry ${retryCount + 1})`);
+        step.events.push(`⏳ Rate limit: waiting ${Math.round(delay/1000)}s (retry ${retryCount + 1})`);
+        // step.result = '🟡 In Progress';
+        recordStep(step);         // ✅ persist this step before sleeping
         await wait(delay);
 
         if (retryCount >= 5) {
           console.error('❌ Too many retries. Exiting.');
           step.events.push('❌ Too many retries. Exiting.');
           step.result = '❌ Failure';
-          steps.push(step);
+          recordStep(step);
           return steps;
         }
-        return await turnLoop(browser, messages, maxTurns, turn, retryCount + 1, step);
+        // return await turnLoop(browser, messages, maxTurns, turn, retryCount + 1, step);
+        // reattempt same turn after delay; keep same loop, bump retryCount
+        retryCount += 1;
+        turn -= 1;
+        continue;
       } else if (err.status === 400) {
         console.error('❌ Bad request:', err.message);
         step.events.push(`❌ Bad request: ${err.message}`);
         step.result = '❌ Failure';
-        steps.push(step);
+        recordStep(step);
         return steps;
       } else {
         throw err;
@@ -296,7 +318,7 @@ export const turnLoop = async (
         } catch { /* non-JSON results ignored */ }
 
         console.log(`[tool ] ← ${fnName} result:`, errorMessage ? '❌ Failed' : '✅ Success');
-        const truncated = result.length > 100 ? result.slice(0, 100) + '…' : result;
+        const truncated = result.length > 1000 ? result.slice(0, 1000) + '…' : result;
         step.events.push(`[tool ] ← ${fnName} result: ${errorMessage ? '❌ Failed' : '✅ Success'}`);
         step.events.push(`[tool ] ← ${truncated}`);
 
@@ -339,7 +361,7 @@ export const turnLoop = async (
       // Merge new assistant + tool responses back into conversation
       messages.push(msg, ...toolResponses);
       step.result = '✅ Passed';
-      steps.push(step);
+      recordStep(step);
       continue;
     }
 
@@ -350,7 +372,7 @@ export const turnLoop = async (
     if (finalResponse !== null) {
       step.events.push(finalResponse.finalMessage);
       step.result = finalResponse.success ? '✅ Mission Success' : '❌ Mission Failure';
-      steps.push(step);
+      recordStep(step);
       return { success: finalResponse.finalMessage, steps };
     }
 
@@ -362,5 +384,8 @@ export const turnLoop = async (
     });
     console.log(`[auto] → Injected DOM for next reasoning step`);
     step.events.push(`[auto] → Injected DOM for next reasoning step`);
+    // This is a meaningful turn even without tool calls — record it.
+    step.result = step.result || '🟡 In Progress';
+    recordStep(step);
   }
 };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "testronaut",
-  "version": "1.2.3",
+  "version": "1.2.5",
   "main": "./runner/testronaut.js",
   "bin": {
     "testronaut": "./bin/cli.js"

package/tools/tokenControl.js

@@ -35,6 +35,7 @@ import { wait } from './turnLoopUtils.js';
  */
 const DEFAULT_LIMITS = [
   // ── OpenAI (newer first) ────────────────────────────────────────────────
+  { test: /^gpt-5.1(-|$)/i,             tpm: 120000 },
   { test: /^gpt-5(-|$)/i,               tpm:  90000 },
   { test: /^gpt-5-mini(-|$)/i,          tpm: 240000 },
   { test: /^gpt-5-nano(-|$)/i,          tpm: 600000 },