testronaut 1.0.27 → 1.1.0

package/bin/cli.js

@@ -26,6 +26,27 @@ const apiBase = 'http://api.testronaut.app'; // Replace with your actual API bas
 
 
 const args = process.argv.slice(2);
+
+// Look for --model=<id> or --model <id>
+let modelOverride;
+const modelFlagIndex = args.findIndex(a => a === '--model' || a.startsWith('--model='));
+if (modelFlagIndex >= 0) {
+  if (args[modelFlagIndex].includes('=')) {
+    modelOverride = args[modelFlagIndex].split('=')[1];
+  } else if (args[modelFlagIndex + 1]) {
+    modelOverride = args[modelFlagIndex + 1];
+  }
+
+  if (modelOverride) {
+    process.env.TESTRONAUT_MODEL = modelOverride.trim();
+    console.log(`🧠 Model override: ${process.env.TESTRONAUT_MODEL}`);
+  }
+
+  // Remove flag & value from args so they don't look like mission filenames
+  args.splice(modelFlagIndex, modelOverride ? 2 : 1);
+}
+
+
 const missionsDir = path.resolve(process.cwd(), 'missions');
 
 const allResults = [];
@@ -165,10 +186,35 @@ const flatMissions = allResults.flatMap(entry => {
   }));
 });
 
+// Read provider/model from config (allow env override for model)
+const cfgPath = path.resolve(process.cwd(), 'testronaut-config.json');
+let llmProvider = 'openai';
+let llmModel = 'gpt-4o';
+
+if (fs.existsSync(cfgPath)) {
+  try {
+    const cfg = JSON.parse(fs.readFileSync(cfgPath, 'utf8'));
+    if (cfg?.provider) llmProvider = String(cfg.provider);
+    if (cfg?.model) llmModel = String(cfg.model);
+  } catch (e) {
+    console.warn('⚠️ Could not read testronaut-config.json:', e.message);
+  }
+}
+
+// Env override takes precedence for the model (matches runtime behavior)
+if (process.env.TESTRONAUT_MODEL?.trim()) {
+  llmModel = process.env.TESTRONAUT_MODEL.trim();
+}
+
+
 const report = {
   runId,
   startTime: startTime.toISOString(),
   endTime: endTime.toISOString(),
+  llm: {
+    provider: llmProvider,
+    model: llmModel,
+  },
   summary: {
     totalMissions: flatMissions.length,
     passed: flatMissions.filter(m => m.status === 'passed').length,

package/bin/init.js

@@ -13,33 +13,76 @@ export async function initializeTestronautProject() {
   let config = {};
   if (fs.existsSync(configPath)) {
     config = JSON.parse(fs.readFileSync(configPath, 'utf-8'));
+
+    // --- Back-compat migration: old configs stored provider in `model`
+    if (!config.provider && config.model === 'openai') {
+      config.provider = 'openai';
+      // don't overwrite model yet; we'll re-ask below only if not initialized
+    }
   }
 
   // If not initialized before, walk through setup
   if (!config.initialized) {
-    // Prompt for LLM
-    const response = await prompts({
+    // 1) Provider
+    const { llmProvider } = await prompts({
       type: 'select',
       name: 'llmProvider',
       message: 'Which LLM provider would you like to use?',
       choices: [
-        { title: 'OpenAI (GPT-4o)', value: 'openai' },
+        { title: 'OpenAI (GPT family)', value: 'openai' },
         // Add others like Anthropic, Mistral, etc. later
       ]
     });
 
-    config.model = response.llmProvider;
+    config.provider = llmProvider; // store provider explicitly
+
+    // 2) If OpenAI, ask for model
+    const { openaiModel } = await prompts({
+      type: 'select',
+      name: 'openaiModel',
+      message: 'Select an OpenAI model for agentic workflows (function/tool calling):',
+      choices: [
+        { title: 'GPT-4.1 (general purpose)', value: 'gpt-4.1' },
+        { title: 'GPT-4.1 mini (faster, cheaper)', value: 'gpt-4.1-mini' },
+        { title: 'GPT-4o (multimodal, tool use)', value: 'gpt-4o' },
+        { title: 'GPT-4o mini (speed/cost optimized)', value: 'gpt-4o-mini' },
+        { title: 'o3 (reasoning w/ native tool use)', value: 'o3' },
+        { title: 'o4-mini (reasoning, cost-effective)', value: 'o4-mini' },
+
+        // 👇 GPT-5 family
+        { title: 'GPT-5 (latest reasoning model)', value: 'gpt-5' },
+        { title: 'GPT-5 mini (faster)', value: 'gpt-5-mini' },
+        { title: 'GPT-5 nano (lightweight)', value: 'gpt-5-nano' },
+      ],
+      initial: 1
+    });
+
+    config.model = openaiModel;
+
+    // Warn about GPT-5 availability
+    if (openaiModel.startsWith('gpt-5')) {
+      console.log(`
+    ⚠️  Warning: GPT-5 support may not be available for all users.
+      - Access depends on your OpenAI account/region
+      - Some capabilities may be gated or rate-limited
+      `);
+    }
+
+    // Default config fields
     config.initialized = true;
-    config.outputDir = 'missions/mission_reports';
-    config.projectName = path.basename(root);
-    config.maxTurns = 20;
+    config.outputDir = config.outputDir || 'missions/mission_reports';
+    config.projectName = config.projectName || path.basename(root);
+    config.maxTurns = config.maxTurns ?? 20;
+
     fs.writeFileSync(configPath, JSON.stringify(config, null, 2));
     console.log('📝 Created testronaut-config.json');
 
-    if (response.llmProvider === 'openai') {
-      // Display .env instructions
+    if (llmProvider === 'openai') {
+      // Display .env instructions for OpenAI
       if (!fs.existsSync(envPath)) {
-        const exampleEnv = `# Add your OpenAI API key below\nOPENAI_API_KEY=sk-...\n`;
+        const exampleEnv = `# Add your OpenAI API key below
+OPENAI_API_KEY=sk-...
+`;
         fs.writeFileSync(envPath, exampleEnv);
         console.log('🔐 Created .env file');
       }
@@ -51,7 +94,7 @@ export async function initializeTestronautProject() {
 
     OPENAI_API_KEY=sk-...
 
-📌 The key must have permissions to call the OpenAI Chat Completions API.
+📌 The key must have permissions to call the OpenAI API.
 
 ✅ You’re now ready to write missions in \`missions/\` and run your first mission using:
 

package/core/turnLoop.js

@@ -11,8 +11,15 @@ import {
   tokenEstimate, 
   tokenUseCoolOff, 
   recordTokenUsage, 
-  pruneOldTokenUsage 
+  pruneOldTokenUsage,
+  getCurrentTokenLimit,
+  updateLimitsFromHeaders
 } from '../tools/tokenControl.js';
+import { resolveModel } from '../openAI/modelResolver.js';
+
+
+const MODEL_ID = resolveModel();
+console.log(`🧠 Using OpenAI model: ${MODEL_ID}`);
 
 const openai = new OpenAI({ apiKey: process.env.OPENAI_API_KEY });
 
@@ -52,7 +59,7 @@ const pushDOMAssistant = async (browser, messages, agentMemory, { skipIfLastTool
   });
 
   const domHtml = await CHROME_TOOL_MAP.get_dom(browser, { limit: 100000, exclude: true }, agentMemory);
-  await tokenEstimate('gpt-4o', domHtml);
+  await tokenEstimate( MODEL_ID, domHtml);
   // fs.writeFileSync(`missions/mission_reports/debug-expanded-${Date.now()}.html`, domHtml);
 
   messages.push({
@@ -89,7 +96,7 @@ export const turnLoop = async (
       ({ turnTimestamps, totalTokensUsed } = pruneOldTokenUsage(turnTimestamps));
 
       // Cool off check
-      ({ totalTokensUsed, turnTimestamps, shouldBackoff } = await tokenUseCoolOff(totalTokensUsed, turnTimestamps));
+      ({ totalTokensUsed, turnTimestamps, shouldBackoff } = await tokenUseCoolOff(totalTokensUsed, turnTimestamps, MODEL_ID));
       if(shouldBackoff) return await turnLoop(browser, messages, maxTurns, turn, currentStep);
 
       //Tool Validator
@@ -123,12 +130,14 @@ export const turnLoop = async (
       
 
       response = await openai.chat.completions.create({
-        model: 'gpt-4o',
+        model: MODEL_ID,
         messages,
         tools: toolsSchema,
       });
     } catch (err) {
       if (err.status === 429) {
+        // If headers include limits, learn them for this model
+        try { updateLimitsFromHeaders(MODEL_ID, err.headers || err.response?.headers || {}); } catch {}
         const delay = Math.min(60000, 2 ** retryCount * 2000);
         console.warn(`⚠️ Rate limited. Retrying in ${delay / 1000}s... (retry ${retryCount + 1})`);
         await wait(delay);
@@ -248,7 +257,7 @@ export const turnLoop = async (
             exclude: true,
             focus: [],
           }, agentMemory);
-          await tokenEstimate('gpt-4o', domHtml);
+          await tokenEstimate(MODEL_ID, domHtml);
           await pushDOMAssistant(browser, messages, agentMemory, {
             skipIfLastTool: ['get_dom', 'check_text']
           });

package/openAI/modelResolver.js

@@ -0,0 +1,33 @@
+import path from 'path';
+import fs from 'fs';
+
+// Resolve model from config (or ENV) with safe fallback
+const root = process.cwd();
+const configPath = path.join(root, 'testronaut-config.json');
+
+export function resolveModel() {
+  // Highest priority: env override
+  if (process.env.TESTRONAUT_MODEL?.trim()) {
+    return process.env.TESTRONAUT_MODEL.trim();
+  }
+
+  try {
+    if (fs.existsSync(configPath)) {
+      const cfg = JSON.parse(fs.readFileSync(configPath, 'utf-8'));
+      // Only use the model if provider is openai (future-proofing for other providers)
+      if (cfg?.provider === 'openai' && typeof cfg?.model === 'string' && cfg.model.trim()) {
+        return cfg.model.trim();
+      }
+      // Back-compat: older configs may have stored 'openai' in model
+      if (!cfg?.provider && cfg?.model === 'openai') {
+        return 'gpt-4o';
+      }
+    }
+  } catch (e) {
+    console.warn('⚠️ Could not read testronaut-config.json; using default model. Error:', e.message);
+  }
+
+  return 'gpt-4o';
+}
+
+

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "testronaut",
-  "version": "1.0.27",
+  "version": "1.1.0",
   "main": "./runner/testronaut.js",
   "bin": {
     "testronaut": "./bin/cli.js"

package/tools/generateHtmlReport.js

@@ -3,7 +3,7 @@ import fs from 'fs';
 import path from 'path';
 
 export function generateHtmlReport(report, outputPath) {
-  const { runId, startTime, endTime, missions = [], summary = {} } = report;
+  const { runId, startTime, endTime, missions = [], summary = {}, llm = {} } = report;
   const durationSec =
     (startTime && endTime)
       ? ((new Date(endTime) - new Date(startTime)) / 1000).toFixed(2)
@@ -241,6 +241,7 @@ export function generateHtmlReport(report, outputPath) {
       <div><strong>Run ID:</strong> ${esc(runId ?? '—')}</div>
       <div><strong>Start:</strong> ${esc(startTime ?? '—')}</div>
       <div><strong>End:</strong> ${esc(endTime ?? '—')} • <strong>Duration:</strong> ${durationSec}s</div>
+      <div><strong>LLM:</strong> ${esc(llm.provider ?? '—')} • <strong>Model:</strong> ${esc(llm.model ?? '—')}</div>
     </div>
   </div>
 
@@ -248,6 +249,7 @@ export function generateHtmlReport(report, outputPath) {
     <div class="pill">Missions: ${esc(summary.totalMissions ?? totals.total)}</div>
     <div class="pill ok">Passed: ${esc(summary.passed ?? totals.passed)}</div>
     <div class="pill bad">Failed: ${esc(summary.failed ?? totals.failed)}</div>
+    <div class="pill">LLM: ${esc(llm.provider ?? '—')} • ${esc(llm.model ?? '')}</div>
   </div>
 
   <div class="container">

package/tools/tokenControl.js

@@ -1,33 +1,164 @@
-import { encoding_for_model } from '@dqbd/tiktoken';
+import { encoding_for_model, get_encoding } from '@dqbd/tiktoken';
 import { wait } from './turnLoopUtils.js';
 
-const TOKEN_BACKOFF_MS = 60000;
-const TOKEN_LIMIT_PER_MIN = 90000;
+/**
+ * Dynamic token-per-minute limits by model family.
+ * - You can tweak these defaults anytime.
+ * - ENV override: TESTRONAUT_TOKENS_PER_MIN forces a single limit for everything.
+ * - At runtime, you can call `updateLimitsFromHeaders(model, headers)` after a 429
+ *   to adopt server-advertised limits (if present).
+ */
+const DEFAULT_LIMITS = [
+  // Newer families first (checked in order)
+  { test: /^gpt-5(-|$)/i,               tpm: 90000 },
+  { test: /^gpt-5-mini(-|$)/i,          tpm: 240000 },
+  { test: /^gpt-5-nano(-|$)/i,          tpm: 600000 },
+
+  { test: /^gpt-4o(-|$)/i,              tpm: 450000 },
+  { test: /^gpt-4\.1(-|$)/i,            tpm: 1000000 },
+
+  { test: /^o3(-|$)/i,                  tpm: 300000 },
+  { test: /^o4-mini(-|$)/i,             tpm: 600000 },
+
+  // Older / fallback
+  { test: /^gpt-4(-|$)/i,               tpm: 150000 },
+  { test: /^gpt-3\.5(-|$)/i,            tpm: 600000 },
+
+  // Ultimate fallback for anything else
+  { test: /.*/,                         tpm: 150000 },
+];
+
+// Live, mutable limits (can be updated by headers at runtime)
+const liveLimits = new Map(); // modelId -> { tpm, source: 'default'|'env'|'header' }
+
+// One-time warning tracking for tokenizer fallback
+const warnedModels = new Set();
+
+/* ---------------- Tokenizer helpers (same idea as before) ---------------- */
+
+function getTokenizer(model) {
+  try {
+    return encoding_for_model(model);
+  } catch (_) {
+    const m = (model || '').toLowerCase();
+    const useO200k =
+      m.startsWith('gpt-5') ||
+      m.startsWith('gpt-4o') ||
+      m.startsWith('gpt-4.1') ||
+      m.startsWith('o') ||
+      m.includes('omni');
+
+    try {
+      return get_encoding(useO200k ? 'o200k_base' : 'cl100k_base');
+    } catch (e2) {
+      if (!warnedModels.has(model)) {
+        console.warn(`⚠️ Could not load tokenizer for "${model}". Will approximate by bytes/4.`, e2?.message || e2);
+        warnedModels.add(model);
+      }
+      return null;
+    }
+  }
+}
 
 export const tokenEstimate = async (model, text) => {
-  const encoding = encoding_for_model(model);
-  const tokenCount = encoding.encode(text).length;
-  console.log(`🧠 Estimated token count: ${tokenCount}`);
-  return tokenCount;
+  const str = typeof text === 'string' ? text : JSON.stringify(text ?? '');
+  const enc = getTokenizer(model);
+
+  if (enc) {
+    try {
+      const tokenCount = enc.encode(str).length;
+      enc.free?.();
+      console.log(`🧠 Estimated token count (${model}): ${tokenCount}`);
+      return tokenCount;
+    } catch (e) {
+      enc.free?.();
+      if (!warnedModels.has(model)) {
+        console.warn(`⚠️ Tokenizer encode failed for "${model}". Falling back to bytes/4.`, e?.message || e);
+        warnedModels.add(model);
+      }
+    }
+  }
+
+  // Final fallback: rough heuristic (UTF-8 bytes / 4)
+  const bytes = Buffer.from(str, 'utf8').length;
+  const approx = Math.ceil(bytes / 4);
+  console.log(`🧠 Estimated token count (approx, ${model}): ${approx}`);
+  return approx;
+};
+
+/* ---------------- Dynamic limit resolution ---------------- */
+
+function resolveDefaultLimitForModel(model) {
+  // Highest priority: ENV override (forces a single limit for all models)
+  const envTPM = process.env.TESTRONAUT_TOKENS_PER_MIN
+    ? Number(process.env.TESTRONAUT_TOKENS_PER_MIN)
+    : undefined;
+  if (envTPM && Number.isFinite(envTPM) && envTPM > 0) {
+    return { tpm: envTPM, source: 'env' };
+  }
+
+  // Otherwise find best-matching default
+  const hit = DEFAULT_LIMITS.find(entry => entry.test.test(model || ''));
+  return { tpm: hit?.tpm ?? 150000, source: 'default' };
+}
+
+/**
+ * Public: get the current token-per-minute limit for a model.
+ * Checks liveOverrides first (header-learned), then ENV, then defaults.
+ */
+export function getCurrentTokenLimit(model) {
+  const m = (model || '').trim() || 'unknown';
+  const live = liveLimits.get(m);
+  if (live?.tpm) return live;
+
+  const resolved = resolveDefaultLimitForModel(m);
+  liveLimits.set(m, resolved);
+  return resolved;
 }
 
-export const tokenUseCoolOff = async (totalTokensUsed, turnTimestamps) => {
-  // console.log("total tokens used: ", totalTokensUsed);
-  // console.log("turnTimestamps: ", turnTimestamps);
-  
-  if (totalTokensUsed > TOKEN_LIMIT_PER_MIN) {
-    const msToWait = await getDynamicBackoffMs(turnTimestamps);
-    console.warn(`⚠️ Token throttle risk → Waiting ${msToWait / 1000}s to cool off...`);
-    await wait(msToWait);
-    // totalTokensUsed = 0; // Reset token count after backoff
-    // turnTimestamps = []; // Reset timestamps after backoff
+/**
+ * Public: update limits from HTTP response headers (e.g., after 429).
+ * Looks for typical headers like:
+ *  - x-ratelimit-limit-tokens: "90000"
+ *  - x-ratelimit-remaining-tokens, x-ratelimit-reset-tokens, etc.
+ * If found, we store the new TPM for this model (source='header').
+ */
+export function updateLimitsFromHeaders(model, headers = {}) {
+  if (!model) return;
+
+  // Normalize header keys to lowercase
+  const lower = {};
+  for (const k of Object.keys(headers || {})) lower[k.toLowerCase()] = headers[k];
+
+  // Common header names OpenAI/Azure may set (varies!)
+  const tokenCap =
+    Number(lower['x-ratelimit-limit-tokens']) ||
+    Number(lower['x-ratelimit-limit-tpm']) ||
+    Number(lower['x-ratelimit-limit-token']) ||
+    undefined;
+
+  if (tokenCap && Number.isFinite(tokenCap) && tokenCap > 0) {
+    const cur = getCurrentTokenLimit(model);
+    if (cur.tpm !== tokenCap || cur.source !== 'header') {
+      liveLimits.set(model, { tpm: tokenCap, source: 'header' });
+      console.log(`📏 Updated TPM for ${model}: ${tokenCap} (from headers)`);
+    }
+  }
+}
+
+/* ---------------- Cooloff / backoff logic ---------------- */
+
+export const tokenUseCoolOff = async (totalTokensUsed, turnTimestamps, model) => {
+  const { tpm } = getCurrentTokenLimit(model);
+  if (totalTokensUsed > tpm) {
+    const msToWait = await getDynamicBackoffMs(turnTimestamps, tpm);
+    console.warn(`⚠️ Token throttle risk (${totalTokensUsed}/${tpm}) → Waiting ${Math.ceil((msToWait || 1000)/1000)}s...`);
+    await wait(msToWait || 1000);
     console.log('✅ Backoff complete, resuming...');
-    // return true;
     return { shouldBackoff: true, totalTokensUsed: 0, turnTimestamps: [] };
   }
-  // return false;
   return { shouldBackoff: false, totalTokensUsed, turnTimestamps };
-}
+};
 
 export const recordTokenUsage = (turnTimestamps, tokensUsed) => {
   const now = Date.now();
@@ -41,22 +172,18 @@ export const pruneOldTokenUsage = (turnTimestamps, windowMs = 60000) => {
   return { turnTimestamps: recentEntries, totalTokensUsed };
 };
 
-const getDynamicBackoffMs = async (turnTimestamps, tokenLimit = TOKEN_LIMIT_PER_MIN) => {
+const getDynamicBackoffMs = async (turnTimestamps, tokenLimit) => {
   const now = Date.now();
   let runningTotal = 0;
 
-  // Sort oldest → newest to find the earliest point to fall below the limit
   const sorted = [...turnTimestamps].sort((a, b) => a[0] - b[0]);
-
   for (let i = 0; i < sorted.length; i++) {
-    runningTotal += sorted[i][1]; // add tokensUsed
-
+    runningTotal += sorted[i][1];
     if (runningTotal > tokenLimit) {
       const [timestampOfExcess] = sorted[i];
       const msUntilSafe = 60000 - (now - timestampOfExcess);
-      return Math.max(msUntilSafe, 1000); // wait at least 1s
+      return Math.max(msUntilSafe, 1000); // at least 1s
     }
   }
-
-  return 0; // Safe to proceed
-}
+  return 0;
+};