testronaut 1.0.6 → 1.0.8

package/README.md

@@ -47,7 +47,8 @@ import { runMissions } from 'testronaut';
 
 export const loginMission = `
 Visit ${process.env.URL}.
-Fill in the username field with ${process.env.USERNAME} and password field with ${process.env.PASSWORD}.
+Fill the username field with ${process.env.USERNAME}. 
+Fill the password field with ${process.env.PASSWORD}.
 Submit the form.
 Wait for the dashboard to appear.
 Report SUCCESS if the dashboard is loaded, otherwise report FAILURE.
@@ -56,7 +57,7 @@ Report SUCCESS if the dashboard is loaded, otherwise report FAILURE.
 export async function executeMission() {
   await runMissions({
     mission: loginMission
-  });
+  }, "Login Mission");
 }
 ```
 
@@ -89,7 +90,7 @@ export async function executeMission() {
     preMission: [loginMission, navigateToContactFormMission],
     mission: fillContactFormMission,
     postMission: logoutMission,
-  });
+  }, "Fill Contact Form Mission");
 }
 ```
 
@@ -97,12 +98,12 @@ export async function executeMission() {
 Run all missions in the missions/ directory:
 
 ```
-testronaut
+npx testronaut
 ```
 Run a specific file:
 
 ```
-testronaut login.mission.js
+npx testronaut login.mission.js
 ```
 
 🧰 Available Helpers
@@ -122,6 +123,9 @@ runSuite(objectives)
 ```
 – Executes all registered test objectives
 
+### 📋 Reports
+HTML and JSON reports automatically created in the ```missions/mission_reports/``` directory for each testronaut run
+
 ### 🧪 Under the Hood
 Uses Playwright for browser automation
 
@@ -141,4 +145,4 @@ Now you can use testronaut in any local project.
 ### 📄 License
 MIT
 
-🤖 Built with ❤️ by Shane Fast
+🤖 Built with ❤️ by [Shane Fast](https://github.com/scfast)

package/bin/cli.js

@@ -4,12 +4,17 @@ import path from 'path';
 import { fileURLToPath } from 'url';
 import { initializeTestronautProject } from './init.js';
 import { createWelcomeMission } from './createWelcomeMission.js';
+import { generateHtmlReport } from '../tools/generateHtmlReport.js';
 
 const __dirname = path.dirname(fileURLToPath(import.meta.url));
 
 const args = process.argv.slice(2);
 const missionsDir = path.resolve(process.cwd(), 'missions');
 
+const allResults = [];
+const runId = `run_${Date.now()}`;
+const startTime = new Date();
+
 const HELP_TEXT = `
 🧑‍🚀 testronaut - Autonomous Agent Mission Runner
 
@@ -50,7 +55,11 @@ const runFile = async (filePath) => {
     const missionsModule = await import(`file://${modulePath}`);
 
     if (typeof missionsModule.executeMission === 'function') {
-      await missionsModule.executeMission(); // or call a named export like `executeMissions()`
+      const result = await missionsModule.executeMission();
+      allResults.push({
+        file: filePath,
+        result
+      });
     }
   } catch (err) {
     console.error(`❌ Error running mission: ${filePath}`);
@@ -70,3 +79,32 @@ if (args.length > 0) {
     await runFile(file);
   }
 }
+
+const endTime = new Date();
+
+const flatMissions = allResults.flatMap(entry => {
+  const result = entry.result;
+  const missions = Array.isArray(result) ? result : [result]; // normalize
+  return missions.map(m => ({
+    ...m,
+    file: entry.file
+  }));
+});
+
+
+const report = {
+  runId,
+  startTime: startTime.toISOString(),
+  endTime: endTime.toISOString(),
+  summary: {
+    totalMissions: flatMissions.length,
+    passed: flatMissions.filter(m => m.status === 'passed').length,
+    failed: flatMissions.filter(m => m.status === 'failed').length,
+  },
+  missions: flatMissions
+};
+
+const outputDir = './missions/mission_reports';
+fs.mkdirSync(outputDir, { recursive: true });
+fs.writeFileSync(`${outputDir}/${runId}.json`, JSON.stringify(report, null, 2));
+generateHtmlReport(report, `${outputDir}/${runId}.html`);

package/bin/createWelcomeMission.js

@@ -23,9 +23,9 @@ If you cannot reach this message, report FAILURE.
 \`;
 
 export async function executeMission() {
-  await runMissions({
+  return await runMissions({
     mission: welcomeGoal
-  });
+  }, 'Welcome Mission!');
 }
 
 `;

package/core/agent.js

@@ -29,14 +29,14 @@ import { turnLoop } from './turnLoop.js';
 //   }
 // }
 
-export async function runAgent(goals, maxTurns = 20) {
+export async function runAgent(goals, missionName, maxTurns = 20, ) {
   const browser = new ChromeBrowser();
   await browser.start();
+  const startTime = Date.now();
 
   try {
     for (const goal of goals) {
-      console.log(`\n🚀 Running Agent with goal:\n${goal}\n`);
-
+      // console.log(`\n🚀 Running Agent with goal:\n${goal}\n`);
       const messages = [
         {
           role: 'system',
@@ -59,12 +59,28 @@ export async function runAgent(goals, maxTurns = 20) {
       const success = await turnLoop(browser, messages, maxTurns);
       if (!success) {
         console.log('🛑 Agent stopped due to failed goal.\n');
-        return false;
+        const endTime = Date.now();
+        return {
+          missionName: missionName,
+          status: 'failed',
+          steps: [ /* tool steps or key moments */ ],
+          startTime: startTime,
+          endTime: endTime,
+        }
+        ;
       }
     }
 
     console.log('✅ All goals completed successfully.\n');
-    return true;
+    const endTime = Date.now();
+    return {
+      missionName: missionName,
+      status: 'passed',
+      steps: [ /* tool steps or key moments */ ],
+      startTime: startTime,
+      endTime: endTime,
+    }
+    ;
 
   } finally {
     await browser.close();

package/core/turnLoop.js

@@ -43,7 +43,7 @@ const pushDOMAssistant = async (browser, messages, agentMemory, { skipIfLastTool
 
   const domHtml = await CHROME_TOOL_MAP.get_dom(browser, { limit: 100000, exclude: true }, agentMemory);
   await tokenEstimate('gpt-4o', domHtml);
-  fs.writeFileSync(`missions/mission_reports/debug-expanded-${Date.now()}.html`, domHtml);
+  // fs.writeFileSync(`missions/mission_reports/debug-expanded-${Date.now()}.html`, domHtml);
 
   messages.push({
     role: 'tool',
@@ -92,6 +92,35 @@ export const turnLoop = async (browser, messages, maxTurns, currentTurn = 0, ret
 
     let response;
     try {
+      // console.log(
+      //   '🔍 Verifying message structure before OpenAI call:\n',
+      //   messages.map(m => ({
+      //     role: m.role,
+      //     name: m.name,
+      //     tool_call_id: m.tool_call_id,
+      //     tool_calls: m.tool_calls?.map(t => t.id),
+      //     content: m.content?.slice?.(0, 100), // truncate for readability
+      //   }))
+      // );
+      
+      const unrespondedCalls = messages.filter(
+        (msg, i) =>
+          msg.role === 'assistant' &&
+          msg.tool_calls?.length &&
+          !msg.tool_calls.every(call =>
+            messages.slice(i + 1).some(
+              m => m.role === 'tool' && m.tool_call_id === call.id
+            )
+          )
+      );
+      
+      if (unrespondedCalls.length) {
+        console.error('🛑 Detected assistant tool calls without matching tool responses:');
+        console.dir(unrespondedCalls, { depth: 5 });
+        return false;
+      }
+      
+
       response = await openai.chat.completions.create({
         model: 'gpt-4o',
         messages,
@@ -112,6 +141,7 @@ export const turnLoop = async (browser, messages, maxTurns, currentTurn = 0, ret
       } else if (err.status === 400) {
         console.error('❌ Bad request:', err.message);
         console.log("current tools", toolsSchema);
+        console.log("messages: ", messages)
         return false;
       } else {
         throw err;
@@ -137,19 +167,31 @@ export const turnLoop = async (browser, messages, maxTurns, currentTurn = 0, ret
     
     const msg = response.choices[0].message;
 
+    // if (!msg.tool_calls?.length) {
+    //   console.error('❌ Assistant message had empty or missing tool_calls array');
+    //   return false;
+    // }   
+
     if (msg.tool_calls?.length) {
       console.log('Processing tool calls...');
-      messages.push(msg);
+      // messages.push(msg);
+      const toolResponses = [];
       let lastToolName = null;
+
       for (const call of msg.tool_calls) {
+        const toolCallId = call.id;
+        // const hasResponse = messages.some(m =>
+        //   m.role === 'tool' && m.tool_call_id === toolCallId
+        // );
         const fnName = call.function.name;
         lastToolName = fnName;
         const args = JSON.parse(call.function.arguments || '{}');
         console.log(`[model] → ${fnName}`, args);
-        console.log('Calling tool:', fnName);
+        // console.log('Calling tool:', fnName);
         let result;
+        let errorMessage = null;
         try {
-          console.log("agentMemory: ", agentMemory);
+          // console.log("agentMemory: ", agentMemory);
           result = await CHROME_TOOL_MAP[fnName](browser, args, agentMemory);
 
           // Smart DOM-ready wait after major actions like click
@@ -165,26 +207,39 @@ export const turnLoop = async (browser, messages, maxTurns, currentTurn = 0, ret
             result = JSON.stringify(result ?? '');
           }
         } catch (err) {
-          result = `ERROR: ${err.message}`;
+          errorMessage = `ERROR: ${err.message}`;
+          result = errorMessage;
         }
 
+        // 🔍 Diagnostic: log whether this tool call succeeded or failed
+        console.log(`[tool ] ← ${fnName} result:`, errorMessage ? '❌ Failed' : '✅ Success');
+
         const truncated = result.length > 100 ? result.slice(0, 100) + '…' : result;
 
-        if (result.length > 100) {
-          const logFile = `missions/mission_reports/tool-result-${Date.now()}.log`;
-          fs.writeFileSync(logFile, result);
-          console.log(`[tool ] ← ${truncated} (full output written to ${logFile})`);
-        } else {
+        // if (result.length > 100) {
+        //   // const logFile = `missions/mission_reports/tool-result-${Date.now()}.log`;
+        //   // fs.writeFileSync(logFile, result);
+        //   console.log(`[tool ] ← ${truncated} (full output written to ${logFile})`);
+        // } else {
           console.log(`[tool ] ← ${truncated}`);
-        }
+        // }
+
+        // messages.push({
+        //   role: 'tool',
+        //   tool_call_id: call.id,
+        //   name: fnName,
+        //   type: 'function',
+        //   content: result,
+        // });
 
-        messages.push({
+        toolResponses.push({
           role: 'tool',
           tool_call_id: call.id,
           name: fnName,
           type: 'function',
           content: result,
         });
+
         if (fnName === 'expand_menu') {
           agentMemory.lastMenuExpanded = true;
         }
@@ -197,13 +252,23 @@ export const turnLoop = async (browser, messages, maxTurns, currentTurn = 0, ret
             focus: [], // or try focusing on ['main'] to keep it light
           }, agentMemory);
           await tokenEstimate('gpt-4o', domHtml);
-          fs.writeFileSync(`missions/mission_reports/debug-expanded-${Date.now()}.html`, domHtml);
+          // fs.writeFileSync(`missions/mission_reports/debug-expanded-${Date.now()}.html`, domHtml);
           await pushDOMAssistant(browser, messages, agentMemory, {
             skipIfLastTool: ['get_dom', 'check_text']
           });
           console.log(`[auto] → DOM size after ${fnName}: ${domHtml.length} chars`);
         }
-      }
+
+
+        
+        // if (!hasResponse) {
+          //   console.error(`❌ Tool call ${toolCallId} for ${call.function.name} was not followed by a response`);
+          //   return false;
+          // }
+        }
+        
+      // Only after ALL tool responses are ready
+      messages.push(msg, ...toolResponses);
 
       continue;
     }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "testronaut",
-  "version": "1.0.6",
+  "version": "1.0.8",
   "main": "./runner/testronaut.js",
   "bin": {
     "testronaut": "./bin/cli.js"

package/runner/testronaut.js

@@ -48,7 +48,7 @@ export async function launch(preMissionSetup) {
 //   console.log('\n✅ Mission flow complete.');
 // }
 
-export async function runMissions({ preMission, mission, postMission }) {
+export async function runMissions({ preMission, mission, postMission }, missionName) {
   const normalizeToArray = (x) => (Array.isArray(x) ? x : x ? [x] : []);
   
   const pre = normalizeToArray(preMission);
@@ -58,17 +58,17 @@ export async function runMissions({ preMission, mission, postMission }) {
   const goals = [];
 
   if (pre.length) {
-    console.log('\n🧭 Running pre-mission setup:', pre);
+    // console.log('\n🧭 Running pre-mission setup:', pre);
     goals.push(...pre.map(fn => ({ goal: fn, label: 'pre-mission' })));
   }
 
   if (main.length) {
-    console.log('\n🧭 Running main mission(s):', main);
+    // console.log('\n🧭 Running main mission(s):', main);
     goals.push(...main.map(fn => ({ goal: fn, label: 'mission' })));
   }
 
   if (post.length) {
-    console.log('\n🧭 Running post-mission cleanup:', post);
+    // console.log('\n🧭 Running post-mission cleanup:', post);
     goals.push(...post.map(fn => ({ goal: fn, label: 'post-mission' })));
   }
 
@@ -77,11 +77,12 @@ export async function runMissions({ preMission, mission, postMission }) {
     goals.map(g => `${g.label}: ${g.goal?.name ?? g.goal}`)
   );
   
-  const success = await runAgent(goals);
+  const success = await runAgent(goals, missionName);
   if (!success) {
     console.log(`❌ Aborting after failed goal.`);
     return;
   }
-
   console.log('\n✅ Mission flow complete.');
+  return success;
+
 }

package/tools/chromeBrowser.js

@@ -90,7 +90,7 @@ export class ChromeBrowser {
       if (innerText.includes(text)) {
         await el.scrollIntoViewIfNeeded().catch(() => {});
         await el.click();
-        await this.page.screenshot({ path: `screenshot_after_click_${Date.now()}.png` });
+        await this.page.screenshot({ path: `missions/mission_reports/screenshot_after_click_${Date.now()}.png` });
         await this.page.waitForTimeout(5000);
         // await this.page.screenshot({ path: `screenshot_after_waiting_click_${Date.now()}.png` });
         return `clicked text "${text}"`;
@@ -119,7 +119,7 @@ export class ChromeBrowser {
         console.log(`[click_text] → Clicking visible element with bounding box`, box);
         await el.click();
         await this.page.waitForTimeout(delayMs);
-        await this.page.screenshot({ path: `screenshot_after_click_${Date.now()}.png` });
+        await this.page.screenshot({ path: `missions/mission_reports/screenshot_after_click_${Date.now()}.png` });
         return `clicked visible text "${text}"`;
       }
     }
@@ -149,9 +149,9 @@ export class ChromeBrowser {
     await this.page.waitForTimeout(delayMs);
 
     const expandedDom = await this.page.content();
-    const filename = `missions/mission_reports/expand_menu_dump_${Date.now()}.html`;
-    fs.writeFileSync(filename, expandedDom);
-    console.log(`[expand_menu] → Dumped DOM after expand click to ${filename}`);
+    // const filename = `missions/mission_reports/expand_menu_dump_${Date.now()}.html`;
+    // fs.writeFileSync(filename, expandedDom);
+    // console.log(`[expand_menu] → Dumped DOM after expand click to ${filename}`);
     await this.page.screenshot({ path: `missions/mission_reports/screenshot_expand_menu_${Date.now()}.png` });
 
     return `expanded menu using ${selector}`;

package/tools/generateHtmlReport.js

@@ -0,0 +1,52 @@
+import fs from 'fs';
+
+export function generateHtmlReport(report, outputPath) {
+  const { runId, startTime, endTime, summary, missions } = report;
+
+  const missionHtml = missions.map(m => {
+    const status = m.status?.toUpperCase?.() || 'UNKNOWN';
+    const missionName = m.missionName || 'Unnamed';
+    const file = m.file || 'N/A';
+    const steps = Array.isArray(m.steps) ? m.steps : [];
+
+    return `
+      <div class="mission ${status.toLowerCase()}">
+        <h3>${file} - ${missionName} - <span>${status}</span></h3>
+        <p><em>${m.startTime || 'N/A'} → ${m.endTime || 'N/A'}</em></p>
+        <ul>
+          ${steps.map(step => `
+            <li>
+              <strong>${step.tool}</strong> → ${step.output}
+            </li>
+          `).join('')}
+        </ul>
+      </div>
+    `;
+  }).join('');
+
+  const html = `
+    <html>
+    <head>
+      <style>
+        body { font-family: sans-serif; padding: 20px; }
+        .passed { background-color: #e6ffed; }
+        .failed { background-color: #ffe6e6; }
+        .unknown { background-color: #f9f9f9; }
+        h3 span { font-weight: bold; }
+        ul { margin-top: 0; }
+      </style>
+    </head>
+    <body>
+      <h1>🧑‍🚀 Testronaut Report</h1>
+      <p><strong>Run ID:</strong> ${runId}</p>
+      <p><strong>Start:</strong> ${startTime}</p>
+      <p><strong>End:</strong> ${endTime}</p>
+      <p><strong>Summary:</strong> ${summary.passed}/${summary.totalMissions} passed</p>
+      ${missionHtml}
+    </body>
+    </html>
+  `;
+
+  fs.writeFileSync(outputPath, html);
+  console.log(`📝 Report generated at: ${outputPath}`);
+}

package/tools/tokenControl.js

@@ -12,8 +12,8 @@ export const tokenEstimate = async (model, text) => {
 }
 
 export const tokenUseCoolOff = async (totalTokensUsed, turnTimestamps) => {
-  console.log("total tokens used: ", totalTokensUsed);
-  console.log("turnTimestamps: ", turnTimestamps);
+  // console.log("total tokens used: ", totalTokensUsed);
+  // console.log("turnTimestamps: ", turnTimestamps);
   
   if (totalTokensUsed > TOKEN_LIMIT_PER_MIN) {
     const msToWait = await getDynamicBackoffMs(turnTimestamps);

package/tools/turnLoopUtils.js

@@ -34,6 +34,9 @@ export function validateAndInsertMissingToolResponses(messages, options = {}) {
         );
 
         if (!hasResponse) {
+          console.warn(
+            `⚠️ Missing response for tool_call_id: ${expectedId}, function: ${toolCall.function.name}, from assistant message index: ${i}`
+          );
           missingResponses.push(expectedId);
 
           if (insertPlaceholders) {