testronaut 1.0.7 → 1.0.9

package/README.md

@@ -47,7 +47,8 @@ import { runMissions } from 'testronaut';
 
 export const loginMission = `
 Visit ${process.env.URL}.
-Fill in the username field with ${process.env.USERNAME} and password field with ${process.env.PASSWORD}.
+Fill the username field with ${process.env.USERNAME}. 
+Fill the password field with ${process.env.PASSWORD}.
 Submit the form.
 Wait for the dashboard to appear.
 Report SUCCESS if the dashboard is loaded, otherwise report FAILURE.
@@ -56,7 +57,7 @@ Report SUCCESS if the dashboard is loaded, otherwise report FAILURE.
 export async function executeMission() {
   await runMissions({
     mission: loginMission
-  });
+  }, "Login Mission");
 }
 ```
 
@@ -89,7 +90,7 @@ export async function executeMission() {
     preMission: [loginMission, navigateToContactFormMission],
     mission: fillContactFormMission,
     postMission: logoutMission,
-  });
+  }, "Fill Contact Form Mission");
 }
 ```
 
@@ -97,12 +98,12 @@ export async function executeMission() {
 Run all missions in the missions/ directory:
 
 ```
-testronaut
+npx testronaut
 ```
 Run a specific file:
 
 ```
-testronaut login.mission.js
+npx testronaut login.mission.js
 ```
 
 🧰 Available Helpers
@@ -122,6 +123,9 @@ runSuite(objectives)
 ```
 – Executes all registered test objectives
 
+### 📋 Reports
+HTML and JSON reports automatically created in the ```missions/mission_reports/``` directory for each testronaut run
+
 ### 🧪 Under the Hood
 Uses Playwright for browser automation
 
@@ -141,4 +145,4 @@ Now you can use testronaut in any local project.
 ### 📄 License
 MIT
 
-🤖 Built with ❤️ by Shane Fast
+🤖 Built with ❤️ by [Shane Fast](https://github.com/scfast)

package/bin/cli.js

@@ -4,12 +4,17 @@ import path from 'path';
 import { fileURLToPath } from 'url';
 import { initializeTestronautProject } from './init.js';
 import { createWelcomeMission } from './createWelcomeMission.js';
+import { generateHtmlReport } from '../tools/generateHtmlReport.js';
 
 const __dirname = path.dirname(fileURLToPath(import.meta.url));
 
 const args = process.argv.slice(2);
 const missionsDir = path.resolve(process.cwd(), 'missions');
 
+const allResults = [];
+const runId = `run_${Date.now()}`;
+const startTime = new Date();
+
 const HELP_TEXT = `
 🧑‍🚀 testronaut - Autonomous Agent Mission Runner
 
@@ -50,7 +55,11 @@ const runFile = async (filePath) => {
     const missionsModule = await import(`file://${modulePath}`);
 
     if (typeof missionsModule.executeMission === 'function') {
-      await missionsModule.executeMission(); // or call a named export like `executeMissions()`
+      const result = await missionsModule.executeMission();
+      allResults.push({
+        file: filePath,
+        result
+      });
     }
   } catch (err) {
     console.error(`❌ Error running mission: ${filePath}`);
@@ -70,3 +79,32 @@ if (args.length > 0) {
     await runFile(file);
   }
 }
+
+const endTime = new Date();
+
+const flatMissions = allResults.flatMap(entry => {
+  const result = entry.result;
+  const missions = Array.isArray(result) ? result : [result]; // normalize
+  return missions.map(m => ({
+    ...m,
+    file: entry.file
+  }));
+});
+
+
+const report = {
+  runId,
+  startTime: startTime.toISOString(),
+  endTime: endTime.toISOString(),
+  summary: {
+    totalMissions: flatMissions.length,
+    passed: flatMissions.filter(m => m.status === 'passed').length,
+    failed: flatMissions.filter(m => m.status === 'failed').length,
+  },
+  missions: flatMissions
+};
+
+const outputDir = './missions/mission_reports';
+fs.mkdirSync(outputDir, { recursive: true });
+fs.writeFileSync(`${outputDir}/${runId}.json`, JSON.stringify(report, null, 2));
+generateHtmlReport(report, `${outputDir}/${runId}.html`);

package/bin/createWelcomeMission.js

@@ -23,9 +23,9 @@ If you cannot reach this message, report FAILURE.
 \`;
 
 export async function executeMission() {
-  await runMissions({
+  return await runMissions({
     mission: welcomeGoal
-  });
+  }, 'Welcome Mission!');
 }
 
 `;

package/core/agent.js

@@ -2,41 +2,14 @@ import 'dotenv/config';
 import { ChromeBrowser } from '../tools/chromeBrowser.js';
 import { turnLoop } from './turnLoop.js';
 
-
-// export async function runAgent(goal, maxTurns = 10) {
-//   console.log(`\n🚀 Running Agent with goal: ${goal}`);
-//   const browser = new ChromeBrowser();
-//   await browser.start();
-
-//   const messages = [
-//     {
-//       role: 'system',
-//       content: `
-//         You are an autonomous web agent. Use function calls to complete the user's goal.
-//         If you are unsure of the selectors for inputs or buttons, call 'get_dom' to retrieve page HTML,
-//         analyze it, then make your best guess based on labels, names, types, and placeholder values.
-//         After completing the goal, respond with a final plain-text message starting with SUCCESS or FAILURE.
-//       `.trim(),
-//     },
-//     { role: 'user', content: goal },
-//   ];
-
-//   try {
-//     const result = await turnLoop(browser, messages, maxTurns);
-//     !result && console.log('🛑 Agent ran out of turns.')
-//   } finally {
-//     await browser.close();
-//   }
-// }
-
-export async function runAgent(goals, maxTurns = 20) {
+export async function runAgent(goals, missionName, maxTurns = 20, ) {
   const browser = new ChromeBrowser();
   await browser.start();
+  const startTime = Date.now();
 
   try {
     for (const goal of goals) {
-      console.log(`\n🚀 Running Agent with goal:\n${goal}\n`);
-
+      // console.log(`\n🚀 Running Agent with goal:\n${goal}\n`);
       const messages = [
         {
           role: 'system',
@@ -59,24 +32,30 @@ export async function runAgent(goals, maxTurns = 20) {
       const success = await turnLoop(browser, messages, maxTurns);
       if (!success) {
         console.log('🛑 Agent stopped due to failed goal.\n');
-        return false;
+        const endTime = Date.now();
+        return {
+          missionName: missionName,
+          status: 'failed',
+          steps: [ /* tool steps or key moments */ ],
+          startTime: startTime,
+          endTime: endTime,
+        }
+        ;
       }
     }
 
     console.log('✅ All goals completed successfully.\n');
-    return true;
+    const endTime = Date.now();
+    return {
+      missionName: missionName,
+      status: 'passed',
+      steps: [ /* tool steps or key moments */ ],
+      startTime: startTime,
+      endTime: endTime,
+    }
+    ;
 
   } finally {
     await browser.close();
   }
 }
-
-
-// (async () => {
-//   await runAgent(`
-//     Visit ${process.env.URL}.
-//     Log in using ${process.env.USERNAME} and password ${process.env.PASSWORD}.
-//     After clicking the login button, use 'check_text' to verify that "${process.env.AFTER_LOGIN_CHECK}" appears on the page.
-//     If found, report SUCCESS. Otherwise, report FAILURE.
-//   `);
-// })();

package/core/turnLoop.js

@@ -18,6 +18,7 @@ const openai = new OpenAI({ apiKey: process.env.OPENAI_API_KEY });
 
 let totalTokensUsed = 0;
 let turnTimestamps = [];
+let shouldBackoff;
 
 const pushDOMAssistant = async (browser, messages, agentMemory, { skipIfLastTool } = {}) => {
   if (skipIfLastTool && skipIfLastTool.includes(messages.at(-1)?.name)) {
@@ -43,7 +44,7 @@ const pushDOMAssistant = async (browser, messages, agentMemory, { skipIfLastTool
 
   const domHtml = await CHROME_TOOL_MAP.get_dom(browser, { limit: 100000, exclude: true }, agentMemory);
   await tokenEstimate('gpt-4o', domHtml);
-  fs.writeFileSync(`missions/mission_reports/debug-expanded-${Date.now()}.html`, domHtml);
+  // fs.writeFileSync(`missions/mission_reports/debug-expanded-${Date.now()}.html`, domHtml);
 
   messages.push({
     role: 'tool',
@@ -61,48 +62,40 @@ export const turnLoop = async (browser, messages, maxTurns, currentTurn = 0, ret
   
   for (let turn = currentTurn; turn < maxTurns; turn++) {
     console.log(`\n🔄 Turn ${turn + 1}/${maxTurns}`);
-    // console.log("current tools", toolsSchema);
-
-    // const now = Date.now();
-    // turnTimestamps = turnTimestamps.filter(ts => now - ts < 60000);
-
-    // Recalculate the rolling window before checking cooldown
-    const pruned = pruneOldTokenUsage(turnTimestamps);
-    turnTimestamps = pruned.turnTimestamps;
-    totalTokensUsed = pruned.totalTokensUsed;
-
-    // Cool off check
-    const cooldownResult = await tokenUseCoolOff(totalTokensUsed, turnTimestamps);
-    totalTokensUsed = cooldownResult.totalTokensUsed;
-    turnTimestamps = cooldownResult.turnTimestamps;
-    if (cooldownResult.shouldBackoff) {
-      return await turnLoop(browser, messages, maxTurns, turn);
-    }
-    // if (await tokenUseCoolOff(totalTokensUsed, turnTimestamps)) {return await turnLoop(browser, messages, maxTurns, turn)}
-
-    const isValid = validateAndInsertMissingToolResponses(messages, {
-      insertPlaceholders: true, // safe fallback behavior
-    });
-    
-    if (!isValid) {
-      console.error('❌ Tool call structure invalid and no placeholders inserted.');
-      return false;
-    }
-    // console.log(JSON.stringify(messages, null, 2));
-
     let response;
     try {
-      // console.log(
-      //   '🔍 Verifying message structure before OpenAI call:\n',
-      //   messages.map(m => ({
-      //     role: m.role,
-      //     name: m.name,
-      //     tool_call_id: m.tool_call_id,
-      //     tool_calls: m.tool_calls?.map(t => t.id),
-      //     content: m.content?.slice?.(0, 100), // truncate for readability
-      //   }))
-      // );
+
+      // Recalculate the rolling window before checking cooldown
+      ({ turnTimestamps, totalTokensUsed } = pruneOldTokenUsage(turnTimestamps));
+
+      // Cool off check
+      ({ totalTokensUsed, turnTimestamps, shouldBackoff } = await tokenUseCoolOff(totalTokensUsed, turnTimestamps));
+      if(shouldBackoff) return await turnLoop(browser, messages, maxTurns, turn);
+
+      //Tool Validator
+      if (!validateAndInsertMissingToolResponses(messages, { insertPlaceholders: true })) {
+        console.error('❌ Tool call structure invalid and no placeholders inserted.');
+        return false;
+      }
+
+      const unrespondedCalls = messages.filter(
+        (msg, i) =>
+          msg.role === 'assistant' &&
+          msg.tool_calls?.length &&
+          !msg.tool_calls.every(call =>
+            messages.slice(i + 1).some(
+              m => m.role === 'tool' && m.tool_call_id === call.id
+            )
+          )
+      );
       
+      if (unrespondedCalls.length) {
+        console.error('🛑 Detected assistant tool calls without matching tool responses:');
+        console.dir(unrespondedCalls, { depth: 5 });
+        return false;
+      }
+      
+
       response = await openai.chat.completions.create({
         model: 'gpt-4o',
         messages,
@@ -122,7 +115,8 @@ export const turnLoop = async (browser, messages, maxTurns, currentTurn = 0, ret
         return await turnLoop(browser, messages, maxTurns, turn, retryCount + 1);
       } else if (err.status === 400) {
         console.error('❌ Bad request:', err.message);
-        console.log("current tools", toolsSchema);
+        // console.log("current tools", toolsSchema);
+        // console.log("messages: ", messages)
         return false;
       } else {
         throw err;
@@ -130,19 +124,12 @@ export const turnLoop = async (browser, messages, maxTurns, currentTurn = 0, ret
     }
     console.log('Response received from OpenAI');
     const usage = response.usage;
-    // if (usage) {
-    //   console.log(`📊 Token Usage This Turn → Prompt: ${usage.prompt_tokens}, Completion: ${usage.completion_tokens}, Total: ${usage.total_tokens}`);
-    //   totalTokensUsed += usage?.total_tokens || 0;
-    //   turnTimestamps.push(now);
-    //   console.log(`📈 Running Total Tokens Used: ${totalTokensUsed}`);
-    // }
     if (usage) {
       const tokensUsed = usage.total_tokens || 0;
       console.log(`📊 Token Usage This Turn → Total: ${tokensUsed}`);
       recordTokenUsage(turnTimestamps, tokensUsed);
-      const pruned = pruneOldTokenUsage(turnTimestamps);
-      turnTimestamps = pruned.turnTimestamps;
-      totalTokensUsed = pruned.totalTokensUsed;
+      ({ turnTimestamps, totalTokensUsed } = pruneOldTokenUsage(turnTimestamps));
+
       console.log(`📈 Running Total Tokens Used (Rolling 60s): ${totalTokensUsed}`);
     }
     
@@ -150,29 +137,21 @@ export const turnLoop = async (browser, messages, maxTurns, currentTurn = 0, ret
 
     if (msg.tool_calls?.length) {
       console.log('Processing tool calls...');
-      messages.push(msg);
+      const toolResponses = [];
       let lastToolName = null;
+
       for (const call of msg.tool_calls) {
         const fnName = call.function.name;
         lastToolName = fnName;
         const args = JSON.parse(call.function.arguments || '{}');
         console.log(`[model] → ${fnName}`, args);
-        console.log('Calling tool:', fnName);
+        // console.log('Calling tool:', fnName);
         let result;
         let errorMessage = null;
         try {
-          console.log("agentMemory: ", agentMemory);
+          // console.log("agentMemory: ", agentMemory);
           result = await CHROME_TOOL_MAP[fnName](browser, args, agentMemory);
 
-          // Smart DOM-ready wait after major actions like click
-          // if (['click', 'click_text', 'fill'].includes(fnName)) {
-          //   try {
-          //     await browser.page.waitForLoadState?.('domcontentloaded', { timeout: 5000 });
-          //     await browser.page.waitForTimeout?.(2000); // allow animations/navigation
-          //   } catch (err) {
-          //     console.warn(`⚠️ DOM readiness wait failed after ${fnName}: ${err.message}`);
-          //   }
-          // }
           if (typeof result !== 'string') {
             result = JSON.stringify(result ?? '');
           }
@@ -181,26 +160,18 @@ export const turnLoop = async (browser, messages, maxTurns, currentTurn = 0, ret
           result = errorMessage;
         }
 
-        // 🔍 Diagnostic: log whether this tool call succeeded or failed
         console.log(`[tool ] ← ${fnName} result:`, errorMessage ? '❌ Failed' : '✅ Success');
-
         const truncated = result.length > 100 ? result.slice(0, 100) + '…' : result;
+        console.log(`[tool ] ← ${truncated}`);
 
-        if (result.length > 100) {
-          const logFile = `missions/mission_reports/tool-result-${Date.now()}.log`;
-          fs.writeFileSync(logFile, result);
-          console.log(`[tool ] ← ${truncated} (full output written to ${logFile})`);
-        } else {
-          console.log(`[tool ] ← ${truncated}`);
-        }
-
-        messages.push({
+        toolResponses.push({
           role: 'tool',
           tool_call_id: call.id,
           name: fnName,
           type: 'function',
           content: result,
         });
+
         if (fnName === 'expand_menu') {
           agentMemory.lastMenuExpanded = true;
         }
@@ -210,17 +181,18 @@ export const turnLoop = async (browser, messages, maxTurns, currentTurn = 0, ret
           const domHtml = await CHROME_TOOL_MAP.get_dom(browser, {
             limit: 100000,
             exclude: true,
-            focus: [], // or try focusing on ['main'] to keep it light
+            focus: [],
           }, agentMemory);
           await tokenEstimate('gpt-4o', domHtml);
-          fs.writeFileSync(`missions/mission_reports/debug-expanded-${Date.now()}.html`, domHtml);
+          // fs.writeFileSync(`missions/mission_reports/debug-expanded-${Date.now()}.html`, domHtml);
           await pushDOMAssistant(browser, messages, agentMemory, {
             skipIfLastTool: ['get_dom', 'check_text']
           });
           console.log(`[auto] → DOM size after ${fnName}: ${domHtml.length} chars`);
         }
+        
       }
-
+      messages.push(msg, ...toolResponses);
       continue;
     }
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "testronaut",
-  "version": "1.0.7",
+  "version": "1.0.9",
   "main": "./runner/testronaut.js",
   "bin": {
     "testronaut": "./bin/cli.js"
@@ -25,5 +25,12 @@
     "openai": "^5.10.2",
     "playwright": "^1.54.1",
     "prompts": "^2.4.2"
-  }
+  },
+  "keywords": [
+    "cli",
+    "testing",
+    "automation",
+    "node",
+    "qa"
+  ]
 }

package/runner/missionLaunch.js

@@ -6,4 +6,4 @@ mission('Collegium Dashboard Login Test', () => {
   objective('should land on dashboard after login', dashboardWorkflow);
 });
 
-await launch(loginWorkflow); // Setup: login
+await launch(loginWorkflow);

package/runner/testronaut.js

@@ -16,39 +16,7 @@ export async function launch(preMissionSetup) {
   await runSuite(objectives, preMissionSetup);
 }
 
-
-// export async function runMissions({ preMission, mission, postMission }) {
-//   const goals = [];
-
-//   if (preMission) {
-//     console.log('\n🧭 Running pre-mission setup:', preMission);
-//     goals.push({ goal: preMission, label: 'pre-mission' });
-//   }
-
-//   if (mission) {
-//     console.log('\n🧭 Running main mission:', mission);
-//     goals.push({ goal: mission, label: 'mission' });
-//   }
-
-//   if (postMission) {
-//     console.log('\n🧭 Running post-mission cleanup:', postMission);
-//     goals.push({ goal: postMission, label: 'post-mission' });
-//   }
-
-// console.log('\n🧭 Running mission flow:\n', goals);
-
-//   // for (const { goal, label } of goals) {
-//     // console.log(`\n🧭 Running ${label}:\n`);
-//   const success = await runAgent(goals);
-//   if (!success) {
-//     console.log(`❌ Aborting after failed goal.`);
-//     return;
-//   }
-//   // }
-//   console.log('\n✅ Mission flow complete.');
-// }
-
-export async function runMissions({ preMission, mission, postMission }) {
+export async function runMissions({ preMission, mission, postMission }, missionName) {
   const normalizeToArray = (x) => (Array.isArray(x) ? x : x ? [x] : []);
   
   const pre = normalizeToArray(preMission);
@@ -58,17 +26,14 @@ export async function runMissions({ preMission, mission, postMission }) {
   const goals = [];
 
   if (pre.length) {
-    console.log('\n🧭 Running pre-mission setup:', pre);
     goals.push(...pre.map(fn => ({ goal: fn, label: 'pre-mission' })));
   }
 
   if (main.length) {
-    console.log('\n🧭 Running main mission(s):', main);
     goals.push(...main.map(fn => ({ goal: fn, label: 'mission' })));
   }
 
   if (post.length) {
-    console.log('\n🧭 Running post-mission cleanup:', post);
     goals.push(...post.map(fn => ({ goal: fn, label: 'post-mission' })));
   }
 
@@ -77,11 +42,11 @@ export async function runMissions({ preMission, mission, postMission }) {
     goals.map(g => `${g.label}: ${g.goal?.name ?? g.goal}`)
   );
   
-  const success = await runAgent(goals);
+  const success = await runAgent(goals, missionName);
   if (!success) {
     console.log(`❌ Aborting after failed goal.`);
     return;
   }
-
   console.log('\n✅ Mission flow complete.');
+  return success;
 }

package/tools/chromeBrowser.js

@@ -92,7 +92,7 @@ export class ChromeBrowser {
         await el.click();
         await this.page.screenshot({ path: `missions/mission_reports/screenshot_after_click_${Date.now()}.png` });
         await this.page.waitForTimeout(5000);
-        // await this.page.screenshot({ path: `screenshot_after_waiting_click_${Date.now()}.png` });
+        await this.page.screenshot({ path: `screenshot_after_waiting_click_${Date.now()}.png` });
         return `clicked text "${text}"`;
         
       }
@@ -147,11 +147,6 @@ export class ChromeBrowser {
     await this.page.waitForSelector(selector, { timeout: 30_000 });
     await this.page.click(selector);
     await this.page.waitForTimeout(delayMs);
-
-    const expandedDom = await this.page.content();
-    const filename = `missions/mission_reports/expand_menu_dump_${Date.now()}.html`;
-    fs.writeFileSync(filename, expandedDom);
-    console.log(`[expand_menu] → Dumped DOM after expand click to ${filename}`);
     await this.page.screenshot({ path: `missions/mission_reports/screenshot_expand_menu_${Date.now()}.png` });
 
     return `expanded menu using ${selector}`;

package/tools/generateHtmlReport.js

@@ -0,0 +1,52 @@
+import fs from 'fs';
+
+export function generateHtmlReport(report, outputPath) {
+  const { runId, startTime, endTime, summary, missions } = report;
+
+  const missionHtml = missions.map(m => {
+    const status = m.status?.toUpperCase?.() || 'UNKNOWN';
+    const missionName = m.missionName || 'Unnamed';
+    const file = m.file || 'N/A';
+    const steps = Array.isArray(m.steps) ? m.steps : [];
+
+    return `
+      <div class="mission ${status.toLowerCase()}">
+        <h3>${file} - ${missionName} - <span>${status}</span></h3>
+        <p><em>${m.startTime || 'N/A'} → ${m.endTime || 'N/A'}</em></p>
+        <ul>
+          ${steps.map(step => `
+            <li>
+              <strong>${step.tool}</strong> → ${step.output}
+            </li>
+          `).join('')}
+        </ul>
+      </div>
+    `;
+  }).join('');
+
+  const html = `
+    <html>
+    <head>
+      <style>
+        body { font-family: sans-serif; padding: 20px; }
+        .passed { background-color: #e6ffed; }
+        .failed { background-color: #ffe6e6; }
+        .unknown { background-color: #f9f9f9; }
+        h3 span { font-weight: bold; }
+        ul { margin-top: 0; }
+      </style>
+    </head>
+    <body>
+      <h1>🧑‍🚀 Testronaut Report</h1>
+      <p><strong>Run ID:</strong> ${runId}</p>
+      <p><strong>Start:</strong> ${startTime}</p>
+      <p><strong>End:</strong> ${endTime}</p>
+      <p><strong>Summary:</strong> ${summary.passed}/${summary.totalMissions} passed</p>
+      ${missionHtml}
+    </body>
+    </html>
+  `;
+
+  fs.writeFileSync(outputPath, html);
+  console.log(`📝 Report generated at: ${outputPath}`);
+}