testronaut 1.0.0

package/tools/chromeBrowser.js

@@ -0,0 +1,184 @@
+import { chromium } from 'playwright';
+import * as cheerio from 'cheerio';
+import { 
+  chunkBySection, 
+  reduceRepeatedDivs, 
+  removeAutoGeneratedClasses, 
+  removeNonContentTags,
+  removeObfuscatedClassNames, 
+} from './domControl.js';
+import fs from 'fs';
+
+export class ChromeBrowser {
+  constructor() {
+    this.browser = null;
+    this.page = null;
+  }
+
+  async start() {
+    this.browser = await chromium.launch({ headless: true });
+    const context = await this.browser.newContext();
+    this.page = await context.newPage();
+  }
+
+  async navigate({ url }) {
+    await this.page.goto(url, { timeout: 30_000 });
+    return `navigated to ${this.page.url()}`;
+  }
+
+  async fill({ selector, text }) {
+    await this.page.waitForSelector(selector, { timeout: 30_000 });
+    await this.page.fill(selector, text);
+    return `filled ${selector}`;
+  }
+
+  async click({ selector, waitFor = 'domcontentloaded', delayMs = 2000 }) {
+    await this.page.waitForSelector(selector, { timeout: 30_000 });
+
+    const [res] = await Promise.all([
+      this.page.waitForLoadState(waitFor, { timeout: 10_000 }).catch(() => null),
+      this.page.click(selector),
+    ]);
+
+    if (delayMs) {
+      await this.page.waitForTimeout(delayMs);
+    }
+
+    return `clicked ${selector} and waited ${delayMs}ms`;
+  }
+
+  async get_dom({ limit = 100000, exclude = true, focus = [] }, agentMemory = {}) {
+    let html;
+    try {
+      html = await this.page.content();
+    } catch (err) {
+      console.warn('⚠️ get_dom retrying due to navigation:', err.message);
+      // Wait for navigation to settle
+      await this.page.waitForLoadState('networkidle', { timeout: 5000 }).catch(() => {});
+      html = await this.page.content();
+    }
+    let $ = cheerio.load(html);
+    
+    // Remove non-content tags
+    if (exclude) {
+      $ = await removeNonContentTags($);
+      // console.log('🔧 Non-content tags removed from DOM.');
+      $ = await reduceRepeatedDivs($);
+      // console.log('🔧 Repeated divs reduced in DOM.');
+      // $ = await removeObfuscatedClassNames($);
+      // console.log('🔧 Obfuscated class names removed from DOM.')
+      // $ = await removeAutoGeneratedClasses($);
+      // console.log('🔧 Auto-generated classes removed from DOM.');
+    }
+    // console.log('🔧 DOM cleaned up:', $);
+    const finalHtml = $.html();
+    const trimmed = finalHtml.slice(0, limit);
+    return trimmed; 
+  }
+  
+  async check_text({ text }) {
+    const html = await this.page.content();
+    return html.includes(text) ? `FOUND: ${text}` : `NOT FOUND: ${text}`;
+  }
+
+  async click_text({ text }) {
+    console.log("attempting to click text with normal click:", text);
+
+    const elements = await this.page.$$('button, a, span, div, li, p'); // expand as needed
+    for (const el of elements) {
+      const innerText = await el.innerText().catch(() => '');
+      if (innerText.includes(text)) {
+        await el.scrollIntoViewIfNeeded().catch(() => {});
+        await el.click();
+        await this.page.screenshot({ path: `screenshot_after_click_${Date.now()}.png` });
+        await this.page.waitForTimeout(5000);
+        await this.page.screenshot({ path: `screenshot_after_waiting_click_${Date.now()}.png` });
+        return `clicked text "${text}"`;
+        
+      }
+    }
+    throw new Error(`Element with text "${text}" not found`);
+  }
+
+  async click_text_xpath({ text, delayMs = 1000 }) {
+    console.log("attempting to click text using XPath:", text);
+    const candidates = await this.page.$x(`//*[normalize-space(text())="${text}"]`);
+    for (const el of candidates) {
+      const visible = await el.isVisible?.() ?? await this.page.evaluate(el => {
+        const rect = el.getBoundingClientRect();
+        return (
+          rect.width > 0 &&
+          rect.height > 0 &&
+          window.getComputedStyle(el).visibility !== 'hidden' &&
+          window.getComputedStyle(el).display !== 'none'
+        );
+      }, el);
+  
+      if (visible) {
+        const box = await el.boundingBox();
+        console.log(`[click_text] → Clicking visible element with bounding box`, box);
+        await el.click();
+        await this.page.waitForTimeout(delayMs);
+        await this.page.screenshot({ path: `screenshot_after_click_${Date.now()}.png` });
+        return `clicked visible text "${text}"`;
+      }
+    }
+  
+    throw new Error(`Text "${text}" not found or not visible`);
+  }
+  
+
+  // async get_dom_focus_hint({ prompt }) {
+  //   const lower = prompt.toLowerCase();
+  //   const focus = [];
+  
+  //   if (lower.includes('logout') || lower.includes('sign out')) {
+  //     focus.push('header', 'nav', 'footer');
+  //   } else if (lower.includes('login') || lower.includes('sign in')) {
+  //     focus.push('main', 'header');
+  //   } else if (lower.includes('settings')) {
+  //     focus.push('nav', 'aside', 'main');
+  //   }
+  
+  //   return { focus };
+  // }
+
+  async expand_menu({ selector, delayMs = 1000 }) {
+    await this.page.waitForSelector(selector, { timeout: 30_000 });
+    await this.page.click(selector);
+    await this.page.waitForTimeout(delayMs);
+
+    const expandedDom = await this.page.content();
+    const filename = `expand_menu_dump_${Date.now()}.html`;
+    fs.writeFileSync(filename, expandedDom);
+    console.log(`[expand_menu] → Dumped DOM after expand click to ${filename}`);
+    await this.page.screenshot({ path: `screenshot_expand_menu_${Date.now()}.png` });
+
+    return `expanded menu using ${selector}`;
+  }
+
+  
+  
+  async close() {
+    await this.browser?.close();
+  }
+}
+
+export const CHROME_TOOL_MAP = {
+  navigate: (b, args) => b.navigate(args),
+  fill: (b, args) => b.fill(args),
+  click: (b, args) => b.click(args),
+  get_dom: (b, args, agentMemory) => b.get_dom(args, agentMemory),
+  check_text: (b, args) => b.check_text(args),
+  click_text: async (browser, args, agentMemory) => {
+    console.log(`[click_text] selecting click type: lastMenuExpanded=${agentMemory.lastMenuExpanded}`);
+    if (agentMemory.lastMenuExpanded) {
+      return await browser.click_text_xpath(args);
+    } else {
+      return await browser.click_text(args);
+    }
+  },
+  
+  expand_menu: (b, args) => b.expand_menu(args),
+  // click_text: (b, args) => b.click_text(args),
+};

package/tools/domControl.js

@@ -0,0 +1,170 @@
+export const removeNonContentTags = async ($) => {
+  $('script, style, meta, link, noscript, iframe, canvas, svg').remove();
+  
+  $('*').each((_, el) => {
+    const $el = $(el);
+    const allowedAttrs = [
+      'href', 'src', 'type', 'alt', 'title',
+      'placeholder', 'name', 'id', 'class', 'aria-label', 'value'
+    ];
+
+    const attrs = el.attribs;
+    // const tagName = $el.get(0)?.tagName?.toLowerCase();
+    // const keepEmpty = ['input', 'button', 'textarea', 'select'];
+
+    // if (!keepEmpty.includes(tagName) && !$el.text().trim() && $el.children().length === 0) {
+    //   $el.remove();
+    // }
+
+    for (const attr in attrs) {
+      if (!allowedAttrs.includes(attr)) {
+        $el.removeAttr(attr);
+      }
+    }
+  });
+
+  $('*')
+    .contents()
+    .each(function () {
+      if (this.type === 'comment') {
+        $(this).remove();
+      }
+    });
+
+  return $;
+}
+
+export const collapseLongContent = async ($, limit = 10) => {
+  $('ul, ol').each((_, el) => {
+    const $el = $(el);
+    const items = $el.children('li');
+
+    if (items.length > limit) {
+      items.slice(limit).remove();
+      $el.append('<li data-collapsed="true">[...truncated]</li>');
+    }
+  });
+
+  $('div, section').each((_, el) => {
+    const $el = $(el);
+    const children = $el.children();
+
+    if (children.length > limit) {
+      children.slice(limit).remove();
+      $el.append('<div data-collapsed="true">[...truncated]</div>');
+    }
+  });
+}
+
+export const chunkBySection = async ($, focus) => {
+  const chunks = {
+    header: $('header').html() || '',
+    nav: $('nav').html() || '',
+    main: $('main').html() || '',
+    footer: $('footer').html() || '',
+    aside: $('aside').html() || '',
+    body: $('body').html() || ''
+  };
+
+  if (Array.isArray(focus) && focus.length > 0) {
+    return Object.fromEntries(
+      Object.entries(chunks).filter(
+        ([k]) => focus.includes(k)
+      )
+    );
+  }
+
+  return chunks;
+
+  // let focusedChunks = Object.entries(chunks)
+  //   .filter(([key]) => focus?.length === 0 || focus?.includes(key))
+  //   .map(([, html]) => html)
+  //   .join('\n');
+
+  // return focusedChunks;
+
+}
+
+export function reduceRepeatedDivs($, maxAllowed = 3) {
+  $('body, div').each((_, parentEl) => {
+    const $parent = $(parentEl);
+    const children = $parent.children('div');
+
+    // Group children divs by class name
+    const classGroups = {};
+    children.each((_, child) => {
+      const $child = $(child);
+      const className = $child.attr('class');
+      if (!className) return;
+      
+      if (!classGroups[className]) {
+        classGroups[className] = [];
+      }
+      classGroups[className].push(child);
+    });
+
+    // Reduce each group to maxAllowed
+    for (const className in classGroups) {
+      const group = classGroups[className];
+      if (group.length > maxAllowed) {
+        // Remove excess nodes from the DOM
+        group.slice(maxAllowed).forEach(el => $(el).remove());
+        $parent.append(`<div data-collapsed="true" data-class="${className}">[...truncated]</div>`);
+      }
+    }
+  });
+  return $;
+}
+
+/**
+ * Remove obfuscated or hashed class names from a DOM string.
+ * Criteria: class names with high entropy (random letters/numbers), or very short length
+ */
+export function removeObfuscatedClassNames($) {
+  $('[class]').each((_, el) => {
+    const $el = $(el);
+    const originalClasses = ($el.attr('class') || '').split(/\s+/);
+    const filtered = originalClasses.filter(cls => {
+      const isShort = cls.length <= 2;
+      const isHashed = /^[a-z0-9_-]{6,}$/.test(cls) && /[0-9]/.test(cls);
+      const isObviousHash = /^[A-Za-z0-9_-]{6,}$/.test(cls) && /[A-Z]/.test(cls) && /[0-9]/.test(cls);
+      return !(isShort || isHashed || isObviousHash);
+    });
+    if (filtered.length) {
+      $el.attr('class', filtered.join(' '));
+    } else {
+      $el.removeAttr('class');
+    }
+  });
+
+  return $;
+}
+
+/**
+ * Remove known auto-generated or utility classes (e.g., Tailwind/MUI/CSS-in-JS)
+ */
+export function removeAutoGeneratedClasses($) {
+  // Regex patterns for utility classes, internal prefixes, and other noise
+  const utilityClassPatterns = [
+    /^text-/, /^bg-/, /^p[trblxy]?-\d+/, /^m[trblxy]?-\d+/, /^w-/, /^h-/, /^rounded/, /^border/, /^hover:/,
+    /^focus:/, /^group/, /^peer/, /^ring/, /^cl-/, /^md:/, /^sm:/, /^lg:/, /^xl:/, /^2xl:/, /^z-\d+/,
+    /^translate/, /^flex/, /^grid/, /^justify-/, /^items-/, /^gap-/, /^shadow/, /^overflow/, /^truncate/,
+    /^relative$/, /^absolute$/, /^inset/, /^top/, /^bottom/, /^left/, /^right/,
+    /^opacity-/, /^transition/, /^duration-/, /^ease-/, /^cursor-/, /^select-/, /^pointer-events-/,
+  ];
+
+  $('[class]').each((_, el) => {
+    const $el = $(el);
+    const classes = ($el.attr('class') || '').split(/\s+/);
+    const filtered = classes.filter(cls => {
+      return !utilityClassPatterns.some(pattern => pattern.test(cls));
+    });
+    if (filtered.length) {
+      $el.attr('class', filtered.join(' '));
+    } else {
+      $el.removeAttr('class');
+    }
+  });
+
+  return $;
+}

package/tools/tokenControl.js

@@ -0,0 +1,62 @@
+import { encoding_for_model } from '@dqbd/tiktoken';
+import { wait } from './turnLoopUtils.js';
+
+const TOKEN_BACKOFF_MS = 60000;
+const TOKEN_LIMIT_PER_MIN = 90000;
+
+export const tokenEstimate = async (model, text) => {
+  const encoding = encoding_for_model(model);
+  const tokenCount = encoding.encode(text).length;
+  console.log(`🧠 Estimated token count: ${tokenCount}`);
+  return tokenCount;
+}
+
+export const tokenUseCoolOff = async (totalTokensUsed, turnTimestamps) => {
+  console.log("total tokens used: ", totalTokensUsed);
+  console.log("turnTimestamps: ", turnTimestamps);
+  
+  if (totalTokensUsed > TOKEN_LIMIT_PER_MIN) {
+    const msToWait = await getDynamicBackoffMs(turnTimestamps);
+    console.warn(`⚠️ Token throttle risk → Waiting ${msToWait / 1000}s to cool off...`);
+    await wait(msToWait);
+    // totalTokensUsed = 0; // Reset token count after backoff
+    // turnTimestamps = []; // Reset timestamps after backoff
+    console.log('✅ Backoff complete, resuming...');
+    // return true;
+    return { shouldBackoff: true, totalTokensUsed: 0, turnTimestamps: [] };
+  }
+  // return false;
+  return { shouldBackoff: false, totalTokensUsed, turnTimestamps };
+}
+
+export const recordTokenUsage = (turnTimestamps, tokensUsed) => {
+  const now = Date.now();
+  turnTimestamps.push([now, tokensUsed]);
+};
+
+export const pruneOldTokenUsage = (turnTimestamps, windowMs = 60000) => {
+  const cutoff = Date.now() - windowMs;
+  const recentEntries = turnTimestamps.filter(([timestamp]) => timestamp > cutoff);
+  const totalTokensUsed = recentEntries.reduce((acc, [, tokens]) => acc + tokens, 0);
+  return { turnTimestamps: recentEntries, totalTokensUsed };
+};
+
+const getDynamicBackoffMs = async (turnTimestamps, tokenLimit = TOKEN_LIMIT_PER_MIN) => {
+  const now = Date.now();
+  let runningTotal = 0;
+
+  // Sort oldest → newest to find the earliest point to fall below the limit
+  const sorted = [...turnTimestamps].sort((a, b) => a[0] - b[0]);
+
+  for (let i = 0; i < sorted.length; i++) {
+    runningTotal += sorted[i][1]; // add tokensUsed
+
+    if (runningTotal > tokenLimit) {
+      const [timestampOfExcess] = sorted[i];
+      const msUntilSafe = 60000 - (now - timestampOfExcess);
+      return Math.max(msUntilSafe, 1000); // wait at least 1s
+    }
+  }
+
+  return 0; // Safe to proceed
+}

package/tools/toolSchema.js

@@ -0,0 +1,162 @@
+const toolsSchema = [
+  {
+    type: 'function',
+    function: {
+      name: 'navigate',
+      description: 'Navigate to a URL',
+      parameters: {
+        type: 'object',
+        properties: {
+          url: { type: 'string' },
+        },
+        required: ['url'],
+      },
+    },
+  },
+  {
+    type: 'function',
+    function: {
+      name: 'fill',
+      description: 'Fill in a field by CSS selector',
+      parameters: {
+        type: 'object',
+        properties: {
+          selector: { type: 'string' },
+          text: { type: 'string' },
+        },
+        required: ['selector', 'text'],
+      },
+    },
+  },
+  {
+    type: 'function',
+    function: {
+      name: 'click',
+      description: 'Click a button or link by CSS selector',
+      parameters: {
+        type: 'object',
+        properties: {
+          selector: { type: 'string' },
+          waitFor: {
+            type: 'string',
+            enum: ['load', 'domcontentloaded', 'networkidle'],
+            default: 'domcontentloaded',
+          },
+          delayMs: {
+            type: 'number',
+            default: 2000,
+            description: 'Milliseconds to wait after click',
+          },
+        },
+        required: ['selector'],
+      },
+    },
+  },
+  {
+    type: 'function',
+    function: {
+      name: 'get_dom',
+      description: 'Return trimmed HTML from the current page',
+      parameters: {
+        type: 'object',
+        properties: {
+          limit: {
+            type: 'number',
+            default: 100000,
+            description: 'Max characters to return',
+          },
+          exclude: {
+            type: 'boolean',
+            default: true,
+            description: 'Whether to exclude noisy tags like script/style/etc.',
+          },
+        },
+      },
+    },
+  },
+  {
+    type: 'function',
+    function: {
+      name: 'check_text',
+      description: 'Search page HTML for specific text',
+      parameters: {
+        type: 'object',
+        properties: {
+          text: { type: 'string' },
+        },
+        required: ['text'],
+      },
+    },
+  },
+  {
+    type: 'function',
+    function: {
+      name: 'expand_menu',
+      description: 'Click an element (like a hamburger menu or profile icon) to reveal a dropdown or hidden menu',
+      parameters: {
+        type: 'object',
+        properties: {
+          selector: { type: 'string' },
+          delayMs: {
+            type: 'number',
+            default: 1000,
+            description: 'Optional delay after expanding menu (in ms)',
+          }
+        },
+        required: ['selector'],
+      },
+    },
+  },
+  {
+    type: 'function',
+    function: {
+      name: 'click_text',
+      description: 'Clicks an element by visible text content, useful for buttons or links when selector is unknown',
+      parameters: {
+        type: 'object',
+        properties: {
+          text: { 
+            type: 'string',
+            description: 'Visible text to search for and click on the page' 
+          }
+        },
+        required: ['text']
+      }
+    }
+  },
+  // {
+  //   type: 'function',
+  //   function: {
+  //     name: 'get_dom_focus_hint',
+  //     description: 'Returns recommended DOM sections (e.g., "header", "main", "footer") to prioritize for the current task.',
+  //     parameters: {
+  //       type: 'object',
+  //       properties: {
+  //         prompt: {
+  //           type: 'string',
+  //           description: 'The current user instruction or task description'
+  //         }
+  //       },
+  //       required: ['prompt']
+  //     }
+  //   }
+  // },
+  {
+    type: 'function',
+    function: {
+      name: 'get_dom_chunk',
+      description: 'Return a specific chunk of the DOM by index',
+      parameters: {
+        type: 'object',
+        properties: {
+          chunkIndex: { type: 'number' },
+          totalChunks: { type: 'number' }
+        },
+        required: ['chunkIndex', 'totalChunks']
+      }
+    }
+  }
+    
+];
+
+export default toolsSchema;

package/tools/turnLoopUtils.js

@@ -0,0 +1,63 @@
+export const finalResponseHandler = (msg) => {
+  const final = msg.content?.trim().toLowerCase();
+  if (final?.startsWith('success')) {
+    console.log('\n┏━ FINAL AGENT RESPONSE ━━━━━━━━━━━━━━━━━━━');
+    console.log(msg.content);
+    console.log('┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━');
+    return true;
+  }
+  if (final?.startsWith('failure')) {
+    console.log('\n┏━ FINAL AGENT RESPONSE ━━━━━━━━━━━━━━━━━━━');
+    console.log(msg.content);
+    console.log('┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━');
+    return false;
+  }
+  return null;
+}
+
+export const wait = (ms) => new Promise(res => setTimeout(res, ms));
+
+export function validateAndInsertMissingToolResponses(messages, options = {}) {
+  const { insertPlaceholders = true } = options;
+  const missingResponses = [];
+
+  for (let i = 0; i < messages.length; i++) {
+    const msg = messages[i];
+    if (msg.role === 'assistant' && msg.tool_calls?.length) {
+      for (const toolCall of msg.tool_calls) {
+        const expectedId = toolCall.id;
+
+        const hasResponse = messages.slice(i + 1).some(
+          (nextMsg) =>
+            nextMsg.role === 'tool' &&
+            nextMsg.tool_call_id === expectedId
+        );
+
+        if (!hasResponse) {
+          missingResponses.push(expectedId);
+
+          if (insertPlaceholders) {
+            const placeholder = {
+              role: 'tool',
+              tool_call_id: expectedId,
+              name: toolCall.function.name,
+              type: 'function',
+              content: `[auto-inserted placeholder] No result returned for ${toolCall.function.name}`,
+            };
+
+            // Insert directly after the assistant tool call
+            messages.splice(i + 1, 0, placeholder);
+            console.warn(`⚠️ Inserted placeholder for missing tool_call_id: ${expectedId}`);
+          }
+        }
+      }
+    }
+  }
+
+  if (missingResponses.length > 0) {
+    console.warn(`⚠️ Missing tool responses for: ${missingResponses.join(', ')}`);
+    return insertPlaceholders;
+  }
+
+  return true;
+}