testronaut 1.3.1 → 1.3.2

package/tools/chromeBrowser.js

@@ -41,7 +41,7 @@ function appendFileEvent(eventObj = {}) {
 }
 
 export class ChromeBrowser {
-  constructor() {
+  constructor(options = {}) {
     this.browser = null;
     this.context = null;
     this.page = null;
@@ -49,6 +49,32 @@ export class ChromeBrowser {
     // simple registry
     this.pages = [];      // [{ id, page }]
     this.currentId = '';  // page id we currently control
+
+    this.domListLimit = options.domListLimit;
+    this.debugEnabled = !!options.debug;
+    this._domLimitLogged = false;
+
+    const defaultResourceGuard = {
+      enabled: true,
+      hrefIncludes: ['/document/', '/file/', '/download', '/attachment/'],
+      dataTypes: ['document', 'file', 'item', 'row'],
+    };
+    const guardCfg = options.resourceGuard || defaultResourceGuard;
+    this.resourceGuard = {
+      enabled: guardCfg.enabled !== false,
+      hrefIncludes: Array.isArray(guardCfg.hrefIncludes)
+        ? guardCfg.hrefIncludes.map(s => s.toLowerCase())
+        : defaultResourceGuard.hrefIncludes,
+      dataTypes: Array.isArray(guardCfg.dataTypes)
+        ? guardCfg.dataTypes.map(s => s.toLowerCase())
+        : defaultResourceGuard.dataTypes,
+    };
+  }
+
+  _debugLog(...args) {
+    if (this.debugEnabled) {
+      console.log('[debug]', ...args);
+    }
   }
 
   _makePageId(page) {
@@ -791,6 +817,57 @@ export class ChromeBrowser {
     { limit = 100000, exclude = true, focus = [], loadState = 'networkidle', timeout = 10000 } = {},
     agentMemory = {}
   ) {
+    const matchesHrefPattern = (href = '') => {
+      const lower = (href || '').toLowerCase();
+      return this.resourceGuard.hrefIncludes.some(p => lower.includes(p));
+    };
+
+    const extractInitialDocumentList = (rawHtml) => {
+      const key = '"initialDocumentList":';
+      const idx = rawHtml.indexOf(key);
+      if (idx < 0) return [];
+      const start = rawHtml.indexOf('[', idx);
+      if (start < 0) return [];
+      let depth = 0;
+      let end = -1;
+      for (let i = start; i < rawHtml.length; i++) {
+        const ch = rawHtml[i];
+        if (ch === '[') depth++;
+        else if (ch === ']') depth--;
+        if (depth === 0) {
+          end = i;
+          break;
+        }
+      }
+      if (end === -1) return [];
+      const jsonSlice = rawHtml.slice(start, end + 1);
+      try {
+        const arr = JSON.parse(jsonSlice);
+        return Array.isArray(arr) ? arr : [];
+      } catch {
+        return [];
+      }
+    };
+
+    const normalizeListLimit = () => {
+      if (this.domListLimit === Infinity) return Infinity;
+      const asNum = Number(this.domListLimit);
+      if (Number.isFinite(asNum)) {
+        return Math.min(100, Math.max(0, asNum));
+      }
+      return 3; // current default behavior (keep first 3 matching items)
+    };
+    const listLimit = normalizeListLimit();
+    if (this.debugEnabled && !this._domLimitLogged) {
+      const pretty = listLimit === Infinity ? 'all' : listLimit;
+      this._debugLog(`DOM list limit resolved`, { raw: this.domListLimit, applied: pretty });
+      this._domLimitLogged = true;
+    }
+    if (this.debugEnabled) {
+      const pretty = listLimit === Infinity ? 'all' : listLimit;
+      this._debugLog(`get_dom invoked`, { limit, exclude, focus, listLimit: pretty });
+    }
+
     // --- 1) Ensure we have a live page ---
     const hasContext = !!this.context;
 
@@ -865,7 +942,7 @@ export class ChromeBrowser {
 
     if (exclude) {
       $ = await removeNonContentTags($);
-      $ = await reduceRepeatedDivs($);
+      $ = await reduceRepeatedDivs($, listLimit);
       // If you later re-enable:
       // $ = await removeObfuscatedClassNames($);
       // $ = await removeAutoGeneratedClasses($);
@@ -874,6 +951,100 @@ export class ChromeBrowser {
     // Minify whitespace *after* structural cleanup
     $ = minifyDomWhitespace($);
 
+    if (this.debugEnabled) {
+      const liCount = $('li').length;
+      const ulCount = $('ul').length;
+      const ulSizes = [];
+      $('ul').each((_, el) => {
+        ulSizes.push($(el).children('li').length);
+      });
+      const maxUlSize = ulSizes.length ? Math.max(...ulSizes) : 0;
+      const anchorDocs = $('a[href]').filter((_, el) => {
+        if (!this.resourceGuard.enabled) return false;
+        const href = $(el).attr('href');
+        return matchesHrefPattern(href);
+      }).map((_, el) => {
+        const $el = $(el);
+        return {
+          id: $el.closest('[data-id]').attr('data-id') || $el.attr('id') || '',
+          title: ($el.attr('data-title') || $el.text() || '').trim().slice(0, 160),
+          href: $el.attr('href') || '',
+        };
+      }).get();
+      const docCards = $('.document-list-view-documents').map((_, el) => {
+        const $el = $(el);
+        const anchor = $el.find('a[href*="/document/"]').first();
+        return {
+          id: $el.attr('data-id') || $el.attr('id') || '',
+          title: ($el.attr('data-title') || anchor.text() || '').trim().slice(0, 160),
+          href: anchor.attr('href') || '',
+        };
+      }).get();
+      const dataDocs = $('[data-type]').filter((_, el) => {
+        if (!this.resourceGuard.enabled) return false;
+        const val = ($(el).attr('data-type') || '').toLowerCase();
+        return this.resourceGuard.dataTypes.includes(val);
+      }).map((_, el) => {
+        const $el = $(el);
+        const href = $el.find('a[href*="/document/"]').attr('href') || '';
+        return {
+          id: $el.attr('data-id') || $el.attr('id') || '',
+          title: ($el.attr('data-title') || '').trim().slice(0, 160),
+          href,
+        };
+      }).get();
+      const mergedDocs = [...anchorDocs, ...dataDocs, ...docCards]
+        .filter(d => d.href || d.title)
+        .map(d => ({
+          id: d.id || '',
+          title: d.title || '',
+          href: d.href || '',
+        }));
+      // dedupe by href+title to avoid spam
+      const seen = new Set();
+      const docItems = [];
+      for (const d of mergedDocs) {
+        const key = `${d.href}|${d.title}`;
+        if (seen.has(key)) continue;
+        seen.add(key);
+        docItems.push(d);
+      }
+      const sampleLi = $('li').slice(0, 5).map((_, el) => $(el).text().trim().slice(0, 80)).get();
+      // Fallback: parse the page's embedded initialDocumentList JSON if present
+      const scriptDocsRaw = extractInitialDocumentList(html);
+      const scriptDocs = scriptDocsRaw.map(d => ({
+        id: d.Id || d.id || '',
+        title: d.Title || d.title || '',
+        href: d.ContentUrl || d.ContentURL || '',
+      }));
+      for (const d of scriptDocs) {
+        const key = `${d.href}|${d.title}`;
+        if (seen.has(key)) continue;
+        seen.add(key);
+        docItems.push(d);
+      }
+      const docLinks = docItems.filter(d => d.href).length;
+      this._debugLog('DOM list stats', {
+        liCount,
+        ulCount,
+        maxUlSize,
+        docLinks,
+        docItems: docItems.length,
+        scriptDocs: scriptDocs.length,
+        anchorDocs: anchorDocs.length,
+        sampleLi,
+      });
+
+      // Provide the agent an explicit, structured list of resources to avoid stopping early.
+      if (this.resourceGuard.enabled && docItems.length) {
+        const summary = [
+          `Testronaut document list (${docItems.length} items):`,
+          ...docItems.map(d => `- [${d.id || 'n/a'}] ${d.title || ''} ${d.href || ''}`.trim()),
+        ].join('\n');
+        $('body').prepend(`<pre data-testronaut-doc-list="true">${summary}</pre>`);
+      }
+    }
+
     // (Optional: If you'd like to use `focus`, you could filter here by selectors/keywords.)
 
     // --- 5) Return trimmed HTML ---
@@ -1060,7 +1231,6 @@ export class ChromeBrowser {
     return JSON.stringify(payload);
   }
 
-
   async download_file({ selector, url, fileName, timeoutMs = 30000 }) {
     ensureDir(FILES_DIR);
 
@@ -1139,6 +1309,30 @@ export class ChromeBrowser {
     return JSON.stringify(payload);
   }
 
+  async list_local_files({ dir = 'missions/files', extensions = [] } = {}) {
+    const resolvedDir = path.resolve(process.cwd(), dir);
+    if (!fs.existsSync(resolvedDir)) {
+      throw new Error(`list_local_files: directory does not exist: ${resolvedDir}`);
+    }
+    const exts = Array.isArray(extensions)
+      ? extensions.map(e => e.toLowerCase())
+      : [];
+    const entries = fs.readdirSync(resolvedDir, { withFileTypes: true });
+    const files = entries
+      .filter(d => d.isFile())
+      .map(d => d.name)
+      .filter(name => {
+        if (!exts.length) return true;
+        const lower = name.toLowerCase();
+        return exts.some(ext => lower.endsWith(ext));
+      });
+    return {
+      dir: resolvedDir,
+      count: files.length,
+      files,
+    };
+  }
+
 
   async close() {
     await this.browser?.close();
@@ -1166,4 +1360,17 @@ export const CHROME_TOOL_MAP = {
   close_current_page: (b, args) => b.close_current_page(args),
   upload_file: (b, args) => b.upload_file(args),
   download_file: (b, args) => b.download_file(args),
-};
+  list_local_files: (b, args) => b.list_local_files(args),
+  resource_progress: (b, args, agentMemory) => {
+    const prog = agentMemory?.docProgress;
+    if (!prog) return JSON.stringify({ enabled: false, total: 0, downloaded: 0, remaining: [] });
+    const remaining = prog.items.filter(i => !prog.downloaded.has(i.id));
+    return JSON.stringify({
+      enabled: true,
+      total: prog.items.length || prog.lastScriptCount || 0,
+      downloaded: prog.downloaded.size || 0,
+      remaining: remaining.map(r => ({ id: r.id, title: r.title, href: r.href })).slice(0, 20),
+      patterns: prog.patterns,
+    });
+  },
+};

package/tools/domControl.js

@@ -37,7 +37,9 @@ export const removeNonContentTags = async ($) => {
     const $el = $(el);
     const allowedAttrs = [
       'href', 'src', 'type', 'alt', 'title',
-      'placeholder', 'name', 'id', 'class', 'aria-label', 'value'
+      'placeholder', 'name', 'id', 'class', 'aria-label', 'value',
+      // keep key data-* attributes that help identify rows/items without bloating too much
+      'data-type', 'data-id', 'data-title', 'data-parentid', 'data-order'
     ];
 
     const attrs = el.attribs;
@@ -155,6 +157,11 @@ export const chunkBySection = async ($, focus) => {
  * @returns {$} same Cheerio instance for chaining
  */
 export function reduceRepeatedDivs($, maxAllowed = 3) {
+  let limit = maxAllowed === Infinity ? Infinity : Number(maxAllowed);
+  if (limit !== Infinity) {
+    limit = Number.isFinite(limit) ? Math.max(0, limit) : 3;
+  }
+
   $('body, div').each((_, parentEl) => {
     const $parent = $(parentEl);
     const children = $parent.children('div');
@@ -175,9 +182,9 @@ export function reduceRepeatedDivs($, maxAllowed = 3) {
     // Reduce each group to maxAllowed
     for (const className in classGroups) {
       const group = classGroups[className];
-      if (group.length > maxAllowed) {
+      if (limit !== Infinity && group.length > limit) {
         // Remove excess nodes from the DOM
-        group.slice(maxAllowed).forEach(el => $(el).remove());
+        group.slice(limit).forEach(el => $(el).remove());
         $parent.append(`<div data-collapsed="true" data-class="${className}">[...truncated]</div>`);
       }
     }

package/tools/toolSchema.js

@@ -1,3 +1,16 @@
+/**
+ * toolsSchema.js
+ * --------------
+ * Canonical list of tool/function definitions exposed to the LLM.
+ * Keep descriptions concise but clear; they are surfaced directly in the
+ * function/tool-calling interface for the model.
+ *
+ * Notes:
+ * - When adding a new tool, include a short description and minimally
+ *   strict parameters schema to avoid over-constraining the model.
+ * - Resource helper `resource_progress` is a lightweight introspection tool
+ *   used to report remaining detected resources (documents/files/rows).
+ */
 const toolsSchema = [
   {
     type: 'function',
@@ -286,6 +299,35 @@ const toolsSchema = [
       }
     }
   },
+  {
+    type: 'function',
+    function: {
+      name: 'list_local_files',
+      description: 'List files from a local directory (defaults to missions/files). Use to discover downloaded PDFs before uploading.',
+      parameters: {
+        type: 'object',
+        properties: {
+          dir: { type: 'string', description: 'Directory relative to project root (default: missions/files)' },
+          extensions: {
+            type: 'array',
+            items: { type: 'string' },
+            description: 'Optional list of extensions to include (e.g., [".pdf"])'
+          }
+        }
+      }
+    }
+  },
+  {
+    type: 'function',
+    function: {
+      name: 'resource_progress',
+      description: 'Report remaining items from the detected resource list (documents/files/rows) to ensure full coverage.',
+      parameters: {
+        type: 'object',
+        properties: {},
+      },
+    },
+  },
   {
     type: 'function',
     function: {
@@ -389,4 +431,4 @@ const toolsSchema = [
   }
 ];
 
-export default toolsSchema;
+export default toolsSchema;