textweb 0.1.2 → 0.2.3

package/src/browser.js

@@ -3,6 +3,10 @@
  */
 
 const { chromium } = require('playwright');
+
+const DEFAULT_VIEWPORT = { width: 1280, height: 800 };
+const DEFAULT_USER_AGENT = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36';
+
 const { render } = require('./renderer');
 
 class AgentBrowser {
@@ -15,32 +19,75 @@ class AgentBrowser {
     this.lastResult = null;
     this.headless = options.headless !== false;
     this.charH = 16; // default, updated after first render
+    this.defaultTimeout = options.timeout || 30000;
+    this.defaultRetries = options.retries ?? 2;
+    this.defaultRetryDelayMs = options.retryDelayMs ?? 250;
   }
 
-  async launch() {
-    this.browser = await chromium.launch({
-      headless: this.headless,
-      args: ['--no-sandbox', '--disable-setuid-sandbox'],
-    });
-    this.context = await this.browser.newContext({
-      viewport: { width: 1280, height: 800 },
-      userAgent: 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
-    });
+  async _withRetries(actionName, fn, options = {}) {
+    const retries = options.retries ?? this.defaultRetries;
+    const retryDelayMs = options.retryDelayMs ?? this.defaultRetryDelayMs;
+
+    let lastError = null;
+    for (let attempt = 0; attempt <= retries; attempt++) {
+      try {
+        return await fn();
+      } catch (err) {
+        lastError = err;
+        if (attempt >= retries) break;
+        await new Promise(r => setTimeout(r, retryDelayMs));
+      }
+    }
+
+    throw new Error(`${actionName} failed after ${retries + 1} attempt(s): ${lastError?.message || 'unknown error'}`);
+  }
+
+  _contextOptions(storageStatePath = null) {
+    const opts = {
+      viewport: DEFAULT_VIEWPORT,
+      userAgent: DEFAULT_USER_AGENT,
+    };
+    if (storageStatePath) {
+      opts.storageState = storageStatePath;
+    }
+    return opts;
+  }
+
+  async _createContext(storageStatePath = null) {
+    this.context = await this.browser.newContext(this._contextOptions(storageStatePath));
     this.page = await this.context.newPage();
-    this.page.setDefaultTimeout(30000);
+    this.page.setDefaultTimeout(this.defaultTimeout);
+  }
+
+  async launch(options = {}) {
+    if (!this.browser) {
+      this.browser = await chromium.launch({
+        headless: this.headless,
+        args: ['--no-sandbox', '--disable-setuid-sandbox'],
+      });
+    }
+
+    if (!this.context) {
+      await this._createContext(options.storageStatePath || null);
+    }
+
     return this;
   }
 
-  async navigate(url) {
+  async navigate(url, options = {}) {
     if (!this.page) await this.launch();
     this.scrollY = 0;
-    // Use domcontentloaded + a short settle, not networkidle (SPAs never go idle)
-    await this.page.goto(url, { waitUntil: 'domcontentloaded', timeout: 30000 });
-    // Wait for network to settle or 3s max — whichever comes first
-    await Promise.race([
-      this.page.waitForLoadState('networkidle').catch(() => {}),
-      new Promise(r => setTimeout(r, 3000)),
-    ]);
+
+    await this._withRetries('navigate', async () => {
+      // Use domcontentloaded + a short settle, not networkidle (SPAs never go idle)
+      await this.page.goto(url, { waitUntil: 'domcontentloaded', timeout: options.timeoutMs || this.defaultTimeout });
+      // Wait for network to settle or 3s max — whichever comes first
+      await Promise.race([
+        this.page.waitForLoadState('networkidle').catch(() => {}),
+        new Promise(r => setTimeout(r, 3000)),
+      ]);
+    }, options);
+
     return await this.snapshot();
   }
 
@@ -57,17 +104,21 @@ class AgentBrowser {
     return this.lastResult;
   }
 
-  async click(ref) {
+  async click(ref, options = {}) {
     const el = this._getElement(ref);
-    await this.page.click(el.selector);
-    await this.page.waitForLoadState('networkidle').catch(() => {});
+    await this._withRetries(`click ref=${ref}`, async () => {
+      await this.page.click(el.selector);
+      await this._settle();
+    }, options);
     return await this.snapshot();
   }
 
-  async type(ref, text) {
+  async type(ref, text, options = {}) {
     const el = this._getElement(ref);
-    await this.page.click(el.selector);
-    await this.page.fill(el.selector, text);
+    await this._withRetries(`type ref=${ref}`, async () => {
+      await this.page.click(el.selector);
+      await this.page.fill(el.selector, text);
+    }, options);
     return await this.snapshot();
   }
 
@@ -101,22 +152,28 @@ class AgentBrowser {
     await this.page.setInputFiles(selector, paths);
   }
 
-  async press(key) {
-    await this.page.keyboard.press(key);
-    await this.page.waitForLoadState('networkidle').catch(() => {});
+  async press(key, options = {}) {
+    await this._withRetries(`press key=${key}`, async () => {
+      await this.page.keyboard.press(key);
+      await this._settle();
+    }, options);
     return await this.snapshot();
   }
 
-  async upload(ref, filePaths) {
+  async upload(ref, filePaths, options = {}) {
     const el = this._getElement(ref);
     const paths = Array.isArray(filePaths) ? filePaths : [filePaths];
-    await this.page.setInputFiles(el.selector, paths);
+    await this._withRetries(`upload ref=${ref}`, async () => {
+      await this.page.setInputFiles(el.selector, paths);
+    }, options);
     return await this.snapshot();
   }
 
-  async select(ref, value) {
+  async select(ref, value, options = {}) {
     const el = this._getElement(ref);
-    await this.page.selectOption(el.selector, value);
+    await this._withRetries(`select ref=${ref}`, async () => {
+      await this.page.selectOption(el.selector, value);
+    }, options);
     return await this.snapshot();
   }
 
@@ -146,8 +203,148 @@ class AgentBrowser {
     return region.join('\n');
   }
 
-  async evaluate(fn) {
-    return await this.page.evaluate(fn);
+  async evaluate(fn, arg) {
+    return await this.page.evaluate(fn, arg);
+  }
+
+  /**
+   * Save cookies/localStorage/sessionStorage state to disk
+   */
+  async saveStorageState(path) {
+    if (!this.context) throw new Error('No browser context open.');
+    await this.context.storageState({ path });
+    return { saved: true, path };
+  }
+
+  /**
+   * Load cookies/localStorage/sessionStorage state from disk into a fresh context
+   */
+  async loadStorageState(path) {
+    if (!this.browser) {
+      await this.launch();
+    }
+
+    if (this.context) {
+      await this.context.close();
+      this.context = null;
+      this.page = null;
+    }
+
+    await this._createContext(path);
+    this.scrollY = 0;
+    this.lastResult = null;
+    return { loaded: true, path };
+  }
+
+  /**
+   * Wait until one or more conditions are true, then return a fresh snapshot.
+   * Supported conditions: selector, text, urlIncludes.
+   */
+  async waitFor(options = {}) {
+    if (!this.page) throw new Error('No page open. Call navigate() first.');
+
+    const timeout = options.timeoutMs || this.defaultTimeout;
+    const pollMs = options.pollMs || 100;
+
+    await this._withRetries('waitFor', async () => {
+      const waits = [];
+
+      if (options.selector) {
+        waits.push(
+          this.page.waitForSelector(options.selector, {
+            state: options.state || 'visible',
+            timeout,
+          })
+        );
+      }
+
+      if (options.text) {
+        waits.push(
+          this.page.waitForFunction(
+            (text) => document.body && document.body.innerText.includes(text),
+            options.text,
+            { timeout, polling: pollMs }
+          )
+        );
+      }
+
+      if (options.urlIncludes) {
+        waits.push(
+          this.page.waitForFunction(
+            (needle) => window.location.href.includes(needle),
+            options.urlIncludes,
+            { timeout, polling: pollMs }
+          )
+        );
+      }
+
+      if (!waits.length) {
+        await this.page.waitForTimeout(timeout);
+      } else {
+        await Promise.all(waits);
+      }
+    }, options);
+
+    await this._settle();
+    return await this.snapshot();
+  }
+
+  /**
+   * Assert a field's value/text by ref.
+   * comparator: equals | includes | regex | not_empty
+   */
+  async assertField(ref, expected, options = {}) {
+    if (!this.page) throw new Error('No page open. Call navigate() first.');
+    const el = this._getElement(ref);
+    const comparator = options.comparator || 'equals';
+    const attribute = options.attribute || null;
+
+    const actual = await this.page.evaluate(({ selector, attributeName }) => {
+      const target = document.querySelector(selector);
+      if (!target) return null;
+
+      if (attributeName) {
+        return target.getAttribute(attributeName);
+      }
+
+      const tag = (target.tagName || '').toLowerCase();
+      if (tag === 'input' || tag === 'textarea' || tag === 'select') {
+        return target.value ?? '';
+      }
+      return (target.textContent || '').trim();
+    }, { selector: el.selector, attributeName: attribute });
+
+    let pass = false;
+    const actualStr = actual == null ? '' : String(actual);
+    const expectedStr = expected == null ? '' : String(expected);
+
+    switch (comparator) {
+      case 'equals':
+        pass = actualStr === expectedStr;
+        break;
+      case 'includes':
+        pass = actualStr.includes(expectedStr);
+        break;
+      case 'regex': {
+        const re = new RegExp(expectedStr);
+        pass = re.test(actualStr);
+        break;
+      }
+      case 'not_empty':
+        pass = actualStr.trim().length > 0;
+        break;
+      default:
+        throw new Error(`Unknown comparator: ${comparator}`);
+    }
+
+    return {
+      pass,
+      ref,
+      selector: el.selector,
+      comparator,
+      expected: expectedStr,
+      actual: actualStr,
+    };
   }
 
   async close() {
@@ -159,6 +356,56 @@ class AgentBrowser {
     }
   }
 
+  /**
+   * Get the current page URL
+   */
+  getCurrentUrl() {
+    return this.page ? this.page.url() : null;
+  }
+
+  /**
+   * Find elements matching a CSS selector
+   * Returns array of {tag, text, selector, visible} objects
+   */
+  async query(selector) {
+    if (!this.page) throw new Error('No page open. Call navigate() first.');
+    return await this.page.evaluate((sel) => {
+      const els = document.querySelectorAll(sel);
+      return Array.from(els).map((el, i) => ({
+        tag: el.tagName.toLowerCase(),
+        text: (el.textContent || '').trim().substring(0, 200),
+        selector: `${sel}:nth-child(${i + 1})`,
+        visible: el.offsetParent !== null,
+        href: el.href || null,
+        value: el.value || null,
+      }));
+    }, selector);
+  }
+
+  /**
+   * Take a screenshot (for debugging)
+   * @param {object} options - Playwright screenshot options (path, fullPage, type, etc.)
+   */
+  async screenshot(options = {}) {
+    if (!this.page) throw new Error('No page open. Call navigate() first.');
+    return await this.page.screenshot({
+      fullPage: true,
+      type: 'png',
+      ...options,
+    });
+  }
+
+  /**
+   * Wait for page to settle after an interaction.
+   * Races networkidle against a short timeout to avoid hanging on SPAs.
+   */
+  async _settle() {
+    await Promise.race([
+      this.page.waitForLoadState('networkidle').catch(() => {}),
+      new Promise(r => setTimeout(r, 3000)),
+    ]);
+  }
+
   _getElement(ref) {
     if (!this.lastResult) throw new Error('No snapshot. Navigate first.');
     const el = this.lastResult.elements[ref];

package/src/cli.js

@@ -17,7 +17,6 @@ function parseArgs() {
     json: false,
     serve: false,
     cols: 100,
-    rows: 30,
     port: 3000,
     help: false
   };
@@ -48,7 +47,9 @@ function parseArgs() {
         
       case '--rows':
       case '-r':
-        options.rows = parseInt(args[++i]) || 30;
+        // Deprecated: height is dynamic (grows to fit content). Ignored.
+        console.error('Warning: --rows is deprecated. Height is dynamic (grows to fit content).');
+        args[++i]; // consume the value
         break;
         
       case '--port':
@@ -85,7 +86,7 @@ USAGE:
 
 OPTIONS:
   --cols, -c <number>                Grid width in characters (default: 100)
-  --rows, -r <number>                Grid height in characters (default: 30)
+  --rows, -r <number>                (deprecated, height is dynamic)
   --port, -p <number>                Server port (default: 3000)
   --interactive, -i                  Interactive REPL mode
   --json, -j                         JSON output format
@@ -117,7 +118,7 @@ INTERACTIVE COMMANDS:
 async function render(url, options) {
   const browser = new AgentBrowser({
     cols: options.cols,
-    rows: options.rows,
+
     headless: true
   });
 
@@ -156,7 +157,7 @@ async function render(url, options) {
 async function interactive(url, options) {
   const browser = new AgentBrowser({
     cols: options.cols,
-    rows: options.rows,
+
     headless: true
   });
 
@@ -367,7 +368,7 @@ async function serve(options) {
   
   const server = createServer({
     cols: options.cols,
-    rows: options.rows
+
   });
   
   server.listen(options.port, () => {

package/src/renderer.js

@@ -7,8 +7,8 @@
  * Key design decisions:
  * - Overflow > truncation (never lose information)
  * - Measure actual font metrics from the page
- * - Table-aware layout
- * - Z-index compositing (back to front)
+ * - Row-grouping layout (elements grouped by Y position)
+ * - Dynamic height (grows to fit all content)
  */
 
 /**
@@ -80,14 +80,48 @@ async function extractElements(page) {
 
     function buildSelector(el) {
       // Build a robust CSS selector for clicking
+      // Priority: id > data-testid > aria > role+name > name > positional
       if (el.id) return '#' + CSS.escape(el.id);
       
-      // Try unique attributes
-      if (el.getAttribute('data-testid')) return `[data-testid="${el.getAttribute('data-testid')}"]`;
-      if (el.getAttribute('name')) return `${el.tagName.toLowerCase()}[name="${el.getAttribute('name')}"]`;
-      
-      // Fallback: positional selector
       const tag = el.tagName.toLowerCase();
+      
+      // Stable test attributes (used by many frameworks)
+      for (const attr of ['data-testid', 'data-test', 'data-cy', 'data-test-id']) {
+        const val = el.getAttribute(attr);
+        if (val) return `[${attr}="${val}"]`;
+      }
+      
+      // Aria-label (very stable, set by developers intentionally)
+      const ariaLabel = el.getAttribute('aria-label');
+      if (ariaLabel) {
+        const sel = `${tag}[aria-label="${CSS.escape(ariaLabel)}"]`;
+        if (document.querySelectorAll(sel).length === 1) return sel;
+      }
+
+      // Role + name combination
+      const role = el.getAttribute('role');
+      if (role) {
+        const name = ariaLabel || el.textContent.trim().substring(0, 50);
+        if (name) {
+          const sel = `[role="${role}"]`;
+          // Only use if unique enough
+          if (document.querySelectorAll(sel).length === 1) return sel;
+        }
+      }
+
+      // Name attribute (forms)
+      if (el.getAttribute('name')) return `${tag}[name="${el.getAttribute('name')}"]`;
+      
+      // href for links (use partial match for stability)
+      if (tag === 'a' && el.href) {
+        const href = el.getAttribute('href');
+        if (href && !href.startsWith('javascript:') && href !== '#') {
+          const sel = `a[href="${CSS.escape(href)}"]`;
+          if (document.querySelectorAll(sel).length === 1) return sel;
+        }
+      }
+
+      // Fallback: positional selector (least stable)
       const parent = el.parentElement;
       if (!parent) return tag;
       const siblings = Array.from(parent.children);
@@ -277,44 +311,6 @@ async function extractElements(page) {
   });
 }
 
-/**
- * Place text onto the grid, allowing overflow (never truncate).
- * Text wraps to the next line at grid edge, continuing at the same start column.
- */
-function placeText(grid, zGrid, z, row, col, text, cols, rows) {
-  let r = row;
-  let c = col;
-
-  for (let i = 0; i < text.length; i++) {
-    // Grow grid vertically if needed (overflow — don't lose data)
-    while (r >= grid.length) {
-      grid.push(Array(cols).fill(' '));
-      zGrid.push(Array(cols).fill(-1));
-    }
-
-    if (c >= cols) {
-      // Wrap to next line at original column position
-      r++;
-      c = col;
-      while (r >= grid.length) {
-        grid.push(Array(cols).fill(' '));
-        zGrid.push(Array(cols).fill(-1));
-      }
-    }
-
-    const ch = text[i];
-    if (ch === '\n') { r++; c = col; continue; }
-
-    if (c >= 0 && c < cols && z >= zGrid[r][c]) {
-      grid[r][c] = ch;
-      zGrid[r][c] = z;
-    }
-    c++;
-  }
-
-  return r; // Return last row written to (useful for tracking grid growth)
-}
-
 /**
  * Detect row boundaries — groups of elements that share the same Y position
  * This prevents text from different elements on the same visual line from overlapping
@@ -464,6 +460,7 @@ function renderGrid(elements, cols, charW, charH, scrollY = 0) {
  */
 async function render(page, options = {}) {
   const { cols = 120, scrollY = 0 } = options;
+  const startMs = Date.now();
   
   // Measure actual font metrics from the page
   const metrics = await measureCharSize(page);
@@ -471,7 +468,16 @@ async function render(page, options = {}) {
   const charH = metrics.charH;
   
   const elements = await extractElements(page);
-  return renderGrid(elements, cols, charW, charH, scrollY);
+  const result = renderGrid(elements, cols, charW, charH, scrollY);
+  
+  // Add stats to meta
+  result.meta.stats = {
+    totalElements: elements.length,
+    interactiveElements: result.meta.totalRefs,
+    renderMs: Date.now() - startMs,
+  };
+  
+  return result;
 }
 
 module.exports = { render, extractElements, renderGrid, measureCharSize };

package/src/server.js

@@ -10,7 +10,7 @@ class TextWebServer {
   constructor(options = {}) {
     this.options = {
       cols: options.cols || 100,
-      rows: options.rows || 30,
+      // rows is deprecated — height is dynamic
       timeout: options.timeout || 30000,
       ...options
     };
@@ -33,7 +33,6 @@ class TextWebServer {
     if (!this.browser) {
       this.browser = new AgentBrowser({
         cols: this.options.cols,
-        rows: this.options.rows,
         headless: true,
         timeout: this.options.timeout
       });