textweb 0.2.0 → 0.2.4

package/README.md

@@ -92,6 +92,12 @@ npx textweb-mcp
 
 Then just ask: *"Go to hacker news and find posts about AI"* — the agent uses text grids instead of screenshots.
 
+**New (v0.2.1-style MCP capabilities):**
+- `session_id` on every tool call for isolated parallel workflows
+- `textweb_storage_save` / `textweb_storage_load` for persistent auth/session state
+- `textweb_wait_for` for multi-step async UI transitions
+- `textweb_assert_field` for flow guards before submit
+
 ### 🛠️ OpenAI / Anthropic Function Calling
 
 Drop-in tool definitions for any function-calling model. See [`tools/tool_definitions.json`](tools/tool_definitions.json).
@@ -174,10 +180,18 @@ const { view, elements, meta } = await browser.navigate('https://example.com');
 
 console.log(view);        // The text grid
 console.log(elements);    // { 0: { selector, tag, text, href }, ... }
+console.log(meta.stats);  // { totalElements, interactiveElements, renderMs }
 
 await browser.click(3);              // Click element [3]
 await browser.type(7, 'hello');      // Type into element [7]
 await browser.scroll('down');        // Scroll down
+await browser.waitFor({ selector: '.step-2.active' }); // Wait for next step
+await browser.assertField(7, 'hello', { comparator: 'equals' }); // Validate field state
+await browser.saveStorageState('/tmp/textweb-state.json');
+await browser.loadStorageState('/tmp/textweb-state.json');
+await browser.query('nav a');        // Find elements by CSS selector
+await browser.screenshot();          // PNG buffer (for debugging)
+console.log(browser.getCurrentUrl());// Current page URL
 await browser.close();
 ```
 
@@ -218,6 +232,67 @@ await browser.close();
 3. **Map** pixel coordinates to character grid positions (spatial layout preserved)
 4. **Annotate** interactive elements with `[ref]` numbers for agent interaction
 
+## Selector Strategy
+
+TextWeb builds stable CSS selectors for each interactive element, preferring resilient strategies over brittle positional ones:
+
+| Priority | Strategy | Example |
+|----------|----------|---------|
+| 1 | `#id` | `#email` |
+| 2 | `[data-testid]` | `[data-testid="submit-btn"]` |
+| 3 | `[aria-label]` | `input[aria-label="Search"]` |
+| 4 | `[role]` (if unique) | `[role="navigation"]` |
+| 5 | `[name]` | `input[name="email"]` |
+| 6 | `a[href]` (if unique) | `a[href="/about"]` |
+| 7 | `nth-child` (fallback) | `div > a:nth-child(3)` |
+
+This means selectors survive DOM changes between snapshots — critical for multi-step agent workflows.
+
+## ATS Workflow Examples (Greenhouse / Lever)
+
+For multi-step ATS flows, use a stable `session_id` and combine wait/assert guards:
+
+```javascript
+// Keep one session for the whole application
+await textweb_navigate({ url: 'https://job-boards.greenhouse.io/acme/jobs/123', session_id: 'apply-acme' });
+
+// Fill + continue
+await textweb_type({ ref: 12, text: 'Christopher', session_id: 'apply-acme' });
+await textweb_type({ ref: 15, text: 'Robison', session_id: 'apply-acme' });
+await textweb_click({ ref: 42, session_id: 'apply-acme', retries: 3, retry_delay_ms: 400 });
+
+// Guard transition
+await textweb_wait_for({ selector: '#step-2.active', timeout_ms: 8000, session_id: 'apply-acme', retries: 2 });
+
+// Validate before submit
+await textweb_assert_field({ ref: 77, expected: 'San Francisco', comparator: 'includes', session_id: 'apply-acme' });
+
+// Persist auth/session for follow-up flow
+await textweb_storage_save({ path: '/tmp/ats-state.json', session_id: 'apply-acme' });
+```
+
+Useful session tools:
+- `textweb_session_list` → inspect active sessions
+- `textweb_session_close` → close one session or all
+
+## Testing
+
+```bash
+# Run all tests (form + live + ATS e2e)
+npm test
+
+# Form fixture tests
+npm run test:form
+
+# Live site tests — example.com, HN, Wikipedia
+npm run test:live
+
+# ATS multi-step fixture test
+npm run test:ats
+```
+
+Test fixtures are in `test/fixtures/` — includes a comprehensive HTML form and an ATS-style multi-step application fixture.
+
 ## Design Principles
 
 1. **Text is native to LLMs** — no vision model middleman

package/logo.svg

@@ -0,0 +1,30 @@
+<?xml version="1.0" encoding="utf-8"?>
+<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 425.697 360.796" xmlns:bx="https://boxy-svg.com">
+  <defs>
+    <style>
+      @keyframes blink {
+        0%, 100% { opacity: 1; }
+        50%       { opacity: 0; }
+      }
+      .cursor {
+        animation: blink 1.1s step-start infinite;
+      }
+    </style>
+    <filter id="ds" x="-20%" y="-20%" width="140%" height="140%">
+      <feDropShadow dx="0" dy="10" stdDeviation="12" flood-color="#000" flood-opacity="0.35"/>
+    </filter>
+    <linearGradient id="g" x1="0" y1="0" x2="1" y2="1">
+      <stop offset="0" stop-color="var(--bg)"/>
+      <stop offset="1" stop-color="var(--bg2)"/>
+    </linearGradient>
+    <bx:export>
+      <bx:file format="png"/>
+    </bx:export>
+  </defs>
+  <rect x="6.482" y="11.817" width="400" height="320" rx="72" ry="72" fill="url(#g)" filter="url(#ds)" style="stroke-width: 1;"/>
+  <rect x="26.482" y="31.817" width="360" height="280" rx="58" ry="58" fill="none" stroke-width="4" style="stroke-width: 4; stroke: rgba(255, 255, 255, 0.255);"/>
+  <g class="mono" fill="var(--green)" style="font-family: ui-monospace, SFMono-Regular, Menlo, Monaco, Consolas, &quot;Liberation Mono&quot;, monospace;" transform="matrix(1, 0, 0, 1, -75.701332, -120.82827)">
+    <text style="fill: rgb(0, 201, 22); font-size: 56px; font-weight: 700; letter-spacing: 0.5px; white-space: pre;" x="112.671" y="313.449">&gt; textweb<tspan class="cursor">_</tspan><tspan x="112.6709976196289" dy="1em">​</tspan></text>
+  </g>
+  <circle cx="90.479" cy="155.817" r="70" fill="var(--green)" opacity="0.05" style="stroke-width: 1;"/>
+</svg>

package/mcp/index.js

@@ -2,21 +2,24 @@
 
 /**
  * TextWeb MCP Server
- * 
+ *
  * Model Context Protocol server that gives any MCP client
  * (Claude Desktop, Cursor, Windsurf, Cline, OpenClaw, etc.)
  * text-based web browsing capabilities.
- * 
+ *
  * Communicates over stdio using JSON-RPC 2.0.
  */
 
 const { AgentBrowser } = require('../src/browser');
+const { ensureBrowser } = require('../src/ensure-browser');
 
 const SERVER_INFO = {
   name: 'textweb',
-  version: '0.1.0',
+  version: '0.2.2',
 };
 
+const SESSION_NOTE = 'Optional session_id to isolate state across flows. Defaults to "default".';
+
 const TOOLS = [
   {
     name: 'textweb_navigate',
@@ -26,6 +29,9 @@ const TOOLS = [
       properties: {
         url: { type: 'string', description: 'The URL to navigate to' },
         cols: { type: 'number', description: 'Grid width in characters (default: 120)' },
+        session_id: { type: 'string', description: SESSION_NOTE },
+        retries: { type: 'number', description: 'Retry attempts for flaky transitions' },
+        retry_delay_ms: { type: 'number', description: 'Delay between retries in ms' },
       },
       required: ['url'],
     },
@@ -37,6 +43,9 @@ const TOOLS = [
       type: 'object',
       properties: {
         ref: { type: 'number', description: 'Element reference number from the text grid (e.g., 3 for [3])' },
+        session_id: { type: 'string', description: SESSION_NOTE },
+        retries: { type: 'number', description: 'Retry attempts for flaky transitions' },
+        retry_delay_ms: { type: 'number', description: 'Delay between retries in ms' },
       },
       required: ['ref'],
     },
@@ -49,6 +58,9 @@ const TOOLS = [
       properties: {
         ref: { type: 'number', description: 'Element reference number of the input field' },
         text: { type: 'string', description: 'Text to type into the field' },
+        session_id: { type: 'string', description: SESSION_NOTE },
+        retries: { type: 'number', description: 'Retry attempts for flaky transitions' },
+        retry_delay_ms: { type: 'number', description: 'Delay between retries in ms' },
       },
       required: ['ref', 'text'],
     },
@@ -61,6 +73,9 @@ const TOOLS = [
       properties: {
         ref: { type: 'number', description: 'Element reference number of the select/dropdown' },
         value: { type: 'string', description: 'Value or visible text of the option to select' },
+        session_id: { type: 'string', description: SESSION_NOTE },
+        retries: { type: 'number', description: 'Retry attempts for flaky transitions' },
+        retry_delay_ms: { type: 'number', description: 'Delay between retries in ms' },
       },
       required: ['ref', 'value'],
     },
@@ -73,6 +88,7 @@ const TOOLS = [
       properties: {
         direction: { type: 'string', enum: ['up', 'down', 'top'], description: 'Scroll direction' },
         amount: { type: 'number', description: 'Number of pages to scroll (default: 1)' },
+        session_id: { type: 'string', description: SESSION_NOTE },
       },
       required: ['direction'],
     },
@@ -82,7 +98,9 @@ const TOOLS = [
     description: 'Re-render the current page as a text grid without navigating. Useful after waiting for dynamic content to load.',
     inputSchema: {
       type: 'object',
-      properties: {},
+      properties: {
+        session_id: { type: 'string', description: SESSION_NOTE },
+      },
     },
   },
   {
@@ -92,10 +110,32 @@ const TOOLS = [
       type: 'object',
       properties: {
         key: { type: 'string', description: 'Key to press (e.g., "Enter", "Tab", "Escape", "ArrowDown")' },
+        session_id: { type: 'string', description: SESSION_NOTE },
+        retries: { type: 'number', description: 'Retry attempts for flaky transitions' },
+        retry_delay_ms: { type: 'number', description: 'Delay between retries in ms' },
       },
       required: ['key'],
     },
   },
+  {
+    name: 'textweb_session_list',
+    description: 'List active textweb sessions and basic metadata (url, age).',
+    inputSchema: {
+      type: 'object',
+      properties: {},
+    },
+  },
+  {
+    name: 'textweb_session_close',
+    description: 'Close one session by session_id, or all sessions when all=true.',
+    inputSchema: {
+      type: 'object',
+      properties: {
+        session_id: { type: 'string', description: 'Session id to close (default: default)' },
+        all: { type: 'boolean', description: 'Close all active sessions' },
+      },
+    },
+  },
   {
     name: 'textweb_upload',
     description: 'Upload a file to a file input element by its reference number.',
@@ -104,22 +144,93 @@ const TOOLS = [
       properties: {
         ref: { type: 'number', description: 'Element reference number of the file input' },
         path: { type: 'string', description: 'Absolute path to the file to upload' },
+        session_id: { type: 'string', description: SESSION_NOTE },
+        retries: { type: 'number', description: 'Retry attempts for flaky transitions' },
+        retry_delay_ms: { type: 'number', description: 'Delay between retries in ms' },
       },
       required: ['ref', 'path'],
     },
   },
+  {
+    name: 'textweb_storage_save',
+    description: 'Save current browser storage state (cookies/localStorage/sessionStorage) to disk for later restore.',
+    inputSchema: {
+      type: 'object',
+      properties: {
+        path: { type: 'string', description: 'Absolute path to write storage state JSON' },
+        session_id: { type: 'string', description: SESSION_NOTE },
+      },
+      required: ['path'],
+    },
+  },
+  {
+    name: 'textweb_storage_load',
+    description: 'Load storage state from disk into a fresh browser context.',
+    inputSchema: {
+      type: 'object',
+      properties: {
+        path: { type: 'string', description: 'Absolute path of previously saved storage state JSON' },
+        cols: { type: 'number', description: 'Grid width in characters (default: 120)' },
+        session_id: { type: 'string', description: SESSION_NOTE },
+      },
+      required: ['path'],
+    },
+  },
+  {
+    name: 'textweb_wait_for',
+    description: 'Wait for UI state in multi-step flows. Supports selector, text, and url_includes checks.',
+    inputSchema: {
+      type: 'object',
+      properties: {
+        selector: { type: 'string', description: 'CSS selector that must appear (or match state)' },
+        text: { type: 'string', description: 'Text that must appear in page body' },
+        url_includes: { type: 'string', description: 'Substring that must appear in current URL' },
+        state: { type: 'string', enum: ['attached', 'detached', 'visible', 'hidden'], description: 'Selector wait state (default: visible)' },
+        timeout_ms: { type: 'number', description: 'Timeout in milliseconds (default: 30000)' },
+        poll_ms: { type: 'number', description: 'Polling interval for text/url waits (default: 100)' },
+        retries: { type: 'number', description: 'Retry attempts for flaky transitions' },
+        retry_delay_ms: { type: 'number', description: 'Delay between retries in ms' },
+        session_id: { type: 'string', description: SESSION_NOTE },
+      },
+    },
+  },
+  {
+    name: 'textweb_assert_field',
+    description: 'Assert a field value/text by element ref. Useful in multi-step forms before submitting.',
+    inputSchema: {
+      type: 'object',
+      properties: {
+        ref: { type: 'number', description: 'Element reference number from current snapshot' },
+        expected: { type: 'string', description: 'Expected value/content' },
+        comparator: { type: 'string', enum: ['equals', 'includes', 'regex', 'not_empty'], description: 'Comparison mode (default: equals)' },
+        attribute: { type: 'string', description: 'Optional DOM attribute name to validate (e.g., aria-invalid)' },
+        session_id: { type: 'string', description: SESSION_NOTE },
+      },
+      required: ['ref', 'expected'],
+    },
+  },
 ];
 
-// ─── Browser Instance ────────────────────────────────────────────────────────
+// ─── Browser Sessions ───────────────────────────────────────────────────────
+
+/** @type {Map<string, AgentBrowser>} */
+const sessions = new Map();
 
-let browser = null;
+function resolveSessionId(args = {}) {
+  return (args.session_id || 'default').trim() || 'default';
+}
+
+async function getBrowser(args = {}) {
+  const sessionId = resolveSessionId(args);
+  let browser = sessions.get(sessionId);
 
-async function getBrowser(cols) {
   if (!browser) {
-    browser = new AgentBrowser({ cols: cols || 120, headless: true });
+    browser = new AgentBrowser({ cols: args.cols || 120, headless: true });
     await browser.launch();
+    sessions.set(sessionId, browser);
   }
-  return browser;
+
+  return { browser, sessionId };
 }
 
 function formatResult(result) {
@@ -130,26 +241,78 @@ function formatResult(result) {
   return `URL: ${result.meta?.url || 'unknown'}\nTitle: ${result.meta?.title || 'unknown'}\nRefs: ${result.meta?.totalRefs || 0}\n\n${result.view}\n\nInteractive elements:\n${refs}`;
 }
 
+function retryOptions(args = {}) {
+  return {
+    retries: args.retries,
+    retryDelayMs: args.retry_delay_ms,
+  };
+}
+
+async function listSessions() {
+  const out = [];
+  for (const [sessionId, browser] of sessions.entries()) {
+    out.push({
+      session_id: sessionId,
+      url: browser.getCurrentUrl() || null,
+      initialized: Boolean(browser.page),
+      refs: browser.lastResult?.meta?.totalRefs ?? null,
+    });
+  }
+  return out;
+}
+
+async function closeSession({ session_id, all } = {}) {
+  if (all) {
+    const closed = [];
+    for (const [sid, browser] of sessions.entries()) {
+      await browser.close();
+      closed.push(sid);
+    }
+    sessions.clear();
+    return { closed };
+  }
+
+  const sid = (session_id || 'default').trim() || 'default';
+  const browser = sessions.get(sid);
+  if (!browser) {
+    return { closed: [], missing: [sid] };
+  }
+
+  await browser.close();
+  sessions.delete(sid);
+  return { closed: [sid] };
+}
+
 // ─── Tool Execution ──────────────────────────────────────────────────────────
 
-async function executeTool(name, args) {
-  const b = await getBrowser(args.cols);
+async function executeTool(name, args = {}) {
+  if (name === 'textweb_session_list') {
+    const active = await listSessions();
+    return JSON.stringify({ count: active.length, sessions: active }, null, 2);
+  }
+
+  if (name === 'textweb_session_close') {
+    const out = await closeSession({ session_id: args.session_id, all: args.all });
+    return JSON.stringify(out, null, 2);
+  }
+
+  const { browser: b, sessionId } = await getBrowser(args);
 
   switch (name) {
     case 'textweb_navigate': {
-      const result = await b.navigate(args.url);
+      const result = await b.navigate(args.url, retryOptions(args));
       return formatResult(result);
     }
     case 'textweb_click': {
-      const result = await b.click(args.ref);
+      const result = await b.click(args.ref, retryOptions(args));
       return formatResult(result);
     }
     case 'textweb_type': {
-      const result = await b.type(args.ref, args.text);
+      const result = await b.type(args.ref, args.text, retryOptions(args));
       return formatResult(result);
     }
     case 'textweb_select': {
-      const result = await b.select(args.ref, args.value);
+      const result = await b.select(args.ref, args.value, retryOptions(args));
       return formatResult(result);
     }
     case 'textweb_scroll': {
@@ -161,13 +324,40 @@ async function executeTool(name, args) {
       return formatResult(result);
     }
     case 'textweb_press': {
-      const result = await b.press(args.key);
+      const result = await b.press(args.key, retryOptions(args));
       return formatResult(result);
     }
     case 'textweb_upload': {
-      const result = await b.upload(args.ref, args.path);
+      const result = await b.upload(args.ref, args.path, retryOptions(args));
+      return formatResult(result);
+    }
+    case 'textweb_storage_save': {
+      const out = await b.saveStorageState(args.path);
+      return `Saved storage state for session "${sessionId}" to ${out.path}`;
+    }
+    case 'textweb_storage_load': {
+      const out = await b.loadStorageState(args.path);
+      return `Loaded storage state for session "${sessionId}" from ${out.path}`;
+    }
+    case 'textweb_wait_for': {
+      const result = await b.waitFor({
+        selector: args.selector,
+        text: args.text,
+        urlIncludes: args.url_includes,
+        timeoutMs: args.timeout_ms,
+        pollMs: args.poll_ms,
+        state: args.state,
+        ...retryOptions(args),
+      });
       return formatResult(result);
     }
+    case 'textweb_assert_field': {
+      const out = await b.assertField(args.ref, args.expected, {
+        comparator: args.comparator,
+        attribute: args.attribute,
+      });
+      return `ASSERT ${out.pass ? 'PASS' : 'FAIL'} | ref=${out.ref} | comparator=${out.comparator} | expected="${out.expected}" | actual="${out.actual}" | selector=${out.selector}`;
+    }
     default:
       throw new Error(`Unknown tool: ${name}`);
   }
@@ -232,7 +422,7 @@ function main() {
   process.stdin.setEncoding('utf8');
   process.stdin.on('data', async (chunk) => {
     buffer += chunk;
-    
+
     // Process complete lines (newline-delimited JSON)
     const lines = buffer.split('\n');
     buffer = lines.pop(); // Keep incomplete line in buffer
@@ -257,19 +447,23 @@ function main() {
   });
 
   process.stdin.on('end', async () => {
-    if (browser) await browser.close();
+    for (const [, browser] of sessions) {
+      await browser.close();
+    }
+    sessions.clear();
     process.exit(0);
   });
 
   process.on('SIGINT', async () => {
-    if (browser) await browser.close();
-    process.exit(0);
-  });
-
-  process.on('SIGTERM', async () => {
-    if (browser) await browser.close();
+    for (const [, browser] of sessions) {
+      await browser.close();
+    }
+    sessions.clear();
     process.exit(0);
   });
 }
 
-main();
+ensureBrowser().then(main).catch((err) => {
+  process.stderr.write(`Fatal: ${err.message}\n`);
+  process.exit(1);
+});

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "textweb",
-  "version": "0.2.0",
+  "version": "0.2.4",
   "description": "A text-grid web renderer for AI agents — see the web without screenshots",
   "main": "src/browser.js",
   "bin": {
@@ -10,8 +10,19 @@
   "scripts": {
     "start": "node src/cli.js",
     "serve": "node src/server.js",
-    "test": "node test/basic.js"
+    "test": "node test/test-form.js && node test/test-live.js && node test/test-ats-e2e.js",
+    "test:form": "node test/test-form.js",
+    "test:live": "node test/test-live.js",
+    "test:ats": "node test/test-ats-e2e.js"
   },
+  "files": [
+    "src/",
+    "mcp/",
+    "tools/",
+    "logo.svg",
+    "README.md",
+    "LICENSE"
+  ],
   "keywords": [
     "ai",
     "agent",