wu-framework 1.1.6 → 1.1.8

package/src/ai/wu-ai-browser.js

@@ -0,0 +1,663 @@
+/**
+ * WU-AI Browser Actions
+ *
+ * Registers browser automation tools into wu.ai so any LLM provider
+ * (OpenAI, Claude, Gemini, Ollama, etc.) can autonomously see and
+ * control the page — no human intervention required.
+ *
+ * Tools registered:
+ *   browser_screenshot  — Capture page/element as PNG (Canvas API)
+ *   browser_click       — Click element by selector or visible text
+ *   browser_type        — Type into inputs (React/Vue/framework compatible)
+ *   browser_snapshot    — Get accessibility tree of the DOM
+ *   browser_navigate    — Navigate SPA routes
+ *   browser_network     — View captured HTTP requests (fetch + XHR)
+ *   browser_console     — View captured console messages
+ *   browser_info        — Get page state: apps, store, URL, viewport
+ *   browser_select      — Select option in dropdowns
+ *   browser_scroll      — Scroll page or element
+ *
+ * @example
+ * // Auto-registered when wu.ai initializes
+ * // Any LLM connected via wu.ai.provider can now use these tools:
+ * const tools = wu.ai.tools();
+ * // → includes browser_screenshot, browser_click, etc.
+ */
+
+// ── Shared capture buffers ──
+const networkLog = [];
+const MAX_NETWORK_LOG = 300;
+const consoleLog = [];
+const MAX_CONSOLE_LOG = 500;
+let interceptorsInstalled = false;
+
+/**
+ * Register all browser automation actions into a WuAI instance.
+ *
+ * @param {object} ai - The WuAI instance (wu.ai)
+ * @param {object} wu - The Wu Framework instance (window.wu)
+ */
+export function registerBrowserActions(ai, wu) {
+  // Install interceptors only once
+  if (!interceptorsInstalled) {
+    _installNetworkInterceptor();
+    _installConsoleInterceptor();
+    interceptorsInstalled = true;
+  }
+
+  // ════════════════════════════════════════════
+  // SCREENSHOT — Canvas API (SVG foreignObject)
+  // ════════════════════════════════════════════
+
+  ai.action('browser_screenshot', {
+    description: 'Take a screenshot of the current page or a specific element. Returns a base64 PNG image. Use this to SEE what the user sees.',
+    parameters: {
+      selector: {
+        type: 'string',
+        description: 'CSS selector of the element to capture. Empty = full visible page.',
+        required: false,
+      },
+    },
+    handler: async (params) => {
+      const target = params.selector
+        ? document.querySelector(params.selector)
+        : document.documentElement;
+
+      if (!target) return { error: `Element not found: ${params.selector}` };
+
+      const rect = target.getBoundingClientRect();
+      const w = Math.ceil(Math.min(rect.width || window.innerWidth, 1920));
+      const h = Math.ceil(Math.min(rect.height || window.innerHeight, 1080));
+
+      // Clone and inline styles for accurate rendering
+      const clone = target.cloneNode(true);
+      _inlineComputedStyles(target, clone);
+
+      const serializer = new XMLSerializer();
+      const xhtml = serializer.serializeToString(clone);
+
+      const svgStr = [
+        `<svg xmlns="http://www.w3.org/2000/svg" width="${w}" height="${h}">`,
+        '<foreignObject width="100%" height="100%">',
+        `<div xmlns="http://www.w3.org/1999/xhtml" style="width:${w}px;height:${h}px;overflow:hidden;">`,
+        xhtml,
+        '</div>',
+        '</foreignObject>',
+        '</svg>',
+      ].join('');
+
+      const svgBlob = new Blob([svgStr], { type: 'image/svg+xml;charset=utf-8' });
+      const url = URL.createObjectURL(svgBlob);
+
+      const dataUrl = await new Promise((resolve) => {
+        const img = new Image();
+        img.onload = () => {
+          const canvas = document.createElement('canvas');
+          canvas.width = w;
+          canvas.height = h;
+          const ctx = canvas.getContext('2d');
+          ctx.drawImage(img, 0, 0);
+          URL.revokeObjectURL(url);
+          resolve(canvas.toDataURL('image/png', 0.8));
+        };
+        img.onerror = () => {
+          URL.revokeObjectURL(url);
+          resolve(null);
+        };
+        img.src = url;
+      });
+
+      if (!dataUrl) return { error: 'Canvas rendering failed' };
+
+      const base64 = dataUrl.split(',')[1];
+      return {
+        width: w,
+        height: h,
+        format: 'png',
+        base64,
+        sizeKB: Math.round((base64.length * 3) / 4 / 1024),
+      };
+    },
+    permissions: [],
+  });
+
+  // ════════════════════════════════════════════
+  // CLICK
+  // ════════════════════════════════════════════
+
+  ai.action('browser_click', {
+    description: 'Click an element on the page. Find by CSS selector or by visible text content. Use this to interact with buttons, links, tabs, etc.',
+    parameters: {
+      selector: {
+        type: 'string',
+        description: 'CSS selector (e.g. "#submit-btn", ".nav-link", "button[type=submit]")',
+        required: false,
+      },
+      text: {
+        type: 'string',
+        description: 'Visible text to find and click (e.g. "Submit", "Next", "Guardar"). Searches buttons, links, and clickable elements.',
+        required: false,
+      },
+    },
+    handler: async (params, api) => {
+      let el = null;
+
+      if (params.selector) {
+        el = document.querySelector(params.selector);
+      }
+
+      if (!el && params.text) {
+        const candidates = document.querySelectorAll(
+          'button, a, [role="button"], input[type="submit"], input[type="button"], [data-click], label, [onclick]'
+        );
+        const searchText = params.text.toLowerCase();
+        for (const candidate of candidates) {
+          if (candidate.textContent?.trim().toLowerCase().includes(searchText)) {
+            el = candidate;
+            break;
+          }
+        }
+      }
+
+      if (!el) return { error: `Element not found: ${params.selector || `text="${params.text}"`}` };
+
+      el.scrollIntoView({ behavior: 'instant', block: 'center' });
+      el.click();
+
+      const tag = el.tagName?.toLowerCase();
+      const id = el.id ? `#${el.id}` : '';
+      api.emit?.('browser:clicked', { selector: params.selector, text: params.text });
+
+      return {
+        clicked: `${tag}${id}`,
+        text: el.textContent?.trim().slice(0, 100) || '',
+      };
+    },
+    permissions: ['emitEvents'],
+  });
+
+  // ════════════════════════════════════════════
+  // TYPE
+  // ════════════════════════════════════════════
+
+  ai.action('browser_type', {
+    description: 'Type text into an input, textarea, or contenteditable element. Works with React, Vue, Angular, and other frameworks. Can optionally clear existing text first and submit the form.',
+    parameters: {
+      selector: {
+        type: 'string',
+        description: 'CSS selector of the input (e.g. "#email", "input[name=search]", "textarea.comment")',
+        required: true,
+      },
+      text: {
+        type: 'string',
+        description: 'Text to type into the element',
+        required: true,
+      },
+      clear: {
+        type: 'boolean',
+        description: 'Clear existing value before typing (default: false)',
+        required: false,
+      },
+      submit: {
+        type: 'boolean',
+        description: 'Submit the form or press Enter after typing (default: false)',
+        required: false,
+      },
+    },
+    handler: async (params, api) => {
+      const el = document.querySelector(params.selector);
+      if (!el) return { error: `Element not found: ${params.selector}` };
+
+      el.focus();
+
+      if (params.clear) {
+        el.value = '';
+        el.dispatchEvent(new Event('input', { bubbles: true }));
+      }
+
+      // Use native setter to trigger framework reactivity (React, Vue, etc.)
+      const nativeSetter = Object.getOwnPropertyDescriptor(
+        window.HTMLInputElement.prototype, 'value'
+      )?.set || Object.getOwnPropertyDescriptor(
+        window.HTMLTextAreaElement.prototype, 'value'
+      )?.set;
+
+      const newValue = params.clear ? params.text : (el.value || '') + params.text;
+      if (nativeSetter) {
+        nativeSetter.call(el, newValue);
+      } else {
+        el.value = newValue;
+      }
+
+      el.dispatchEvent(new Event('input', { bubbles: true }));
+      el.dispatchEvent(new Event('change', { bubbles: true }));
+
+      if (params.submit) {
+        const form = el.closest('form');
+        if (form) {
+          form.dispatchEvent(new Event('submit', { bubbles: true, cancelable: true }));
+        } else {
+          el.dispatchEvent(new KeyboardEvent('keydown', { key: 'Enter', code: 'Enter', bubbles: true }));
+        }
+      }
+
+      api.emit?.('browser:typed', { selector: params.selector, length: params.text.length });
+
+      return {
+        selector: params.selector,
+        typed: params.text,
+        currentValue: el.value?.slice(0, 200),
+        submitted: !!params.submit,
+      };
+    },
+    permissions: ['emitEvents'],
+  });
+
+  // ════════════════════════════════════════════
+  // SELECT (dropdowns)
+  // ════════════════════════════════════════════
+
+  ai.action('browser_select', {
+    description: 'Select an option in a <select> dropdown or a custom dropdown component.',
+    parameters: {
+      selector: {
+        type: 'string',
+        description: 'CSS selector of the <select> element',
+        required: true,
+      },
+      value: {
+        type: 'string',
+        description: 'The value attribute of the option to select. Use "text:" prefix to match by visible text (e.g. "text:Mexico")',
+        required: true,
+      },
+    },
+    handler: async (params, api) => {
+      const el = document.querySelector(params.selector);
+      if (!el) return { error: `Element not found: ${params.selector}` };
+
+      if (el.tagName?.toLowerCase() === 'select') {
+        const options = Array.from(el.options);
+        let option;
+
+        if (params.value.startsWith('text:')) {
+          const searchText = params.value.slice(5).toLowerCase();
+          option = options.find((o) => o.textContent.trim().toLowerCase().includes(searchText));
+        } else {
+          option = options.find((o) => o.value === params.value);
+        }
+
+        if (!option) return { error: `Option not found: ${params.value}` };
+
+        el.value = option.value;
+        el.dispatchEvent(new Event('change', { bubbles: true }));
+        el.dispatchEvent(new Event('input', { bubbles: true }));
+
+        api.emit?.('browser:selected', { selector: params.selector, value: option.value });
+        return { selected: option.value, text: option.textContent.trim() };
+      }
+
+      // Custom dropdown: try clicking the trigger, then the option
+      el.click();
+      return { clicked: params.selector, note: 'Custom dropdown — clicked trigger. Use browser_click to select an option from the opened menu.' };
+    },
+    permissions: ['emitEvents'],
+  });
+
+  // ════════════════════════════════════════════
+  // SCROLL
+  // ════════════════════════════════════════════
+
+  ai.action('browser_scroll', {
+    description: 'Scroll the page or a specific element. Use to reveal content that is not visible.',
+    parameters: {
+      direction: {
+        type: 'string',
+        description: 'Direction: "up", "down", "top", "bottom"',
+        required: true,
+      },
+      selector: {
+        type: 'string',
+        description: 'CSS selector of scrollable container (empty = page)',
+        required: false,
+      },
+      amount: {
+        type: 'number',
+        description: 'Pixels to scroll (default: 500). Ignored for "top"/"bottom".',
+        required: false,
+      },
+    },
+    handler: async (params) => {
+      const target = params.selector
+        ? document.querySelector(params.selector)
+        : window;
+      const amount = params.amount || 500;
+
+      if (params.selector && !target) return { error: `Element not found: ${params.selector}` };
+
+      const scrollEl = target === window ? document.documentElement : target;
+
+      switch (params.direction) {
+        case 'up': scrollEl.scrollBy({ top: -amount, behavior: 'smooth' }); break;
+        case 'down': scrollEl.scrollBy({ top: amount, behavior: 'smooth' }); break;
+        case 'top': scrollEl.scrollTo({ top: 0, behavior: 'smooth' }); break;
+        case 'bottom': scrollEl.scrollTo({ top: scrollEl.scrollHeight, behavior: 'smooth' }); break;
+        default: return { error: `Invalid direction: ${params.direction}` };
+      }
+
+      return {
+        scrolled: params.direction,
+        amount: params.direction === 'top' || params.direction === 'bottom' ? 'max' : amount,
+        currentScroll: scrollEl.scrollTop,
+      };
+    },
+    permissions: [],
+  });
+
+  // ════════════════════════════════════════════
+  // SNAPSHOT — Accessibility tree
+  // ════════════════════════════════════════════
+
+  ai.action('browser_snapshot', {
+    description: 'Get a text representation of the visible DOM structure (accessibility tree). Use this to understand what elements are on the page, their roles, IDs, and text content. Cheaper and faster than a screenshot.',
+    parameters: {
+      selector: {
+        type: 'string',
+        description: 'CSS selector to snapshot (empty = full page). Use "[data-wu-app=appName]" for a specific micro-app.',
+        required: false,
+      },
+      depth: {
+        type: 'number',
+        description: 'Max depth to traverse (default: 5)',
+        required: false,
+      },
+    },
+    handler: async (params) => {
+      const target = params.selector
+        ? document.querySelector(params.selector)
+        : document.body;
+
+      if (!target) return { error: `Element not found: ${params.selector}` };
+
+      const tree = _buildA11yTree(target, 0, params.depth || 5);
+      return { snapshot: tree };
+    },
+    permissions: [],
+  });
+
+  // ════════════════════════════════════════════
+  // NAVIGATE
+  // ════════════════════════════════════════════
+
+  ai.action('browser_navigate', {
+    description: 'Navigate to a route within the SPA application. Emits a shell:navigate event and updates the store.',
+    parameters: {
+      route: {
+        type: 'string',
+        description: 'Route path (e.g. "/dashboard", "/users", "/pos/cotizador")',
+        required: true,
+      },
+    },
+    handler: async (params, api) => {
+      api.emit?.('shell:navigate', { route: params.route });
+      api.setState?.('currentPath', params.route);
+      return { navigated: params.route };
+    },
+    permissions: ['emitEvents', 'writeStore'],
+  });
+
+  // ════════════════════════════════════════════
+  // NETWORK — Captured HTTP requests
+  // ════════════════════════════════════════════
+
+  ai.action('browser_network', {
+    description: 'View captured HTTP network requests (fetch and XHR). Shows URL, method, status code, duration, and size. Use to debug API calls, check for errors, or monitor performance.',
+    parameters: {
+      method: {
+        type: 'string',
+        description: 'Filter by HTTP method: GET, POST, PUT, DELETE (empty = all)',
+        required: false,
+      },
+      status: {
+        type: 'string',
+        description: 'Filter: "2" (2xx success), "4" (4xx errors), "5" (5xx errors), "error" (all failures)',
+        required: false,
+      },
+      limit: {
+        type: 'number',
+        description: 'Max requests to return (default: 30)',
+        required: false,
+      },
+    },
+    handler: async (params) => {
+      let filtered = networkLog;
+      if (params.method) {
+        filtered = filtered.filter((r) => r.method === params.method.toUpperCase());
+      }
+      if (params.status) {
+        if (params.status === 'error') {
+          filtered = filtered.filter((r) => r.status === 0 || r.status >= 400);
+        } else {
+          filtered = filtered.filter((r) => String(r.status).startsWith(params.status));
+        }
+      }
+      const limit = params.limit || 30;
+      return {
+        requests: filtered.slice(-limit),
+        total: networkLog.length,
+        showing: Math.min(filtered.length, limit),
+      };
+    },
+    permissions: [],
+  });
+
+  // ════════════════════════════════════════════
+  // CONSOLE — Captured logs
+  // ════════════════════════════════════════════
+
+  ai.action('browser_console', {
+    description: 'View captured browser console messages (log, warn, error). Use to check for errors, warnings, or debug output.',
+    parameters: {
+      level: {
+        type: 'string',
+        description: 'Filter by level: "log", "warn", "error" (empty = all)',
+        required: false,
+      },
+      limit: {
+        type: 'number',
+        description: 'Max messages to return (default: 30)',
+        required: false,
+      },
+    },
+    handler: async (params) => {
+      const filtered = params.level
+        ? consoleLog.filter((m) => m.level === params.level)
+        : consoleLog;
+      const limit = params.limit || 30;
+      return {
+        messages: filtered.slice(-limit),
+        total: consoleLog.length,
+        showing: Math.min(filtered.length, limit),
+      };
+    },
+    permissions: [],
+  });
+
+  // ════════════════════════════════════════════
+  // INFO — Page state overview
+  // ════════════════════════════════════════════
+
+  ai.action('browser_info', {
+    description: 'Get an overview of the current page state: URL, viewport size, mounted micro-apps, store keys, visible elements summary. Use this FIRST to understand the page before taking actions.',
+    parameters: {},
+    handler: async (params, api) => {
+      const apps = [];
+
+      // Discover mounted apps
+      if (wu._apps) {
+        for (const [name, app] of Object.entries(wu._apps)) {
+          apps.push({
+            name,
+            mounted: app.mounted || app.isMounted || false,
+            status: app.status || 'unknown',
+          });
+        }
+      }
+      if (apps.length === 0) {
+        document.querySelectorAll('[data-wu-app]').forEach((el) => {
+          apps.push({ name: el.getAttribute('data-wu-app'), mounted: true });
+        });
+      }
+
+      const storeData = api.getState?.('') || {};
+      const storeKeys = typeof storeData === 'object' ? Object.keys(storeData) : [];
+
+      return {
+        url: window.location.href,
+        title: document.title,
+        viewport: { width: window.innerWidth, height: window.innerHeight },
+        apps,
+        storeKeys,
+        networkRequests: networkLog.length,
+        consoleMessages: consoleLog.length,
+        consoleErrors: consoleLog.filter((m) => m.level === 'error').length,
+      };
+    },
+    permissions: ['readStore'],
+  });
+}
+
+// ── Private: Inline computed styles for Canvas rendering ──
+
+function _inlineComputedStyles(source, clone) {
+  const props = ['color', 'background', 'background-color', 'font-family',
+    'font-size', 'font-weight', 'border', 'border-radius', 'padding', 'margin',
+    'display', 'flex-direction', 'align-items', 'justify-content', 'gap',
+    'width', 'height', 'max-width', 'max-height', 'overflow', 'opacity',
+    'box-shadow', 'text-align', 'line-height', 'position', 'top', 'left',
+    'right', 'bottom', 'z-index', 'transform', 'visibility'];
+
+  try {
+    const style = window.getComputedStyle(source);
+    for (const prop of props) {
+      const val = style.getPropertyValue(prop);
+      if (val) clone.style?.setProperty(prop, val);
+    }
+  } catch (_) { /* skip */ }
+
+  const srcKids = source.children || [];
+  const cloneKids = clone.children || [];
+  const max = Math.min(srcKids.length, cloneKids.length, 200);
+  for (let i = 0; i < max; i++) {
+    _inlineComputedStyles(srcKids[i], cloneKids[i]);
+  }
+}
+
+// ── Private: Build accessibility tree ──
+
+function _buildA11yTree(el, depth, maxDepth) {
+  if (depth > maxDepth || !el) return '';
+
+  const indent = '  '.repeat(depth);
+  const tag = el.tagName?.toLowerCase() || '';
+  const role = el.getAttribute?.('role') || '';
+  const ariaLabel = el.getAttribute?.('aria-label') || '';
+  const text = el.childNodes?.length === 1 && el.childNodes[0].nodeType === 3
+    ? el.textContent?.trim().slice(0, 80) : '';
+
+  let line = `${indent}<${tag}`;
+  if (el.id) line += ` id="${el.id}"`;
+  if (role) line += ` role="${role}"`;
+  if (ariaLabel) line += ` aria-label="${ariaLabel}"`;
+  if (el.className && typeof el.className === 'string') {
+    const cls = el.className.trim().slice(0, 60);
+    if (cls) line += ` class="${cls}"`;
+  }
+  line += '>';
+  if (text) line += ` "${text}"`;
+
+  let result = line + '\n';
+  const root = el.shadowRoot || el;
+  const children = root.children || [];
+
+  for (let i = 0; i < children.length && i < 50; i++) {
+    result += _buildA11yTree(children[i], depth + 1, maxDepth);
+  }
+  return result;
+}
+
+// ── Private: Network interceptor ──
+
+function _installNetworkInterceptor() {
+  // Intercept fetch
+  const originalFetch = window.fetch;
+  window.fetch = async function (...args) {
+    const start = Date.now();
+    const req = args[0];
+    const url = typeof req === 'string' ? req : req?.url || '';
+    const method = (args[1]?.method || req?.method || 'GET').toUpperCase();
+
+    try {
+      const response = await originalFetch.apply(window, args);
+      const size = parseInt(response.headers?.get('content-length') || '0', 10);
+      networkLog.push({
+        type: 'fetch', method, url,
+        status: response.status, statusText: response.statusText,
+        duration: Date.now() - start, size, timestamp: start,
+      });
+      if (networkLog.length > MAX_NETWORK_LOG) networkLog.shift();
+      return response;
+    } catch (err) {
+      networkLog.push({
+        type: 'fetch', method, url,
+        status: 0, error: err.message,
+        duration: Date.now() - start, timestamp: start,
+      });
+      if (networkLog.length > MAX_NETWORK_LOG) networkLog.shift();
+      throw err;
+    }
+  };
+
+  // Intercept XMLHttpRequest
+  const origOpen = XMLHttpRequest.prototype.open;
+  const origSend = XMLHttpRequest.prototype.send;
+
+  XMLHttpRequest.prototype.open = function (method, url, ...rest) {
+    this._wuAi = { method: (method || 'GET').toUpperCase(), url: String(url) };
+    return origOpen.call(this, method, url, ...rest);
+  };
+
+  XMLHttpRequest.prototype.send = function (...args) {
+    if (this._wuAi) {
+      this._wuAi.start = Date.now();
+      this.addEventListener('loadend', () => {
+        networkLog.push({
+          type: 'xhr', method: this._wuAi.method, url: this._wuAi.url,
+          status: this.status, statusText: this.statusText,
+          duration: Date.now() - this._wuAi.start,
+          size: parseInt(this.getResponseHeader('content-length') || '0', 10),
+          timestamp: this._wuAi.start,
+        });
+        if (networkLog.length > MAX_NETWORK_LOG) networkLog.shift();
+      });
+    }
+    return origSend.apply(this, args);
+  };
+}
+
+// ── Private: Console interceptor ──
+
+function _installConsoleInterceptor() {
+  const levels = ['log', 'warn', 'error'];
+  for (const level of levels) {
+    const original = console[level];
+    console[level] = (...args) => {
+      consoleLog.push({
+        level,
+        message: args.map((a) => (typeof a === 'object' ? JSON.stringify(a) : String(a))).join(' '),
+        timestamp: Date.now(),
+      });
+      if (consoleLog.length > MAX_CONSOLE_LOG) consoleLog.shift();
+      original.apply(console, args);
+    };
+  }
+}