tab-agent 0.3.4 → 0.4.1

package/extension/content-script.js

@@ -246,7 +246,7 @@ if (window.__tabAgent_contentScriptLoaded) {
       };
     }
 
-    return { getElementByRef, snapshot };
+    return { getElementByRef, snapshot, isVisible, getRole, getName, nextRef, storeRef };
   })();
 
   function getElementByRef(ref) {
@@ -283,17 +283,25 @@ if (window.__tabAgent_contentScriptLoaded) {
 
     element.focus();
 
-    for (const char of text) {
-      element.dispatchEvent(new KeyboardEvent('keydown', { key: char, bubbles: true }));
-      element.dispatchEvent(new KeyboardEvent('keypress', { key: char, bubbles: true }));
+    const isContentEditable = element.contentEditable === 'true' || element.isContentEditable;
 
-      if (element.tagName === 'INPUT' || element.tagName === 'TEXTAREA') {
-        element.value += char;
-        element.dispatchEvent(new Event('input', { bubbles: true }));
-      }
+    if (isContentEditable) {
+      // For contentEditable elements (rich text editors like Gemini, Notion, etc.)
+      // Use document.execCommand which triggers proper input events
+      document.execCommand('insertText', false, text);
+    } else {
+      for (const char of text) {
+        element.dispatchEvent(new KeyboardEvent('keydown', { key: char, bubbles: true }));
+        element.dispatchEvent(new KeyboardEvent('keypress', { key: char, bubbles: true }));
+
+        if (element.tagName === 'INPUT' || element.tagName === 'TEXTAREA') {
+          element.value += char;
+          element.dispatchEvent(new Event('input', { bubbles: true }));
+        }
 
-      element.dispatchEvent(new KeyboardEvent('keyup', { key: char, bubbles: true }));
-      await new Promise(r => setTimeout(r, 10));
+        element.dispatchEvent(new KeyboardEvent('keyup', { key: char, bubbles: true }));
+        await new Promise(r => setTimeout(r, 10));
+      }
     }
 
     // Handle submit if requested
@@ -318,12 +326,20 @@ if (window.__tabAgent_contentScriptLoaded) {
     }
 
     element.focus();
-    element.value = '';
-    element.dispatchEvent(new Event('input', { bubbles: true }));
 
-    element.value = value;
-    element.dispatchEvent(new Event('input', { bubbles: true }));
-    element.dispatchEvent(new Event('change', { bubbles: true }));
+    const isContentEditable = element.contentEditable === 'true' || element.isContentEditable;
+
+    if (isContentEditable) {
+      // Clear and fill contentEditable elements
+      element.textContent = '';
+      document.execCommand('insertText', false, value);
+    } else {
+      element.value = '';
+      element.dispatchEvent(new Event('input', { bubbles: true }));
+      element.value = value;
+      element.dispatchEvent(new Event('input', { bubbles: true }));
+      element.dispatchEvent(new Event('change', { bubbles: true }));
+    }
 
     return { ok: true, ref, filled: value };
   }
@@ -394,9 +410,9 @@ if (window.__tabAgent_contentScriptLoaded) {
     return { ok: true, direction, scrollY: window.scrollY };
   }
 
-  // Wait for condition (text, selector, or timeout)
+  // Wait for condition (text, selector, url pattern, or visible ref)
   async function executeWait(params) {
-    const { text, selector, timeout = 30000 } = params;
+    const { text, selector, urlPattern, visibleRef, timeout = 30000 } = params;
     const start = Date.now();
 
     while (Date.now() - start < timeout) {
@@ -406,6 +422,15 @@ if (window.__tabAgent_contentScriptLoaded) {
       if (selector && document.querySelector(selector)) {
         return { ok: true, found: 'selector' };
       }
+      if (urlPattern && window.location.href.includes(urlPattern)) {
+        return { ok: true, found: 'url', url: window.location.href };
+      }
+      if (visibleRef) {
+        const el = getElementByRef(visibleRef);
+        if (el && snapshotState.isVisible(el)) {
+          return { ok: true, found: 'visible', ref: visibleRef };
+        }
+      }
       await new Promise(r => setTimeout(r, 100));
     }
 
@@ -440,6 +465,167 @@ if (window.__tabAgent_contentScriptLoaded) {
     return { ok: results.every(r => r.ok), results };
   }
 
+  async function executeDrag(params) {
+    const { fromRef, toRef } = params;
+    const fromEl = getElementByRef(fromRef);
+    const toEl = getElementByRef(toRef);
+
+    if (!fromEl) return { ok: false, error: `Element ${fromRef} not found` };
+    if (!toEl) return { ok: false, error: `Element ${toRef} not found` };
+
+    const fromRect = fromEl.getBoundingClientRect();
+    const toRect = toEl.getBoundingClientRect();
+    const fromX = fromRect.left + fromRect.width / 2;
+    const fromY = fromRect.top + fromRect.height / 2;
+    const toX = toRect.left + toRect.width / 2;
+    const toY = toRect.top + toRect.height / 2;
+
+    fromEl.dispatchEvent(new MouseEvent('mousedown', { clientX: fromX, clientY: fromY, bubbles: true }));
+    await new Promise(r => setTimeout(r, 50));
+    fromEl.dispatchEvent(new MouseEvent('mousemove', { clientX: fromX + 5, clientY: fromY + 5, bubbles: true }));
+    await new Promise(r => setTimeout(r, 50));
+    toEl.dispatchEvent(new MouseEvent('mousemove', { clientX: toX, clientY: toY, bubbles: true }));
+    await new Promise(r => setTimeout(r, 50));
+    toEl.dispatchEvent(new MouseEvent('mouseup', { clientX: toX, clientY: toY, bubbles: true }));
+
+    // Also fire dragstart/drop for HTML5 drag-and-drop
+    fromEl.dispatchEvent(new DragEvent('dragstart', { bubbles: true }));
+    toEl.dispatchEvent(new DragEvent('drop', { bubbles: true }));
+    fromEl.dispatchEvent(new DragEvent('dragend', { bubbles: true }));
+
+    return { ok: true, from: fromRef, to: toRef };
+  }
+
+  async function executeGet(params) {
+    const { subcommand, ref, attr } = params;
+
+    if (subcommand === 'url') return { ok: true, result: window.location.href };
+    if (subcommand === 'title') return { ok: true, result: document.title };
+
+    if (!ref) return { ok: false, error: 'No ref provided' };
+    const element = getElementByRef(ref);
+    if (!element) return { ok: false, error: `Element ${ref} not found` };
+
+    switch (subcommand) {
+      case 'text':
+        return { ok: true, result: element.textContent.trim() };
+      case 'html':
+        return { ok: true, result: element.innerHTML };
+      case 'value':
+        return { ok: true, result: element.value || '' };
+      case 'attr':
+        return { ok: true, result: element.getAttribute(attr) };
+      default:
+        return { ok: false, error: `Unknown get subcommand: ${subcommand}` };
+    }
+  }
+
+  async function executeFind(params) {
+    const { by, query } = params;
+    const results = [];
+
+    const matches = [];
+    if (by === 'text') {
+      const lowerQuery = query.toLowerCase();
+      const walker = document.createTreeWalker(document.body, NodeFilter.SHOW_ELEMENT);
+      while (walker.nextNode()) {
+        const el = walker.currentNode;
+        if (!snapshotState.isVisible(el)) continue;
+        // Prefer elements with direct text match (not just inherited from children)
+        let directText = '';
+        for (const node of el.childNodes) {
+          if (node.nodeType === Node.TEXT_NODE) directText += node.textContent;
+        }
+        const hasDirectMatch = directText.trim().toLowerCase().includes(lowerQuery);
+        const isLeafLike = el.children.length <= 2;
+        if (hasDirectMatch || (isLeafLike && el.textContent.trim().toLowerCase().includes(lowerQuery))) {
+          matches.push(el);
+          if (matches.length >= 20) break;
+        }
+      }
+    } else if (by === 'role') {
+      const els = document.querySelectorAll(`[role="${query}"]`);
+      const tagRoles = { button: 'button', a: 'link', input: 'textbox', textarea: 'textbox', select: 'combobox', img: 'img' };
+      const tagMatch = Object.entries(tagRoles).find(([, r]) => r === query);
+      if (tagMatch) {
+        document.querySelectorAll(tagMatch[0]).forEach(el => { if (snapshotState.isVisible(el)) matches.push(el); });
+      }
+      els.forEach(el => { if (snapshotState.isVisible(el) && !matches.includes(el)) matches.push(el); });
+    } else if (by === 'label') {
+      const labels = document.querySelectorAll('label');
+      labels.forEach(label => {
+        if (label.textContent.trim().toLowerCase().includes(query.toLowerCase())) {
+          const input = label.control || (label.htmlFor && document.getElementById(label.htmlFor));
+          if (input) matches.push(input);
+        }
+      });
+      // Also search aria-label
+      document.querySelectorAll(`[aria-label*="${query}" i]`).forEach(el => {
+        if (!matches.includes(el)) matches.push(el);
+      });
+    } else if (by === 'placeholder') {
+      document.querySelectorAll(`[placeholder*="${query}" i]`).forEach(el => matches.push(el));
+    } else if (by === 'selector') {
+      document.querySelectorAll(query).forEach(el => { if (snapshotState.isVisible(el)) matches.push(el); });
+    } else {
+      return { ok: false, error: `Unknown find method: ${by}` };
+    }
+
+    // Assign refs to found elements
+    for (const el of matches.slice(0, 20)) {
+      const ref = snapshotState.nextRef();
+      snapshotState.storeRef(ref, el);
+      const role = snapshotState.getRole(el);
+      const name = snapshotState.getName(el);
+      results.push({ ref, role, name: name.substring(0, 100) });
+    }
+
+    return { ok: true, results, count: results.length };
+  }
+
+  async function executeCookies(params) {
+    const { subcommand } = params;
+
+    if (subcommand === 'get') {
+      return { ok: true, result: document.cookie };
+    } else if (subcommand === 'clear') {
+      document.cookie.split(';').forEach(c => {
+        const name = c.split('=')[0].trim();
+        document.cookie = `${name}=; expires=Thu, 01 Jan 1970 00:00:00 UTC; path=/;`;
+      });
+      return { ok: true, cleared: true };
+    } else {
+      return { ok: false, error: `Unknown cookies subcommand: ${subcommand}` };
+    }
+  }
+
+  async function executeStorage(params) {
+    const { subcommand, storageType = 'local', key, value } = params;
+    const store = storageType === 'session' ? sessionStorage : localStorage;
+
+    switch (subcommand) {
+      case 'get':
+        if (key) return { ok: true, result: store.getItem(key) };
+        const all = {};
+        for (let i = 0; i < store.length; i++) {
+          const k = store.key(i);
+          all[k] = store.getItem(k);
+        }
+        return { ok: true, result: all };
+      case 'set':
+        store.setItem(key, value);
+        return { ok: true, key, value };
+      case 'remove':
+        store.removeItem(key);
+        return { ok: true, removed: key };
+      case 'clear':
+        store.clear();
+        return { ok: true, cleared: true };
+      default:
+        return { ok: false, error: `Unknown storage subcommand: ${subcommand}` };
+    }
+  }
+
   async function executeNavigate(params) {
     const { url } = params;
     window.location.href = url;
@@ -496,6 +682,21 @@ if (window.__tabAgent_contentScriptLoaded) {
         case 'navigate':
           result = await executeNavigate(params);
           break;
+        case 'drag':
+          result = await executeDrag(params);
+          break;
+        case 'get':
+          result = await executeGet(params);
+          break;
+        case 'find':
+          result = await executeFind(params);
+          break;
+        case 'cookies':
+          result = await executeCookies(params);
+          break;
+        case 'storage':
+          result = await executeStorage(params);
+          break;
         default:
           result = { ok: false, error: `Unknown action: ${action}` };
       }

package/extension/manifest.json

@@ -1,8 +1,8 @@
 {
   "manifest_version": 3,
   "name": "Tab Agent",
-  "version": "0.1.0",
-  "description": "Browser control for Claude Code and Codex via WebSocket",
+  "version": "0.4.1",
+  "description": "Secure browser control for Claude, Codex, ChatGPT, and other AI tools",
   "permissions": [
     "activeTab",
     "scripting",
@@ -19,7 +19,8 @@
     "type": "module"
   },
   "action": {
-    "default_title": "Tab Agent - Click to toggle",
+    "default_title": "Tab Agent - Click to manage",
+    "default_popup": "popup/popup.html",
     "default_icon": {
       "16": "icons/icon16.png",
       "48": "icons/icon48.png",

package/extension/manifest.safari.json

@@ -0,0 +1,45 @@
+{
+  "manifest_version": 3,
+  "name": "Tab Agent",
+  "version": "0.4.1",
+  "description": "Secure browser control for Claude, Codex, ChatGPT, and other AI tools",
+  "permissions": [
+    "activeTab",
+    "scripting",
+    "storage",
+    "tabs",
+    "nativeMessaging"
+  ],
+  "host_permissions": [
+    "<all_urls>"
+  ],
+  "background": {
+    "service_worker": "service-worker.js",
+    "type": "module"
+  },
+  "action": {
+    "default_title": "Tab Agent - Click to manage",
+    "default_popup": "popup/popup.html",
+    "default_icon": {
+      "16": "icons/icon16.png",
+      "48": "icons/icon48.png",
+      "128": "icons/icon128.png"
+    }
+  },
+  "icons": {
+    "16": "icons/icon16.png",
+    "48": "icons/icon48.png",
+    "128": "icons/icon128.png"
+  },
+  "web_accessible_resources": [
+    {
+      "resources": ["snapshot.js"],
+      "matches": ["<all_urls>"]
+    }
+  ],
+  "browser_specific_settings": {
+    "safari": {
+      "strict_min_version": "17.0"
+    }
+  }
+}

package/extension/popup/popup.html

@@ -115,6 +115,55 @@
       color: #666;
       text-align: center;
     }
+
+    .auto-activate {
+      margin-bottom: 12px;
+      padding: 10px;
+      background: #2a2a3e;
+      border-radius: 6px;
+    }
+
+    .toggle {
+      display: flex;
+      align-items: center;
+      gap: 10px;
+      cursor: pointer;
+      font-size: 13px;
+    }
+
+    .toggle input { display: none; }
+
+    .toggle-slider {
+      width: 36px;
+      height: 20px;
+      background: #444;
+      border-radius: 10px;
+      position: relative;
+      transition: background 0.2s;
+      flex-shrink: 0;
+    }
+
+    .toggle-slider::after {
+      content: '';
+      position: absolute;
+      width: 16px;
+      height: 16px;
+      background: white;
+      border-radius: 50%;
+      top: 2px;
+      left: 2px;
+      transition: transform 0.2s;
+    }
+
+    .toggle input:checked + .toggle-slider {
+      background: #3b82f6;
+    }
+
+    .toggle input:checked + .toggle-slider::after {
+      transform: translateX(16px);
+    }
+
+    .toggle-label { color: #ccc; }
   </style>
 </head>
 <body>
@@ -122,6 +171,14 @@
 
   <div id="status" class="status disconnected">Checking connection...</div>
 
+  <div class="auto-activate">
+    <label class="toggle">
+      <input type="checkbox" id="autoActivateToggle">
+      <span class="toggle-slider"></span>
+      <span class="toggle-label">Auto-activate all tabs</span>
+    </label>
+  </div>
+
   <div class="current-tab">
     <div class="label">Current Tab</div>
     <div id="currentUrl" class="url">Loading...</div>
@@ -135,7 +192,7 @@
     </div>
   </div>
 
-  <div class="footer">ws://localhost:9876 | v0.1.0</div>
+  <div class="footer" id="footer">ws://localhost:9876</div>
 
   <script src="popup.js"></script>
 </body>

package/extension/popup/popup.js

@@ -8,10 +8,28 @@ async function init() {
   currentTabId = tab.id;
 
   document.getElementById('currentUrl').textContent = tab.url;
+  document.getElementById('footer').textContent =
+    `ws://localhost:9876 | v${chrome.runtime.getManifest().version}`;
 
   await refreshActivatedTabs();
   updateActivateButton();
   checkConnection();
+
+  // Load auto-activate state
+  const autoResponse = await chrome.runtime.sendMessage({ action: 'getAutoActivate' });
+  const toggle = document.getElementById('autoActivateToggle');
+  if (autoResponse && autoResponse.ok) {
+    toggle.checked = autoResponse.autoActivateAll;
+  }
+
+  toggle.addEventListener('change', async () => {
+    await chrome.runtime.sendMessage({
+      action: 'setAutoActivate',
+      enabled: toggle.checked
+    });
+    await refreshActivatedTabs();
+    updateActivateButton();
+  });
 }
 
 async function refreshActivatedTabs() {

package/extension/service-worker.js

@@ -2,13 +2,32 @@
 // Tab Agent - Service Worker
 // Manages activated tabs and routes commands to content scripts
 
+// Browser detection
+const IS_SAFARI = typeof browser !== 'undefined' &&
+  navigator.userAgent.includes('Safari') &&
+  !navigator.userAgent.includes('Chrome');
+const IS_CHROME = typeof chrome !== 'undefined' && !IS_SAFARI;
+
+// Safari uses 'browser' namespace, Chrome uses 'chrome'
+const browserAPI = IS_SAFARI ? browser : chrome;
+
 const state = {
   activatedTabs: new Map(), // tabId -> { url, title, activatedAt }
   auditLog: [],
   nativeConnected: false,
   lastNativeError: null,
+  autoActivateAll: false,
 };
 
+// Load auto-activate setting from storage on startup
+chrome.storage.local.get(['autoActivateAll'], (result) => {
+  if (result.autoActivateAll) {
+    state.autoActivateAll = true;
+    autoActivateExistingTabs();
+    updateAutoActivateBadge();
+  }
+});
+
 // Dialog handling with chrome.debugger
 const pendingDialogs = new Map();
 const attachedDebuggerTabs = new Set();
@@ -53,15 +72,41 @@ async function handleDialog(tabId, accept, promptText = '') {
 
 // Update badge for a tab
 function updateBadge(tabId) {
+  if (state.autoActivateAll) return;
   const isActive = state.activatedTabs.has(tabId);
   chrome.action.setBadgeText({ tabId, text: isActive ? 'ON' : '' });
   chrome.action.setBadgeBackgroundColor({ tabId, color: isActive ? '#22c55e' : '#666' });
   chrome.action.setTitle({
     tabId,
-    title: isActive ? 'Tab Agent - Active (click to deactivate)' : 'Tab Agent - Click to activate'
+    title: isActive ? 'Tab Agent - Active' : 'Tab Agent - Click to manage'
   });
 }
 
+// Update badge to show AUTO mode
+function updateAutoActivateBadge() {
+  if (state.autoActivateAll) {
+    chrome.action.setBadgeText({ text: 'AUTO' });
+    chrome.action.setBadgeBackgroundColor({ color: '#3b82f6' });
+    chrome.action.setTitle({ title: 'Tab Agent - Auto-activate ON (click to manage)' });
+  } else {
+    chrome.action.setBadgeText({ text: '' });
+    chrome.action.setTitle({ title: 'Tab Agent - Click to manage' });
+    for (const [tabId] of state.activatedTabs) {
+      updateBadge(tabId);
+    }
+  }
+}
+
+// Activate all existing tabs
+async function autoActivateExistingTabs() {
+  const tabs = await chrome.tabs.query({});
+  for (const tab of tabs) {
+    if (!state.activatedTabs.has(tab.id) && tab.url && !tab.url.startsWith('chrome://')) {
+      await activateTab(tab.id);
+    }
+  }
+}
+
 // Log all actions for audit trail
 function audit(action, data, result) {
   const entry = {
@@ -254,6 +299,26 @@ async function routeCommand(tabId, command) {
       };
     }
 
+    // Handle PDF generation - must be done in service worker via debugger
+    if (command.action === 'pdf') {
+      try {
+        try { await chrome.debugger.detach({ tabId }); } catch {}
+        await chrome.debugger.attach({ tabId }, '1.3');
+        const pdf = await chrome.debugger.sendCommand({ tabId }, 'Page.printToPDF', {
+          printBackground: true,
+          preferCSSPageSize: true,
+        });
+        await chrome.debugger.detach({ tabId });
+        audit('pdf', { tabId }, { ok: true });
+        return { ok: true, pdf: pdf.data, format: 'pdf', encoding: 'base64' };
+      } catch (error) {
+        try { await chrome.debugger.detach({ tabId }); } catch {}
+        const result = { ok: false, error: error.message };
+        audit('pdf', { tabId }, result);
+        return result;
+      }
+    }
+
     const injectResult = await ensureContentScript(tabId);
     if (!injectResult.ok) {
       const result = { ok: false, error: injectResult.error };
@@ -302,6 +367,22 @@ chrome.runtime.onMessage.addListener((message, sender, sendResponse) => {
         result = listActivatedTabs();
         break;
 
+      case 'getAutoActivate':
+        result = { ok: true, autoActivateAll: state.autoActivateAll };
+        break;
+
+      case 'setAutoActivate': {
+        state.autoActivateAll = !!params.enabled;
+        chrome.storage.local.set({ autoActivateAll: state.autoActivateAll });
+        updateAutoActivateBadge();
+        if (state.autoActivateAll) {
+          await autoActivateExistingTabs();
+        }
+        audit('setAutoActivate', { enabled: state.autoActivateAll }, { ok: true });
+        result = { ok: true, autoActivateAll: state.autoActivateAll };
+        break;
+      }
+
       default:
         result = { ok: false, error: `Unknown action: ${action}` };
     }
@@ -312,15 +393,6 @@ chrome.runtime.onMessage.addListener((message, sender, sendResponse) => {
   return true;
 });
 
-// Handle extension icon click - toggle activation
-chrome.action.onClicked.addListener(async (tab) => {
-  if (state.activatedTabs.has(tab.id)) {
-    deactivateTab(tab.id);
-  } else {
-    await activateTab(tab.id);
-  }
-});
-
 // Update badge when switching tabs
 chrome.tabs.onActivated.addListener(({ tabId }) => {
   updateBadge(tabId);
@@ -348,9 +420,13 @@ chrome.tabs.onUpdated.addListener((tabId, changeInfo, tab) => {
     info.url = changeInfo.url;
     info.title = tab.title;
   }
-  if (changeInfo.status === 'complete' && state.activatedTabs.has(tabId)) {
-    ensureContentScript(tabId);
-    updateBadge(tabId);
+  if (changeInfo.status === 'complete') {
+    if (state.autoActivateAll && !state.activatedTabs.has(tabId) && tab.url && !tab.url.startsWith('chrome://')) {
+      activateTab(tabId);
+    } else if (state.activatedTabs.has(tabId)) {
+      ensureContentScript(tabId);
+      updateBadge(tabId);
+    }
   }
 });
 
@@ -361,6 +437,17 @@ function connectNativeHost() {
   console.log('Attempting to connect to native host...');
 
   try {
+    if (IS_SAFARI) {
+      // Safari: native messaging is handled by the containing app
+      // The app will inject messages via browser.runtime messaging
+      console.log('Safari detected - native messaging handled by container app');
+      state.nativeConnected = true;
+      state.lastNativeError = null;
+      // Safari extension will receive commands via runtime.onMessage from the app
+      return;
+    }
+
+    // Chrome: use connectNative
     nativePort = chrome.runtime.connectNative('com.tabagent.relay');
     console.log('connectNative called, port created');
 
@@ -424,6 +511,12 @@ function connectNativeHost() {
           case 'wait':
           case 'scrollintoview':
           case 'batchfill':
+          case 'drag':
+          case 'get':
+          case 'find':
+          case 'cookies':
+          case 'storage':
+          case 'pdf':
             result = await routeCommand(tabId, { action, ...params });
             break;
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "tab-agent",
-  "version": "0.3.4",
+  "version": "0.4.1",
   "description": "Give LLMs full control of your browser - secure, click-to-activate automation for Claude, ChatGPT, Codex, and any AI",
   "bin": {
     "tab-agent": "./bin/tab-agent.js"
@@ -12,9 +12,20 @@
   "files": [
     "bin/",
     "cli/",
-    "relay/",
+    "extension/content-script.js",
+    "extension/icons/",
+    "extension/manifest.json",
+    "extension/manifest.safari.json",
+    "extension/popup/",
+    "extension/service-worker.js",
+    "extension/snapshot.js",
+    "relay/install-native-host.sh",
+    "relay/native-host-wrapper.cmd",
+    "relay/native-host-wrapper.sh",
+    "relay/native-host.js",
+    "relay/server.js",
     "skills/",
-    "extension/"
+    "CHANGELOG.md"
   ],
   "dependencies": {
     "ws": "^8.16.0"