promethios-bridge 1.9.0 → 2.0.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "promethios-bridge",
-  "version": "1.9.0",
+  "version": "2.0.0",
   "description": "Run Promethios agent frameworks locally on your computer with full file, terminal, browser access, ambient context capture, and the always-on-top floating chat overlay. Native Framework Mode supports OpenClaw and other frameworks via the bridge.",
   "main": "src/index.js",
   "bin": {
@@ -53,7 +53,9 @@
   },
   "optionalDependencies": {
     "playwright": "^1.42.0",
-    "electron": "^29.0.0"
+    "electron": "^29.0.0",
+    "screenshot-desktop": "^1.12.7",
+    "sharp": "^0.33.0"
   },
   "engines": {
     "node": ">=18.0.0"

package/src/bridge.js

@@ -22,7 +22,18 @@ const fetch = require('node-fetch');
 const { executeLocalTool } = require('./executor');
 const { captureContext } = require('./contextCapture');
 const { startMcpServer } = require('./mcp-server');
-const { setPinnedRegion, setPinnedApps } = require('./tools/desktop');
+const { setPinnedRegion, setPinnedApps, registerBrowserPageAccessor } = require('./tools/desktop');
+
+// Wire the browser-dom tools to the shared Playwright context.
+// The context is created lazily in executor.js when browser_control is first used.
+// We expose a getter so browser-dom tools can access the live page.
+registerBrowserPageAccessor(async () => {
+  if (!global.__playwrightContext) {
+    throw new Error('Browser not open. Use the browser_control tool to navigate to a page first, then retry.');
+  }
+  const pages = global.__playwrightContext.pages();
+  return pages.length > 0 ? pages[pages.length - 1] : await global.__playwrightContext.newPage();
+});
 const { initAndroidTools } = require('./tools/android');
 
 // Optional: Electron overlay window (bundled in src/overlay — gracefully skipped if Electron not available)

package/src/tools/accessibility.js

@@ -0,0 +1,408 @@
+/**
+ * accessibility.js — Read live text content from native app windows
+ *
+ * Uses platform accessibility APIs to extract structured text from any
+ * focused or pinned window — without needing a saved file.
+ *
+ * Windows: PowerShell + UIAutomation (built-in, no install needed)
+ * macOS:   osascript + AXUIElement accessibility API
+ * Linux:   AT-SPI via xdotool + atspi-dump (best-effort)
+ *
+ * Exported tools:
+ *   read_window_text   — full text content of a window
+ *   get_text_at_cursor — text around the current cursor/caret position
+ *   get_selected_text  — currently selected/highlighted text
+ */
+
+'use strict';
+
+const { exec, execFile } = require('child_process');
+const os = require('os');
+const fs = require('fs');
+const path = require('path');
+
+// ── Windows: UIAutomation via PowerShell ──────────────────────────────────────
+const WIN_READ_WINDOW_PS = (windowTitle) => `
+Add-Type -AssemblyName UIAutomationClient
+Add-Type -AssemblyName UIAutomationTypes
+
+$root = [System.Windows.Automation.AutomationElement]::RootElement
+$condition = New-Object System.Windows.Automation.PropertyCondition(
+  [System.Windows.Automation.AutomationElement]::NameProperty, "${windowTitle.replace(/"/g, '`"')}"
+)
+$window = $root.FindFirst([System.Windows.Automation.TreeScope]::Children, $condition)
+
+if (-not $window) {
+  # Try partial match
+  $allWindows = $root.FindAll([System.Windows.Automation.TreeScope]::Children,
+    [System.Windows.Automation.Condition]::TrueCondition)
+  foreach ($w in $allWindows) {
+    if ($w.Current.Name -like "*${windowTitle.replace(/"/g, '`"')}*") {
+      $window = $w; break
+    }
+  }
+}
+
+if (-not $window) {
+  Write-Output "ERROR: Window not found: ${windowTitle.replace(/"/g, '`"')}"
+  exit 1
+}
+
+# Collect all text elements
+$textCondition = New-Object System.Windows.Automation.PropertyCondition(
+  [System.Windows.Automation.AutomationElement]::ControlTypeProperty,
+  [System.Windows.Automation.ControlType]::Text
+)
+$editCondition = New-Object System.Windows.Automation.PropertyCondition(
+  [System.Windows.Automation.AutomationElement]::ControlTypeProperty,
+  [System.Windows.Automation.ControlType]::Edit
+)
+$docCondition = New-Object System.Windows.Automation.PropertyCondition(
+  [System.Windows.Automation.AutomationElement]::ControlTypeProperty,
+  [System.Windows.Automation.ControlType]::Document
+)
+
+$orCond = New-Object System.Windows.Automation.OrCondition($textCondition, $editCondition, $docCondition)
+$elements = $window.FindAll([System.Windows.Automation.TreeScope]::Descendants, $orCond)
+
+$lines = @()
+foreach ($el in $elements) {
+  try {
+    $valPat = $el.GetCurrentPattern([System.Windows.Automation.ValuePattern]::Pattern)
+    $val = $valPat.Current.Value
+    if ($val -and $val.Trim()) { $lines += $val }
+  } catch {
+    $name = $el.Current.Name
+    if ($name -and $name.Trim()) { $lines += $name }
+  }
+}
+
+$lines | ForEach-Object { Write-Output $_ }
+`;
+
+const WIN_GET_SELECTED_TEXT_PS = `
+Add-Type -AssemblyName UIAutomationClient
+Add-Type -AssemblyName UIAutomationTypes
+Add-Type -AssemblyName System.Windows.Forms
+
+# Get focused element
+$focused = [System.Windows.Automation.AutomationElement]::FocusedElement
+if (-not $focused) { Write-Output ""; exit 0 }
+
+try {
+  $textPat = $focused.GetCurrentPattern([System.Windows.Automation.TextPattern]::Pattern)
+  $sel = $textPat.GetSelection()
+  if ($sel -and $sel.Length -gt 0) {
+    Write-Output $sel[0].GetText(-1)
+  } else {
+    Write-Output ""
+  }
+} catch {
+  # Fallback: clipboard method
+  $prev = [System.Windows.Forms.Clipboard]::GetText()
+  Add-Type -AssemblyName System.Windows.Forms
+  [System.Windows.Forms.SendKeys]::SendWait("^c")
+  Start-Sleep -Milliseconds 150
+  $sel = [System.Windows.Forms.Clipboard]::GetText()
+  # Restore clipboard
+  if ($prev) { [System.Windows.Forms.Clipboard]::SetText($prev) }
+  Write-Output $sel
+}
+`;
+
+const WIN_GET_ACTIVE_DOCUMENT_PS = `
+Add-Type -AssemblyName UIAutomationClient
+Add-Type -AssemblyName UIAutomationTypes
+
+$focused = [System.Windows.Automation.AutomationElement]::FocusedElement
+if (-not $focused) { Write-Output "{}"; exit 0 }
+
+# Walk up to find the window
+$walker = [System.Windows.Automation.TreeWalker]::ControlViewWalker
+$current = $focused
+$window = $null
+while ($current -ne $null) {
+  if ($current.Current.ControlType -eq [System.Windows.Automation.ControlType]::Window) {
+    $window = $current; break
+  }
+  try { $current = $walker.GetParent($current) } catch { break }
+}
+
+$result = @{
+  focused_app   = $focused.Current.ClassName
+  focused_name  = $focused.Current.Name
+  window_title  = if ($window) { $window.Current.Name } else { "" }
+  process_name  = $focused.Current.ProcessId
+}
+
+# Try to get text content of focused element
+try {
+  $valPat = $focused.GetCurrentPattern([System.Windows.Automation.ValuePattern]::Pattern)
+  $result["text_content"] = $valPat.Current.Value
+} catch {
+  try {
+    $textPat = $focused.GetCurrentPattern([System.Windows.Automation.TextPattern]::Pattern)
+    $docRange = $textPat.DocumentRange
+    $result["text_content"] = $docRange.GetText(50000)
+  } catch {
+    $result["text_content"] = ""
+  }
+}
+
+$result | ConvertTo-Json
+`;
+
+// ── macOS: osascript AXUIElement ──────────────────────────────────────────────
+const MAC_READ_WINDOW_SCRIPT = (appName) => `
+tell application "System Events"
+  tell process "${appName.replace(/"/g, '\\"')}"
+    set allText to ""
+    try
+      set frontWin to front window
+      set allText to value of text area 1 of scroll area 1 of frontWin
+    end try
+    if allText is "" then
+      try
+        set allText to value of text field 1 of front window
+      end try
+    end if
+    return allText
+  end tell
+end tell
+`;
+
+const MAC_GET_SELECTED_TEXT_SCRIPT = `
+tell application "System Events"
+  set frontApp to name of first process whose frontmost is true
+end tell
+
+tell application frontApp
+  try
+    return (get selection as text)
+  end try
+end tell
+
+-- Fallback: clipboard
+set prevClip to the clipboard
+tell application "System Events" to keystroke "c" using command down
+delay 0.15
+set selText to the clipboard
+set the clipboard to prevClip
+return selText
+`;
+
+// ── Helpers ───────────────────────────────────────────────────────────────────
+function runPowerShell(script, timeout = 10000) {
+  return new Promise((resolve, reject) => {
+    const tmpPs = path.join(os.tmpdir(), `pmth_acc_${Date.now()}.ps1`);
+    fs.writeFileSync(tmpPs, script, 'utf8');
+    execFile('powershell', ['-NoProfile', '-NonInteractive', '-File', tmpPs],
+      { timeout },
+      (err, stdout, stderr) => {
+        try { fs.unlinkSync(tmpPs); } catch {}
+        if (err && !stdout) return reject(new Error(stderr || err.message));
+        resolve(stdout.trim());
+      }
+    );
+  });
+}
+
+function runOsascript(script, timeout = 8000) {
+  return new Promise((resolve, reject) => {
+    const tmpAs = path.join(os.tmpdir(), `pmth_acc_${Date.now()}.scpt`);
+    fs.writeFileSync(tmpAs, script, 'utf8');
+    execFile('osascript', [tmpAs], { timeout }, (err, stdout, stderr) => {
+      try { fs.unlinkSync(tmpAs); } catch {}
+      if (err && !stdout) return reject(new Error(stderr || err.message));
+      resolve(stdout.trim());
+    });
+  });
+}
+
+// ── Tool implementations ──────────────────────────────────────────────────────
+
+/**
+ * Read the full text content of a window by title or app name.
+ */
+async function readWindowText(args) {
+  const windowTitle = args.window_title || args.app || '';
+  const maxChars    = args.max_chars || 50000;
+
+  if (process.platform === 'win32') {
+    try {
+      const output = await runPowerShell(WIN_READ_WINDOW_PS(windowTitle));
+      if (output.startsWith('ERROR:')) {
+        return { ok: false, error: output, text: null };
+      }
+      const text = output.slice(0, maxChars);
+      return {
+        ok:          true,
+        text,
+        char_count:  text.length,
+        truncated:   output.length > maxChars,
+        source:      'UIAutomation',
+        window:      windowTitle,
+        captured_at: new Date().toISOString(),
+      };
+    } catch (err) {
+      return { ok: false, error: err.message, text: null };
+    }
+  } else if (process.platform === 'darwin') {
+    try {
+      const text = await runOsascript(MAC_READ_WINDOW_SCRIPT(windowTitle));
+      return {
+        ok:          true,
+        text:        text.slice(0, maxChars),
+        char_count:  text.length,
+        truncated:   text.length > maxChars,
+        source:      'AppleScript',
+        window:      windowTitle,
+        captured_at: new Date().toISOString(),
+      };
+    } catch (err) {
+      return { ok: false, error: err.message, text: null };
+    }
+  } else {
+    // Linux: best-effort via xdotool + xclip
+    return new Promise((resolve) => {
+      exec(`xdotool getactivewindow getwindowname`, { timeout: 3000 }, (err, stdout) => {
+        resolve({
+          ok:          !err,
+          text:        stdout.trim() || null,
+          source:      'xdotool',
+          note:        'Linux accessibility support is limited. Full text extraction requires AT-SPI.',
+          captured_at: new Date().toISOString(),
+        });
+      });
+    });
+  }
+}
+
+/**
+ * Get the currently selected/highlighted text in the focused window.
+ */
+async function getSelectedText() {
+  if (process.platform === 'win32') {
+    try {
+      const text = await runPowerShell(WIN_GET_SELECTED_TEXT_PS, 5000);
+      return { ok: true, selected_text: text, captured_at: new Date().toISOString() };
+    } catch (err) {
+      return { ok: false, error: err.message, selected_text: null };
+    }
+  } else if (process.platform === 'darwin') {
+    try {
+      const text = await runOsascript(MAC_GET_SELECTED_TEXT_SCRIPT, 5000);
+      return { ok: true, selected_text: text, captured_at: new Date().toISOString() };
+    } catch (err) {
+      return { ok: false, error: err.message, selected_text: null };
+    }
+  } else {
+    return { ok: false, error: 'Linux selected text not yet supported', selected_text: null };
+  }
+}
+
+/**
+ * Get the active document context: focused app, window title, and text content
+ * of the focused element (works great for Word, Notepad, VS Code, etc.)
+ */
+async function getActiveDocumentContext() {
+  if (process.platform === 'win32') {
+    try {
+      const json = await runPowerShell(WIN_GET_ACTIVE_DOCUMENT_PS, 10000);
+      let parsed;
+      try { parsed = JSON.parse(json); } catch { parsed = { raw: json }; }
+      return {
+        ok:          true,
+        source:      'UIAutomation',
+        captured_at: new Date().toISOString(),
+        ...parsed,
+      };
+    } catch (err) {
+      return { ok: false, error: err.message };
+    }
+  } else if (process.platform === 'darwin') {
+    try {
+      const script = `
+tell application "System Events"
+  set frontApp to name of first process whose frontmost is true
+  set frontTitle to ""
+  try
+    set frontTitle to name of front window of process frontApp
+  end try
+  return frontApp & "|" & frontTitle
+end tell
+`;
+      const out = await runOsascript(script, 5000);
+      const [app, title] = out.split('|');
+      const textResult = await readWindowText({ app: app?.trim(), max_chars: 50000 });
+      return {
+        ok:           true,
+        focused_app:  app?.trim(),
+        window_title: title?.trim(),
+        text_content: textResult.text || '',
+        source:       'AppleScript',
+        captured_at:  new Date().toISOString(),
+      };
+    } catch (err) {
+      return { ok: false, error: err.message };
+    }
+  } else {
+    return { ok: false, error: 'Linux not yet fully supported' };
+  }
+}
+
+// ── TOOL_MANIFEST entries (added to desktop.js TOOL_MANIFEST) ─────────────────
+const ACCESSIBILITY_TOOLS = [
+  {
+    name: 'read_window_text',
+    description: 'Read the full text content of any open window or application using the OS accessibility API. Works on unsaved documents (Word, Notepad, VS Code, etc.) — no file path needed. Returns the live text as the user sees it.',
+    inputSchema: {
+      type: 'object',
+      properties: {
+        window_title: {
+          type: 'string',
+          description: 'Title of the window to read (partial match supported). Use list_open_windows to find window titles.',
+        },
+        app: {
+          type: 'string',
+          description: 'Application name (e.g. "Word", "Notepad", "Code"). Used if window_title is not provided.',
+        },
+        max_chars: {
+          type: 'number',
+          description: 'Maximum characters to return (default: 50000)',
+        },
+      },
+    },
+    handler: readWindowText,
+  },
+  {
+    name: 'get_selected_text',
+    description: 'Get the text the user currently has selected/highlighted in any application. Useful for "help me with this specific part" workflows.',
+    inputSchema: {
+      type: 'object',
+      properties: {},
+    },
+    handler: getSelectedText,
+  },
+  {
+    name: 'get_active_document',
+    description: 'Get the full context of what the user is actively working on: the focused application, window title, and complete text content of the focused document or editor. This is the primary tool for understanding what the user is working on right now.',
+    inputSchema: {
+      type: 'object',
+      properties: {
+        max_chars: {
+          type: 'number',
+          description: 'Maximum characters to return from the document (default: 50000)',
+        },
+      },
+    },
+    handler: getActiveDocumentContext,
+  },
+];
+
+module.exports = {
+  ACCESSIBILITY_TOOLS,
+  readWindowText,
+  getSelectedText,
+  getActiveDocumentContext,
+};