promethios-bridge 1.6.0 → 1.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +9 -5
- package/src/bridge.js +136 -4
- package/src/contextCapture.js +324 -0
- package/src/executor.js +38 -10
package/package.json
CHANGED
|
@@ -1,14 +1,15 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "promethios-bridge",
|
|
3
|
-
"version": "1.
|
|
4
|
-
"description": "Run Promethios agent frameworks locally on your computer with full file, terminal, browser access, and Native Framework Mode
|
|
3
|
+
"version": "1.7.0",
|
|
4
|
+
"description": "Run Promethios agent frameworks locally on your computer with full file, terminal, browser access, ambient context capture, and the always-on-top floating chat overlay. Native Framework Mode supports OpenClaw and other frameworks via the bridge.",
|
|
5
5
|
"main": "src/index.js",
|
|
6
6
|
"bin": {
|
|
7
7
|
"promethios-bridge": "src/cli.js"
|
|
8
8
|
},
|
|
9
9
|
"scripts": {
|
|
10
10
|
"start": "node src/cli.js",
|
|
11
|
-
"dev": "node src/cli.js --dev"
|
|
11
|
+
"dev": "node src/cli.js --dev",
|
|
12
|
+
"postinstall": "node node_modules/playwright/cli.js install chromium || npx playwright install chromium || true"
|
|
12
13
|
},
|
|
13
14
|
"keywords": [
|
|
14
15
|
"promethios",
|
|
@@ -19,7 +20,9 @@
|
|
|
19
20
|
"framework",
|
|
20
21
|
"openclaw",
|
|
21
22
|
"native",
|
|
22
|
-
"clawhub"
|
|
23
|
+
"clawhub",
|
|
24
|
+
"overlay",
|
|
25
|
+
"ambient"
|
|
23
26
|
],
|
|
24
27
|
"author": "Promethios <hello@promethios.ai>",
|
|
25
28
|
"license": "MIT",
|
|
@@ -50,7 +53,8 @@
|
|
|
50
53
|
"playwright": "^1.42.0"
|
|
51
54
|
},
|
|
52
55
|
"optionalDependencies": {
|
|
53
|
-
"playwright": "^1.42.0"
|
|
56
|
+
"playwright": "^1.42.0",
|
|
57
|
+
"electron": "^29.0.0"
|
|
54
58
|
},
|
|
55
59
|
"engines": {
|
|
56
60
|
"node": ">=18.0.0"
|
package/src/bridge.js
CHANGED
|
@@ -20,9 +20,15 @@ const chalk = require('chalk');
|
|
|
20
20
|
const ora = require('ora');
|
|
21
21
|
const fetch = require('node-fetch');
|
|
22
22
|
const { executeLocalTool } = require('./executor');
|
|
23
|
+
const { captureContext } = require('./contextCapture');
|
|
24
|
+
|
|
25
|
+
// Optional: Electron overlay window (gracefully skipped if not installed)
|
|
26
|
+
let launchOverlay = null;
|
|
27
|
+
try { launchOverlay = require('promethios-overlay/src/launcher').launchOverlay; } catch { /* overlay not installed */ }
|
|
23
28
|
|
|
24
29
|
const HEARTBEAT_INTERVAL = 30_000; // 30s
|
|
25
30
|
const POLL_INTERVAL = 1_000; // 1s — poll for pending tool calls
|
|
31
|
+
const CONTEXT_PUSH_INTERVAL = 5_000; // 5s — push ambient context snapshot
|
|
26
32
|
|
|
27
33
|
async function startBridge({ setupToken, apiBase, port, dev }) {
|
|
28
34
|
const log = dev
|
|
@@ -169,6 +175,21 @@ async function startBridge({ setupToken, apiBase, port, dev }) {
|
|
|
169
175
|
console.log(chalk.gray(' Press Ctrl+C to disconnect.'));
|
|
170
176
|
console.log('');
|
|
171
177
|
|
|
178
|
+
// ── Step 4c: Launch Electron overlay window (if available) ──────────────────
|
|
179
|
+
// The overlay is an always-on-top floating chat pill that works across all
|
|
180
|
+
// apps on Windows, Mac, and Linux. It auto-launches alongside the bridge.
|
|
181
|
+
// Users can toggle watching on/off from the pill without restarting the bridge.
|
|
182
|
+
if (launchOverlay) {
|
|
183
|
+
try {
|
|
184
|
+
launchOverlay({ authToken, apiBase, dev });
|
|
185
|
+
console.log(chalk.cyan(' ⬡ Promethios overlay launched — floating chat is ready'));
|
|
186
|
+
console.log(chalk.gray(' Hotkey: Ctrl+Shift+P (Win/Linux) or Cmd+Shift+P (Mac)'));
|
|
187
|
+
console.log('');
|
|
188
|
+
} catch (err) {
|
|
189
|
+
log('Overlay launch failed (non-critical):', err.message);
|
|
190
|
+
}
|
|
191
|
+
}
|
|
192
|
+
|
|
172
193
|
// Heartbeat loop
|
|
173
194
|
const heartbeatTimer = setInterval(async () => {
|
|
174
195
|
try {
|
|
@@ -182,6 +203,27 @@ async function startBridge({ setupToken, apiBase, port, dev }) {
|
|
|
182
203
|
}
|
|
183
204
|
}, HEARTBEAT_INTERVAL);
|
|
184
205
|
|
|
206
|
+
// ── Step 4b: Ambient context push loop ──────────────────────────────────────
|
|
207
|
+
// Every 5 seconds, capture what the user is working on (active window, browser
|
|
208
|
+
// URL, clipboard) and push it to the API. The agent uses this to give
|
|
209
|
+
// context-aware help without the user having to explain what they're doing.
|
|
210
|
+
const contextPushLoop = async () => {
|
|
211
|
+
while (pollActive) {
|
|
212
|
+
try {
|
|
213
|
+
const snapshot = await captureContext(process.platform, dev);
|
|
214
|
+
await fetch(`${apiBase}/api/local-bridge/context`, {
|
|
215
|
+
method: 'POST',
|
|
216
|
+
headers: { Authorization: `Bearer ${authToken}`, 'Content-Type': 'application/json' },
|
|
217
|
+
body: JSON.stringify(snapshot),
|
|
218
|
+
});
|
|
219
|
+
log('Context pushed:', snapshot.active_app, snapshot.active_url || '');
|
|
220
|
+
} catch (err) {
|
|
221
|
+
log('Context push failed:', err.message);
|
|
222
|
+
}
|
|
223
|
+
await sleep(CONTEXT_PUSH_INTERVAL);
|
|
224
|
+
}
|
|
225
|
+
};
|
|
226
|
+
|
|
185
227
|
// ── Step 5: Poll for pending tool calls (Firestore relay) ─────────────────
|
|
186
228
|
// The cloud backend queues tool calls in Firestore when the agent uses
|
|
187
229
|
// local_shell, local_file_read, or local_file_write. We poll for them here,
|
|
@@ -219,6 +261,11 @@ async function startBridge({ setupToken, apiBase, port, dev }) {
|
|
|
219
261
|
console.error(chalk.red(' ✗ Poll loop crashed:'), err.message);
|
|
220
262
|
});
|
|
221
263
|
|
|
264
|
+
// Start context push loop in background
|
|
265
|
+
contextPushLoop().catch(err => {
|
|
266
|
+
log('Context push loop crashed:', err.message);
|
|
267
|
+
});
|
|
268
|
+
|
|
222
269
|
// Graceful shutdown
|
|
223
270
|
const shutdown = async (signal) => {
|
|
224
271
|
console.log('');
|
|
@@ -321,6 +368,27 @@ async function registerBridge({ authToken, apiBase, callbackUrl, port, dev }) {
|
|
|
321
368
|
const homeDir = osModule.homedir();
|
|
322
369
|
const username = osModule.userInfo().username;
|
|
323
370
|
|
|
371
|
+
// ── Desktop path probe ────────────────────────────────────────────────────
|
|
372
|
+
// On Windows, the visual Desktop can be redirected anywhere by OneDrive,
|
|
373
|
+
// Group Policy, or manual folder redirection. We probe the real path at
|
|
374
|
+
// connect time so the agent always uses the correct location.
|
|
375
|
+
// Resolution order (first that exists wins):
|
|
376
|
+
// 1. Windows registry: HKCU\Software\Microsoft\Windows\CurrentVersion\Explorer\User Shell Folders\Desktop
|
|
377
|
+
// This is the authoritative source — it reflects OneDrive redirection,
|
|
378
|
+
// GPO folder redirection, and any custom Desktop path.
|
|
379
|
+
// 2. %USERPROFILE%\OneDrive\Desktop (common OneDrive default)
|
|
380
|
+
// 3. %USERPROFILE%\Desktop (standard local Desktop)
|
|
381
|
+
let desktopPath = null;
|
|
382
|
+
if (platform === 'win32') {
|
|
383
|
+
desktopPath = await probeWindowsDesktopPath(homeDir, dev);
|
|
384
|
+
} else if (platform === 'darwin') {
|
|
385
|
+
desktopPath = `${homeDir}/Desktop`;
|
|
386
|
+
} else {
|
|
387
|
+
desktopPath = `${homeDir}/Desktop`;
|
|
388
|
+
}
|
|
389
|
+
|
|
390
|
+
if (dev) console.log('[debug] Resolved desktopPath:', desktopPath);
|
|
391
|
+
|
|
324
392
|
const res = await fetch(`${apiBase}/api/local-bridge/register`, {
|
|
325
393
|
method: 'POST',
|
|
326
394
|
headers: {
|
|
@@ -332,10 +400,11 @@ async function registerBridge({ authToken, apiBase, callbackUrl, port, dev }) {
|
|
|
332
400
|
callbackUrl,
|
|
333
401
|
capabilities,
|
|
334
402
|
bridgeVersion: require('../package.json').version,
|
|
335
|
-
os: platform,
|
|
336
|
-
shell,
|
|
337
|
-
homeDir,
|
|
338
|
-
username,
|
|
403
|
+
os: platform, // 'win32' | 'darwin' | 'linux'
|
|
404
|
+
shell, // 'cmd' | 'zsh' | 'bash' etc.
|
|
405
|
+
homeDir, // e.g. 'C:\\Users\\ted' or '/Users/ted'
|
|
406
|
+
username, // e.g. 'ted'
|
|
407
|
+
desktopPath, // resolved real Desktop path — may differ from homeDir\Desktop
|
|
339
408
|
}),
|
|
340
409
|
});
|
|
341
410
|
|
|
@@ -345,6 +414,69 @@ async function registerBridge({ authToken, apiBase, callbackUrl, port, dev }) {
|
|
|
345
414
|
}
|
|
346
415
|
}
|
|
347
416
|
|
|
417
|
+
// ─────────────────────────────────────────────────────────────────────────────
|
|
418
|
+
// Probe the real Windows Desktop path for any user configuration
|
|
419
|
+
// ─────────────────────────────────────────────────────────────────────────────
|
|
420
|
+
async function probeWindowsDesktopPath(homeDir, dev) {
|
|
421
|
+
const { execSync } = require('child_process');
|
|
422
|
+
const fs = require('fs');
|
|
423
|
+
const path = require('path');
|
|
424
|
+
const log = dev ? (...a) => console.log('[debug]', ...a) : () => {};
|
|
425
|
+
|
|
426
|
+
// Strategy 1: Read the Windows registry — the authoritative source.
|
|
427
|
+
// The 'User Shell Folders' key stores the ACTUAL Desktop path after any
|
|
428
|
+
// OneDrive or GPO folder redirection. It may contain %USERPROFILE% which
|
|
429
|
+
// we expand manually.
|
|
430
|
+
try {
|
|
431
|
+
const regOutput = execSync(
|
|
432
|
+
'reg query "HKCU\\Software\\Microsoft\\Windows\\CurrentVersion\\Explorer\\User Shell Folders" /v Desktop',
|
|
433
|
+
{ encoding: 'utf8', timeout: 5000 }
|
|
434
|
+
);
|
|
435
|
+
// Output format: " Desktop REG_EXPAND_SZ C:\Users\ted\OneDrive\Desktop"
|
|
436
|
+
const match = regOutput.match(/Desktop\s+REG_EXPAND_SZ\s+(.+)/);
|
|
437
|
+
if (match) {
|
|
438
|
+
let regPath = match[1].trim();
|
|
439
|
+
// Expand %USERPROFILE% manually (Windows env vars in registry values)
|
|
440
|
+
regPath = regPath.replace(/%USERPROFILE%/gi, homeDir);
|
|
441
|
+
regPath = regPath.replace(/%USERNAME%/gi, path.basename(homeDir));
|
|
442
|
+
log('Registry Desktop path:', regPath);
|
|
443
|
+
if (fs.existsSync(regPath)) {
|
|
444
|
+
return regPath;
|
|
445
|
+
}
|
|
446
|
+
log('Registry path does not exist on disk, falling back:', regPath);
|
|
447
|
+
}
|
|
448
|
+
} catch (e) {
|
|
449
|
+
log('Registry query failed:', e.message);
|
|
450
|
+
}
|
|
451
|
+
|
|
452
|
+
// Strategy 2: PowerShell [Environment]::GetFolderPath('Desktop')
|
|
453
|
+
// This resolves the shell namespace Desktop correctly including OneDrive.
|
|
454
|
+
try {
|
|
455
|
+
const psOutput = execSync(
|
|
456
|
+
'powershell -NoProfile -Command "[Environment]::GetFolderPath(\'Desktop\')"',
|
|
457
|
+
{ encoding: 'utf8', timeout: 8000 }
|
|
458
|
+
).trim();
|
|
459
|
+
if (psOutput && fs.existsSync(psOutput)) {
|
|
460
|
+
log('PowerShell Desktop path:', psOutput);
|
|
461
|
+
return psOutput;
|
|
462
|
+
}
|
|
463
|
+
} catch (e) {
|
|
464
|
+
log('PowerShell Desktop probe failed:', e.message);
|
|
465
|
+
}
|
|
466
|
+
|
|
467
|
+
// Strategy 3: Check common OneDrive Desktop path
|
|
468
|
+
const oneDriveDesktop = path.join(homeDir, 'OneDrive', 'Desktop');
|
|
469
|
+
if (fs.existsSync(oneDriveDesktop)) {
|
|
470
|
+
log('OneDrive Desktop found:', oneDriveDesktop);
|
|
471
|
+
return oneDriveDesktop;
|
|
472
|
+
}
|
|
473
|
+
|
|
474
|
+
// Strategy 4: Standard local Desktop fallback
|
|
475
|
+
const localDesktop = path.join(homeDir, 'Desktop');
|
|
476
|
+
log('Falling back to local Desktop:', localDesktop);
|
|
477
|
+
return localDesktop;
|
|
478
|
+
}
|
|
479
|
+
|
|
348
480
|
// ─────────────────────────────────────────────────────────────────────────────
|
|
349
481
|
// What this bridge can do (declared to the cloud for permission manifest display)
|
|
350
482
|
// ─────────────────────────────────────────────────────────────────────────────
|
|
@@ -0,0 +1,324 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Promethios Local Bridge — Ambient Context Capture
|
|
3
|
+
*
|
|
4
|
+
* Captures what the user is currently working on:
|
|
5
|
+
* - Active window title + process name
|
|
6
|
+
* - Active browser URL (Chrome, Edge, Firefox via accessibility / window title)
|
|
7
|
+
* - Selected / clipboard text (last copied text)
|
|
8
|
+
* - Active form field content (via clipboard snapshot)
|
|
9
|
+
*
|
|
10
|
+
* Designed to run every 5 seconds and push a lightweight context snapshot
|
|
11
|
+
* to the Promethios API so the AI always knows what the user is doing.
|
|
12
|
+
*
|
|
13
|
+
* Platform support:
|
|
14
|
+
* - Windows: PowerShell Get-Process / UIAutomation / Get-Clipboard
|
|
15
|
+
* - macOS: AppleScript / pbpaste
|
|
16
|
+
* - Linux: xdotool / xclip (if available)
|
|
17
|
+
*/
|
|
18
|
+
|
|
19
|
+
const { execSync, exec } = require('child_process');
|
|
20
|
+
|
|
21
|
+
// ─────────────────────────────────────────────────────────────────────────────
|
|
22
|
+
// Main capture function — returns a context snapshot object
|
|
23
|
+
// ─────────────────────────────────────────────────────────────────────────────
|
|
24
|
+
async function captureContext(platform, dev) {
|
|
25
|
+
const log = dev ? (...a) => console.log('[context]', ...a) : () => {};
|
|
26
|
+
|
|
27
|
+
const snapshot = {
|
|
28
|
+
captured_at: new Date().toISOString(),
|
|
29
|
+
active_window: null, // { title, process }
|
|
30
|
+
active_url: null, // string — browser URL if active
|
|
31
|
+
active_app: null, // friendly app name
|
|
32
|
+
clipboard_text: null, // last copied text (truncated to 500 chars)
|
|
33
|
+
browser_tab_title: null, // browser tab title if browser is active
|
|
34
|
+
};
|
|
35
|
+
|
|
36
|
+
try {
|
|
37
|
+
if (platform === 'win32') {
|
|
38
|
+
await captureWindows(snapshot, log);
|
|
39
|
+
} else if (platform === 'darwin') {
|
|
40
|
+
await captureMac(snapshot, log);
|
|
41
|
+
} else {
|
|
42
|
+
await captureLinux(snapshot, log);
|
|
43
|
+
}
|
|
44
|
+
} catch (err) {
|
|
45
|
+
log('Context capture error:', err.message);
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
return snapshot;
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
// ─────────────────────────────────────────────────────────────────────────────
|
|
52
|
+
// Windows capture via PowerShell
|
|
53
|
+
// ─────────────────────────────────────────────────────────────────────────────
|
|
54
|
+
async function captureWindows(snapshot, log) {
|
|
55
|
+
// Active window title + process name
|
|
56
|
+
try {
|
|
57
|
+
const psScript = `
|
|
58
|
+
Add-Type @"
|
|
59
|
+
using System;
|
|
60
|
+
using System.Runtime.InteropServices;
|
|
61
|
+
using System.Text;
|
|
62
|
+
public class Win32 {
|
|
63
|
+
[DllImport("user32.dll")] public static extern IntPtr GetForegroundWindow();
|
|
64
|
+
[DllImport("user32.dll")] public static extern int GetWindowText(IntPtr hWnd, StringBuilder text, int count);
|
|
65
|
+
[DllImport("user32.dll")] public static extern int GetWindowThreadProcessId(IntPtr hWnd, out int lpdwProcessId);
|
|
66
|
+
}
|
|
67
|
+
"@
|
|
68
|
+
$hwnd = [Win32]::GetForegroundWindow()
|
|
69
|
+
$sb = New-Object System.Text.StringBuilder 512
|
|
70
|
+
[Win32]::GetWindowText($hwnd, $sb, 512) | Out-Null
|
|
71
|
+
$pid = 0
|
|
72
|
+
[Win32]::GetWindowThreadProcessId($hwnd, [ref]$pid) | Out-Null
|
|
73
|
+
$proc = Get-Process -Id $pid -ErrorAction SilentlyContinue
|
|
74
|
+
$result = @{
|
|
75
|
+
title = $sb.ToString()
|
|
76
|
+
process = if ($proc) { $proc.Name } else { "unknown" }
|
|
77
|
+
} | ConvertTo-Json -Compress
|
|
78
|
+
Write-Output $result
|
|
79
|
+
`.trim();
|
|
80
|
+
|
|
81
|
+
const output = execSync(
|
|
82
|
+
`powershell -NoProfile -NonInteractive -Command "${psScript.replace(/"/g, '\\"').replace(/\n/g, ' ')}"`,
|
|
83
|
+
{ encoding: 'utf8', timeout: 4000, windowsHide: true }
|
|
84
|
+
).trim();
|
|
85
|
+
|
|
86
|
+
const parsed = JSON.parse(output);
|
|
87
|
+
snapshot.active_window = { title: parsed.title, process: parsed.process };
|
|
88
|
+
snapshot.active_app = friendlyAppName(parsed.process, parsed.title);
|
|
89
|
+
log('Active window:', snapshot.active_window);
|
|
90
|
+
} catch (err) {
|
|
91
|
+
log('Windows active window capture failed:', err.message);
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
// Extract browser URL from window title (works for Chrome, Edge, Firefox)
|
|
95
|
+
// Chrome/Edge format: "<page title> - <browser name>"
|
|
96
|
+
// Firefox format: "<page title> — Mozilla Firefox"
|
|
97
|
+
if (snapshot.active_window) {
|
|
98
|
+
const url = extractUrlFromBrowserTitle(snapshot.active_window.title, snapshot.active_window.process);
|
|
99
|
+
if (url) {
|
|
100
|
+
snapshot.active_url = url;
|
|
101
|
+
snapshot.browser_tab_title = extractTabTitle(snapshot.active_window.title, snapshot.active_window.process);
|
|
102
|
+
}
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
// Clipboard text (last copied)
|
|
106
|
+
try {
|
|
107
|
+
const clip = execSync(
|
|
108
|
+
'powershell -NoProfile -NonInteractive -Command "Get-Clipboard -ErrorAction SilentlyContinue"',
|
|
109
|
+
{ encoding: 'utf8', timeout: 2000, windowsHide: true }
|
|
110
|
+
).trim();
|
|
111
|
+
if (clip && clip.length > 0 && clip.length < 5000) {
|
|
112
|
+
snapshot.clipboard_text = clip.slice(0, 500);
|
|
113
|
+
log('Clipboard:', snapshot.clipboard_text.slice(0, 80));
|
|
114
|
+
}
|
|
115
|
+
} catch (err) {
|
|
116
|
+
log('Clipboard capture failed:', err.message);
|
|
117
|
+
}
|
|
118
|
+
}
|
|
119
|
+
|
|
120
|
+
// ─────────────────────────────────────────────────────────────────────────────
|
|
121
|
+
// macOS capture via AppleScript
|
|
122
|
+
// ─────────────────────────────────────────────────────────────────────────────
|
|
123
|
+
async function captureMac(snapshot, log) {
|
|
124
|
+
// Active app + window title
|
|
125
|
+
try {
|
|
126
|
+
const script = `
|
|
127
|
+
tell application "System Events"
|
|
128
|
+
set frontApp to first application process whose frontmost is true
|
|
129
|
+
set appName to name of frontApp
|
|
130
|
+
set windowTitle to ""
|
|
131
|
+
try
|
|
132
|
+
set windowTitle to name of front window of frontApp
|
|
133
|
+
end try
|
|
134
|
+
return appName & "|" & windowTitle
|
|
135
|
+
end tell`;
|
|
136
|
+
const output = execSync(`osascript -e '${script.replace(/'/g, "\\'")}'`,
|
|
137
|
+
{ encoding: 'utf8', timeout: 4000 }).trim();
|
|
138
|
+
const [appName, windowTitle] = output.split('|');
|
|
139
|
+
snapshot.active_window = { title: windowTitle || '', process: appName || '' };
|
|
140
|
+
snapshot.active_app = appName;
|
|
141
|
+
log('Active app:', appName, 'Window:', windowTitle);
|
|
142
|
+
} catch (err) {
|
|
143
|
+
log('macOS active window capture failed:', err.message);
|
|
144
|
+
}
|
|
145
|
+
|
|
146
|
+
// Browser URL via AppleScript (Chrome, Safari, Firefox)
|
|
147
|
+
if (snapshot.active_window) {
|
|
148
|
+
const proc = (snapshot.active_window.process || '').toLowerCase();
|
|
149
|
+
try {
|
|
150
|
+
let urlScript = '';
|
|
151
|
+
if (proc.includes('chrome') || proc.includes('chromium')) {
|
|
152
|
+
urlScript = 'tell application "Google Chrome" to return URL of active tab of front window';
|
|
153
|
+
} else if (proc.includes('safari')) {
|
|
154
|
+
urlScript = 'tell application "Safari" to return URL of current tab of front window';
|
|
155
|
+
} else if (proc.includes('firefox')) {
|
|
156
|
+
urlScript = 'tell application "Firefox" to return URL of active tab of front window';
|
|
157
|
+
} else if (proc.includes('edge')) {
|
|
158
|
+
urlScript = 'tell application "Microsoft Edge" to return URL of active tab of front window';
|
|
159
|
+
}
|
|
160
|
+
if (urlScript) {
|
|
161
|
+
const url = execSync(`osascript -e '${urlScript}'`,
|
|
162
|
+
{ encoding: 'utf8', timeout: 3000 }).trim();
|
|
163
|
+
snapshot.active_url = url;
|
|
164
|
+
snapshot.browser_tab_title = snapshot.active_window.title;
|
|
165
|
+
log('Browser URL:', url);
|
|
166
|
+
}
|
|
167
|
+
} catch (err) {
|
|
168
|
+
log('macOS browser URL capture failed:', err.message);
|
|
169
|
+
}
|
|
170
|
+
}
|
|
171
|
+
|
|
172
|
+
// Clipboard
|
|
173
|
+
try {
|
|
174
|
+
const clip = execSync('pbpaste', { encoding: 'utf8', timeout: 2000 }).trim();
|
|
175
|
+
if (clip && clip.length > 0 && clip.length < 5000) {
|
|
176
|
+
snapshot.clipboard_text = clip.slice(0, 500);
|
|
177
|
+
}
|
|
178
|
+
} catch (err) {
|
|
179
|
+
log('macOS clipboard capture failed:', err.message);
|
|
180
|
+
}
|
|
181
|
+
}
|
|
182
|
+
|
|
183
|
+
// ─────────────────────────────────────────────────────────────────────────────
|
|
184
|
+
// Linux capture via xdotool / xclip (best-effort)
|
|
185
|
+
// ─────────────────────────────────────────────────────────────────────────────
|
|
186
|
+
async function captureLinux(snapshot, log) {
|
|
187
|
+
try {
|
|
188
|
+
const title = execSync('xdotool getactivewindow getwindowname 2>/dev/null',
|
|
189
|
+
{ encoding: 'utf8', timeout: 2000 }).trim();
|
|
190
|
+
snapshot.active_window = { title, process: 'unknown' };
|
|
191
|
+
snapshot.active_app = title;
|
|
192
|
+
} catch (err) {
|
|
193
|
+
log('Linux active window capture failed:', err.message);
|
|
194
|
+
}
|
|
195
|
+
|
|
196
|
+
try {
|
|
197
|
+
const clip = execSync('xclip -selection clipboard -o 2>/dev/null || xsel --clipboard --output 2>/dev/null',
|
|
198
|
+
{ encoding: 'utf8', timeout: 2000 }).trim();
|
|
199
|
+
if (clip && clip.length > 0 && clip.length < 5000) {
|
|
200
|
+
snapshot.clipboard_text = clip.slice(0, 500);
|
|
201
|
+
}
|
|
202
|
+
} catch (err) {
|
|
203
|
+
log('Linux clipboard capture failed:', err.message);
|
|
204
|
+
}
|
|
205
|
+
}
|
|
206
|
+
|
|
207
|
+
// ─────────────────────────────────────────────────────────────────────────────
|
|
208
|
+
// Helpers
|
|
209
|
+
// ─────────────────────────────────────────────────────────────────────────────
|
|
210
|
+
|
|
211
|
+
/**
|
|
212
|
+
* Try to extract a URL from a browser window title.
|
|
213
|
+
* Chrome/Edge show: "<page title> - Google Chrome"
|
|
214
|
+
* Firefox shows: "<page title> — Mozilla Firefox"
|
|
215
|
+
* We can't get the actual URL from the title alone, but we can detect
|
|
216
|
+
* that a browser is active and parse the tab title.
|
|
217
|
+
*
|
|
218
|
+
* For richer URL capture on Windows, we use the UIAutomation address bar
|
|
219
|
+
* approach via PowerShell — but that's expensive, so we only do it when
|
|
220
|
+
* the active process is a known browser.
|
|
221
|
+
*/
|
|
222
|
+
function extractUrlFromBrowserTitle(title, process) {
|
|
223
|
+
const proc = (process || '').toLowerCase();
|
|
224
|
+
const isBrowser = proc.includes('chrome') || proc.includes('msedge') ||
|
|
225
|
+
proc.includes('firefox') || proc.includes('opera') || proc.includes('brave');
|
|
226
|
+
if (!isBrowser) return null;
|
|
227
|
+
|
|
228
|
+
// Try to get URL from Chrome/Edge via PowerShell UIAutomation (Windows only)
|
|
229
|
+
// This reads the address bar text directly — works even for SPAs
|
|
230
|
+
if (proc.includes('chrome') || proc.includes('msedge')) {
|
|
231
|
+
try {
|
|
232
|
+
const psScript = `
|
|
233
|
+
Add-Type -AssemblyName UIAutomationClient
|
|
234
|
+
Add-Type -AssemblyName UIAutomationTypes
|
|
235
|
+
$proc = Get-Process -Name "${proc.includes('msedge') ? 'msedge' : 'chrome'}" -ErrorAction SilentlyContinue | Select-Object -First 1
|
|
236
|
+
if ($proc) {
|
|
237
|
+
$root = [System.Windows.Automation.AutomationElement]::FromHandle($proc.MainWindowHandle)
|
|
238
|
+
$cond = New-Object System.Windows.Automation.PropertyCondition([System.Windows.Automation.AutomationElement]::NameProperty, "Address and search bar")
|
|
239
|
+
$bar = $root.FindFirst([System.Windows.Automation.TreeScope]::Descendants, $cond)
|
|
240
|
+
if ($bar) {
|
|
241
|
+
$vp = $bar.GetCurrentPattern([System.Windows.Automation.ValuePattern]::Pattern)
|
|
242
|
+
Write-Output $vp.Current.Value
|
|
243
|
+
}
|
|
244
|
+
}`.trim();
|
|
245
|
+
const url = execSync(
|
|
246
|
+
`powershell -NoProfile -NonInteractive -Command "${psScript.replace(/"/g, '\\"').replace(/\n/g, ' ')}"`,
|
|
247
|
+
{ encoding: 'utf8', timeout: 5000, windowsHide: true }
|
|
248
|
+
).trim();
|
|
249
|
+
if (url && (url.startsWith('http') || url.startsWith('chrome'))) {
|
|
250
|
+
return url;
|
|
251
|
+
}
|
|
252
|
+
} catch (err) {
|
|
253
|
+
// UIAutomation not available — fall back to title parsing
|
|
254
|
+
}
|
|
255
|
+
}
|
|
256
|
+
|
|
257
|
+
return null; // URL not extractable from title alone
|
|
258
|
+
}
|
|
259
|
+
|
|
260
|
+
function extractTabTitle(windowTitle, process) {
|
|
261
|
+
const proc = (process || '').toLowerCase();
|
|
262
|
+
// Strip browser suffix from title
|
|
263
|
+
return windowTitle
|
|
264
|
+
.replace(/ - Google Chrome$/, '')
|
|
265
|
+
.replace(/ - Microsoft Edge$/, '')
|
|
266
|
+
.replace(/ — Mozilla Firefox$/, '')
|
|
267
|
+
.replace(/ - Brave$/, '')
|
|
268
|
+
.replace(/ - Opera$/, '')
|
|
269
|
+
.trim();
|
|
270
|
+
}
|
|
271
|
+
|
|
272
|
+
function friendlyAppName(processName, windowTitle) {
|
|
273
|
+
const proc = (processName || '').toLowerCase();
|
|
274
|
+
if (proc.includes('chrome')) return 'Google Chrome';
|
|
275
|
+
if (proc.includes('msedge')) return 'Microsoft Edge';
|
|
276
|
+
if (proc.includes('firefox')) return 'Firefox';
|
|
277
|
+
if (proc.includes('brave')) return 'Brave Browser';
|
|
278
|
+
if (proc.includes('code')) return 'VS Code';
|
|
279
|
+
if (proc.includes('excel')) return 'Microsoft Excel';
|
|
280
|
+
if (proc.includes('word')) return 'Microsoft Word';
|
|
281
|
+
if (proc.includes('outlook')) return 'Microsoft Outlook';
|
|
282
|
+
if (proc.includes('slack')) return 'Slack';
|
|
283
|
+
if (proc.includes('teams')) return 'Microsoft Teams';
|
|
284
|
+
if (proc.includes('zoom')) return 'Zoom';
|
|
285
|
+
if (proc.includes('notepad')) return 'Notepad';
|
|
286
|
+
if (proc.includes('explorer')) return 'File Explorer';
|
|
287
|
+
if (proc.includes('cmd') || proc.includes('powershell') || proc.includes('windowsterminal')) return 'Terminal';
|
|
288
|
+
return processName || 'Unknown';
|
|
289
|
+
}
|
|
290
|
+
|
|
291
|
+
// ─────────────────────────────────────────────────────────────────────────────
|
|
292
|
+
// Format context snapshot as a human-readable string for the agent system prompt
|
|
293
|
+
// ─────────────────────────────────────────────────────────────────────────────
|
|
294
|
+
function formatContextForPrompt(snapshot) {
|
|
295
|
+
if (!snapshot) return '';
|
|
296
|
+
|
|
297
|
+
const lines = [];
|
|
298
|
+
lines.push('── USER CONTEXT (live, captured just now) ──────────────────');
|
|
299
|
+
|
|
300
|
+
if (snapshot.active_app) {
|
|
301
|
+
lines.push(`Active app: ${snapshot.active_app}`);
|
|
302
|
+
}
|
|
303
|
+
if (snapshot.active_window && snapshot.active_window.title) {
|
|
304
|
+
lines.push(`Window title: ${snapshot.active_window.title}`);
|
|
305
|
+
}
|
|
306
|
+
if (snapshot.active_url) {
|
|
307
|
+
lines.push(`Browser URL: ${snapshot.active_url}`);
|
|
308
|
+
}
|
|
309
|
+
if (snapshot.browser_tab_title && snapshot.browser_tab_title !== snapshot.active_window?.title) {
|
|
310
|
+
lines.push(`Tab: ${snapshot.browser_tab_title}`);
|
|
311
|
+
}
|
|
312
|
+
if (snapshot.clipboard_text) {
|
|
313
|
+
lines.push(`Clipboard: "${snapshot.clipboard_text.slice(0, 200)}${snapshot.clipboard_text.length > 200 ? '...' : ''}"`);
|
|
314
|
+
}
|
|
315
|
+
if (snapshot.captured_at) {
|
|
316
|
+
lines.push(`Captured: ${snapshot.captured_at}`);
|
|
317
|
+
}
|
|
318
|
+
|
|
319
|
+
lines.push('────────────────────────────────────────────────────────────');
|
|
320
|
+
|
|
321
|
+
return lines.join('\n');
|
|
322
|
+
}
|
|
323
|
+
|
|
324
|
+
module.exports = { captureContext, formatContextForPrompt };
|
package/src/executor.js
CHANGED
|
@@ -234,12 +234,17 @@ async function executeLocalTool({ toolName, args, frameworkId, dev }) {
|
|
|
234
234
|
try {
|
|
235
235
|
playwright = require('playwright');
|
|
236
236
|
} catch (e) {
|
|
237
|
-
// Playwright not installed — install it
|
|
237
|
+
// Playwright not installed — install it locally into the bridge's own node_modules
|
|
238
|
+
// so require() can find it without relying on global npm or npx resolution.
|
|
238
239
|
const chalk = require('chalk');
|
|
240
|
+
const bridgeDir = path.join(__dirname, '..');
|
|
239
241
|
console.log(chalk.yellow('\n Playwright not found — installing automatically (one-time setup, ~2 min)...\n'));
|
|
240
242
|
try {
|
|
241
|
-
execSync('npm install -
|
|
242
|
-
|
|
243
|
+
execSync('npm install playwright --save-optional', { stdio: 'inherit', cwd: bridgeDir });
|
|
244
|
+
// Use the locally installed playwright binary directly to avoid npx resolution issues
|
|
245
|
+
const isWin = process.platform === 'win32';
|
|
246
|
+
const playwrightBin = path.join(bridgeDir, 'node_modules', '.bin', isWin ? 'playwright.cmd' : 'playwright');
|
|
247
|
+
execSync(`"${playwrightBin}" install chromium`, { stdio: 'inherit', cwd: bridgeDir });
|
|
243
248
|
playwright = require('playwright');
|
|
244
249
|
console.log(chalk.green('\n Playwright installed. Browser automation is ready.\n'));
|
|
245
250
|
} catch (installErr) {
|
|
@@ -847,33 +852,56 @@ async function executeLocalTool({ toolName, args, frameworkId, dev }) {
|
|
|
847
852
|
*
|
|
848
853
|
* On macOS/Linux the standard ~/Desktop is used.
|
|
849
854
|
*/
|
|
855
|
+
// Cache the resolved Desktop path so registry queries don't repeat every call
|
|
856
|
+
let _resolvedDesktopPath = null;
|
|
857
|
+
|
|
850
858
|
function resolveDesktopPath() {
|
|
859
|
+
if (_resolvedDesktopPath) return _resolvedDesktopPath;
|
|
860
|
+
|
|
851
861
|
if (process.platform !== 'win32') {
|
|
852
|
-
|
|
862
|
+
_resolvedDesktopPath = path.join(require('os').homedir(), 'Desktop');
|
|
863
|
+
console.log(`[Executor] Desktop path resolved (non-Windows): ${_resolvedDesktopPath}`);
|
|
864
|
+
return _resolvedDesktopPath;
|
|
853
865
|
}
|
|
854
|
-
|
|
866
|
+
|
|
867
|
+
// Try registry first (most reliable — works even with custom Desktop locations,
|
|
868
|
+
// including OneDrive Desktop Backup which silently redirects the Desktop folder)
|
|
855
869
|
try {
|
|
856
870
|
const { execSync } = require('child_process');
|
|
857
871
|
const regOut = execSync(
|
|
858
872
|
'reg query "HKCU\\Software\\Microsoft\\Windows\\CurrentVersion\\Explorer\\User Shell Folders" /v Desktop',
|
|
859
|
-
{ encoding: 'utf8', timeout:
|
|
873
|
+
{ encoding: 'utf8', timeout: 5000 }
|
|
860
874
|
);
|
|
861
875
|
const match = regOut.match(/Desktop\s+REG_(?:SZ|EXPAND_SZ)\s+(.+)/i);
|
|
862
876
|
if (match) {
|
|
863
877
|
// Expand environment variables like %USERPROFILE%
|
|
864
878
|
let desktopPath = match[1].trim();
|
|
865
879
|
desktopPath = desktopPath.replace(/%([^%]+)%/g, (_, varName) => process.env[varName] || `%${varName}%`);
|
|
866
|
-
if (require('fs').existsSync(desktopPath))
|
|
880
|
+
if (require('fs').existsSync(desktopPath)) {
|
|
881
|
+
_resolvedDesktopPath = desktopPath;
|
|
882
|
+
console.log(`[Executor] Desktop path resolved via registry: ${_resolvedDesktopPath}`);
|
|
883
|
+
return _resolvedDesktopPath;
|
|
884
|
+
} else {
|
|
885
|
+
console.warn(`[Executor] Registry Desktop path does not exist on disk: ${desktopPath} — falling through`);
|
|
886
|
+
}
|
|
867
887
|
}
|
|
868
|
-
} catch {
|
|
888
|
+
} catch (regErr) {
|
|
889
|
+
console.warn(`[Executor] Registry Desktop query failed (${regErr.message}) — falling through to OneDrive check`);
|
|
890
|
+
}
|
|
869
891
|
|
|
870
892
|
// Fallback: check OneDrive Desktop first (most common on Windows 11)
|
|
871
893
|
const userProfile = process.env.USERPROFILE || require('os').homedir();
|
|
872
894
|
const oneDriveDesktop = path.join(userProfile, 'OneDrive', 'Desktop');
|
|
873
|
-
if (require('fs').existsSync(oneDriveDesktop))
|
|
895
|
+
if (require('fs').existsSync(oneDriveDesktop)) {
|
|
896
|
+
_resolvedDesktopPath = oneDriveDesktop;
|
|
897
|
+
console.log(`[Executor] Desktop path resolved via OneDrive fallback: ${_resolvedDesktopPath}`);
|
|
898
|
+
return _resolvedDesktopPath;
|
|
899
|
+
}
|
|
874
900
|
|
|
875
901
|
// Final fallback: local Desktop
|
|
876
|
-
|
|
902
|
+
_resolvedDesktopPath = path.join(userProfile, 'Desktop');
|
|
903
|
+
console.log(`[Executor] Desktop path resolved via local fallback: ${_resolvedDesktopPath}`);
|
|
904
|
+
return _resolvedDesktopPath;
|
|
877
905
|
}
|
|
878
906
|
|
|
879
907
|
function resolveSafePath(inputPath) {
|