pi-browser-use 0.6.0 → 0.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +39 -11
- package/dist/auth-verifiers.d.ts +60 -0
- package/dist/auth-verifiers.d.ts.map +1 -0
- package/dist/auth-verifiers.js +92 -0
- package/dist/auth-verifiers.js.map +1 -0
- package/dist/chrome-launcher.d.ts +100 -0
- package/dist/chrome-launcher.d.ts.map +1 -0
- package/dist/chrome-launcher.js +268 -0
- package/dist/chrome-launcher.js.map +1 -0
- package/dist/config.d.ts +3 -1
- package/dist/config.d.ts.map +1 -1
- package/dist/config.js +28 -5
- package/dist/config.js.map +1 -1
- package/dist/doctor.d.ts +8 -1
- package/dist/doctor.d.ts.map +1 -1
- package/dist/doctor.js +16 -6
- package/dist/doctor.js.map +1 -1
- package/dist/existing-flow.d.ts +68 -0
- package/dist/existing-flow.d.ts.map +1 -0
- package/dist/existing-flow.js +128 -0
- package/dist/existing-flow.js.map +1 -0
- package/dist/focus-policy.d.ts +28 -0
- package/dist/focus-policy.d.ts.map +1 -0
- package/dist/focus-policy.js +28 -0
- package/dist/focus-policy.js.map +1 -0
- package/dist/index.d.ts +2 -2
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +497 -39
- package/dist/index.js.map +1 -1
- package/dist/named-profile.d.ts +47 -0
- package/dist/named-profile.d.ts.map +1 -0
- package/dist/named-profile.js +123 -0
- package/dist/named-profile.js.map +1 -0
- package/dist/persistent-backend.d.ts +77 -0
- package/dist/persistent-backend.d.ts.map +1 -0
- package/dist/persistent-backend.js +166 -0
- package/dist/persistent-backend.js.map +1 -0
- package/dist/persistent-store.d.ts +35 -0
- package/dist/persistent-store.d.ts.map +1 -0
- package/dist/persistent-store.js +98 -0
- package/dist/persistent-store.js.map +1 -0
- package/dist/profile-lock.d.ts +40 -0
- package/dist/profile-lock.d.ts.map +1 -0
- package/dist/profile-lock.js +165 -0
- package/dist/profile-lock.js.map +1 -0
- package/dist/profile.d.ts +3 -1
- package/dist/profile.d.ts.map +1 -1
- package/dist/profile.js +15 -6
- package/dist/profile.js.map +1 -1
- package/dist/session-manager.d.ts +125 -0
- package/dist/session-manager.d.ts.map +1 -0
- package/dist/session-manager.js +275 -0
- package/dist/session-manager.js.map +1 -0
- package/dist/session.d.ts +92 -0
- package/dist/session.d.ts.map +1 -0
- package/dist/session.js +29 -0
- package/dist/session.js.map +1 -0
- package/dist/setup-flow.d.ts +78 -0
- package/dist/setup-flow.d.ts.map +1 -0
- package/dist/setup-flow.js +97 -0
- package/dist/setup-flow.js.map +1 -0
- package/dist/tab-bridge.d.ts +71 -0
- package/dist/tab-bridge.d.ts.map +1 -0
- package/dist/tab-bridge.js +198 -0
- package/dist/tab-bridge.js.map +1 -0
- package/extension/README.md +33 -0
- package/extension/background.js +185 -0
- package/extension/manifest.json +12 -0
- package/package.json +2 -1
- package/skills/auth-bootstrap/SKILL.md +13 -0
- package/skills/browser-policy/SKILL.md +29 -4
- package/skills/gmail-auth/SKILL.md +51 -0
package/dist/index.js
CHANGED
|
@@ -3,13 +3,21 @@ import { homedir } from 'node:os';
|
|
|
3
3
|
import { dirname, join } from 'node:path';
|
|
4
4
|
import { Type } from 'typebox';
|
|
5
5
|
import { DevToolsClient } from './client.js';
|
|
6
|
-
import { resolveConfig, resolveModeTarget, } from './config.js';
|
|
6
|
+
import { DEFAULT_PROFILE_DIR, resolveConfig, resolveModeTarget, } from './config.js';
|
|
7
7
|
import { isProjectTrusted, loadConfig } from './settings.js';
|
|
8
8
|
import { augmentToolDescription, classifyPageState, extractTextContent, looksLikeLoginWall, looksOverlayBlocked, OVERLAY_RECOVERABLE, postProcessToolResult, } from './tool-augment.js';
|
|
9
9
|
import { pickImageData, resolveArtifactTarget } from './artifacts.js';
|
|
10
10
|
import { CLEANUP_ANNOTATIONS, formatAnnotatedMap, INJECT_ANNOTATIONS, parseAnnotatedElements, } from './annotate.js';
|
|
11
11
|
import { diagnose, formatDoctorReport } from './doctor.js';
|
|
12
|
+
import { checkExistingCloseAllowed, normalizeTabUrl, openExistingPage, parseMcpPageList, } from './existing-flow.js';
|
|
13
|
+
import { applyNewPageDefaults, applySelectPageDefaults } from './focus-policy.js';
|
|
14
|
+
import { frontProcessByPid } from './chrome-launcher.js';
|
|
15
|
+
import { PersistentBackend, shouldSelfLaunch } from './persistent-backend.js';
|
|
16
|
+
import { loadPersistentMetadata, loadSitePreferences, markAutomationResult, saveSitePreferences, } from './persistent-store.js';
|
|
12
17
|
import { prepareBrowserProfile } from './profile.js';
|
|
18
|
+
import { runBootstrap, runReauth } from './setup-flow.js';
|
|
19
|
+
import { normalizeOrigin, rememberExecutionPreference, resolveExecutionForOrigin, } from './session-manager.js';
|
|
20
|
+
import { DEFAULT_BRIDGE_PORT, TabBridge } from './tab-bridge.js';
|
|
13
21
|
import { createRegistryVisionCaller, handleAnalyzeScreenshot, } from './vision.js';
|
|
14
22
|
export { configToArgs, resolveConfig } from './config.js';
|
|
15
23
|
// All upstream tools are re-exported with this prefix to avoid name collisions.
|
|
@@ -60,13 +68,27 @@ function toToolContent(result, originalName) {
|
|
|
60
68
|
* discovers upstream tools, and registers each as browser_*. On
|
|
61
69
|
* session_shutdown tears the subprocess down. Nothing runs persistently.
|
|
62
70
|
*
|
|
63
|
-
* Defaults are
|
|
64
|
-
*
|
|
71
|
+
* Defaults are persistent headless (Pi-owned profile, no window, no consent
|
|
72
|
+
* popups). Set mode "fresh" for an isolated clean room, or "existing" with
|
|
65
73
|
* autoConnect/browserUrl to drive an already-running Chrome.
|
|
66
74
|
*/
|
|
67
75
|
export default function browserUseExtension(pi) {
|
|
68
76
|
let config;
|
|
69
77
|
let client;
|
|
78
|
+
// Pi-owned persistent Chrome (self-launched, MCP attached via browserUrl).
|
|
79
|
+
// Set only for persistent mode when the legacy MCP-launch path is off.
|
|
80
|
+
let ownBackend;
|
|
81
|
+
// Existing-mode tab broker bridge. Lazy; lives for the whole session.
|
|
82
|
+
let bridge;
|
|
83
|
+
// Last navigated origin: drives the per-origin headed-background fallback.
|
|
84
|
+
let lastOrigin;
|
|
85
|
+
// URLs Pi opened or navigated to in Existing mode (raw + normalized): the
|
|
86
|
+
// only close_page targets allowed there without explicit force:true.
|
|
87
|
+
const piOwnedUrls = new Set();
|
|
88
|
+
function trackPiUrl(url) {
|
|
89
|
+
piOwnedUrls.add(url);
|
|
90
|
+
piOwnedUrls.add(normalizeTabUrl(url));
|
|
91
|
+
}
|
|
70
92
|
// Tracks which identity the live backend holds, so results can suggest
|
|
71
93
|
// escalation. 'custom' covers user-configured attach setups we did not pick.
|
|
72
94
|
let currentMode = 'fresh';
|
|
@@ -75,14 +97,15 @@ export default function browserUseExtension(pi) {
|
|
|
75
97
|
return 'fresh';
|
|
76
98
|
if (config?.sessionMode === 'persistent')
|
|
77
99
|
return 'persistent';
|
|
100
|
+
if (config?.sessionMode === 'existing')
|
|
101
|
+
return 'existing';
|
|
78
102
|
return 'custom';
|
|
79
103
|
}
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
async function
|
|
85
|
-
const next = resolveConfig(resolveModeTarget(config ?? {}, mode, headed));
|
|
104
|
+
function persistentProfileDir(cfg) {
|
|
105
|
+
return cfg.userDataDir ?? DEFAULT_PROFILE_DIR;
|
|
106
|
+
}
|
|
107
|
+
/** Close the MCP transport and any Pi-owned Chrome. The bridge survives. */
|
|
108
|
+
async function teardownBackend() {
|
|
86
109
|
if (client) {
|
|
87
110
|
try {
|
|
88
111
|
await client.close();
|
|
@@ -92,12 +115,78 @@ export default function browserUseExtension(pi) {
|
|
|
92
115
|
}
|
|
93
116
|
client = undefined;
|
|
94
117
|
}
|
|
95
|
-
|
|
96
|
-
|
|
118
|
+
if (ownBackend) {
|
|
119
|
+
try {
|
|
120
|
+
await ownBackend.stop();
|
|
121
|
+
}
|
|
122
|
+
catch {
|
|
123
|
+
// Shutdown is best-effort; the profile lock release inside never
|
|
124
|
+
// throws fatally, so a new backend can still start.
|
|
125
|
+
}
|
|
126
|
+
ownBackend = undefined;
|
|
127
|
+
}
|
|
128
|
+
}
|
|
129
|
+
function pinCurrentSite(profileDir, headed) {
|
|
130
|
+
if (!lastOrigin)
|
|
131
|
+
return false;
|
|
132
|
+
const prefs = rememberExecutionPreference(loadSitePreferences(profileDir), lastOrigin, headed ? 'headed-background' : 'headless');
|
|
133
|
+
saveSitePreferences(profileDir, prefs);
|
|
134
|
+
return true;
|
|
135
|
+
}
|
|
136
|
+
/**
|
|
137
|
+
* Rebuild the backend for a mode switch. Shared by the switch tool and
|
|
138
|
+
* automatic escalation so both paths behave identically. Persistent
|
|
139
|
+
* self-launches Pi-owned Chrome (MCP attaches via browserUrl) unless
|
|
140
|
+
* PI_BROWSER_USE_LEGACY_PERSISTENT=1. A per-origin headed-background pin
|
|
141
|
+
* wins over a headless request so one headless-hostile site never
|
|
142
|
+
* downgrades every site.
|
|
143
|
+
*/
|
|
144
|
+
async function switchBackend(mode, headed, signal, opts) {
|
|
145
|
+
const next = resolveConfig(resolveModeTarget(config ?? {}, mode, headed));
|
|
146
|
+
await teardownBackend();
|
|
147
|
+
let effectiveHeaded = headed;
|
|
148
|
+
let backendNote = '';
|
|
149
|
+
if (mode === 'persistent' && shouldSelfLaunch(next)) {
|
|
150
|
+
const profileDir = persistentProfileDir(next);
|
|
151
|
+
if (opts?.rememberSite === true)
|
|
152
|
+
pinCurrentSite(profileDir, headed);
|
|
153
|
+
if (!headed && lastOrigin) {
|
|
154
|
+
const pinned = resolveExecutionForOrigin(loadSitePreferences(profileDir), lastOrigin, 'headless');
|
|
155
|
+
if (pinned === 'headed-background') {
|
|
156
|
+
effectiveHeaded = true;
|
|
157
|
+
backendNote = ` (${normalizeOrigin(lastOrigin)} prefers the visible fallback)`;
|
|
158
|
+
}
|
|
159
|
+
}
|
|
160
|
+
ownBackend = new PersistentBackend({ config: next, headed: effectiveHeaded });
|
|
161
|
+
const attach = await ownBackend.start(signal);
|
|
162
|
+
client = new DevToolsClient(attach);
|
|
163
|
+
markAutomationResult(profileDir, effectiveHeaded ? 'headed' : 'headless');
|
|
164
|
+
}
|
|
165
|
+
else {
|
|
166
|
+
if (mode === 'persistent' && opts?.rememberSite === true) {
|
|
167
|
+
pinCurrentSite(persistentProfileDir(next), headed);
|
|
168
|
+
}
|
|
169
|
+
prepareBrowserProfile(next);
|
|
170
|
+
client = new DevToolsClient(next);
|
|
171
|
+
}
|
|
97
172
|
await client.ensureReady(signal);
|
|
173
|
+
// Record what actually launched (a per-origin pin may have upgraded
|
|
174
|
+
// headless to headed-background) so status/doctor tell the truth.
|
|
175
|
+
next.headless = !effectiveHeaded;
|
|
98
176
|
config = next;
|
|
99
177
|
currentMode = mode;
|
|
100
|
-
return next;
|
|
178
|
+
return { next, effectiveHeaded, backendNote };
|
|
179
|
+
}
|
|
180
|
+
/** Start the Existing-mode tab broker bridge on demand. */
|
|
181
|
+
async function ensureBridge() {
|
|
182
|
+
if (bridge)
|
|
183
|
+
return bridge;
|
|
184
|
+
const port = config?.tabBridgePort ?? DEFAULT_BRIDGE_PORT;
|
|
185
|
+
if (port === 0)
|
|
186
|
+
throw new Error('The tab bridge is disabled (tabBridgePort: 0).');
|
|
187
|
+
bridge = new TabBridge({ port });
|
|
188
|
+
await bridge.start();
|
|
189
|
+
return bridge;
|
|
101
190
|
}
|
|
102
191
|
function loginWallHint(url, text) {
|
|
103
192
|
if (currentMode !== 'fresh')
|
|
@@ -106,22 +195,58 @@ export default function browserUseExtension(pi) {
|
|
|
106
195
|
return '';
|
|
107
196
|
return '\n\nHint: this looks like a login wall in a fresh (logged-out) session. If the page needs your identity, call browser_switch_mode({"mode": "persistent"}) — a human must complete any SSO, 2FA, or passkey step, ideally headed.';
|
|
108
197
|
}
|
|
198
|
+
function sameOrigin(a, b) {
|
|
199
|
+
try {
|
|
200
|
+
if (!a || !b)
|
|
201
|
+
return false;
|
|
202
|
+
return new URL(a).origin === new URL(b).origin;
|
|
203
|
+
}
|
|
204
|
+
catch {
|
|
205
|
+
return false;
|
|
206
|
+
}
|
|
207
|
+
}
|
|
109
208
|
/**
|
|
110
209
|
* Automatic escalation for hard blocks the agent cannot clear alone.
|
|
111
210
|
* Returns prompt text for the agent to relay, or empty when nothing
|
|
112
211
|
* applies. Escalates at most once per call; never loops, never retries
|
|
113
212
|
* a challenge page, and never switches away from an attached session.
|
|
213
|
+
*
|
|
214
|
+
* Challenges only escalate with navigation context: a stale "Just a
|
|
215
|
+
* moment..." shortcut tile on a New Tab snapshot must not rebuild the
|
|
216
|
+
* backend, and a challenge on another origin than requested means the
|
|
217
|
+
* navigation never landed there.
|
|
114
218
|
*/
|
|
115
|
-
async function escalateBlockedPage(url, text, signal) {
|
|
219
|
+
async function escalateBlockedPage(url, text, signal, context) {
|
|
116
220
|
const state = classifyPageState(url, text);
|
|
117
221
|
if (state === 'ok')
|
|
118
222
|
return '';
|
|
223
|
+
if (state === 'challenge') {
|
|
224
|
+
if (context?.tool !== 'navigate_page')
|
|
225
|
+
return '';
|
|
226
|
+
if (context.requestedUrl && url && !sameOrigin(context.requestedUrl, url)) {
|
|
227
|
+
// Challenge-provider handoffs (dedicated challenge domains) still
|
|
228
|
+
// count; anything else means the navigation never landed there.
|
|
229
|
+
const host = (() => {
|
|
230
|
+
try {
|
|
231
|
+
return new URL(url).hostname;
|
|
232
|
+
}
|
|
233
|
+
catch {
|
|
234
|
+
return '';
|
|
235
|
+
}
|
|
236
|
+
})();
|
|
237
|
+
if (!/challenge|turnstile|captcha|cf-chl|kasada|perimeterx|datadome/i.test(host))
|
|
238
|
+
return '';
|
|
239
|
+
}
|
|
240
|
+
}
|
|
119
241
|
if (state === 'login-wall') {
|
|
120
242
|
if (currentMode === 'fresh')
|
|
121
243
|
return loginWallHint(url, text);
|
|
122
|
-
if (currentMode === 'custom') {
|
|
244
|
+
if (currentMode === 'custom' || currentMode === 'existing') {
|
|
123
245
|
return '\n\nThis page needs an identity this session does not have. Sign in is required — complete it in the visible browser, then retry.';
|
|
124
246
|
}
|
|
247
|
+
if (config?.headless === false) {
|
|
248
|
+
return '\n\nThis page needs a login and the browser is already visible. Sign in in that window, then retry — no relaunch, nothing closed.';
|
|
249
|
+
}
|
|
125
250
|
return await escalateToHeaded(url ?? 'this page');
|
|
126
251
|
}
|
|
127
252
|
// Challenge (bot check): identity never helps; only a human-gated
|
|
@@ -129,7 +254,7 @@ export default function browserUseExtension(pi) {
|
|
|
129
254
|
if (config?.headless === false) {
|
|
130
255
|
return '\n\nA bot challenge is blocking this page and the browser is already visible. Complete the challenge in the window, then retry.';
|
|
131
256
|
}
|
|
132
|
-
if (currentMode === 'custom') {
|
|
257
|
+
if (currentMode === 'custom' || currentMode === 'existing') {
|
|
133
258
|
return '\n\nA bot challenge is blocking this page. Complete it in the visible browser, then retry — do not loop against the challenge.';
|
|
134
259
|
}
|
|
135
260
|
return await escalateToHeaded(url ?? 'this page');
|
|
@@ -137,7 +262,9 @@ export default function browserUseExtension(pi) {
|
|
|
137
262
|
/**
|
|
138
263
|
* Rebuild the current backend headed so a human can act (log in, clear
|
|
139
264
|
* a challenge), then tell the agent exactly what to relay. Attach
|
|
140
|
-
* sessions are already visible and owned by the user: prompt only.
|
|
265
|
+
* sessions are already visible and owned by the user: prompt only. The
|
|
266
|
+
* headed window is navigated to the blocked page and fronted: auth
|
|
267
|
+
* handoff is the one case where taking foreground is the job, not a bug.
|
|
141
268
|
*/
|
|
142
269
|
async function escalateToHeaded(url, signal) {
|
|
143
270
|
const mode = currentMode === 'persistent' ? 'persistent' : 'fresh';
|
|
@@ -147,11 +274,28 @@ export default function browserUseExtension(pi) {
|
|
|
147
274
|
catch (error) {
|
|
148
275
|
return `\n\nBlocked on ${url} and the headed browser failed to launch (${error instanceof Error ? error.message : String(error)}). Ask the user to proceed manually.`;
|
|
149
276
|
}
|
|
150
|
-
|
|
277
|
+
if (/^https?:\/\//.test(url)) {
|
|
278
|
+
// Best effort: the window is already open for manual navigation.
|
|
279
|
+
try {
|
|
280
|
+
await callUpstream(client, 'new_page', { url, background: false }, signal);
|
|
281
|
+
}
|
|
282
|
+
catch {
|
|
283
|
+
// Manual navigation in the opened window covers this.
|
|
284
|
+
}
|
|
285
|
+
}
|
|
286
|
+
// Front Pi-owned Chrome so the handoff window is actually visible.
|
|
287
|
+
// Fresh MCP-launched Chrome fronts itself; only Pi-owned needs help.
|
|
288
|
+
if (ownBackend)
|
|
289
|
+
frontProcessByPid(ownBackend.pid());
|
|
290
|
+
return `\n\nBlocked on ${url}: a browser window just opened on that page (same ${mode} session — previous tabs are gone, re-list pages after). Please complete the login or challenge in that window, then tell the agent to continue. Do not close the window until done.`;
|
|
151
291
|
}
|
|
152
292
|
function pageUrlFromSnapshot(text) {
|
|
153
293
|
return text.match(/\burl="([^"]+)"/)?.[1];
|
|
154
294
|
}
|
|
295
|
+
/** Best-effort MCP page list → entries (shared parser, never throws). */
|
|
296
|
+
function mcpPageEntries(result) {
|
|
297
|
+
return parseMcpPageList(result);
|
|
298
|
+
}
|
|
155
299
|
async function ensureConnected(signal) {
|
|
156
300
|
if (!client)
|
|
157
301
|
throw new Error('browser-use: session not started');
|
|
@@ -166,15 +310,71 @@ export default function browserUseExtension(pi) {
|
|
|
166
310
|
const prefixedName = `${TOOL_PREFIX}${tool.name}`;
|
|
167
311
|
const originalName = tool.name;
|
|
168
312
|
const description = augmentToolDescription(prefixedName, tool.description ?? '');
|
|
313
|
+
// close_page carries an extra force gate so Existing mode can refuse
|
|
314
|
+
// to close tabs Pi did not open (spec 19); stripped before upstream.
|
|
315
|
+
const parameters = originalName === 'close_page'
|
|
316
|
+
? Type.Object({
|
|
317
|
+
pageId: Type.Number({
|
|
318
|
+
description: 'The ID of the page to close. Call browser_list_pages first; IDs shift when tabs close.',
|
|
319
|
+
}),
|
|
320
|
+
force: Type.Optional(Type.Boolean({
|
|
321
|
+
description: 'Existing mode only: Pi refuses to close tabs it did not open unless force is true and the user explicitly asked for that exact tab.',
|
|
322
|
+
})),
|
|
323
|
+
})
|
|
324
|
+
: Type.Unsafe(tool.inputSchema);
|
|
169
325
|
pi.registerTool({
|
|
170
326
|
name: prefixedName,
|
|
171
327
|
label: prefixedName,
|
|
172
328
|
description,
|
|
173
|
-
parameters
|
|
329
|
+
parameters,
|
|
174
330
|
async execute(_toolCallId, params, signal) {
|
|
175
331
|
await ensureConnected(signal);
|
|
176
332
|
const browser = client;
|
|
177
|
-
|
|
333
|
+
// Focus policy (headed-background / existing): Pi-created pages
|
|
334
|
+
// open in the background and selections never take foreground
|
|
335
|
+
// unless the caller explicitly asked. Explicit values always win.
|
|
336
|
+
const effectiveParams = originalName === 'new_page'
|
|
337
|
+
? applyNewPageDefaults(params)
|
|
338
|
+
: originalName === 'select_page'
|
|
339
|
+
? applySelectPageDefaults(params)
|
|
340
|
+
: params;
|
|
341
|
+
if ((originalName === 'navigate_page' || originalName === 'new_page') &&
|
|
342
|
+
typeof effectiveParams.url === 'string') {
|
|
343
|
+
// Remember the origin for the per-origin headed-background
|
|
344
|
+
// fallback; normalizeOrigin never throws (falls back to raw).
|
|
345
|
+
lastOrigin = normalizeOrigin(effectiveParams.url);
|
|
346
|
+
// In Existing mode a Pi-driven navigation marks the destination
|
|
347
|
+
// as Pi-touched for the close guard below.
|
|
348
|
+
if (currentMode === 'existing')
|
|
349
|
+
trackPiUrl(effectiveParams.url);
|
|
350
|
+
}
|
|
351
|
+
// Existing mode never closes user tabs: close_page carries a
|
|
352
|
+
// force gate and a Pi-ownership check (spec 19).
|
|
353
|
+
if (originalName === 'close_page') {
|
|
354
|
+
const { force: _force, ...closeArgs } = effectiveParams;
|
|
355
|
+
void _force;
|
|
356
|
+
if (currentMode === 'existing' && effectiveParams.force !== true) {
|
|
357
|
+
if (typeof effectiveParams.pageId !== 'number') {
|
|
358
|
+
return {
|
|
359
|
+
content: [{ type: 'text', text: 'close_page needs a numeric pageId.' }],
|
|
360
|
+
isError: true,
|
|
361
|
+
details: undefined,
|
|
362
|
+
};
|
|
363
|
+
}
|
|
364
|
+
const entries = parseMcpPageList(await callUpstream(browser, 'list_pages', {}, signal));
|
|
365
|
+
const verdict = checkExistingCloseAllowed(entries, effectiveParams.pageId, piOwnedUrls);
|
|
366
|
+
if (!verdict.ok) {
|
|
367
|
+
return {
|
|
368
|
+
content: [{ type: 'text', text: verdict.reason }],
|
|
369
|
+
isError: true,
|
|
370
|
+
details: undefined,
|
|
371
|
+
};
|
|
372
|
+
}
|
|
373
|
+
}
|
|
374
|
+
const result = await callUpstream(browser, originalName, closeArgs, signal);
|
|
375
|
+
return { ...toToolContent(result, originalName), details: undefined };
|
|
376
|
+
}
|
|
377
|
+
let result = await callUpstream(browser, originalName, effectiveParams, signal);
|
|
178
378
|
if (result.isError &&
|
|
179
379
|
OVERLAY_RECOVERABLE.has(originalName) &&
|
|
180
380
|
looksOverlayBlocked(extractTextContent(result.content))) {
|
|
@@ -182,11 +382,11 @@ export default function browserUseExtension(pi) {
|
|
|
182
382
|
// original call. Any failure here falls through to the
|
|
183
383
|
// original error, which already carries a hint.
|
|
184
384
|
try {
|
|
185
|
-
const escapeArgs = typeof
|
|
186
|
-
? { pageId:
|
|
385
|
+
const escapeArgs = typeof effectiveParams.pageId === 'number'
|
|
386
|
+
? { pageId: effectiveParams.pageId, key: 'Escape' }
|
|
187
387
|
: { key: 'Escape' };
|
|
188
388
|
await callUpstream(browser, 'press_key', escapeArgs, signal);
|
|
189
|
-
result = await callUpstream(browser, originalName,
|
|
389
|
+
result = await callUpstream(browser, originalName, effectiveParams, signal);
|
|
190
390
|
}
|
|
191
391
|
catch {
|
|
192
392
|
// Fall through to the original result below.
|
|
@@ -195,10 +395,14 @@ export default function browserUseExtension(pi) {
|
|
|
195
395
|
const toolContent = toToolContent(result, originalName);
|
|
196
396
|
if (!toolContent.isError &&
|
|
197
397
|
(originalName === 'navigate_page' || originalName === 'take_snapshot')) {
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
const
|
|
398
|
+
// Prefer the page's real URL from the snapshot; fall back to the
|
|
399
|
+
// requested URL only when the snapshot carries none.
|
|
400
|
+
const snapshotUrl = pageUrlFromSnapshot(extractTextContent(result.content));
|
|
401
|
+
const requestedUrl = originalName === 'navigate_page' && typeof effectiveParams.url === 'string'
|
|
402
|
+
? effectiveParams.url
|
|
403
|
+
: undefined;
|
|
404
|
+
const url = snapshotUrl ?? requestedUrl;
|
|
405
|
+
const escalation = await escalateBlockedPage(url, extractTextContent(result.content), signal, { tool: originalName, requestedUrl });
|
|
202
406
|
const first = toolContent.content[0];
|
|
203
407
|
if (escalation && first && first.text !== undefined) {
|
|
204
408
|
first.text += escalation;
|
|
@@ -299,23 +503,257 @@ export default function browserUseExtension(pi) {
|
|
|
299
503
|
pi.registerTool({
|
|
300
504
|
name: `${TOOL_PREFIX}switch_mode`,
|
|
301
505
|
label: `${TOOL_PREFIX}switch_mode`,
|
|
302
|
-
description: 'Switch the browser backend without restarting: "fresh" is an isolated clean room, "
|
|
506
|
+
description: 'Switch the browser backend without restarting: "persistent" is Pi\'s own browser (saved profile with your logins, default), "fresh" is an isolated clean room for anonymous checks, "existing" attaches to your running Chrome (tabs go to the collapsed pi-browser-use group via browser_open_background_tab, consent popup each session). Fresh and persistent default to headless; pass headed true to watch. Tabs do not transfer; call browser_list_pages after switching. Prefer persistent; drop to fresh for clean-room checks.',
|
|
303
507
|
parameters: Type.Object({
|
|
304
|
-
mode: Type.Union([
|
|
508
|
+
mode: Type.Union([
|
|
509
|
+
Type.Literal('fresh'),
|
|
510
|
+
Type.Literal('persistent'),
|
|
511
|
+
Type.Literal('existing'),
|
|
512
|
+
]),
|
|
305
513
|
headed: Type.Optional(Type.Boolean({
|
|
306
514
|
description: 'Show the browser window. Default is headless — everything works with no popups.',
|
|
307
515
|
})),
|
|
516
|
+
rememberSite: Type.Optional(Type.Boolean({
|
|
517
|
+
description: "Persistent only: remember the last-visited site's visibility (headless or headed-background) for next time.",
|
|
518
|
+
})),
|
|
308
519
|
}),
|
|
309
520
|
async execute(_toolCallId, params, signal) {
|
|
310
|
-
const mode = params.mode === 'persistent'
|
|
311
|
-
|
|
521
|
+
const mode = params.mode === 'persistent'
|
|
522
|
+
? 'persistent'
|
|
523
|
+
: params.mode === 'existing'
|
|
524
|
+
? 'existing'
|
|
525
|
+
: 'fresh';
|
|
526
|
+
const { next, effectiveHeaded, backendNote } = await switchBackend(mode, params.headed === true, signal, { rememberSite: params.rememberSite === true });
|
|
527
|
+
const visibility = mode === 'existing' ? 'headed (your Chrome)' : effectiveHeaded ? 'headed' : 'headless';
|
|
528
|
+
const what = mode === 'fresh'
|
|
529
|
+
? 'a fresh isolated browser'
|
|
530
|
+
: mode === 'persistent'
|
|
531
|
+
? 'the persistent Pi profile'
|
|
532
|
+
: 'your running Chrome';
|
|
533
|
+
const extra = mode === 'existing'
|
|
534
|
+
? ' Open Pi tabs with browser_open_background_tab so they land in the collapsed pi-browser-use group.'
|
|
535
|
+
: '';
|
|
536
|
+
void next;
|
|
312
537
|
return {
|
|
313
538
|
content: [
|
|
314
539
|
{
|
|
315
540
|
type: 'text',
|
|
316
|
-
text:
|
|
317
|
-
|
|
318
|
-
|
|
541
|
+
text: `Switched to ${what} (${visibility})${backendNote}. Previous tabs are gone; call browser_list_pages to start.${extra}`,
|
|
542
|
+
},
|
|
543
|
+
],
|
|
544
|
+
details: undefined,
|
|
545
|
+
};
|
|
546
|
+
},
|
|
547
|
+
});
|
|
548
|
+
}
|
|
549
|
+
function registerSetupTool() {
|
|
550
|
+
pi.registerTool({
|
|
551
|
+
name: `${TOOL_PREFIX}setup`,
|
|
552
|
+
label: `${TOOL_PREFIX}setup`,
|
|
553
|
+
description: 'First-run setup for the persistent Pi browser profile: opens a plain headed Chrome window (no automation attached) for a human to sign into Google and any sites. Completes when the window is closed. Run once; afterwards Pi automates headless.',
|
|
554
|
+
parameters: Type.Object({}),
|
|
555
|
+
async execute() {
|
|
556
|
+
const profileDir = persistentProfileDir(config ?? {});
|
|
557
|
+
const meta = loadPersistentMetadata(profileDir);
|
|
558
|
+
if (meta.initialized) {
|
|
559
|
+
return {
|
|
560
|
+
content: [
|
|
561
|
+
{
|
|
562
|
+
type: 'text',
|
|
563
|
+
text: `Pi browser profile is already initialized (${profileDir}). If a login expired, use browser_reauth instead.`,
|
|
564
|
+
},
|
|
565
|
+
],
|
|
566
|
+
details: undefined,
|
|
567
|
+
};
|
|
568
|
+
}
|
|
569
|
+
// No Chrome may hold the profile while the setup window runs.
|
|
570
|
+
await teardownBackend();
|
|
571
|
+
await runBootstrap({
|
|
572
|
+
profileDir,
|
|
573
|
+
executablePath: config?.executablePath,
|
|
574
|
+
chromeArgs: config?.chromeArgs,
|
|
575
|
+
});
|
|
576
|
+
return {
|
|
577
|
+
content: [
|
|
578
|
+
{
|
|
579
|
+
type: 'text',
|
|
580
|
+
text: 'Pi browser profile initialized. Pi now works in the background — no Chrome window will appear during normal automation.',
|
|
581
|
+
},
|
|
582
|
+
],
|
|
583
|
+
details: undefined,
|
|
584
|
+
};
|
|
585
|
+
},
|
|
586
|
+
});
|
|
587
|
+
}
|
|
588
|
+
function registerStatusTool() {
|
|
589
|
+
pi.registerTool({
|
|
590
|
+
name: `${TOOL_PREFIX}status`,
|
|
591
|
+
label: `${TOOL_PREFIX}status`,
|
|
592
|
+
description: 'Plain-language Pi browser status: profile readiness, execution mode, and what to do next. No page is touched.',
|
|
593
|
+
parameters: Type.Object({}),
|
|
594
|
+
async execute() {
|
|
595
|
+
const mode = currentMode;
|
|
596
|
+
const profileDir = persistentProfileDir(config ?? {});
|
|
597
|
+
const meta = loadPersistentMetadata(profileDir);
|
|
598
|
+
const sitePins = loadSitePreferences(profileDir).length;
|
|
599
|
+
const lines = ['Pi Browser', '──────────'];
|
|
600
|
+
if (mode === 'fresh') {
|
|
601
|
+
lines.push('Profile: Ephemeral (nothing persists)');
|
|
602
|
+
lines.push('Execution: Headless');
|
|
603
|
+
}
|
|
604
|
+
else if (mode === 'persistent') {
|
|
605
|
+
lines.push(`Profile: ${meta.initialized ? 'Ready' : 'Setup required'}`);
|
|
606
|
+
if (!meta.initialized) {
|
|
607
|
+
lines.push('Next step: run browser_setup and sign in, then close the window.');
|
|
608
|
+
}
|
|
609
|
+
else {
|
|
610
|
+
const headed = config?.headless === false;
|
|
611
|
+
lines.push(`Execution: ${headed ? 'Visible fallback (background)' : 'Headless'}${ownBackend?.running() ? '' : ' (backend stopped)'}`);
|
|
612
|
+
if (meta.lastSuccessfulMode)
|
|
613
|
+
lines.push(`Last working mode: ${meta.lastSuccessfulMode}`);
|
|
614
|
+
if (sitePins > 0)
|
|
615
|
+
lines.push(`Sites pinned to visible fallback: ${sitePins}`);
|
|
616
|
+
}
|
|
617
|
+
}
|
|
618
|
+
else if (mode === 'existing') {
|
|
619
|
+
lines.push('Profile: Your browser');
|
|
620
|
+
lines.push('Execution: Background tabs in the collapsed pi-browser-use group');
|
|
621
|
+
lines.push(`Tab bridge: ${bridge ? bridge.baseUrl() : 'not running'}`);
|
|
622
|
+
}
|
|
623
|
+
else {
|
|
624
|
+
lines.push('Profile: Externally attached browser');
|
|
625
|
+
lines.push('Execution: Visible (owned by its launcher)');
|
|
626
|
+
}
|
|
627
|
+
return { content: [{ type: 'text', text: lines.join('\n') }], details: undefined };
|
|
628
|
+
},
|
|
629
|
+
});
|
|
630
|
+
}
|
|
631
|
+
function registerReauthTool() {
|
|
632
|
+
pi.registerTool({
|
|
633
|
+
name: `${TOOL_PREFIX}reauth`,
|
|
634
|
+
label: `${TOOL_PREFIX}reauth`,
|
|
635
|
+
description: 'Reauthenticate the persistent Pi profile after a login/challenge wall: shuts the headless browser down cleanly, opens a headed window for the human to verify, then resumes headless. The plain variant (no automation attached) is for providers that reject instrumented browsers.',
|
|
636
|
+
parameters: Type.Object({
|
|
637
|
+
url: Type.Optional(Type.String({ description: 'Page that needs authentication. Defaults to last origin.' })),
|
|
638
|
+
variant: Type.Optional(Type.Union([Type.Literal('instrumented'), Type.Literal('plain')], {
|
|
639
|
+
description: 'Headed variant: instrumented (Pi navigates first) or plain (maximum compatibility).',
|
|
640
|
+
})),
|
|
641
|
+
}),
|
|
642
|
+
async execute(_toolCallId, params, signal) {
|
|
643
|
+
if (currentMode !== 'persistent') {
|
|
644
|
+
return {
|
|
645
|
+
content: [
|
|
646
|
+
{
|
|
647
|
+
type: 'text',
|
|
648
|
+
text: 'Reauth applies to the persistent Pi profile. Switch to it first with browser_switch_mode({"mode": "persistent"}).',
|
|
649
|
+
},
|
|
650
|
+
],
|
|
651
|
+
details: undefined,
|
|
652
|
+
};
|
|
653
|
+
}
|
|
654
|
+
const url = typeof params.url === 'string' && params.url.length > 0
|
|
655
|
+
? params.url
|
|
656
|
+
: (lastOrigin ?? 'this page');
|
|
657
|
+
const variant = params.variant === 'plain' ? 'plain' : 'instrumented';
|
|
658
|
+
if (!ownBackend) {
|
|
659
|
+
// Legacy MCP-launched persistent: headed switch is the reauth path.
|
|
660
|
+
await switchBackend('persistent', true, signal);
|
|
661
|
+
return {
|
|
662
|
+
content: [
|
|
663
|
+
{
|
|
664
|
+
type: 'text',
|
|
665
|
+
text: `A browser window just opened (legacy persistent backend). ${url}: please complete the login there, then tell the agent to continue.`,
|
|
666
|
+
},
|
|
667
|
+
],
|
|
668
|
+
details: undefined,
|
|
669
|
+
};
|
|
670
|
+
}
|
|
671
|
+
// Spec §7: close headless Chrome cleanly before any headed reauth.
|
|
672
|
+
await teardownBackend();
|
|
673
|
+
const backend = new PersistentBackend({
|
|
674
|
+
config: config ?? {},
|
|
675
|
+
headed: variant === 'instrumented',
|
|
676
|
+
});
|
|
677
|
+
ownBackend = backend;
|
|
678
|
+
const message = await runReauth({
|
|
679
|
+
backend,
|
|
680
|
+
url,
|
|
681
|
+
variant,
|
|
682
|
+
restartBackend: (headed) => backend.restart(headed),
|
|
683
|
+
});
|
|
684
|
+
if (variant === 'plain') {
|
|
685
|
+
// Plain window closed by the human: resume headless automation.
|
|
686
|
+
const attach = await backend.restart(false);
|
|
687
|
+
client = new DevToolsClient(attach);
|
|
688
|
+
await client.ensureReady(signal);
|
|
689
|
+
return {
|
|
690
|
+
content: [
|
|
691
|
+
{ type: 'text', text: `${message}\n\nVerification recorded — Pi resumed headless.` },
|
|
692
|
+
],
|
|
693
|
+
details: undefined,
|
|
694
|
+
};
|
|
695
|
+
}
|
|
696
|
+
client = new DevToolsClient(backend.attachConfig());
|
|
697
|
+
await client.ensureReady(signal);
|
|
698
|
+
return {
|
|
699
|
+
content: [
|
|
700
|
+
{
|
|
701
|
+
type: 'text',
|
|
702
|
+
text: `${message}\n\nAfter verifying, tell the agent to continue; it resumes with browser_switch_mode({"mode": "persistent"}) back to headless.`,
|
|
703
|
+
},
|
|
704
|
+
],
|
|
705
|
+
details: undefined,
|
|
706
|
+
};
|
|
707
|
+
},
|
|
708
|
+
});
|
|
709
|
+
}
|
|
710
|
+
function registerOpenBackgroundTabTool() {
|
|
711
|
+
pi.registerTool({
|
|
712
|
+
name: `${TOOL_PREFIX}open_background_tab`,
|
|
713
|
+
label: `${TOOL_PREFIX}open_background_tab`,
|
|
714
|
+
description: 'Existing mode only: open a URL as an inactive tab in the collapsed pi-browser-use group via the Pi extension — never a foreground tab. Fails clearly when the extension bridge is unavailable.',
|
|
715
|
+
parameters: Type.Object({
|
|
716
|
+
url: Type.String({ description: 'URL to open in a background Pi tab.' }),
|
|
717
|
+
timeoutMs: Type.Optional(Type.Number({
|
|
718
|
+
description: 'How long to wait for the extension (default 90000: a suspended worker wakes on the ~1min alarm cadence).',
|
|
719
|
+
})),
|
|
720
|
+
}),
|
|
721
|
+
async execute(_toolCallId, params, signal) {
|
|
722
|
+
if (currentMode !== 'existing') {
|
|
723
|
+
return {
|
|
724
|
+
content: [
|
|
725
|
+
{
|
|
726
|
+
type: 'text',
|
|
727
|
+
text: 'Background Pi tabs need Existing mode (your Chrome). Switch first with browser_switch_mode({"mode": "existing"}).',
|
|
728
|
+
isError: true,
|
|
729
|
+
},
|
|
730
|
+
],
|
|
731
|
+
details: undefined,
|
|
732
|
+
isError: true,
|
|
733
|
+
};
|
|
734
|
+
}
|
|
735
|
+
if (typeof params.url !== 'string' || params.url.length === 0) {
|
|
736
|
+
throw new Error('A URL is required.');
|
|
737
|
+
}
|
|
738
|
+
const activeBridge = await ensureBridge();
|
|
739
|
+
const timeoutMs = typeof params.timeoutMs === 'number' && params.timeoutMs > 0 ? params.timeoutMs : 90_000;
|
|
740
|
+
const result = await openExistingPage(params.url, {
|
|
741
|
+
bridge: activeBridge,
|
|
742
|
+
listPages: async () => {
|
|
743
|
+
await ensureConnected(signal);
|
|
744
|
+
const pages = await client.callTool('list_pages', {}, signal);
|
|
745
|
+
return mcpPageEntries(pages);
|
|
746
|
+
},
|
|
747
|
+
}, { timeoutMs, signal });
|
|
748
|
+
trackPiUrl(params.url);
|
|
749
|
+
const selectHint = result.pageId !== undefined
|
|
750
|
+
? ` Select it with browser_select_page (it stays in the background).`
|
|
751
|
+
: ' Call browser_list_pages to find it (it stays in the background).';
|
|
752
|
+
return {
|
|
753
|
+
content: [
|
|
754
|
+
{
|
|
755
|
+
type: 'text',
|
|
756
|
+
text: `Opened ${params.url} as an inactive tab in the collapsed pi-browser-use group.${selectHint}`,
|
|
319
757
|
},
|
|
320
758
|
],
|
|
321
759
|
details: undefined,
|
|
@@ -331,7 +769,10 @@ export default function browserUseExtension(pi) {
|
|
|
331
769
|
parameters: Type.Object({}),
|
|
332
770
|
async execute() {
|
|
333
771
|
await ensureConnected();
|
|
334
|
-
const report = await diagnose(config ?? {}, async () => (await client.listAllTools()).map((tool) => tool.name)
|
|
772
|
+
const report = await diagnose(config ?? {}, async () => (await client.listAllTools()).map((tool) => tool.name), {
|
|
773
|
+
backend: ownBackend ? 'pi-owned' : undefined,
|
|
774
|
+
bridgeUrl: bridge?.baseUrl() ?? null,
|
|
775
|
+
});
|
|
335
776
|
return { content: [{ type: 'text', text: formatDoctorReport(report) }], details: undefined };
|
|
336
777
|
},
|
|
337
778
|
});
|
|
@@ -368,20 +809,37 @@ export default function browserUseExtension(pi) {
|
|
|
368
809
|
pi.on('session_start', async (_event, ctx) => {
|
|
369
810
|
config = resolveConfig(loadConfig({ cwd: ctx.cwd, projectTrusted: isProjectTrusted(ctx) }));
|
|
370
811
|
currentMode = describeMode();
|
|
371
|
-
|
|
372
|
-
|
|
812
|
+
if (currentMode === 'persistent' && shouldSelfLaunch(config)) {
|
|
813
|
+
// Phase 2: Pi owns the persistent Chrome process; MCP attaches.
|
|
814
|
+
ownBackend = new PersistentBackend({ config, headed: config.headless === false });
|
|
815
|
+
client = new DevToolsClient(await ownBackend.start());
|
|
816
|
+
}
|
|
817
|
+
else {
|
|
818
|
+
prepareBrowserProfile(config);
|
|
819
|
+
client = new DevToolsClient(config);
|
|
820
|
+
}
|
|
373
821
|
await registerUpstreamTools();
|
|
374
822
|
registerSaveArtifactTool();
|
|
375
823
|
registerDoctorTool();
|
|
376
824
|
registerSwitchModeTool();
|
|
825
|
+
registerSetupTool();
|
|
826
|
+
registerStatusTool();
|
|
827
|
+
registerReauthTool();
|
|
828
|
+
registerOpenBackgroundTabTool();
|
|
377
829
|
if (config.visionModel) {
|
|
378
830
|
await registerVisionTool(config.visionModel);
|
|
379
831
|
}
|
|
380
832
|
});
|
|
381
833
|
pi.on('session_shutdown', async () => {
|
|
382
|
-
|
|
383
|
-
|
|
384
|
-
|
|
834
|
+
await teardownBackend();
|
|
835
|
+
if (bridge) {
|
|
836
|
+
try {
|
|
837
|
+
await bridge.stop();
|
|
838
|
+
}
|
|
839
|
+
catch {
|
|
840
|
+
// Session teardown is best-effort.
|
|
841
|
+
}
|
|
842
|
+
bridge = undefined;
|
|
385
843
|
}
|
|
386
844
|
});
|
|
387
845
|
}
|