pi-browser-use 0.6.1 → 0.8.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (76) hide show
  1. package/README.md +39 -11
  2. package/dist/auth-verifiers.d.ts +60 -0
  3. package/dist/auth-verifiers.d.ts.map +1 -0
  4. package/dist/auth-verifiers.js +92 -0
  5. package/dist/auth-verifiers.js.map +1 -0
  6. package/dist/chrome-launcher.d.ts +100 -0
  7. package/dist/chrome-launcher.d.ts.map +1 -0
  8. package/dist/chrome-launcher.js +268 -0
  9. package/dist/chrome-launcher.js.map +1 -0
  10. package/dist/config.d.ts +3 -1
  11. package/dist/config.d.ts.map +1 -1
  12. package/dist/config.js +28 -5
  13. package/dist/config.js.map +1 -1
  14. package/dist/doctor.d.ts +11 -1
  15. package/dist/doctor.d.ts.map +1 -1
  16. package/dist/doctor.js +23 -7
  17. package/dist/doctor.js.map +1 -1
  18. package/dist/existing-flow.d.ts +68 -0
  19. package/dist/existing-flow.d.ts.map +1 -0
  20. package/dist/existing-flow.js +128 -0
  21. package/dist/existing-flow.js.map +1 -0
  22. package/dist/focus-policy.d.ts +28 -0
  23. package/dist/focus-policy.d.ts.map +1 -0
  24. package/dist/focus-policy.js +28 -0
  25. package/dist/focus-policy.js.map +1 -0
  26. package/dist/index.d.ts +2 -2
  27. package/dist/index.d.ts.map +1 -1
  28. package/dist/index.js +565 -39
  29. package/dist/index.js.map +1 -1
  30. package/dist/named-profile.d.ts +47 -0
  31. package/dist/named-profile.d.ts.map +1 -0
  32. package/dist/named-profile.js +123 -0
  33. package/dist/named-profile.js.map +1 -0
  34. package/dist/persistent-backend.d.ts +90 -0
  35. package/dist/persistent-backend.d.ts.map +1 -0
  36. package/dist/persistent-backend.js +229 -0
  37. package/dist/persistent-backend.js.map +1 -0
  38. package/dist/persistent-store.d.ts +35 -0
  39. package/dist/persistent-store.d.ts.map +1 -0
  40. package/dist/persistent-store.js +98 -0
  41. package/dist/persistent-store.js.map +1 -0
  42. package/dist/profile-lock.d.ts +40 -0
  43. package/dist/profile-lock.d.ts.map +1 -0
  44. package/dist/profile-lock.js +165 -0
  45. package/dist/profile-lock.js.map +1 -0
  46. package/dist/profile.d.ts +3 -1
  47. package/dist/profile.d.ts.map +1 -1
  48. package/dist/profile.js +15 -6
  49. package/dist/profile.js.map +1 -1
  50. package/dist/session-manager.d.ts +125 -0
  51. package/dist/session-manager.d.ts.map +1 -0
  52. package/dist/session-manager.js +275 -0
  53. package/dist/session-manager.js.map +1 -0
  54. package/dist/session.d.ts +92 -0
  55. package/dist/session.d.ts.map +1 -0
  56. package/dist/session.js +29 -0
  57. package/dist/session.js.map +1 -0
  58. package/dist/setup-flow.d.ts +78 -0
  59. package/dist/setup-flow.d.ts.map +1 -0
  60. package/dist/setup-flow.js +97 -0
  61. package/dist/setup-flow.js.map +1 -0
  62. package/dist/shared-backend.d.ts +70 -0
  63. package/dist/shared-backend.d.ts.map +1 -0
  64. package/dist/shared-backend.js +152 -0
  65. package/dist/shared-backend.js.map +1 -0
  66. package/dist/tab-bridge.d.ts +71 -0
  67. package/dist/tab-bridge.d.ts.map +1 -0
  68. package/dist/tab-bridge.js +198 -0
  69. package/dist/tab-bridge.js.map +1 -0
  70. package/extension/README.md +33 -0
  71. package/extension/background.js +185 -0
  72. package/extension/manifest.json +12 -0
  73. package/package.json +2 -1
  74. package/skills/auth-bootstrap/SKILL.md +13 -0
  75. package/skills/browser-policy/SKILL.md +39 -4
  76. package/skills/gmail-auth/SKILL.md +51 -0
package/dist/index.js CHANGED
@@ -3,13 +3,22 @@ import { homedir } from 'node:os';
3
3
  import { dirname, join } from 'node:path';
4
4
  import { Type } from 'typebox';
5
5
  import { DevToolsClient } from './client.js';
6
- import { resolveConfig, resolveModeTarget, } from './config.js';
6
+ import { DEFAULT_PROFILE_DIR, resolveConfig, resolveModeTarget, } from './config.js';
7
7
  import { isProjectTrusted, loadConfig } from './settings.js';
8
8
  import { augmentToolDescription, classifyPageState, extractTextContent, looksLikeLoginWall, looksOverlayBlocked, OVERLAY_RECOVERABLE, postProcessToolResult, } from './tool-augment.js';
9
9
  import { pickImageData, resolveArtifactTarget } from './artifacts.js';
10
10
  import { CLEANUP_ANNOTATIONS, formatAnnotatedMap, INJECT_ANNOTATIONS, parseAnnotatedElements, } from './annotate.js';
11
11
  import { diagnose, formatDoctorReport } from './doctor.js';
12
+ import { checkExistingCloseAllowed, normalizeTabUrl, openExistingPage, parseMcpPageList, } from './existing-flow.js';
13
+ import { checkSharedCloseAllowed, claimPage, livePeerSessions, newSessionId, pruneDeadOwners, releasePages, } from './shared-backend.js';
14
+ import { applyNewPageDefaults, applySelectPageDefaults } from './focus-policy.js';
15
+ import { frontProcessByPid } from './chrome-launcher.js';
16
+ import { PersistentBackend, shouldSelfLaunch } from './persistent-backend.js';
17
+ import { loadPersistentMetadata, loadSitePreferences, markAutomationResult, saveSitePreferences, } from './persistent-store.js';
12
18
  import { prepareBrowserProfile } from './profile.js';
19
+ import { runBootstrap, runReauth } from './setup-flow.js';
20
+ import { normalizeOrigin, rememberExecutionPreference, resolveExecutionForOrigin, } from './session-manager.js';
21
+ import { DEFAULT_BRIDGE_PORT, TabBridge } from './tab-bridge.js';
13
22
  import { createRegistryVisionCaller, handleAnalyzeScreenshot, } from './vision.js';
14
23
  export { configToArgs, resolveConfig } from './config.js';
15
24
  // All upstream tools are re-exported with this prefix to avoid name collisions.
@@ -60,13 +69,35 @@ function toToolContent(result, originalName) {
60
69
  * discovers upstream tools, and registers each as browser_*. On
61
70
  * session_shutdown tears the subprocess down. Nothing runs persistently.
62
71
  *
63
- * Defaults are fresh headless (isolated ephemeral profile, no window). Set
64
- * sessionMode "persistent" for the authenticated profile, or "existing" with
72
+ * Defaults are persistent headless (Pi-owned profile, no window, no consent
73
+ * popups). Set mode "fresh" for an isolated clean room, or "existing" with
65
74
  * autoConnect/browserUrl to drive an already-running Chrome.
66
75
  */
67
76
  export default function browserUseExtension(pi) {
68
77
  let config;
69
78
  let client;
79
+ // Pi-owned persistent Chrome (self-launched, MCP attached via browserUrl).
80
+ // Set only for persistent mode when the legacy MCP-launch path is off.
81
+ let ownBackend;
82
+ // Existing-mode tab broker bridge. Lazy; lives for the whole session.
83
+ let bridge;
84
+ // Last navigated origin: drives the per-origin headed-background fallback.
85
+ let lastOrigin;
86
+ // URLs Pi opened or navigated to in Existing mode (raw + normalized): the
87
+ // only close_page targets allowed there without explicit force:true.
88
+ const piOwnedUrls = new Set();
89
+ // This extension load's agent-session identity for the shared registry.
90
+ const sessionId = newSessionId();
91
+ const myOwner = { sessionId, pid: process.pid };
92
+ const shortSession = sessionId.slice(0, 8);
93
+ /** Registry home: the persistent profile, or the default home for existing. */
94
+ function registryDir() {
95
+ return currentMode === 'persistent' ? persistentProfileDir(config ?? {}) : DEFAULT_PROFILE_DIR;
96
+ }
97
+ function trackPiUrl(url) {
98
+ piOwnedUrls.add(url);
99
+ piOwnedUrls.add(normalizeTabUrl(url));
100
+ }
70
101
  // Tracks which identity the live backend holds, so results can suggest
71
102
  // escalation. 'custom' covers user-configured attach setups we did not pick.
72
103
  let currentMode = 'fresh';
@@ -75,14 +106,15 @@ export default function browserUseExtension(pi) {
75
106
  return 'fresh';
76
107
  if (config?.sessionMode === 'persistent')
77
108
  return 'persistent';
109
+ if (config?.sessionMode === 'existing')
110
+ return 'existing';
78
111
  return 'custom';
79
112
  }
80
- /**
81
- * Rebuild the backend for a mode switch. Shared by the switch tool and
82
- * automatic escalation so both paths behave identically.
83
- */
84
- async function switchBackend(mode, headed, signal) {
85
- const next = resolveConfig(resolveModeTarget(config ?? {}, mode, headed));
113
+ function persistentProfileDir(cfg) {
114
+ return cfg.userDataDir ?? DEFAULT_PROFILE_DIR;
115
+ }
116
+ /** Close the MCP transport and any Pi-owned Chrome. The bridge survives. */
117
+ async function teardownBackend() {
86
118
  if (client) {
87
119
  try {
88
120
  await client.close();
@@ -92,12 +124,78 @@ export default function browserUseExtension(pi) {
92
124
  }
93
125
  client = undefined;
94
126
  }
95
- prepareBrowserProfile(next);
96
- client = new DevToolsClient(next);
127
+ if (ownBackend) {
128
+ try {
129
+ await ownBackend.stop();
130
+ }
131
+ catch {
132
+ // Shutdown is best-effort; the profile lock release inside never
133
+ // throws fatally, so a new backend can still start.
134
+ }
135
+ ownBackend = undefined;
136
+ }
137
+ }
138
+ function pinCurrentSite(profileDir, headed) {
139
+ if (!lastOrigin)
140
+ return false;
141
+ const prefs = rememberExecutionPreference(loadSitePreferences(profileDir), lastOrigin, headed ? 'headed-background' : 'headless');
142
+ saveSitePreferences(profileDir, prefs);
143
+ return true;
144
+ }
145
+ /**
146
+ * Rebuild the backend for a mode switch. Shared by the switch tool and
147
+ * automatic escalation so both paths behave identically. Persistent
148
+ * self-launches Pi-owned Chrome (MCP attaches via browserUrl) unless
149
+ * PI_BROWSER_USE_LEGACY_PERSISTENT=1. A per-origin headed-background pin
150
+ * wins over a headless request so one headless-hostile site never
151
+ * downgrades every site.
152
+ */
153
+ async function switchBackend(mode, headed, signal, opts) {
154
+ const next = resolveConfig(resolveModeTarget(config ?? {}, mode, headed));
155
+ await teardownBackend();
156
+ let effectiveHeaded = headed;
157
+ let backendNote = '';
158
+ if (mode === 'persistent' && shouldSelfLaunch(next)) {
159
+ const profileDir = persistentProfileDir(next);
160
+ if (opts?.rememberSite === true)
161
+ pinCurrentSite(profileDir, headed);
162
+ if (!headed && lastOrigin) {
163
+ const pinned = resolveExecutionForOrigin(loadSitePreferences(profileDir), lastOrigin, 'headless');
164
+ if (pinned === 'headed-background') {
165
+ effectiveHeaded = true;
166
+ backendNote = ` (${normalizeOrigin(lastOrigin)} prefers the visible fallback)`;
167
+ }
168
+ }
169
+ ownBackend = new PersistentBackend({ config: next, headed: effectiveHeaded, sessionId });
170
+ const attach = await ownBackend.start(signal);
171
+ client = new DevToolsClient(attach);
172
+ markAutomationResult(profileDir, effectiveHeaded ? 'headed' : 'headless');
173
+ }
174
+ else {
175
+ if (mode === 'persistent' && opts?.rememberSite === true) {
176
+ pinCurrentSite(persistentProfileDir(next), headed);
177
+ }
178
+ prepareBrowserProfile(next);
179
+ client = new DevToolsClient(next);
180
+ }
97
181
  await client.ensureReady(signal);
182
+ // Record what actually launched (a per-origin pin may have upgraded
183
+ // headless to headed-background) so status/doctor tell the truth.
184
+ next.headless = !effectiveHeaded;
98
185
  config = next;
99
186
  currentMode = mode;
100
- return next;
187
+ return { next, effectiveHeaded, backendNote };
188
+ }
189
+ /** Start the Existing-mode tab broker bridge on demand. */
190
+ async function ensureBridge() {
191
+ if (bridge)
192
+ return bridge;
193
+ const port = config?.tabBridgePort ?? DEFAULT_BRIDGE_PORT;
194
+ if (port === 0)
195
+ throw new Error('The tab bridge is disabled (tabBridgePort: 0).');
196
+ bridge = new TabBridge({ port });
197
+ await bridge.start();
198
+ return bridge;
101
199
  }
102
200
  function loginWallHint(url, text) {
103
201
  if (currentMode !== 'fresh')
@@ -106,20 +204,53 @@ export default function browserUseExtension(pi) {
106
204
  return '';
107
205
  return '\n\nHint: this looks like a login wall in a fresh (logged-out) session. If the page needs your identity, call browser_switch_mode({"mode": "persistent"}) — a human must complete any SSO, 2FA, or passkey step, ideally headed.';
108
206
  }
207
+ function sameOrigin(a, b) {
208
+ try {
209
+ if (!a || !b)
210
+ return false;
211
+ return new URL(a).origin === new URL(b).origin;
212
+ }
213
+ catch {
214
+ return false;
215
+ }
216
+ }
109
217
  /**
110
218
  * Automatic escalation for hard blocks the agent cannot clear alone.
111
219
  * Returns prompt text for the agent to relay, or empty when nothing
112
220
  * applies. Escalates at most once per call; never loops, never retries
113
221
  * a challenge page, and never switches away from an attached session.
222
+ *
223
+ * Challenges only escalate with navigation context: a stale "Just a
224
+ * moment..." shortcut tile on a New Tab snapshot must not rebuild the
225
+ * backend, and a challenge on another origin than requested means the
226
+ * navigation never landed there.
114
227
  */
115
- async function escalateBlockedPage(url, text, signal) {
228
+ async function escalateBlockedPage(url, text, signal, context) {
116
229
  const state = classifyPageState(url, text);
117
230
  if (state === 'ok')
118
231
  return '';
232
+ if (state === 'challenge') {
233
+ if (context?.tool !== 'navigate_page')
234
+ return '';
235
+ if (context.requestedUrl && url && !sameOrigin(context.requestedUrl, url)) {
236
+ // Challenge-provider handoffs (dedicated challenge domains) still
237
+ // count; anything else means the navigation never landed there.
238
+ const host = (() => {
239
+ try {
240
+ return new URL(url).hostname;
241
+ }
242
+ catch {
243
+ return '';
244
+ }
245
+ })();
246
+ if (!/challenge|turnstile|captcha|cf-chl|kasada|perimeterx|datadome/i.test(host))
247
+ return '';
248
+ }
249
+ }
119
250
  if (state === 'login-wall') {
120
251
  if (currentMode === 'fresh')
121
252
  return loginWallHint(url, text);
122
- if (currentMode === 'custom') {
253
+ if (currentMode === 'custom' || currentMode === 'existing') {
123
254
  return '\n\nThis page needs an identity this session does not have. Sign in is required — complete it in the visible browser, then retry.';
124
255
  }
125
256
  if (config?.headless === false) {
@@ -132,7 +263,7 @@ export default function browserUseExtension(pi) {
132
263
  if (config?.headless === false) {
133
264
  return '\n\nA bot challenge is blocking this page and the browser is already visible. Complete the challenge in the window, then retry.';
134
265
  }
135
- if (currentMode === 'custom') {
266
+ if (currentMode === 'custom' || currentMode === 'existing') {
136
267
  return '\n\nA bot challenge is blocking this page. Complete it in the visible browser, then retry — do not loop against the challenge.';
137
268
  }
138
269
  return await escalateToHeaded(url ?? 'this page');
@@ -140,7 +271,9 @@ export default function browserUseExtension(pi) {
140
271
  /**
141
272
  * Rebuild the current backend headed so a human can act (log in, clear
142
273
  * a challenge), then tell the agent exactly what to relay. Attach
143
- * sessions are already visible and owned by the user: prompt only.
274
+ * sessions are already visible and owned by the user: prompt only. The
275
+ * headed window is navigated to the blocked page and fronted: auth
276
+ * handoff is the one case where taking foreground is the job, not a bug.
144
277
  */
145
278
  async function escalateToHeaded(url, signal) {
146
279
  const mode = currentMode === 'persistent' ? 'persistent' : 'fresh';
@@ -150,11 +283,28 @@ export default function browserUseExtension(pi) {
150
283
  catch (error) {
151
284
  return `\n\nBlocked on ${url} and the headed browser failed to launch (${error instanceof Error ? error.message : String(error)}). Ask the user to proceed manually.`;
152
285
  }
153
- return `\n\nBlocked on ${url}: a browser window just opened (same ${mode} session — previous tabs are gone, re-list pages after). Please complete the login or challenge in that window, then tell the agent to continue. Do not close the window until done.`;
286
+ if (/^https?:\/\//.test(url)) {
287
+ // Best effort: the window is already open for manual navigation.
288
+ try {
289
+ await callUpstream(client, 'new_page', { url, background: false }, signal);
290
+ }
291
+ catch {
292
+ // Manual navigation in the opened window covers this.
293
+ }
294
+ }
295
+ // Front Pi-owned Chrome so the handoff window is actually visible.
296
+ // Fresh MCP-launched Chrome fronts itself; only Pi-owned needs help.
297
+ if (ownBackend)
298
+ frontProcessByPid(ownBackend.pid());
299
+ return `\n\nBlocked on ${url}: a browser window just opened on that page (same ${mode} session — previous tabs are gone, re-list pages after). Please complete the login or challenge in that window, then tell the agent to continue. Do not close the window until done.`;
154
300
  }
155
301
  function pageUrlFromSnapshot(text) {
156
302
  return text.match(/\burl="([^"]+)"/)?.[1];
157
303
  }
304
+ /** Best-effort MCP page list → entries (shared parser, never throws). */
305
+ function mcpPageEntries(result) {
306
+ return parseMcpPageList(result);
307
+ }
158
308
  async function ensureConnected(signal) {
159
309
  if (!client)
160
310
  throw new Error('browser-use: session not started');
@@ -169,15 +319,96 @@ export default function browserUseExtension(pi) {
169
319
  const prefixedName = `${TOOL_PREFIX}${tool.name}`;
170
320
  const originalName = tool.name;
171
321
  const description = augmentToolDescription(prefixedName, tool.description ?? '');
322
+ // close_page carries an extra force gate so Existing mode can refuse
323
+ // to close tabs Pi did not open (spec 19); stripped before upstream.
324
+ const parameters = originalName === 'close_page'
325
+ ? Type.Object({
326
+ pageId: Type.Number({
327
+ description: 'The ID of the page to close. Call browser_list_pages first; IDs shift when tabs close.',
328
+ }),
329
+ force: Type.Optional(Type.Boolean({
330
+ description: 'Existing mode only: Pi refuses to close tabs it did not open unless force is true and the user explicitly asked for that exact tab.',
331
+ })),
332
+ })
333
+ : Type.Unsafe(tool.inputSchema);
172
334
  pi.registerTool({
173
335
  name: prefixedName,
174
336
  label: prefixedName,
175
337
  description,
176
- parameters: Type.Unsafe(tool.inputSchema),
338
+ parameters,
177
339
  async execute(_toolCallId, params, signal) {
178
340
  await ensureConnected(signal);
179
341
  const browser = client;
180
- let result = await callUpstream(browser, originalName, params, signal);
342
+ // Focus policy (headed-background / existing): Pi-created pages
343
+ // open in the background and selections never take foreground
344
+ // unless the caller explicitly asked. Explicit values always win.
345
+ const effectiveParams = originalName === 'new_page'
346
+ ? applyNewPageDefaults(params)
347
+ : originalName === 'select_page'
348
+ ? applySelectPageDefaults(params)
349
+ : params;
350
+ if ((originalName === 'navigate_page' || originalName === 'new_page') &&
351
+ typeof effectiveParams.url === 'string') {
352
+ // Remember the origin for the per-origin headed-background
353
+ // fallback; normalizeOrigin never throws (falls back to raw).
354
+ lastOrigin = normalizeOrigin(effectiveParams.url);
355
+ // In Existing mode a Pi-driven navigation marks the destination
356
+ // as Pi-touched for the close guard below.
357
+ if (currentMode === 'existing')
358
+ trackPiUrl(effectiveParams.url);
359
+ // Claim navigated pages for this session so peer agents in a
360
+ // shared browser can tell ours apart (best effort, never fatal).
361
+ if (originalName === 'navigate_page' &&
362
+ (currentMode === 'persistent' || currentMode === 'existing') &&
363
+ typeof effectiveParams.pageId === 'number') {
364
+ try {
365
+ claimPage(registryDir(), { pageId: effectiveParams.pageId, url: effectiveParams.url }, myOwner);
366
+ }
367
+ catch {
368
+ // Ownership is coordination metadata, not the task itself.
369
+ }
370
+ }
371
+ }
372
+ // Never close another session's tabs: Existing mode checks Pi URL
373
+ // ownership (spec 19); a shared persistent backend checks the page
374
+ // registry instead. Both yield to explicit force:true.
375
+ if (originalName === 'close_page') {
376
+ const { force: _force, ...closeArgs } = effectiveParams;
377
+ void _force;
378
+ const wantsForce = effectiveParams.force === true;
379
+ const sharedPersistent = currentMode === 'persistent' && !!ownBackend && !ownBackend.owned;
380
+ if ((currentMode === 'existing' || sharedPersistent) && !wantsForce) {
381
+ if (typeof effectiveParams.pageId !== 'number') {
382
+ return {
383
+ content: [{ type: 'text', text: 'close_page needs a numeric pageId.' }],
384
+ isError: true,
385
+ details: undefined,
386
+ };
387
+ }
388
+ const entries = parseMcpPageList(await callUpstream(browser, 'list_pages', {}, signal));
389
+ const verdict = currentMode === 'existing'
390
+ ? checkExistingCloseAllowed(entries, effectiveParams.pageId, piOwnedUrls)
391
+ : checkSharedCloseAllowed(entries, effectiveParams.pageId, registryDir(), myOwner, piOwnedUrls);
392
+ if (!verdict.ok) {
393
+ return {
394
+ content: [{ type: 'text', text: verdict.reason }],
395
+ isError: true,
396
+ details: undefined,
397
+ };
398
+ }
399
+ }
400
+ const result = await callUpstream(browser, originalName, closeArgs, signal);
401
+ if (!result.isError && typeof effectiveParams.pageId === 'number') {
402
+ try {
403
+ releasePages(registryDir(), { pageId: effectiveParams.pageId });
404
+ }
405
+ catch {
406
+ // Registry hygiene never fails the close itself.
407
+ }
408
+ }
409
+ return { ...toToolContent(result, originalName), details: undefined };
410
+ }
411
+ let result = await callUpstream(browser, originalName, effectiveParams, signal);
181
412
  if (result.isError &&
182
413
  OVERLAY_RECOVERABLE.has(originalName) &&
183
414
  looksOverlayBlocked(extractTextContent(result.content))) {
@@ -185,11 +416,11 @@ export default function browserUseExtension(pi) {
185
416
  // original call. Any failure here falls through to the
186
417
  // original error, which already carries a hint.
187
418
  try {
188
- const escapeArgs = typeof params.pageId === 'number'
189
- ? { pageId: params.pageId, key: 'Escape' }
419
+ const escapeArgs = typeof effectiveParams.pageId === 'number'
420
+ ? { pageId: effectiveParams.pageId, key: 'Escape' }
190
421
  : { key: 'Escape' };
191
422
  await callUpstream(browser, 'press_key', escapeArgs, signal);
192
- result = await callUpstream(browser, originalName, params, signal);
423
+ result = await callUpstream(browser, originalName, effectiveParams, signal);
193
424
  }
194
425
  catch {
195
426
  // Fall through to the original result below.
@@ -198,10 +429,14 @@ export default function browserUseExtension(pi) {
198
429
  const toolContent = toToolContent(result, originalName);
199
430
  if (!toolContent.isError &&
200
431
  (originalName === 'navigate_page' || originalName === 'take_snapshot')) {
201
- const url = originalName === 'navigate_page' && typeof params.url === 'string'
202
- ? params.url
203
- : pageUrlFromSnapshot(extractTextContent(result.content));
204
- const escalation = await escalateBlockedPage(url, extractTextContent(result.content), signal);
432
+ // Prefer the page's real URL from the snapshot; fall back to the
433
+ // requested URL only when the snapshot carries none.
434
+ const snapshotUrl = pageUrlFromSnapshot(extractTextContent(result.content));
435
+ const requestedUrl = originalName === 'navigate_page' && typeof effectiveParams.url === 'string'
436
+ ? effectiveParams.url
437
+ : undefined;
438
+ const url = snapshotUrl ?? requestedUrl;
439
+ const escalation = await escalateBlockedPage(url, extractTextContent(result.content), signal, { tool: originalName, requestedUrl });
205
440
  const first = toolContent.content[0];
206
441
  if (escalation && first && first.text !== undefined) {
207
442
  first.text += escalation;
@@ -302,23 +537,277 @@ export default function browserUseExtension(pi) {
302
537
  pi.registerTool({
303
538
  name: `${TOOL_PREFIX}switch_mode`,
304
539
  label: `${TOOL_PREFIX}switch_mode`,
305
- description: 'Switch the browser backend without restarting: "fresh" is an isolated clean room, "persistent" keeps the saved profile with your logins. Both default to headless; pass headed true to watch. Tabs do not transfer; call browser_list_pages after switching. Prefer fresh; escalate to persistent only on login walls.',
540
+ description: 'Switch the browser backend without restarting: "persistent" is Pi\'s own browser (saved profile with your logins, default), "fresh" is an isolated clean room for anonymous checks, "existing" attaches to your running Chrome (tabs go to the collapsed pi-browser-use group via browser_open_background_tab, consent popup each session). Fresh and persistent default to headless; pass headed true to watch. Tabs do not transfer; call browser_list_pages after switching. Prefer persistent; drop to fresh for clean-room checks.',
306
541
  parameters: Type.Object({
307
- mode: Type.Union([Type.Literal('fresh'), Type.Literal('persistent')]),
542
+ mode: Type.Union([
543
+ Type.Literal('fresh'),
544
+ Type.Literal('persistent'),
545
+ Type.Literal('existing'),
546
+ ]),
308
547
  headed: Type.Optional(Type.Boolean({
309
548
  description: 'Show the browser window. Default is headless — everything works with no popups.',
310
549
  })),
550
+ rememberSite: Type.Optional(Type.Boolean({
551
+ description: "Persistent only: remember the last-visited site's visibility (headless or headed-background) for next time.",
552
+ })),
553
+ }),
554
+ async execute(_toolCallId, params, signal) {
555
+ const mode = params.mode === 'persistent'
556
+ ? 'persistent'
557
+ : params.mode === 'existing'
558
+ ? 'existing'
559
+ : 'fresh';
560
+ const { next, effectiveHeaded, backendNote } = await switchBackend(mode, params.headed === true, signal, { rememberSite: params.rememberSite === true });
561
+ const visibility = mode === 'existing' ? 'headed (your Chrome)' : effectiveHeaded ? 'headed' : 'headless';
562
+ const what = mode === 'fresh'
563
+ ? 'a fresh isolated browser'
564
+ : mode === 'persistent'
565
+ ? 'the persistent Pi profile'
566
+ : 'your running Chrome';
567
+ const extra = mode === 'existing'
568
+ ? ' Open Pi tabs with browser_open_background_tab so they land in the collapsed pi-browser-use group.'
569
+ : '';
570
+ void next;
571
+ return {
572
+ content: [
573
+ {
574
+ type: 'text',
575
+ text: `Switched to ${what} (${visibility})${backendNote}. Previous tabs are gone; call browser_list_pages to start.${extra}`,
576
+ },
577
+ ],
578
+ details: undefined,
579
+ };
580
+ },
581
+ });
582
+ }
583
+ function registerSetupTool() {
584
+ pi.registerTool({
585
+ name: `${TOOL_PREFIX}setup`,
586
+ label: `${TOOL_PREFIX}setup`,
587
+ description: 'First-run setup for the persistent Pi browser profile: opens a plain headed Chrome window (no automation attached) for a human to sign into Google and any sites. Completes when the window is closed. Run once; afterwards Pi automates headless.',
588
+ parameters: Type.Object({}),
589
+ async execute() {
590
+ const profileDir = persistentProfileDir(config ?? {});
591
+ const meta = loadPersistentMetadata(profileDir);
592
+ if (meta.initialized) {
593
+ return {
594
+ content: [
595
+ {
596
+ type: 'text',
597
+ text: `Pi browser profile is already initialized (${profileDir}). If a login expired, use browser_reauth instead.`,
598
+ },
599
+ ],
600
+ details: undefined,
601
+ };
602
+ }
603
+ // No Chrome may hold the profile while the setup window runs.
604
+ await teardownBackend();
605
+ await runBootstrap({
606
+ profileDir,
607
+ executablePath: config?.executablePath,
608
+ chromeArgs: config?.chromeArgs,
609
+ });
610
+ return {
611
+ content: [
612
+ {
613
+ type: 'text',
614
+ text: 'Pi browser profile initialized. Pi now works in the background — no Chrome window will appear during normal automation.',
615
+ },
616
+ ],
617
+ details: undefined,
618
+ };
619
+ },
620
+ });
621
+ }
622
+ function registerStatusTool() {
623
+ pi.registerTool({
624
+ name: `${TOOL_PREFIX}status`,
625
+ label: `${TOOL_PREFIX}status`,
626
+ description: 'Plain-language Pi browser status: profile readiness, execution mode, and what to do next. No page is touched.',
627
+ parameters: Type.Object({}),
628
+ async execute() {
629
+ const mode = currentMode;
630
+ const profileDir = persistentProfileDir(config ?? {});
631
+ const meta = loadPersistentMetadata(profileDir);
632
+ const sitePins = loadSitePreferences(profileDir).length;
633
+ const lines = ['Pi Browser', '──────────', `Session: ${shortSession}`];
634
+ if (mode === 'fresh') {
635
+ lines.push('Profile: Ephemeral (nothing persists)');
636
+ lines.push('Execution: Headless');
637
+ }
638
+ else if (mode === 'persistent') {
639
+ lines.push(`Profile: ${meta.initialized ? 'Ready' : 'Setup required'}`);
640
+ if (!meta.initialized) {
641
+ lines.push('Next step: run browser_setup and sign in, then close the window.');
642
+ }
643
+ else {
644
+ const headed = config?.headless === false;
645
+ lines.push(`Execution: ${headed ? 'Visible fallback (background)' : 'Headless'}${ownBackend?.running() ? '' : ' (backend stopped)'}`);
646
+ if (meta.lastSuccessfulMode)
647
+ lines.push(`Last working mode: ${meta.lastSuccessfulMode}`);
648
+ if (sitePins > 0)
649
+ lines.push(`Sites pinned to visible fallback: ${sitePins}`);
650
+ }
651
+ }
652
+ else if (mode === 'existing') {
653
+ lines.push('Profile: Your browser');
654
+ lines.push('Execution: Background tabs in the collapsed pi-browser-use group');
655
+ lines.push(`Tab bridge: ${bridge ? bridge.baseUrl() : 'not running'}`);
656
+ }
657
+ else {
658
+ lines.push('Profile: Externally attached browser');
659
+ lines.push('Execution: Visible (owned by its launcher)');
660
+ }
661
+ return { content: [{ type: 'text', text: lines.join('\n') }], details: undefined };
662
+ },
663
+ });
664
+ }
665
+ function registerReauthTool() {
666
+ pi.registerTool({
667
+ name: `${TOOL_PREFIX}reauth`,
668
+ label: `${TOOL_PREFIX}reauth`,
669
+ description: 'Reauthenticate the persistent Pi profile after a login/challenge wall: shuts the headless browser down cleanly, opens a headed window for the human to verify, then resumes headless. The plain variant (no automation attached) is for providers that reject instrumented browsers.',
670
+ parameters: Type.Object({
671
+ url: Type.Optional(Type.String({ description: 'Page that needs authentication. Defaults to last origin.' })),
672
+ variant: Type.Optional(Type.Union([Type.Literal('instrumented'), Type.Literal('plain')], {
673
+ description: 'Headed variant: instrumented (Pi navigates first) or plain (maximum compatibility).',
674
+ })),
311
675
  }),
312
676
  async execute(_toolCallId, params, signal) {
313
- const mode = params.mode === 'persistent' ? 'persistent' : 'fresh';
314
- const next = await switchBackend(mode, params.headed === true, signal);
677
+ if (currentMode !== 'persistent') {
678
+ return {
679
+ content: [
680
+ {
681
+ type: 'text',
682
+ text: 'Reauth applies to the persistent Pi profile. Switch to it first with browser_switch_mode({"mode": "persistent"}).',
683
+ },
684
+ ],
685
+ details: undefined,
686
+ };
687
+ }
688
+ const url = typeof params.url === 'string' && params.url.length > 0
689
+ ? params.url
690
+ : (lastOrigin ?? 'this page');
691
+ const variant = params.variant === 'plain' ? 'plain' : 'instrumented';
692
+ if (ownBackend && !ownBackend.owned) {
693
+ return {
694
+ content: [
695
+ {
696
+ type: 'text',
697
+ text: 'Another live agent session owns the persistent browser (shared mode). Reauth must happen there — ask that session to run browser_reauth, or wait for it to exit and retry.',
698
+ },
699
+ ],
700
+ details: undefined,
701
+ };
702
+ }
703
+ if (!ownBackend) {
704
+ // Legacy MCP-launched persistent: headed switch is the reauth path.
705
+ await switchBackend('persistent', true, signal);
706
+ return {
707
+ content: [
708
+ {
709
+ type: 'text',
710
+ text: `A browser window just opened (legacy persistent backend). ${url}: please complete the login there, then tell the agent to continue.`,
711
+ },
712
+ ],
713
+ details: undefined,
714
+ };
715
+ }
716
+ // Spec §7: close headless Chrome cleanly before any headed reauth.
717
+ await teardownBackend();
718
+ const backend = new PersistentBackend({
719
+ config: config ?? {},
720
+ headed: variant === 'instrumented',
721
+ sessionId,
722
+ });
723
+ ownBackend = backend;
724
+ const message = await runReauth({
725
+ backend,
726
+ url,
727
+ variant,
728
+ restartBackend: (headed) => backend.restart(headed),
729
+ });
730
+ if (variant === 'plain') {
731
+ // Plain window closed by the human: resume headless automation.
732
+ const attach = await backend.restart(false);
733
+ client = new DevToolsClient(attach);
734
+ await client.ensureReady(signal);
735
+ return {
736
+ content: [
737
+ { type: 'text', text: `${message}\n\nVerification recorded — Pi resumed headless.` },
738
+ ],
739
+ details: undefined,
740
+ };
741
+ }
742
+ client = new DevToolsClient(backend.attachConfig());
743
+ await client.ensureReady(signal);
315
744
  return {
316
745
  content: [
317
746
  {
318
747
  type: 'text',
319
- text: mode === 'fresh'
320
- ? `Switched to a fresh isolated browser (${next.headless === false ? 'headed' : 'headless'}). Previous tabs are gone; call browser_list_pages to start.`
321
- : `Switched to the persistent profile (${next.headless === false ? 'headed' : 'headless'}). Previous tabs are gone; call browser_list_pages to start.`,
748
+ text: `${message}\n\nAfter verifying, tell the agent to continue; it resumes with browser_switch_mode({"mode": "persistent"}) back to headless.`,
749
+ },
750
+ ],
751
+ details: undefined,
752
+ };
753
+ },
754
+ });
755
+ }
756
+ function registerOpenBackgroundTabTool() {
757
+ pi.registerTool({
758
+ name: `${TOOL_PREFIX}open_background_tab`,
759
+ label: `${TOOL_PREFIX}open_background_tab`,
760
+ description: 'Existing mode only: open a URL as an inactive tab in the collapsed pi-browser-use group via the Pi extension — never a foreground tab. Fails clearly when the extension bridge is unavailable.',
761
+ parameters: Type.Object({
762
+ url: Type.String({ description: 'URL to open in a background Pi tab.' }),
763
+ timeoutMs: Type.Optional(Type.Number({
764
+ description: 'How long to wait for the extension (default 90000: a suspended worker wakes on the ~1min alarm cadence).',
765
+ })),
766
+ }),
767
+ async execute(_toolCallId, params, signal) {
768
+ if (currentMode !== 'existing') {
769
+ return {
770
+ content: [
771
+ {
772
+ type: 'text',
773
+ text: 'Background Pi tabs need Existing mode (your Chrome). Switch first with browser_switch_mode({"mode": "existing"}).',
774
+ isError: true,
775
+ },
776
+ ],
777
+ details: undefined,
778
+ isError: true,
779
+ };
780
+ }
781
+ if (typeof params.url !== 'string' || params.url.length === 0) {
782
+ throw new Error('A URL is required.');
783
+ }
784
+ const activeBridge = await ensureBridge();
785
+ const timeoutMs = typeof params.timeoutMs === 'number' && params.timeoutMs > 0 ? params.timeoutMs : 90_000;
786
+ const result = await openExistingPage(params.url, {
787
+ bridge: activeBridge,
788
+ listPages: async () => {
789
+ await ensureConnected(signal);
790
+ const pages = await client.callTool('list_pages', {}, signal);
791
+ return mcpPageEntries(pages);
792
+ },
793
+ }, { timeoutMs, signal });
794
+ trackPiUrl(params.url);
795
+ if (result.pageId !== undefined) {
796
+ try {
797
+ claimPage(registryDir(), { pageId: result.pageId, url: params.url }, myOwner);
798
+ }
799
+ catch {
800
+ // Ownership is coordination metadata, not the task itself.
801
+ }
802
+ }
803
+ const selectHint = result.pageId !== undefined
804
+ ? ` Select it with browser_select_page (it stays in the background).`
805
+ : ' Call browser_list_pages to find it (it stays in the background).';
806
+ return {
807
+ content: [
808
+ {
809
+ type: 'text',
810
+ text: `Opened ${params.url} as an inactive tab in the collapsed pi-browser-use group.${selectHint}`,
322
811
  },
323
812
  ],
324
813
  details: undefined,
@@ -334,7 +823,18 @@ export default function browserUseExtension(pi) {
334
823
  parameters: Type.Object({}),
335
824
  async execute() {
336
825
  await ensureConnected();
337
- const report = await diagnose(config ?? {}, async () => (await client.listAllTools()).map((tool) => tool.name));
826
+ let peers = 0;
827
+ try {
828
+ peers = livePeerSessions(registryDir()).length;
829
+ }
830
+ catch {
831
+ peers = 0;
832
+ }
833
+ const report = await diagnose(config ?? {}, async () => (await client.listAllTools()).map((tool) => tool.name), {
834
+ backend: ownBackend ? (ownBackend.owned ? 'pi-owned' : 'shared') : undefined,
835
+ bridgeUrl: bridge?.baseUrl() ?? null,
836
+ peers,
837
+ });
338
838
  return { content: [{ type: 'text', text: formatDoctorReport(report) }], details: undefined };
339
839
  },
340
840
  });
@@ -371,20 +871,46 @@ export default function browserUseExtension(pi) {
371
871
  pi.on('session_start', async (_event, ctx) => {
372
872
  config = resolveConfig(loadConfig({ cwd: ctx.cwd, projectTrusted: isProjectTrusted(ctx) }));
373
873
  currentMode = describeMode();
374
- prepareBrowserProfile(config);
375
- client = new DevToolsClient(config);
874
+ if (currentMode === 'persistent' && shouldSelfLaunch(config)) {
875
+ // Phase 2: Pi owns the persistent Chrome process; MCP attaches.
876
+ ownBackend = new PersistentBackend({ config, headed: config.headless === false, sessionId });
877
+ client = new DevToolsClient(await ownBackend.start());
878
+ }
879
+ else {
880
+ prepareBrowserProfile(config);
881
+ client = new DevToolsClient(config);
882
+ }
376
883
  await registerUpstreamTools();
377
884
  registerSaveArtifactTool();
378
885
  registerDoctorTool();
379
886
  registerSwitchModeTool();
887
+ registerSetupTool();
888
+ registerStatusTool();
889
+ registerReauthTool();
890
+ registerOpenBackgroundTabTool();
380
891
  if (config.visionModel) {
381
892
  await registerVisionTool(config.visionModel);
382
893
  }
383
894
  });
384
895
  pi.on('session_shutdown', async () => {
385
- if (client) {
386
- await client.close();
387
- client = undefined;
896
+ await teardownBackend();
897
+ // Release this session's page claims in every scope it may have used.
898
+ for (const scope of new Set([persistentProfileDir(config ?? {}), DEFAULT_PROFILE_DIR])) {
899
+ try {
900
+ releasePages(scope, { sessionId });
901
+ }
902
+ catch {
903
+ // Best effort.
904
+ }
905
+ }
906
+ if (bridge) {
907
+ try {
908
+ await bridge.stop();
909
+ }
910
+ catch {
911
+ // Session teardown is best-effort.
912
+ }
913
+ bridge = undefined;
388
914
  }
389
915
  });
390
916
  }