turno-browser 0.6.0 → 0.8.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (76) hide show
  1. package/CHANGELOG.md +40 -0
  2. package/README.md +13 -5
  3. package/dist/act.js +4 -29
  4. package/dist/agent-cursor-overlay.d.ts +95 -0
  5. package/dist/agent-cursor-overlay.js +358 -0
  6. package/dist/agent-cursor.d.ts +191 -0
  7. package/dist/agent-cursor.js +494 -0
  8. package/dist/api.d.ts +8 -3
  9. package/dist/api.js +6 -3
  10. package/dist/argv.d.ts +2 -1
  11. package/dist/argv.js +7 -1
  12. package/dist/bin/mcp.js +5 -0
  13. package/dist/cli/commands.js +12 -3
  14. package/dist/cli/completion.js +3 -0
  15. package/dist/cli/context.d.ts +2 -0
  16. package/dist/cli/context.js +8 -4
  17. package/dist/cli/do-background.d.ts +2 -0
  18. package/dist/cli/do-background.js +15 -0
  19. package/dist/cli/do.d.ts +4 -0
  20. package/dist/cli/do.js +17 -1
  21. package/dist/cli/flows.js +3 -0
  22. package/dist/cli/identity.d.ts +2 -0
  23. package/dist/cli/identity.js +67 -11
  24. package/dist/cli/init.d.ts +12 -0
  25. package/dist/cli/init.js +219 -0
  26. package/dist/cli/login-summary.d.ts +6 -0
  27. package/dist/cli/login-summary.js +11 -0
  28. package/dist/cli/main.js +3 -0
  29. package/dist/cli/setup.d.ts +32 -0
  30. package/dist/cli/setup.js +32 -23
  31. package/dist/doctor.js +18 -2
  32. package/dist/engine-camoufox.js +1 -0
  33. package/dist/engine-lightpanda.js +1 -0
  34. package/dist/host.d.ts +7 -0
  35. package/dist/host.js +330 -68
  36. package/dist/human-input.d.ts +5 -0
  37. package/dist/human-input.js +1 -1
  38. package/dist/human-step-overlay.d.ts +73 -0
  39. package/dist/human-step-overlay.js +285 -0
  40. package/dist/human-step.d.ts +128 -0
  41. package/dist/human-step.js +521 -0
  42. package/dist/index.d.ts +2 -2
  43. package/dist/index.js +2 -2
  44. package/dist/live-server.d.ts +7 -3
  45. package/dist/live-server.js +13 -15
  46. package/dist/live.d.ts +20 -1
  47. package/dist/live.js +52 -10
  48. package/dist/login.d.ts +22 -2
  49. package/dist/login.js +95 -7
  50. package/dist/network-redact.d.ts +28 -0
  51. package/dist/network-redact.js +202 -0
  52. package/dist/recorder.d.ts +0 -21
  53. package/dist/recorder.js +1 -41
  54. package/dist/recordings.d.ts +5 -0
  55. package/dist/recordings.js +4 -2
  56. package/dist/redact.d.ts +7 -0
  57. package/dist/redact.js +32 -1
  58. package/dist/replay.d.ts +2 -0
  59. package/dist/replay.js +33 -14
  60. package/dist/reputation.d.ts +5 -0
  61. package/dist/reputation.js +12 -7
  62. package/dist/sensitive.d.ts +79 -0
  63. package/dist/sensitive.js +275 -0
  64. package/dist/session.d.ts +12 -0
  65. package/dist/session.js +24 -0
  66. package/dist/settings.d.ts +12 -2
  67. package/dist/settings.js +14 -2
  68. package/dist/tools.d.ts +5 -0
  69. package/dist/tools.js +9 -1
  70. package/dist/tuning.d.ts +4 -1
  71. package/dist/tuning.js +1 -0
  72. package/dist/upstream.d.ts +5 -0
  73. package/dist/watch.d.ts +60 -0
  74. package/dist/watch.js +114 -0
  75. package/npm-shrinkwrap.json +2 -2
  76. package/package.json +1 -1
package/dist/host.js CHANGED
@@ -10,17 +10,22 @@ import { coveredBeforeCall, explainFailure } from "./diagnose.js";
10
10
  import { createEngine, engineChoice, } from "./engine.js";
11
11
  import { TabGovernor } from "./governor.js";
12
12
  import { NEEDS_AUTH_HOLD_MS, healthDue, needsAuthMessage, readHealth, runHealthCheck, writeHealth, } from "./health.js";
13
+ import { AgentCursor, CursorInput, cursorTools, resolveCursor } from "./agent-cursor.js";
13
14
  import { HumanInput, adaptiveSites, humanizeTools } from "./human-input.js";
14
15
  import { identityStateDir } from "./ids.js";
15
16
  import { idleMs } from "./idle-timer.js";
16
- import { describeChanges, resolveLive, resumeNote, timeoutMessage, viewRefusal } from "./live.js";
17
+ import { HUMAN_STEP_TOOL, parseStepParams, runHumanStep } from "./human-step.js";
18
+ import { describeChanges, pausedWhileRunningNote, resolveLive, resumeNote, timeoutMessage, viewRefusal, } from "./live.js";
17
19
  import { LiveServer } from "./live-server.js";
18
- import { FlowRecorder, secretValues } from "./recorder.js";
20
+ import { names } from "./names.js";
21
+ import { NETWORK_TOOLS, redactNetworkText } from "./network-redact.js";
22
+ import { FlowRecorder } from "./recorder.js";
19
23
  import { Recording, tracesTmpDir } from "./recordings.js";
20
24
  import { Redactor, SCRIPT_TOOLS, VaultGuard } from "./redact.js";
21
25
  import { recordChallenge, watchChallenges } from "./reputation.js";
22
26
  import { installResources } from "./resources.js";
23
- import { SESSION_CAPABILITY, SESSION_ENDED_METHOD, SESSION_META_KEY, sessionFromMeta } from "./session.js";
27
+ import { SENSITIVE_MARK, TYPING_TOOLS, maskSnapshots, secretValues, sensitiveFields, sensitiveRefusal, typesIntoSensitive, } from "./sensitive.js";
28
+ import { SESSION_CAPABILITY, SESSION_ENDED_METHOD, SESSION_META_KEY, clientFromMeta, sessionFromMeta, } from "./session.js";
24
29
  import { TabSharing, viewConfig } from "./sharing.js";
25
30
  import { DEFAULT_TAB_LOCKS, DIRECT_SESSION } from "./tab-locks.js";
26
31
  import { readIdentitySettings } from "./settings.js";
@@ -28,6 +33,7 @@ import { TURNO_TOOLS, checkTurnoTools, optInTools, parseArguments, } from "./too
28
33
  import { resolveTuning } from "./tuning.js";
29
34
  import { vaultHasSites } from "./vault.js";
30
35
  import { backendContext, toMcpTool, upstream, } from "./upstream.js";
36
+ import { PlainWatch } from "./watch.js";
31
37
  /**
32
38
  * turno-browser as the MCP server host.
33
39
  *
@@ -54,7 +60,13 @@ import { backendContext, toMcpTool, upstream, } from "./upstream.js";
54
60
  * identity.json turns them on when the server starts, and hands those tools MCP sampling,
55
61
  * routed to the calling session, when the client offers it;
56
62
  * - with the live view on (identity.json `live`, `--live`), holds every browser call while the
57
- * user has paused the agent, and tells the held call what changed (`live.ts`, `live-server.ts`).
63
+ * user has paused the agent, and tells the held call what changed (`live.ts`, `live-server.ts`);
64
+ * a person step (`browser_request_human`, `human-step.ts`) holds them the same way, with the live
65
+ * view or without it (`watch.ts`);
66
+ * - masks password and one-time code fields in every page snapshot and hides their values from
67
+ * every result, whoever typed them, refuses the agent's typing into them (`sensitive.ts`), drops
68
+ * credential values from network output (`network-redact.ts`), and says when a result ends on
69
+ * a challenge page (`reputation.ts`).
58
70
  *
59
71
  * stdio, the transport mcpd and every agent use, is served by turno with the MCP SDK.
60
72
  * `--port` keeps upstream's HTTP server (Host check, streamable HTTP and SSE) serving the
@@ -142,6 +154,45 @@ function errorResult(message, json) {
142
154
  isError: true,
143
155
  };
144
156
  }
157
+ /** How long a one-time code read from a page stays redacted, as a vault one-time code. */
158
+ const FIELD_CODE_MS = 10 * 60_000;
159
+ /** Longest a result waits for the challenge check of the page it ended on. */
160
+ const CHALLENGE_WAIT_MS = 2_000;
161
+ /** A note after the rest of a result (under `turno.<key>` in JSON). */
162
+ function withTrailer(result, title, note, key, json) {
163
+ const first = result.content[0];
164
+ if (!first || first.type !== "text")
165
+ return { ...result, content: [...result.content, { type: "text", text: `### ${title}\n${note}` }] };
166
+ const text = first.text;
167
+ if (json) {
168
+ try {
169
+ const payload = JSON.parse(text);
170
+ const turno = (payload.turno ?? {});
171
+ const out = JSON.stringify({ ...payload, turno: { ...turno, [key]: note } }, null, 2);
172
+ return { ...result, content: [{ ...first, text: out }, ...result.content.slice(1)] };
173
+ }
174
+ catch {
175
+ /* not JSON after all: add a section */
176
+ }
177
+ }
178
+ return { ...result, content: [{ ...first, text: `${text}\n### ${title}\n${note}` }, ...result.content.slice(1)] };
179
+ }
180
+ /** What the agent reads when the page it ended on is a challenge (`reputation.ts`). */
181
+ export function challengeNote(event) {
182
+ const what = event.kind === "captcha"
183
+ ? "a captcha"
184
+ : event.kind === "verify"
185
+ ? "a check that asks whether a person is there"
186
+ : "a page that blocks this browser";
187
+ return `This page (${event.site}) shows ${what}. A person should handle it: ask them to (with browser_request_human, if this server offers it), and go on once they are done.`;
188
+ }
189
+ /** A result's text parts with credentials replaced (`network-redact.ts`). */
190
+ function redactNetworkResult(result) {
191
+ return {
192
+ ...result,
193
+ content: result.content.map((part) => part.type === "text" ? { ...part, text: redactNetworkText(part.text) } : part),
194
+ };
195
+ }
145
196
  /**
146
197
  * One MCP session. Implements upstream's server backend contract, so upstream's HTTP server
147
198
  * can serve it as well as turno's stdio server.
@@ -198,10 +249,19 @@ class SessionBackend extends EventEmitter {
198
249
  recordSettings;
199
250
  /** Each client session's flow recorder: a recording belongs to the session that started it. */
200
251
  recorders = new Map();
252
+ /** The agent name each client session's calls came with (mcpd, a shared host), for the cursor. */
253
+ clients = new Map();
254
+ /** When the sensitive fields of a page were last read (a snapshot reads them too). */
255
+ fieldsRead = new WeakMap();
256
+ /** The challenge check of each page's last load, until a result reports it. */
257
+ challenges = new WeakMap();
258
+ /** The pause gate and the tabs of this browser: the live server, or a plain watch without it. */
259
+ watch;
201
260
  constructor(host, title) {
202
261
  super();
203
262
  this.host = host;
204
263
  this.title = title;
264
+ this.watch = host.live ?? new PlainWatch();
205
265
  }
206
266
  async initialize(clientInfo) {
207
267
  this.clientInfo = clientInfo;
@@ -288,7 +348,7 @@ class SessionBackend extends EventEmitter {
288
348
  const profile = launched.engine !== "lightpanda" && launched.engine !== "attach";
289
349
  if (identity && profile) {
290
350
  try {
291
- await afterIdentityLaunch(identity, launched, (event) => input?.noteChallenge(event.site));
351
+ await afterIdentityLaunch(identity, launched, (event) => input?.noteChallenge(event.site), (page, found) => this.challenges.set(page, found));
292
352
  }
293
353
  catch (error) {
294
354
  await launched.close().catch(() => { });
@@ -313,8 +373,19 @@ class SessionBackend extends EventEmitter {
313
373
  };
314
374
  const sessionConfig = configForSession(config, cliOptions, tuning);
315
375
  const idle = launched.idleTimer ? new CallIdleTimer(launched.idleTimer, () => launched.isOpen()) : undefined;
376
+ // The agent cursor (on when someone can see this browser): the input tools move it. It wraps
377
+ // the humanized tools, so it is the layer next to the page: humanize's pointer path runs
378
+ // through the cursor's mouse and the arrow follows it step by step.
379
+ const cursor = resolveCursor(tuning.cursor, {
380
+ headless: config.browser?.launchOptions?.headless === true && !launched.userBrowser,
381
+ live: this.host.live !== undefined,
382
+ engine: launched.engine,
383
+ })
384
+ ? new AgentCursor({ title, ...(identity ? { identity: identity.id } : {}) })
385
+ : undefined;
386
+ const humanized = input ? humanizeTools(stock, input) : stock;
316
387
  // Every backend of this browser (each session's too, with "sharing": "tabs") runs these tools.
317
- const tools = input ? humanizeTools(stock, input) : stock;
388
+ const tools = cursor ? cursorTools(humanized, new CursorInput(cursor, input !== undefined)) : humanized;
318
389
  const backend = new BrowserBackend(sessionConfig, launched.context, tools, {
319
390
  ...(idle ? { idleTimer: idle } : {}),
320
391
  dispose: async () => {
@@ -323,8 +394,20 @@ class SessionBackend extends EventEmitter {
323
394
  await launched.close();
324
395
  },
325
396
  });
397
+ // Every snapshot of every page masks its password and one-time code fields.
398
+ const masked = new WeakSet();
399
+ const mask = (page) => {
400
+ if (masked.has(page))
401
+ return;
402
+ masked.add(page);
403
+ maskSnapshots(page, (fields) => this.noteFields(page, fields));
404
+ };
405
+ for (const page of launched.context.pages())
406
+ mask(page);
407
+ launched.context.on("page", mask);
326
408
  try {
327
409
  await backend.initialize(clientInfo);
410
+ this.hookUpstream(backend);
328
411
  // After upstream's own routes (--allowed-origins, --blocked-origins), so this one runs
329
412
  // first and hands every request it does not block or serve on to them.
330
413
  if (identity && profile)
@@ -341,6 +424,9 @@ class SessionBackend extends EventEmitter {
341
424
  title,
342
425
  ...(noTraces ? { noTraces } : {}),
343
426
  ...(idleLimit ? { idleMs: idleLimit } : {}),
427
+ beforeSave: async () => {
428
+ await Promise.all(launched.context.pages().map((page) => this.readFields(page)));
429
+ },
344
430
  });
345
431
  }
346
432
  catch (error) {
@@ -357,6 +443,7 @@ class SessionBackend extends EventEmitter {
357
443
  const config = viewConfig(forViews);
358
444
  const view = new BrowserBackend(config, launched.context, tools, {});
359
445
  await view.initialize(clientInfo);
446
+ this.hookUpstream(view);
360
447
  return { backend: view, config };
361
448
  },
362
449
  });
@@ -391,10 +478,12 @@ class SessionBackend extends EventEmitter {
391
478
  throw error;
392
479
  }
393
480
  }
394
- if (this.host.live) {
481
+ // The watch (the live server, or a plain one without the live view) holds this browser
482
+ // during a pause: the user's own, or a person step's.
483
+ {
395
484
  const tabs = governor;
396
485
  let endIdle;
397
- unbindLive = this.host.live.bind({
486
+ unbindLive = this.watch.bind({
398
487
  context: launched.context,
399
488
  engine: launched.engine,
400
489
  userBrowser: launched.userBrowser === true,
@@ -405,12 +494,14 @@ class SessionBackend extends EventEmitter {
405
494
  onPause: async () => {
406
495
  tabs.beginCall();
407
496
  endIdle = idle?.begin();
497
+ await cursor?.setPaused(launched.context.pages(), true);
408
498
  await recording?.pauseForUser();
409
499
  },
410
500
  onResume: () => {
411
501
  endIdle?.();
412
502
  endIdle = undefined;
413
503
  tabs.endCall();
504
+ void cursor?.setPaused(launched.context.pages(), false);
414
505
  },
415
506
  });
416
507
  }
@@ -434,6 +525,7 @@ class SessionBackend extends EventEmitter {
434
525
  ...(sharing ? { sharing } : {}),
435
526
  ...(recording ? { recording } : {}),
436
527
  ...(input ? { input } : {}),
528
+ ...(cursor ? { cursor } : {}),
437
529
  };
438
530
  })();
439
531
  promise.catch(() => {
@@ -447,9 +539,89 @@ class SessionBackend extends EventEmitter {
447
539
  * `session` is the client session mcpd put in the call's `_meta`. Only turno tools
448
540
  * see it; stock tools run exactly as before. Upstream's HTTP server (`--port`) never passes it.
449
541
  */
450
- async callTool(name, args, signal, session, sample) {
542
+ async callTool(name, args, signal, session, sample, client, progress) {
543
+ if (client && session)
544
+ this.clients.set(session, client);
545
+ const started = performance.now();
546
+ let result = await this.callLive(name, args, signal, session, sample, progress);
547
+ if (this.touchesBrowser(name))
548
+ result = await this.afterCall(name, args, result, session, started);
451
549
  // Every result and error of this server leaves through the redactor.
452
- return this.host.redactor.result(await this.callLive(name, args, signal, session, sample));
550
+ return this.host.redactor.result(result);
551
+ }
552
+ /**
553
+ * After a browser call, before the redactor: the sensitive fields of the tab it ended on are
554
+ * read unless a snapshot of the call read them (the values join the redactor, whoever typed
555
+ * them), credentials leave a network tool's output, and a challenge page is pointed out.
556
+ */
557
+ async afterCall(name, args, result, session, started) {
558
+ const json = !!args._meta?.json;
559
+ const page = await this.currentPageOf(session ?? DIRECT_SESSION);
560
+ if (NETWORK_TOOLS.has(name))
561
+ result = redactNetworkResult(result);
562
+ if (!page)
563
+ return result;
564
+ if ((this.fieldsRead.get(page) ?? -1) < started)
565
+ await this.readFields(page);
566
+ const check = this.challenges.get(page);
567
+ if (!check)
568
+ return result;
569
+ this.challenges.delete(page);
570
+ let timer;
571
+ const event = await Promise.race([
572
+ check,
573
+ new Promise((r) => {
574
+ timer = setTimeout(() => r(undefined), CHALLENGE_WAIT_MS);
575
+ }),
576
+ ]).finally(() => clearTimeout(timer));
577
+ return event ? withTrailer(result, "Challenge", challengeNote(event), "challenge", json) : result;
578
+ }
579
+ /** Values of a page's sensitive fields join the redactor, shown as `[sensitive field]`. */
580
+ noteFields(page, fields) {
581
+ this.fieldsRead.set(page, performance.now());
582
+ for (const f of fields)
583
+ this.host.redactor.addField(f.value, SENSITIVE_MARK, f.kind === "one-time code" ? FIELD_CODE_MS : undefined);
584
+ }
585
+ async readFields(page) {
586
+ if (page.isClosed())
587
+ return;
588
+ this.noteFields(page, await sensitiveFields(page).catch(() => []));
589
+ }
590
+ /**
591
+ * Chains the redactor into upstream's own (`redactSecrets`), which every text result, written
592
+ * file (a snapshot or a network request saved with `filename`) and session log line passes
593
+ * through, and, while a network tool runs, the credential rules of `network-redact.ts`.
594
+ */
595
+ hookUpstream(backend) {
596
+ const context = backendContext(backend);
597
+ if (!context || typeof context.redactSecrets !== "function")
598
+ return;
599
+ const own = context.redactSecrets.bind(context);
600
+ let network = 0;
601
+ context.redactSecrets = (text) => this.host.redactor.text(network ? redactNetworkText(own(text)) : own(text));
602
+ const call = backend.callTool.bind(backend);
603
+ backend.callTool = async (name, args, signal) => {
604
+ const counted = NETWORK_TOOLS.has(name);
605
+ if (counted)
606
+ network++;
607
+ try {
608
+ return await call(name, args, signal);
609
+ }
610
+ finally {
611
+ if (counted)
612
+ network--;
613
+ }
614
+ };
615
+ }
616
+ /**
617
+ * Typing into a password or one-time code field is the person's or the vault's: the call is
618
+ * refused, unless the text was read from the secret store (a flow replay's `keychain:` value).
619
+ */
620
+ async refuseSensitive(name, args, page, options) {
621
+ if (!page || options.fromSecretStore || !TYPING_TOOLS.has(name))
622
+ return undefined;
623
+ const kind = await typesIntoSensitive(page, name, args).catch(() => undefined);
624
+ return kind ? sensitiveRefusal(name, kind) : undefined;
453
625
  }
454
626
  /** True when a call needs the browser, so a pause holds it. */
455
627
  touchesBrowser(name) {
@@ -457,47 +629,51 @@ class SessionBackend extends EventEmitter {
457
629
  return tool ? tool.browser !== false : true;
458
630
  }
459
631
  /**
460
- * A call through the live view's pause gate (`live.ts`): it runs at once unless the user
461
- * paused the agent; then it waits, and on resume it runs, or returns an error with a fresh
462
- * snapshot when the browser changed and the call does more than read.
632
+ * A call through the pause gate (`live.ts`): it runs at once unless the user paused the agent
633
+ * (the live view) or a person step holds the browser; then it waits, and on resume it runs, or
634
+ * returns an error with a fresh snapshot when the browser changed and the call does more than
635
+ * read. A person step itself is not counted as a call in flight: it waits for those.
463
636
  */
464
- async callLive(name, args, signal, session, sample) {
465
- const live = this.host.live;
466
- if (!live || !this.touchesBrowser(name))
467
- return await this.callAny(name, args, signal, session, sample);
637
+ async callLive(name, args, signal, session, sample, progress) {
638
+ if (!this.touchesBrowser(name))
639
+ return await this.callAny(name, args, signal, session, sample, progress);
640
+ const watch = this.watch;
641
+ const step = name === HUMAN_STEP_TOOL;
468
642
  const json = !!args._meta?.json;
469
643
  const key = session ?? DIRECT_SESSION;
470
- const outcome = await live.gate.wait(live.waitMs, signal);
644
+ const outcome = await watch.gate.wait(watch.waitMs, signal);
471
645
  if (outcome.kind === "timeout")
472
- return errorResult(timeoutMessage(this.host.identity?.id ?? "", name, outcome.pausedMs, outcome.by, live.waitMs), json);
646
+ return errorResult(timeoutMessage(this.host.identity?.id ?? "", name, outcome.pausedMs, outcome.by, watch.waitMs), json);
473
647
  if (outcome.kind === "aborted")
474
648
  return errorResult(`${name} was cancelled while the user had the browser`, json);
475
- const end = live.begin();
649
+ const end = step ? () => { } : watch.begin();
476
650
  try {
477
651
  if (outcome.kind === "open") {
478
- const result = await this.callAny(name, args, signal, session, sample);
479
- if (!live.paused)
652
+ const result = await this.callAny(name, args, signal, session, sample, progress);
653
+ const paused = watch.gate.paused;
654
+ if (!paused || step)
480
655
  return result;
481
- return withNote(result, "### Paused by the user\nThe user paused this browser while this call ran. Your next call waits until they resume.", json);
656
+ return withNote(result, pausedWhileRunningNote(paused.step), json);
482
657
  }
483
- const beforePage = live.lastPage(key);
658
+ const beforePage = watch.lastPage(key);
484
659
  const afterPage = beforePage ? await this.currentPageOf(key) : undefined;
485
- const after = await live.tabStates();
486
- const { changed, lines } = describeChanges(outcome.before, after, beforePage ? live.idOf(beforePage) : undefined, afterPage ? live.idOf(afterPage) : undefined, outcome.inputs);
660
+ const after = await watch.tabStates();
661
+ const { changed, lines } = describeChanges(outcome.before, after, beforePage ? watch.idOf(beforePage) : undefined, afterPage ? watch.idOf(afterPage) : undefined, outcome.inputs);
487
662
  const note = {
488
663
  pausedMs: outcome.pausedMs,
489
664
  by: outcome.by,
665
+ step: outcome.step,
490
666
  lines,
491
667
  tool: name,
492
668
  recordingFlow: this.recorderFor(session)?.recording !== undefined,
493
669
  };
494
670
  if (!changed || this.readsOnly(name)) {
495
- const result = await this.callAny(name, args, signal, session, sample);
671
+ const result = await this.callAny(name, args, signal, session, sample, progress);
496
672
  return withNote(result, resumeNote({ ...note, ran: true }), json);
497
673
  }
498
674
  // A call that was not run is an error, so a client that reads only the outcome (a script,
499
675
  // turno browser do) does not take it for done.
500
- const notRun = `### Error\n${name} was not run: the user changed the browser while you waited (below).`;
676
+ const notRun = `### Error\n${name} was not run: ${outcome.step ? "the person" : "the user"} changed the browser while you waited (below).`;
501
677
  if (!this.launched?.isOpen()) {
502
678
  const text = `${notRun}\n\n${resumeNote({ ...note, ran: false, noBrowser: true })}`;
503
679
  return errorResultOf(json ? JSON.stringify({ isError: true, error: text }, null, 2) : text);
@@ -524,7 +700,7 @@ class SessionBackend extends EventEmitter {
524
700
  const view = await running.sharing.view(key).catch(() => undefined);
525
701
  return view ? running.sharing.tabView(view).current() : undefined;
526
702
  }
527
- async callAny(name, args, signal, session, sample) {
703
+ async callAny(name, args, signal, session, sample, progress) {
528
704
  const tool = this.host.turno.get(name);
529
705
  if (!tool)
530
706
  return await this.callStock(name, args, signal, {}, session);
@@ -547,6 +723,7 @@ class SessionBackend extends EventEmitter {
547
723
  let recording;
548
724
  let engine;
549
725
  let input;
726
+ let human;
550
727
  if (tool.browser !== false) {
551
728
  const running = await this.browser();
552
729
  engine = running.launched.engine;
@@ -576,6 +753,10 @@ class SessionBackend extends EventEmitter {
576
753
  }
577
754
  stock = this.stockAccess(view, signal, session);
578
755
  recording = running.recording;
756
+ if (name === HUMAN_STEP_TOOL) {
757
+ const access = stock;
758
+ human = (params) => this.requestHuman(running, access, params, json, session, signal, progress);
759
+ }
579
760
  }
580
761
  const recorder = this.recorderFor(session);
581
762
  const secretHook = this.secretHook(recording);
@@ -593,6 +774,7 @@ class SessionBackend extends EventEmitter {
593
774
  ...(sample ? { sample } : {}),
594
775
  ...(engine ? { engine } : {}),
595
776
  ...(this.host.providers ? { providers: this.host.providers } : {}),
777
+ ...(human ? { human } : {}),
596
778
  ...(this.host.identity
597
779
  ? {
598
780
  vault: {
@@ -608,7 +790,7 @@ class SessionBackend extends EventEmitter {
608
790
  };
609
791
  const result = await tool.handle(context, parsed.params);
610
792
  if (stock)
611
- this.host.live?.follow(session ?? DIRECT_SESSION, stock.currentPage());
793
+ this.watch.follow(session ?? DIRECT_SESSION, stock.currentPage());
612
794
  if (name === "browser_login") {
613
795
  if (!result.isError)
614
796
  recorder?.record("browser_login", { site: parsed.params.site });
@@ -658,22 +840,31 @@ class SessionBackend extends EventEmitter {
658
840
  let result;
659
841
  try {
660
842
  await governor.quiet();
661
- const pending = await this.beforeStock(running, session, name, args, backendContext(backend)?.currentTab()?.page);
843
+ // A parked tab loads again first, so the field a call would type into is the one it reaches.
662
844
  await governor.prepare(name, args);
663
- if (tuning.backgroundThrottling && !launched.userBrowser && name !== "browser_tabs")
664
- await backendContext(backend)
665
- ?.currentTab()
666
- ?.page.bringToFront()
667
- .catch(() => { });
668
- const page = backendContext(backend)?.currentTab()?.page;
669
- result =
670
- (launched.engine === "lightpanda" ? undefined : await coveredBeforeCall(page, name, args, json)) ??
671
- (await backend.callTool(name, args, signal));
672
- this.host.live?.follow(session ?? DIRECT_SESSION, backendContext(backend)?.currentTab()?.page);
673
- if (result.isError)
674
- result = await explainFailure(backendContext(backend)?.currentTab()?.page, name, args, result, json);
675
- await this.afterStock(running, session, name, args, pending, result, backendContext(backend)?.currentTab()?.page);
676
- await governor.afterCall();
845
+ const refusedField = await this.refuseSensitive(name, args, backendContext(backend)?.currentTab()?.page, options);
846
+ if (refusedField)
847
+ result = errorResult(refusedField, json);
848
+ else {
849
+ const pending = await this.beforeStock(running, session, name, args, backendContext(backend)?.currentTab()?.page);
850
+ if (tuning.backgroundThrottling && !launched.userBrowser && name !== "browser_tabs")
851
+ await backendContext(backend)
852
+ ?.currentTab()
853
+ ?.page.bringToFront()
854
+ .catch(() => { });
855
+ const page = backendContext(backend)?.currentTab()?.page;
856
+ const key = session ?? DIRECT_SESSION;
857
+ await running.cursor?.begin(page, key, this.clients.get(key));
858
+ result =
859
+ (launched.engine === "lightpanda" ? undefined : await coveredBeforeCall(page, name, args, json)) ??
860
+ (await this.withCursor(running, name, page, () => backend.callTool(name, args, signal)));
861
+ void running.cursor?.end(backendContext(backend)?.currentTab()?.page, key);
862
+ this.watch.follow(key, backendContext(backend)?.currentTab()?.page);
863
+ if (result.isError)
864
+ result = await explainFailure(backendContext(backend)?.currentTab()?.page, name, args, result, json);
865
+ await this.afterStock(running, session, name, args, pending, result, backendContext(backend)?.currentTab()?.page);
866
+ await governor.afterCall();
867
+ }
677
868
  }
678
869
  finally {
679
870
  running.recording?.callEnded();
@@ -721,25 +912,32 @@ class SessionBackend extends EventEmitter {
721
912
  };
722
913
  }
723
914
  const tabs = sharing.tabView(view);
724
- const pending = await this.beforeStock(running, key, name, args, tabs.current());
725
915
  await governor.prepare(name, args, tabs);
726
- if (tuning.backgroundThrottling && name !== "browser_tabs")
727
- await tabs
728
- .current()
729
- ?.bringToFront()
730
- .catch(() => { });
731
- result =
732
- (running.launched.engine === "lightpanda"
733
- ? undefined
734
- : await coveredBeforeCall(tabs.current(), name, args, json)) ??
735
- (await backend.callTool(name, args, signal));
736
- this.host.live?.follow(key, tabs.current());
737
- if (result.isError)
738
- result = await explainFailure(tabs.current(), name, args, result, json);
739
- await this.afterStock(running, key, name, args, pending, result, tabs.current());
740
- // The tab the call left current is the session's before the cap counts its tabs.
741
- sharing.after(key, view);
742
- await governor.afterCall();
916
+ const refusedField = await this.refuseSensitive(name, args, tabs.current(), options);
917
+ if (refusedField)
918
+ result = errorResult(refusedField, json);
919
+ else {
920
+ const pending = await this.beforeStock(running, key, name, args, tabs.current());
921
+ if (tuning.backgroundThrottling && name !== "browser_tabs")
922
+ await tabs
923
+ .current()
924
+ ?.bringToFront()
925
+ .catch(() => { });
926
+ await running.cursor?.begin(tabs.current(), key, this.clients.get(key));
927
+ result =
928
+ (running.launched.engine === "lightpanda"
929
+ ? undefined
930
+ : await coveredBeforeCall(tabs.current(), name, args, json)) ??
931
+ (await this.withCursor(running, name, tabs.current(), () => backend.callTool(name, args, signal)));
932
+ void running.cursor?.end(tabs.current(), key);
933
+ this.watch.follow(key, tabs.current());
934
+ if (result.isError)
935
+ result = await explainFailure(tabs.current(), name, args, result, json);
936
+ await this.afterStock(running, key, name, args, pending, result, tabs.current());
937
+ // The tab the call left current is the session's before the cap counts its tabs.
938
+ sharing.after(key, view);
939
+ await governor.afterCall();
940
+ }
743
941
  }
744
942
  }
745
943
  }
@@ -762,6 +960,12 @@ class SessionBackend extends EventEmitter {
762
960
  const text = sharing.annotate(first.text, key, view);
763
961
  return { ...annotated, content: [{ ...first, text }, ...annotated.content.slice(1)] };
764
962
  }
963
+ /** A stock call, with the agent cursor hidden while it takes the model's screenshot. */
964
+ async withCursor(running, name, page, call) {
965
+ if (!running.cursor || name !== "browser_take_screenshot")
966
+ return await call();
967
+ return await running.cursor.hiddenDuring(page, call);
968
+ }
765
969
  /**
766
970
  * Before a stock call: the flow step it may become (`recorder.ts`), and a trace stopped and
767
971
  * saved before a value goes into a secret field, as before a vault fill (`recordings.ts`).
@@ -769,6 +973,9 @@ class SessionBackend extends EventEmitter {
769
973
  async beforeStock(running, session, name, args, page) {
770
974
  const { recording } = running;
771
975
  await recording?.beforeCall();
976
+ // While a trace records, a value the person typed since the last call is known before it is traced.
977
+ if (recording && page)
978
+ await this.readFields(page);
772
979
  const recorder = this.recorderFor(session);
773
980
  const pending = recorder?.active ? await recorder.capture(name, args, page).catch(() => undefined) : undefined;
774
981
  const secretHook = this.secretHook(recording);
@@ -814,6 +1021,33 @@ class SessionBackend extends EventEmitter {
814
1021
  screenshot: true,
815
1022
  });
816
1023
  }
1024
+ /**
1025
+ * A person step on the session's current tab (`human-step.ts`). The person must be able to see
1026
+ * the browser: a Chrome window (headed, or attach mode), or the live view open on a headless one.
1027
+ */
1028
+ async requestHuman(running, stock, raw, json, session, signal, progress) {
1029
+ const params = parseStepParams(raw);
1030
+ if ("error" in params)
1031
+ return errorResult(params.error, json);
1032
+ const page = stock.currentPage();
1033
+ if (!page)
1034
+ return errorResult("no page is open; navigate to the page of the step first", json);
1035
+ const { launched } = running;
1036
+ const headless = this.host.config.browser?.launchOptions?.headless === true && !launched.userBrowser;
1037
+ const unseen = headless ? unseenReason(this.host.live, this.host.identity?.id) : undefined;
1038
+ return await runHumanStep(params, {
1039
+ page,
1040
+ watch: this.watch,
1041
+ guard: this.host.guard,
1042
+ session: session ?? DIRECT_SESSION,
1043
+ ...(unseen ? { unseen } : {}),
1044
+ window: !headless && !launched.userBrowser,
1045
+ json,
1046
+ ...(signal ? { signal } : {}),
1047
+ ...(progress ? { progress } : {}),
1048
+ currentPage: () => stock.currentPage(),
1049
+ });
1050
+ }
817
1051
  /**
818
1052
  * The script tools near the vault: `browser_evaluate` and `browser_run_code_unsafe`
819
1053
  * are refused on a page browser_login filled, until it navigates; `browser_run_code_unsafe`
@@ -838,10 +1072,10 @@ class SessionBackend extends EventEmitter {
838
1072
  schemas: this.host.stockSchemas,
839
1073
  formatError: (error) => upstream().utils.z.prettifyError(error),
840
1074
  // A batch, a replay or an act run stops at its next step once the user pauses the agent.
841
- call: (name, args, options) => this.host.live?.paused
1075
+ call: (name, args, options) => this.watch.paused
842
1076
  ? Promise.resolve(errorResult(`${PAUSED_STEP}: this step and the ones after it were not run. Your next call waits until they resume.`, !!args._meta?.json))
843
1077
  : this.callStock(name, args, signal, options, session),
844
- held: () => (this.host.live?.paused ? PAUSED_STEP : undefined),
1078
+ held: () => (this.watch.paused ? PAUSED_STEP : undefined),
845
1079
  page: async () => {
846
1080
  const page = backendContext(view.backend)?.currentTab()?.page;
847
1081
  if (!page)
@@ -856,6 +1090,7 @@ class SessionBackend extends EventEmitter {
856
1090
  // A flow the session was recording and never stopped is dropped with it.
857
1091
  this.recorders.get(session)?.cancel();
858
1092
  this.recorders.delete(session);
1093
+ this.clients.delete(session);
859
1094
  const current = this.current;
860
1095
  if (!current)
861
1096
  return;
@@ -915,7 +1150,7 @@ function refuseWhileNeedsAuth(identity) {
915
1150
  * log) and, when identity.json has a health check that is due, run it; a page without the
916
1151
  * proof of a signed-in session throws NEEDS_AUTH, and the call fails with that message.
917
1152
  */
918
- async function afterIdentityLaunch(identity, launched, onChallenge) {
1153
+ async function afterIdentityLaunch(identity, launched, onChallenge, onCheck) {
919
1154
  const { id, env } = identity;
920
1155
  watchChallenges(launched.context, (event) => {
921
1156
  onChallenge?.(event);
@@ -925,7 +1160,7 @@ async function afterIdentityLaunch(identity, launched, onChallenge) {
925
1160
  catch {
926
1161
  /* the log is best effort; a full disk must not break a tool call */
927
1162
  }
928
- });
1163
+ }, onCheck ? { onCheck: (page, found) => onCheck(page, found) } : {});
929
1164
  const check = readIdentitySettings(id, env)?.health;
930
1165
  if (!check || !healthDue(readHealth(id, env), check))
931
1166
  return;
@@ -1086,6 +1321,17 @@ function samplerFor(server, requestId) {
1086
1321
  };
1087
1322
  };
1088
1323
  }
1324
+ /** Progress notifications for a call whose client gave a progress token. */
1325
+ function progressFor(token, send) {
1326
+ if (token === undefined)
1327
+ return undefined;
1328
+ return (elapsedMs, message) => {
1329
+ void send({
1330
+ method: "notifications/progress",
1331
+ params: { progressToken: token, progress: Math.round(elapsedMs / 1000), message },
1332
+ }).catch(() => { });
1333
+ };
1334
+ }
1089
1335
  /**
1090
1336
  * An MCP server for one connection of this host, on any transport: with its own session
1091
1337
  * backend (stdio, one browser per connection), or on the backend of a shared host.
@@ -1121,7 +1367,11 @@ function createServer(host, shared) {
1121
1367
  try {
1122
1368
  const b = await session();
1123
1369
  const args = (request.params.arguments ?? {});
1124
- const result = await b.callTool(request.params.name, args, extra.signal, sessionFromMeta(request.params._meta), samplerFor(server, extra.requestId));
1370
+ const meta = request.params._meta;
1371
+ const sessionId = sessionFromMeta(meta);
1372
+ // The agent behind the call: mcpd says it; on a shared host each connection is one agent.
1373
+ const client = clientFromMeta(meta) ?? (shared && sessionId ? server.getClientVersion()?.name : undefined);
1374
+ const result = await b.callTool(request.params.name, args, extra.signal, sessionId, samplerFor(server, extra.requestId), client, progressFor(meta?.progressToken, extra.sendNotification));
1125
1375
  return mergeTextParts(result);
1126
1376
  }
1127
1377
  catch (error) {
@@ -1238,4 +1488,16 @@ export async function runHost(args, options = {}) {
1238
1488
  core.utils.gracefullyProcessExitDoNotHang(1);
1239
1489
  }
1240
1490
  }
1491
+ /** Why the person cannot see a headless Chrome now, or undefined when the live view shows it. */
1492
+ export function unseenReason(live, id) {
1493
+ const b = names().browser;
1494
+ if (live && live.viewerCount > 0)
1495
+ return undefined;
1496
+ if (live && id)
1497
+ return `Chrome runs without a window here, so the person sees the page only in the live view, and it is not open. Ask the person to run \`${b} live ${id}\` (it opens the view in their browser), then call ${HUMAN_STEP_TOOL} again.`;
1498
+ const start = id
1499
+ ? `\`${b} set ${id} --live true\` and a new session, then \`${b} live ${id}\``
1500
+ : "an identity with the live view on";
1501
+ return `Chrome runs without a window here (--headless) and the live view is off, so the person cannot see the page. Ask the person to run the browser with a window (start turno-browser-mcp without --headless), or to turn the live view on (${start}), then call ${HUMAN_STEP_TOOL} again.`;
1502
+ }
1241
1503
  //# sourceMappingURL=host.js.map