turno-browser 0.7.0 → 0.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +13 -0
- package/README.md +5 -1
- package/dist/act.js +4 -29
- package/dist/agent-cursor-overlay.d.ts +10 -0
- package/dist/agent-cursor-overlay.js +50 -5
- package/dist/agent-cursor.d.ts +39 -7
- package/dist/agent-cursor.js +106 -20
- package/dist/cli/flows.js +3 -0
- package/dist/cli/identity.js +3 -4
- package/dist/cli/init.js +4 -2
- package/dist/engine-camoufox.js +1 -0
- package/dist/engine-lightpanda.js +1 -0
- package/dist/host.d.ts +7 -0
- package/dist/host.js +313 -76
- package/dist/human-step-overlay.d.ts +73 -0
- package/dist/human-step-overlay.js +285 -0
- package/dist/human-step.d.ts +128 -0
- package/dist/human-step.js +521 -0
- package/dist/index.d.ts +1 -1
- package/dist/index.js +1 -1
- package/dist/live-server.d.ts +7 -3
- package/dist/live-server.js +13 -15
- package/dist/live.d.ts +20 -1
- package/dist/live.js +52 -10
- package/dist/network-redact.d.ts +28 -0
- package/dist/network-redact.js +202 -0
- package/dist/recorder.d.ts +0 -21
- package/dist/recorder.js +1 -41
- package/dist/recordings.d.ts +5 -0
- package/dist/recordings.js +4 -2
- package/dist/redact.d.ts +7 -0
- package/dist/redact.js +32 -1
- package/dist/replay.d.ts +2 -0
- package/dist/replay.js +33 -14
- package/dist/reputation.d.ts +5 -0
- package/dist/reputation.js +12 -7
- package/dist/sensitive.d.ts +79 -0
- package/dist/sensitive.js +275 -0
- package/dist/session.d.ts +12 -0
- package/dist/session.js +24 -0
- package/dist/tools.d.ts +5 -0
- package/dist/tools.js +9 -1
- package/dist/upstream.d.ts +5 -0
- package/dist/watch.d.ts +60 -0
- package/dist/watch.js +114 -0
- package/npm-shrinkwrap.json +2 -2
- package/package.json +1 -1
package/dist/host.js
CHANGED
|
@@ -14,14 +14,18 @@ import { AgentCursor, CursorInput, cursorTools, resolveCursor } from "./agent-cu
|
|
|
14
14
|
import { HumanInput, adaptiveSites, humanizeTools } from "./human-input.js";
|
|
15
15
|
import { identityStateDir } from "./ids.js";
|
|
16
16
|
import { idleMs } from "./idle-timer.js";
|
|
17
|
-
import {
|
|
17
|
+
import { HUMAN_STEP_TOOL, parseStepParams, runHumanStep } from "./human-step.js";
|
|
18
|
+
import { describeChanges, pausedWhileRunningNote, resolveLive, resumeNote, timeoutMessage, viewRefusal, } from "./live.js";
|
|
18
19
|
import { LiveServer } from "./live-server.js";
|
|
19
|
-
import {
|
|
20
|
+
import { names } from "./names.js";
|
|
21
|
+
import { NETWORK_TOOLS, redactNetworkText } from "./network-redact.js";
|
|
22
|
+
import { FlowRecorder } from "./recorder.js";
|
|
20
23
|
import { Recording, tracesTmpDir } from "./recordings.js";
|
|
21
24
|
import { Redactor, SCRIPT_TOOLS, VaultGuard } from "./redact.js";
|
|
22
25
|
import { recordChallenge, watchChallenges } from "./reputation.js";
|
|
23
26
|
import { installResources } from "./resources.js";
|
|
24
|
-
import {
|
|
27
|
+
import { SENSITIVE_MARK, TYPING_TOOLS, maskSnapshots, secretValues, sensitiveFields, sensitiveRefusal, typesIntoSensitive, } from "./sensitive.js";
|
|
28
|
+
import { SESSION_CAPABILITY, SESSION_ENDED_METHOD, SESSION_META_KEY, clientFromMeta, sessionFromMeta, } from "./session.js";
|
|
25
29
|
import { TabSharing, viewConfig } from "./sharing.js";
|
|
26
30
|
import { DEFAULT_TAB_LOCKS, DIRECT_SESSION } from "./tab-locks.js";
|
|
27
31
|
import { readIdentitySettings } from "./settings.js";
|
|
@@ -29,6 +33,7 @@ import { TURNO_TOOLS, checkTurnoTools, optInTools, parseArguments, } from "./too
|
|
|
29
33
|
import { resolveTuning } from "./tuning.js";
|
|
30
34
|
import { vaultHasSites } from "./vault.js";
|
|
31
35
|
import { backendContext, toMcpTool, upstream, } from "./upstream.js";
|
|
36
|
+
import { PlainWatch } from "./watch.js";
|
|
32
37
|
/**
|
|
33
38
|
* turno-browser as the MCP server host.
|
|
34
39
|
*
|
|
@@ -55,7 +60,13 @@ import { backendContext, toMcpTool, upstream, } from "./upstream.js";
|
|
|
55
60
|
* identity.json turns them on when the server starts, and hands those tools MCP sampling,
|
|
56
61
|
* routed to the calling session, when the client offers it;
|
|
57
62
|
* - with the live view on (identity.json `live`, `--live`), holds every browser call while the
|
|
58
|
-
* user has paused the agent, and tells the held call what changed (`live.ts`, `live-server.ts`)
|
|
63
|
+
* user has paused the agent, and tells the held call what changed (`live.ts`, `live-server.ts`);
|
|
64
|
+
* a person step (`browser_request_human`, `human-step.ts`) holds them the same way, with the live
|
|
65
|
+
* view or without it (`watch.ts`);
|
|
66
|
+
* - masks password and one-time code fields in every page snapshot and hides their values from
|
|
67
|
+
* every result, whoever typed them, refuses the agent's typing into them (`sensitive.ts`), drops
|
|
68
|
+
* credential values from network output (`network-redact.ts`), and says when a result ends on
|
|
69
|
+
* a challenge page (`reputation.ts`).
|
|
59
70
|
*
|
|
60
71
|
* stdio, the transport mcpd and every agent use, is served by turno with the MCP SDK.
|
|
61
72
|
* `--port` keeps upstream's HTTP server (Host check, streamable HTTP and SSE) serving the
|
|
@@ -143,6 +154,45 @@ function errorResult(message, json) {
|
|
|
143
154
|
isError: true,
|
|
144
155
|
};
|
|
145
156
|
}
|
|
157
|
+
/** How long a one-time code read from a page stays redacted, as a vault one-time code. */
|
|
158
|
+
const FIELD_CODE_MS = 10 * 60_000;
|
|
159
|
+
/** Longest a result waits for the challenge check of the page it ended on. */
|
|
160
|
+
const CHALLENGE_WAIT_MS = 2_000;
|
|
161
|
+
/** A note after the rest of a result (under `turno.<key>` in JSON). */
|
|
162
|
+
function withTrailer(result, title, note, key, json) {
|
|
163
|
+
const first = result.content[0];
|
|
164
|
+
if (!first || first.type !== "text")
|
|
165
|
+
return { ...result, content: [...result.content, { type: "text", text: `### ${title}\n${note}` }] };
|
|
166
|
+
const text = first.text;
|
|
167
|
+
if (json) {
|
|
168
|
+
try {
|
|
169
|
+
const payload = JSON.parse(text);
|
|
170
|
+
const turno = (payload.turno ?? {});
|
|
171
|
+
const out = JSON.stringify({ ...payload, turno: { ...turno, [key]: note } }, null, 2);
|
|
172
|
+
return { ...result, content: [{ ...first, text: out }, ...result.content.slice(1)] };
|
|
173
|
+
}
|
|
174
|
+
catch {
|
|
175
|
+
/* not JSON after all: add a section */
|
|
176
|
+
}
|
|
177
|
+
}
|
|
178
|
+
return { ...result, content: [{ ...first, text: `${text}\n### ${title}\n${note}` }, ...result.content.slice(1)] };
|
|
179
|
+
}
|
|
180
|
+
/** What the agent reads when the page it ended on is a challenge (`reputation.ts`). */
|
|
181
|
+
export function challengeNote(event) {
|
|
182
|
+
const what = event.kind === "captcha"
|
|
183
|
+
? "a captcha"
|
|
184
|
+
: event.kind === "verify"
|
|
185
|
+
? "a check that asks whether a person is there"
|
|
186
|
+
: "a page that blocks this browser";
|
|
187
|
+
return `This page (${event.site}) shows ${what}. A person should handle it: ask them to (with browser_request_human, if this server offers it), and go on once they are done.`;
|
|
188
|
+
}
|
|
189
|
+
/** A result's text parts with credentials replaced (`network-redact.ts`). */
|
|
190
|
+
function redactNetworkResult(result) {
|
|
191
|
+
return {
|
|
192
|
+
...result,
|
|
193
|
+
content: result.content.map((part) => part.type === "text" ? { ...part, text: redactNetworkText(part.text) } : part),
|
|
194
|
+
};
|
|
195
|
+
}
|
|
146
196
|
/**
|
|
147
197
|
* One MCP session. Implements upstream's server backend contract, so upstream's HTTP server
|
|
148
198
|
* can serve it as well as turno's stdio server.
|
|
@@ -199,10 +249,19 @@ class SessionBackend extends EventEmitter {
|
|
|
199
249
|
recordSettings;
|
|
200
250
|
/** Each client session's flow recorder: a recording belongs to the session that started it. */
|
|
201
251
|
recorders = new Map();
|
|
252
|
+
/** The agent name each client session's calls came with (mcpd, a shared host), for the cursor. */
|
|
253
|
+
clients = new Map();
|
|
254
|
+
/** When the sensitive fields of a page were last read (a snapshot reads them too). */
|
|
255
|
+
fieldsRead = new WeakMap();
|
|
256
|
+
/** The challenge check of each page's last load, until a result reports it. */
|
|
257
|
+
challenges = new WeakMap();
|
|
258
|
+
/** The pause gate and the tabs of this browser: the live server, or a plain watch without it. */
|
|
259
|
+
watch;
|
|
202
260
|
constructor(host, title) {
|
|
203
261
|
super();
|
|
204
262
|
this.host = host;
|
|
205
263
|
this.title = title;
|
|
264
|
+
this.watch = host.live ?? new PlainWatch();
|
|
206
265
|
}
|
|
207
266
|
async initialize(clientInfo) {
|
|
208
267
|
this.clientInfo = clientInfo;
|
|
@@ -289,7 +348,7 @@ class SessionBackend extends EventEmitter {
|
|
|
289
348
|
const profile = launched.engine !== "lightpanda" && launched.engine !== "attach";
|
|
290
349
|
if (identity && profile) {
|
|
291
350
|
try {
|
|
292
|
-
await afterIdentityLaunch(identity, launched, (event) => input?.noteChallenge(event.site));
|
|
351
|
+
await afterIdentityLaunch(identity, launched, (event) => input?.noteChallenge(event.site), (page, found) => this.challenges.set(page, found));
|
|
293
352
|
}
|
|
294
353
|
catch (error) {
|
|
295
354
|
await launched.close().catch(() => { });
|
|
@@ -314,8 +373,9 @@ class SessionBackend extends EventEmitter {
|
|
|
314
373
|
};
|
|
315
374
|
const sessionConfig = configForSession(config, cliOptions, tuning);
|
|
316
375
|
const idle = launched.idleTimer ? new CallIdleTimer(launched.idleTimer, () => launched.isOpen()) : undefined;
|
|
317
|
-
// The agent cursor (on when someone can see this browser): the input tools move it
|
|
318
|
-
//
|
|
376
|
+
// The agent cursor (on when someone can see this browser): the input tools move it. It wraps
|
|
377
|
+
// the humanized tools, so it is the layer next to the page: humanize's pointer path runs
|
|
378
|
+
// through the cursor's mouse and the arrow follows it step by step.
|
|
319
379
|
const cursor = resolveCursor(tuning.cursor, {
|
|
320
380
|
headless: config.browser?.launchOptions?.headless === true && !launched.userBrowser,
|
|
321
381
|
live: this.host.live !== undefined,
|
|
@@ -323,9 +383,9 @@ class SessionBackend extends EventEmitter {
|
|
|
323
383
|
})
|
|
324
384
|
? new AgentCursor({ title, ...(identity ? { identity: identity.id } : {}) })
|
|
325
385
|
: undefined;
|
|
326
|
-
const
|
|
386
|
+
const humanized = input ? humanizeTools(stock, input) : stock;
|
|
327
387
|
// Every backend of this browser (each session's too, with "sharing": "tabs") runs these tools.
|
|
328
|
-
const tools =
|
|
388
|
+
const tools = cursor ? cursorTools(humanized, new CursorInput(cursor, input !== undefined)) : humanized;
|
|
329
389
|
const backend = new BrowserBackend(sessionConfig, launched.context, tools, {
|
|
330
390
|
...(idle ? { idleTimer: idle } : {}),
|
|
331
391
|
dispose: async () => {
|
|
@@ -334,8 +394,20 @@ class SessionBackend extends EventEmitter {
|
|
|
334
394
|
await launched.close();
|
|
335
395
|
},
|
|
336
396
|
});
|
|
397
|
+
// Every snapshot of every page masks its password and one-time code fields.
|
|
398
|
+
const masked = new WeakSet();
|
|
399
|
+
const mask = (page) => {
|
|
400
|
+
if (masked.has(page))
|
|
401
|
+
return;
|
|
402
|
+
masked.add(page);
|
|
403
|
+
maskSnapshots(page, (fields) => this.noteFields(page, fields));
|
|
404
|
+
};
|
|
405
|
+
for (const page of launched.context.pages())
|
|
406
|
+
mask(page);
|
|
407
|
+
launched.context.on("page", mask);
|
|
337
408
|
try {
|
|
338
409
|
await backend.initialize(clientInfo);
|
|
410
|
+
this.hookUpstream(backend);
|
|
339
411
|
// After upstream's own routes (--allowed-origins, --blocked-origins), so this one runs
|
|
340
412
|
// first and hands every request it does not block or serve on to them.
|
|
341
413
|
if (identity && profile)
|
|
@@ -352,6 +424,9 @@ class SessionBackend extends EventEmitter {
|
|
|
352
424
|
title,
|
|
353
425
|
...(noTraces ? { noTraces } : {}),
|
|
354
426
|
...(idleLimit ? { idleMs: idleLimit } : {}),
|
|
427
|
+
beforeSave: async () => {
|
|
428
|
+
await Promise.all(launched.context.pages().map((page) => this.readFields(page)));
|
|
429
|
+
},
|
|
355
430
|
});
|
|
356
431
|
}
|
|
357
432
|
catch (error) {
|
|
@@ -368,6 +443,7 @@ class SessionBackend extends EventEmitter {
|
|
|
368
443
|
const config = viewConfig(forViews);
|
|
369
444
|
const view = new BrowserBackend(config, launched.context, tools, {});
|
|
370
445
|
await view.initialize(clientInfo);
|
|
446
|
+
this.hookUpstream(view);
|
|
371
447
|
return { backend: view, config };
|
|
372
448
|
},
|
|
373
449
|
});
|
|
@@ -402,10 +478,12 @@ class SessionBackend extends EventEmitter {
|
|
|
402
478
|
throw error;
|
|
403
479
|
}
|
|
404
480
|
}
|
|
405
|
-
|
|
481
|
+
// The watch (the live server, or a plain one without the live view) holds this browser
|
|
482
|
+
// during a pause: the user's own, or a person step's.
|
|
483
|
+
{
|
|
406
484
|
const tabs = governor;
|
|
407
485
|
let endIdle;
|
|
408
|
-
unbindLive = this.
|
|
486
|
+
unbindLive = this.watch.bind({
|
|
409
487
|
context: launched.context,
|
|
410
488
|
engine: launched.engine,
|
|
411
489
|
userBrowser: launched.userBrowser === true,
|
|
@@ -461,9 +539,89 @@ class SessionBackend extends EventEmitter {
|
|
|
461
539
|
* `session` is the client session mcpd put in the call's `_meta`. Only turno tools
|
|
462
540
|
* see it; stock tools run exactly as before. Upstream's HTTP server (`--port`) never passes it.
|
|
463
541
|
*/
|
|
464
|
-
async callTool(name, args, signal, session, sample) {
|
|
542
|
+
async callTool(name, args, signal, session, sample, client, progress) {
|
|
543
|
+
if (client && session)
|
|
544
|
+
this.clients.set(session, client);
|
|
545
|
+
const started = performance.now();
|
|
546
|
+
let result = await this.callLive(name, args, signal, session, sample, progress);
|
|
547
|
+
if (this.touchesBrowser(name))
|
|
548
|
+
result = await this.afterCall(name, args, result, session, started);
|
|
465
549
|
// Every result and error of this server leaves through the redactor.
|
|
466
|
-
return this.host.redactor.result(
|
|
550
|
+
return this.host.redactor.result(result);
|
|
551
|
+
}
|
|
552
|
+
/**
|
|
553
|
+
* After a browser call, before the redactor: the sensitive fields of the tab it ended on are
|
|
554
|
+
* read unless a snapshot of the call read them (the values join the redactor, whoever typed
|
|
555
|
+
* them), credentials leave a network tool's output, and a challenge page is pointed out.
|
|
556
|
+
*/
|
|
557
|
+
async afterCall(name, args, result, session, started) {
|
|
558
|
+
const json = !!args._meta?.json;
|
|
559
|
+
const page = await this.currentPageOf(session ?? DIRECT_SESSION);
|
|
560
|
+
if (NETWORK_TOOLS.has(name))
|
|
561
|
+
result = redactNetworkResult(result);
|
|
562
|
+
if (!page)
|
|
563
|
+
return result;
|
|
564
|
+
if ((this.fieldsRead.get(page) ?? -1) < started)
|
|
565
|
+
await this.readFields(page);
|
|
566
|
+
const check = this.challenges.get(page);
|
|
567
|
+
if (!check)
|
|
568
|
+
return result;
|
|
569
|
+
this.challenges.delete(page);
|
|
570
|
+
let timer;
|
|
571
|
+
const event = await Promise.race([
|
|
572
|
+
check,
|
|
573
|
+
new Promise((r) => {
|
|
574
|
+
timer = setTimeout(() => r(undefined), CHALLENGE_WAIT_MS);
|
|
575
|
+
}),
|
|
576
|
+
]).finally(() => clearTimeout(timer));
|
|
577
|
+
return event ? withTrailer(result, "Challenge", challengeNote(event), "challenge", json) : result;
|
|
578
|
+
}
|
|
579
|
+
/** Values of a page's sensitive fields join the redactor, shown as `[sensitive field]`. */
|
|
580
|
+
noteFields(page, fields) {
|
|
581
|
+
this.fieldsRead.set(page, performance.now());
|
|
582
|
+
for (const f of fields)
|
|
583
|
+
this.host.redactor.addField(f.value, SENSITIVE_MARK, f.kind === "one-time code" ? FIELD_CODE_MS : undefined);
|
|
584
|
+
}
|
|
585
|
+
async readFields(page) {
|
|
586
|
+
if (page.isClosed())
|
|
587
|
+
return;
|
|
588
|
+
this.noteFields(page, await sensitiveFields(page).catch(() => []));
|
|
589
|
+
}
|
|
590
|
+
/**
|
|
591
|
+
* Chains the redactor into upstream's own (`redactSecrets`), which every text result, written
|
|
592
|
+
* file (a snapshot or a network request saved with `filename`) and session log line passes
|
|
593
|
+
* through, and, while a network tool runs, the credential rules of `network-redact.ts`.
|
|
594
|
+
*/
|
|
595
|
+
hookUpstream(backend) {
|
|
596
|
+
const context = backendContext(backend);
|
|
597
|
+
if (!context || typeof context.redactSecrets !== "function")
|
|
598
|
+
return;
|
|
599
|
+
const own = context.redactSecrets.bind(context);
|
|
600
|
+
let network = 0;
|
|
601
|
+
context.redactSecrets = (text) => this.host.redactor.text(network ? redactNetworkText(own(text)) : own(text));
|
|
602
|
+
const call = backend.callTool.bind(backend);
|
|
603
|
+
backend.callTool = async (name, args, signal) => {
|
|
604
|
+
const counted = NETWORK_TOOLS.has(name);
|
|
605
|
+
if (counted)
|
|
606
|
+
network++;
|
|
607
|
+
try {
|
|
608
|
+
return await call(name, args, signal);
|
|
609
|
+
}
|
|
610
|
+
finally {
|
|
611
|
+
if (counted)
|
|
612
|
+
network--;
|
|
613
|
+
}
|
|
614
|
+
};
|
|
615
|
+
}
|
|
616
|
+
/**
|
|
617
|
+
* Typing into a password or one-time code field is the person's or the vault's: the call is
|
|
618
|
+
* refused, unless the text was read from the secret store (a flow replay's `keychain:` value).
|
|
619
|
+
*/
|
|
620
|
+
async refuseSensitive(name, args, page, options) {
|
|
621
|
+
if (!page || options.fromSecretStore || !TYPING_TOOLS.has(name))
|
|
622
|
+
return undefined;
|
|
623
|
+
const kind = await typesIntoSensitive(page, name, args).catch(() => undefined);
|
|
624
|
+
return kind ? sensitiveRefusal(name, kind) : undefined;
|
|
467
625
|
}
|
|
468
626
|
/** True when a call needs the browser, so a pause holds it. */
|
|
469
627
|
touchesBrowser(name) {
|
|
@@ -471,47 +629,51 @@ class SessionBackend extends EventEmitter {
|
|
|
471
629
|
return tool ? tool.browser !== false : true;
|
|
472
630
|
}
|
|
473
631
|
/**
|
|
474
|
-
* A call through the
|
|
475
|
-
*
|
|
476
|
-
* snapshot when the browser changed and the call does more than
|
|
632
|
+
* A call through the pause gate (`live.ts`): it runs at once unless the user paused the agent
|
|
633
|
+
* (the live view) or a person step holds the browser; then it waits, and on resume it runs, or
|
|
634
|
+
* returns an error with a fresh snapshot when the browser changed and the call does more than
|
|
635
|
+
* read. A person step itself is not counted as a call in flight: it waits for those.
|
|
477
636
|
*/
|
|
478
|
-
async callLive(name, args, signal, session, sample) {
|
|
479
|
-
|
|
480
|
-
|
|
481
|
-
|
|
637
|
+
async callLive(name, args, signal, session, sample, progress) {
|
|
638
|
+
if (!this.touchesBrowser(name))
|
|
639
|
+
return await this.callAny(name, args, signal, session, sample, progress);
|
|
640
|
+
const watch = this.watch;
|
|
641
|
+
const step = name === HUMAN_STEP_TOOL;
|
|
482
642
|
const json = !!args._meta?.json;
|
|
483
643
|
const key = session ?? DIRECT_SESSION;
|
|
484
|
-
const outcome = await
|
|
644
|
+
const outcome = await watch.gate.wait(watch.waitMs, signal);
|
|
485
645
|
if (outcome.kind === "timeout")
|
|
486
|
-
return errorResult(timeoutMessage(this.host.identity?.id ?? "", name, outcome.pausedMs, outcome.by,
|
|
646
|
+
return errorResult(timeoutMessage(this.host.identity?.id ?? "", name, outcome.pausedMs, outcome.by, watch.waitMs), json);
|
|
487
647
|
if (outcome.kind === "aborted")
|
|
488
648
|
return errorResult(`${name} was cancelled while the user had the browser`, json);
|
|
489
|
-
const end =
|
|
649
|
+
const end = step ? () => { } : watch.begin();
|
|
490
650
|
try {
|
|
491
651
|
if (outcome.kind === "open") {
|
|
492
|
-
const result = await this.callAny(name, args, signal, session, sample);
|
|
493
|
-
|
|
652
|
+
const result = await this.callAny(name, args, signal, session, sample, progress);
|
|
653
|
+
const paused = watch.gate.paused;
|
|
654
|
+
if (!paused || step)
|
|
494
655
|
return result;
|
|
495
|
-
return withNote(result,
|
|
656
|
+
return withNote(result, pausedWhileRunningNote(paused.step), json);
|
|
496
657
|
}
|
|
497
|
-
const beforePage =
|
|
658
|
+
const beforePage = watch.lastPage(key);
|
|
498
659
|
const afterPage = beforePage ? await this.currentPageOf(key) : undefined;
|
|
499
|
-
const after = await
|
|
500
|
-
const { changed, lines } = describeChanges(outcome.before, after, beforePage ?
|
|
660
|
+
const after = await watch.tabStates();
|
|
661
|
+
const { changed, lines } = describeChanges(outcome.before, after, beforePage ? watch.idOf(beforePage) : undefined, afterPage ? watch.idOf(afterPage) : undefined, outcome.inputs);
|
|
501
662
|
const note = {
|
|
502
663
|
pausedMs: outcome.pausedMs,
|
|
503
664
|
by: outcome.by,
|
|
665
|
+
step: outcome.step,
|
|
504
666
|
lines,
|
|
505
667
|
tool: name,
|
|
506
668
|
recordingFlow: this.recorderFor(session)?.recording !== undefined,
|
|
507
669
|
};
|
|
508
670
|
if (!changed || this.readsOnly(name)) {
|
|
509
|
-
const result = await this.callAny(name, args, signal, session, sample);
|
|
671
|
+
const result = await this.callAny(name, args, signal, session, sample, progress);
|
|
510
672
|
return withNote(result, resumeNote({ ...note, ran: true }), json);
|
|
511
673
|
}
|
|
512
674
|
// A call that was not run is an error, so a client that reads only the outcome (a script,
|
|
513
675
|
// turno browser do) does not take it for done.
|
|
514
|
-
const notRun = `### Error\n${name} was not run: the user changed the browser while you waited (below).`;
|
|
676
|
+
const notRun = `### Error\n${name} was not run: ${outcome.step ? "the person" : "the user"} changed the browser while you waited (below).`;
|
|
515
677
|
if (!this.launched?.isOpen()) {
|
|
516
678
|
const text = `${notRun}\n\n${resumeNote({ ...note, ran: false, noBrowser: true })}`;
|
|
517
679
|
return errorResultOf(json ? JSON.stringify({ isError: true, error: text }, null, 2) : text);
|
|
@@ -538,7 +700,7 @@ class SessionBackend extends EventEmitter {
|
|
|
538
700
|
const view = await running.sharing.view(key).catch(() => undefined);
|
|
539
701
|
return view ? running.sharing.tabView(view).current() : undefined;
|
|
540
702
|
}
|
|
541
|
-
async callAny(name, args, signal, session, sample) {
|
|
703
|
+
async callAny(name, args, signal, session, sample, progress) {
|
|
542
704
|
const tool = this.host.turno.get(name);
|
|
543
705
|
if (!tool)
|
|
544
706
|
return await this.callStock(name, args, signal, {}, session);
|
|
@@ -561,6 +723,7 @@ class SessionBackend extends EventEmitter {
|
|
|
561
723
|
let recording;
|
|
562
724
|
let engine;
|
|
563
725
|
let input;
|
|
726
|
+
let human;
|
|
564
727
|
if (tool.browser !== false) {
|
|
565
728
|
const running = await this.browser();
|
|
566
729
|
engine = running.launched.engine;
|
|
@@ -590,6 +753,10 @@ class SessionBackend extends EventEmitter {
|
|
|
590
753
|
}
|
|
591
754
|
stock = this.stockAccess(view, signal, session);
|
|
592
755
|
recording = running.recording;
|
|
756
|
+
if (name === HUMAN_STEP_TOOL) {
|
|
757
|
+
const access = stock;
|
|
758
|
+
human = (params) => this.requestHuman(running, access, params, json, session, signal, progress);
|
|
759
|
+
}
|
|
593
760
|
}
|
|
594
761
|
const recorder = this.recorderFor(session);
|
|
595
762
|
const secretHook = this.secretHook(recording);
|
|
@@ -607,6 +774,7 @@ class SessionBackend extends EventEmitter {
|
|
|
607
774
|
...(sample ? { sample } : {}),
|
|
608
775
|
...(engine ? { engine } : {}),
|
|
609
776
|
...(this.host.providers ? { providers: this.host.providers } : {}),
|
|
777
|
+
...(human ? { human } : {}),
|
|
610
778
|
...(this.host.identity
|
|
611
779
|
? {
|
|
612
780
|
vault: {
|
|
@@ -622,7 +790,7 @@ class SessionBackend extends EventEmitter {
|
|
|
622
790
|
};
|
|
623
791
|
const result = await tool.handle(context, parsed.params);
|
|
624
792
|
if (stock)
|
|
625
|
-
this.
|
|
793
|
+
this.watch.follow(session ?? DIRECT_SESSION, stock.currentPage());
|
|
626
794
|
if (name === "browser_login") {
|
|
627
795
|
if (!result.isError)
|
|
628
796
|
recorder?.record("browser_login", { site: parsed.params.site });
|
|
@@ -672,25 +840,31 @@ class SessionBackend extends EventEmitter {
|
|
|
672
840
|
let result;
|
|
673
841
|
try {
|
|
674
842
|
await governor.quiet();
|
|
675
|
-
|
|
843
|
+
// A parked tab loads again first, so the field a call would type into is the one it reaches.
|
|
676
844
|
await governor.prepare(name, args);
|
|
677
|
-
|
|
678
|
-
|
|
679
|
-
|
|
680
|
-
|
|
681
|
-
|
|
682
|
-
|
|
683
|
-
|
|
684
|
-
|
|
685
|
-
|
|
686
|
-
|
|
687
|
-
|
|
688
|
-
|
|
689
|
-
|
|
690
|
-
|
|
691
|
-
|
|
692
|
-
|
|
693
|
-
|
|
845
|
+
const refusedField = await this.refuseSensitive(name, args, backendContext(backend)?.currentTab()?.page, options);
|
|
846
|
+
if (refusedField)
|
|
847
|
+
result = errorResult(refusedField, json);
|
|
848
|
+
else {
|
|
849
|
+
const pending = await this.beforeStock(running, session, name, args, backendContext(backend)?.currentTab()?.page);
|
|
850
|
+
if (tuning.backgroundThrottling && !launched.userBrowser && name !== "browser_tabs")
|
|
851
|
+
await backendContext(backend)
|
|
852
|
+
?.currentTab()
|
|
853
|
+
?.page.bringToFront()
|
|
854
|
+
.catch(() => { });
|
|
855
|
+
const page = backendContext(backend)?.currentTab()?.page;
|
|
856
|
+
const key = session ?? DIRECT_SESSION;
|
|
857
|
+
await running.cursor?.begin(page, key, this.clients.get(key));
|
|
858
|
+
result =
|
|
859
|
+
(launched.engine === "lightpanda" ? undefined : await coveredBeforeCall(page, name, args, json)) ??
|
|
860
|
+
(await this.withCursor(running, name, page, () => backend.callTool(name, args, signal)));
|
|
861
|
+
void running.cursor?.end(backendContext(backend)?.currentTab()?.page, key);
|
|
862
|
+
this.watch.follow(key, backendContext(backend)?.currentTab()?.page);
|
|
863
|
+
if (result.isError)
|
|
864
|
+
result = await explainFailure(backendContext(backend)?.currentTab()?.page, name, args, result, json);
|
|
865
|
+
await this.afterStock(running, session, name, args, pending, result, backendContext(backend)?.currentTab()?.page);
|
|
866
|
+
await governor.afterCall();
|
|
867
|
+
}
|
|
694
868
|
}
|
|
695
869
|
finally {
|
|
696
870
|
running.recording?.callEnded();
|
|
@@ -738,27 +912,32 @@ class SessionBackend extends EventEmitter {
|
|
|
738
912
|
};
|
|
739
913
|
}
|
|
740
914
|
const tabs = sharing.tabView(view);
|
|
741
|
-
const pending = await this.beforeStock(running, key, name, args, tabs.current());
|
|
742
915
|
await governor.prepare(name, args, tabs);
|
|
743
|
-
|
|
744
|
-
|
|
745
|
-
|
|
746
|
-
|
|
747
|
-
|
|
748
|
-
|
|
749
|
-
|
|
750
|
-
|
|
751
|
-
|
|
752
|
-
|
|
753
|
-
|
|
754
|
-
|
|
755
|
-
|
|
756
|
-
|
|
757
|
-
|
|
758
|
-
|
|
759
|
-
|
|
760
|
-
|
|
761
|
-
|
|
916
|
+
const refusedField = await this.refuseSensitive(name, args, tabs.current(), options);
|
|
917
|
+
if (refusedField)
|
|
918
|
+
result = errorResult(refusedField, json);
|
|
919
|
+
else {
|
|
920
|
+
const pending = await this.beforeStock(running, key, name, args, tabs.current());
|
|
921
|
+
if (tuning.backgroundThrottling && name !== "browser_tabs")
|
|
922
|
+
await tabs
|
|
923
|
+
.current()
|
|
924
|
+
?.bringToFront()
|
|
925
|
+
.catch(() => { });
|
|
926
|
+
await running.cursor?.begin(tabs.current(), key, this.clients.get(key));
|
|
927
|
+
result =
|
|
928
|
+
(running.launched.engine === "lightpanda"
|
|
929
|
+
? undefined
|
|
930
|
+
: await coveredBeforeCall(tabs.current(), name, args, json)) ??
|
|
931
|
+
(await this.withCursor(running, name, tabs.current(), () => backend.callTool(name, args, signal)));
|
|
932
|
+
void running.cursor?.end(tabs.current(), key);
|
|
933
|
+
this.watch.follow(key, tabs.current());
|
|
934
|
+
if (result.isError)
|
|
935
|
+
result = await explainFailure(tabs.current(), name, args, result, json);
|
|
936
|
+
await this.afterStock(running, key, name, args, pending, result, tabs.current());
|
|
937
|
+
// The tab the call left current is the session's before the cap counts its tabs.
|
|
938
|
+
sharing.after(key, view);
|
|
939
|
+
await governor.afterCall();
|
|
940
|
+
}
|
|
762
941
|
}
|
|
763
942
|
}
|
|
764
943
|
}
|
|
@@ -794,6 +973,9 @@ class SessionBackend extends EventEmitter {
|
|
|
794
973
|
async beforeStock(running, session, name, args, page) {
|
|
795
974
|
const { recording } = running;
|
|
796
975
|
await recording?.beforeCall();
|
|
976
|
+
// While a trace records, a value the person typed since the last call is known before it is traced.
|
|
977
|
+
if (recording && page)
|
|
978
|
+
await this.readFields(page);
|
|
797
979
|
const recorder = this.recorderFor(session);
|
|
798
980
|
const pending = recorder?.active ? await recorder.capture(name, args, page).catch(() => undefined) : undefined;
|
|
799
981
|
const secretHook = this.secretHook(recording);
|
|
@@ -839,6 +1021,33 @@ class SessionBackend extends EventEmitter {
|
|
|
839
1021
|
screenshot: true,
|
|
840
1022
|
});
|
|
841
1023
|
}
|
|
1024
|
+
/**
|
|
1025
|
+
* A person step on the session's current tab (`human-step.ts`). The person must be able to see
|
|
1026
|
+
* the browser: a Chrome window (headed, or attach mode), or the live view open on a headless one.
|
|
1027
|
+
*/
|
|
1028
|
+
async requestHuman(running, stock, raw, json, session, signal, progress) {
|
|
1029
|
+
const params = parseStepParams(raw);
|
|
1030
|
+
if ("error" in params)
|
|
1031
|
+
return errorResult(params.error, json);
|
|
1032
|
+
const page = stock.currentPage();
|
|
1033
|
+
if (!page)
|
|
1034
|
+
return errorResult("no page is open; navigate to the page of the step first", json);
|
|
1035
|
+
const { launched } = running;
|
|
1036
|
+
const headless = this.host.config.browser?.launchOptions?.headless === true && !launched.userBrowser;
|
|
1037
|
+
const unseen = headless ? unseenReason(this.host.live, this.host.identity?.id) : undefined;
|
|
1038
|
+
return await runHumanStep(params, {
|
|
1039
|
+
page,
|
|
1040
|
+
watch: this.watch,
|
|
1041
|
+
guard: this.host.guard,
|
|
1042
|
+
session: session ?? DIRECT_SESSION,
|
|
1043
|
+
...(unseen ? { unseen } : {}),
|
|
1044
|
+
window: !headless && !launched.userBrowser,
|
|
1045
|
+
json,
|
|
1046
|
+
...(signal ? { signal } : {}),
|
|
1047
|
+
...(progress ? { progress } : {}),
|
|
1048
|
+
currentPage: () => stock.currentPage(),
|
|
1049
|
+
});
|
|
1050
|
+
}
|
|
842
1051
|
/**
|
|
843
1052
|
* The script tools near the vault: `browser_evaluate` and `browser_run_code_unsafe`
|
|
844
1053
|
* are refused on a page browser_login filled, until it navigates; `browser_run_code_unsafe`
|
|
@@ -863,10 +1072,10 @@ class SessionBackend extends EventEmitter {
|
|
|
863
1072
|
schemas: this.host.stockSchemas,
|
|
864
1073
|
formatError: (error) => upstream().utils.z.prettifyError(error),
|
|
865
1074
|
// A batch, a replay or an act run stops at its next step once the user pauses the agent.
|
|
866
|
-
call: (name, args, options) => this.
|
|
1075
|
+
call: (name, args, options) => this.watch.paused
|
|
867
1076
|
? Promise.resolve(errorResult(`${PAUSED_STEP}: this step and the ones after it were not run. Your next call waits until they resume.`, !!args._meta?.json))
|
|
868
1077
|
: this.callStock(name, args, signal, options, session),
|
|
869
|
-
held: () => (this.
|
|
1078
|
+
held: () => (this.watch.paused ? PAUSED_STEP : undefined),
|
|
870
1079
|
page: async () => {
|
|
871
1080
|
const page = backendContext(view.backend)?.currentTab()?.page;
|
|
872
1081
|
if (!page)
|
|
@@ -881,6 +1090,7 @@ class SessionBackend extends EventEmitter {
|
|
|
881
1090
|
// A flow the session was recording and never stopped is dropped with it.
|
|
882
1091
|
this.recorders.get(session)?.cancel();
|
|
883
1092
|
this.recorders.delete(session);
|
|
1093
|
+
this.clients.delete(session);
|
|
884
1094
|
const current = this.current;
|
|
885
1095
|
if (!current)
|
|
886
1096
|
return;
|
|
@@ -940,7 +1150,7 @@ function refuseWhileNeedsAuth(identity) {
|
|
|
940
1150
|
* log) and, when identity.json has a health check that is due, run it; a page without the
|
|
941
1151
|
* proof of a signed-in session throws NEEDS_AUTH, and the call fails with that message.
|
|
942
1152
|
*/
|
|
943
|
-
async function afterIdentityLaunch(identity, launched, onChallenge) {
|
|
1153
|
+
async function afterIdentityLaunch(identity, launched, onChallenge, onCheck) {
|
|
944
1154
|
const { id, env } = identity;
|
|
945
1155
|
watchChallenges(launched.context, (event) => {
|
|
946
1156
|
onChallenge?.(event);
|
|
@@ -950,7 +1160,7 @@ async function afterIdentityLaunch(identity, launched, onChallenge) {
|
|
|
950
1160
|
catch {
|
|
951
1161
|
/* the log is best effort; a full disk must not break a tool call */
|
|
952
1162
|
}
|
|
953
|
-
});
|
|
1163
|
+
}, onCheck ? { onCheck: (page, found) => onCheck(page, found) } : {});
|
|
954
1164
|
const check = readIdentitySettings(id, env)?.health;
|
|
955
1165
|
if (!check || !healthDue(readHealth(id, env), check))
|
|
956
1166
|
return;
|
|
@@ -1111,6 +1321,17 @@ function samplerFor(server, requestId) {
|
|
|
1111
1321
|
};
|
|
1112
1322
|
};
|
|
1113
1323
|
}
|
|
1324
|
+
/** Progress notifications for a call whose client gave a progress token. */
|
|
1325
|
+
function progressFor(token, send) {
|
|
1326
|
+
if (token === undefined)
|
|
1327
|
+
return undefined;
|
|
1328
|
+
return (elapsedMs, message) => {
|
|
1329
|
+
void send({
|
|
1330
|
+
method: "notifications/progress",
|
|
1331
|
+
params: { progressToken: token, progress: Math.round(elapsedMs / 1000), message },
|
|
1332
|
+
}).catch(() => { });
|
|
1333
|
+
};
|
|
1334
|
+
}
|
|
1114
1335
|
/**
|
|
1115
1336
|
* An MCP server for one connection of this host, on any transport: with its own session
|
|
1116
1337
|
* backend (stdio, one browser per connection), or on the backend of a shared host.
|
|
@@ -1146,7 +1367,11 @@ function createServer(host, shared) {
|
|
|
1146
1367
|
try {
|
|
1147
1368
|
const b = await session();
|
|
1148
1369
|
const args = (request.params.arguments ?? {});
|
|
1149
|
-
const
|
|
1370
|
+
const meta = request.params._meta;
|
|
1371
|
+
const sessionId = sessionFromMeta(meta);
|
|
1372
|
+
// The agent behind the call: mcpd says it; on a shared host each connection is one agent.
|
|
1373
|
+
const client = clientFromMeta(meta) ?? (shared && sessionId ? server.getClientVersion()?.name : undefined);
|
|
1374
|
+
const result = await b.callTool(request.params.name, args, extra.signal, sessionId, samplerFor(server, extra.requestId), client, progressFor(meta?.progressToken, extra.sendNotification));
|
|
1150
1375
|
return mergeTextParts(result);
|
|
1151
1376
|
}
|
|
1152
1377
|
catch (error) {
|
|
@@ -1263,4 +1488,16 @@ export async function runHost(args, options = {}) {
|
|
|
1263
1488
|
core.utils.gracefullyProcessExitDoNotHang(1);
|
|
1264
1489
|
}
|
|
1265
1490
|
}
|
|
1491
|
+
/** Why the person cannot see a headless Chrome now, or undefined when the live view shows it. */
|
|
1492
|
+
export function unseenReason(live, id) {
|
|
1493
|
+
const b = names().browser;
|
|
1494
|
+
if (live && live.viewerCount > 0)
|
|
1495
|
+
return undefined;
|
|
1496
|
+
if (live && id)
|
|
1497
|
+
return `Chrome runs without a window here, so the person sees the page only in the live view, and it is not open. Ask the person to run \`${b} live ${id}\` (it opens the view in their browser), then call ${HUMAN_STEP_TOOL} again.`;
|
|
1498
|
+
const start = id
|
|
1499
|
+
? `\`${b} set ${id} --live true\` and a new session, then \`${b} live ${id}\``
|
|
1500
|
+
: "an identity with the live view on";
|
|
1501
|
+
return `Chrome runs without a window here (--headless) and the live view is off, so the person cannot see the page. Ask the person to run the browser with a window (start turno-browser-mcp without --headless), or to turn the live view on (${start}), then call ${HUMAN_STEP_TOOL} again.`;
|
|
1502
|
+
}
|
|
1266
1503
|
//# sourceMappingURL=host.js.map
|