turno-browser 0.6.0 → 0.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +40 -0
- package/README.md +13 -5
- package/dist/act.js +4 -29
- package/dist/agent-cursor-overlay.d.ts +95 -0
- package/dist/agent-cursor-overlay.js +358 -0
- package/dist/agent-cursor.d.ts +191 -0
- package/dist/agent-cursor.js +494 -0
- package/dist/api.d.ts +8 -3
- package/dist/api.js +6 -3
- package/dist/argv.d.ts +2 -1
- package/dist/argv.js +7 -1
- package/dist/bin/mcp.js +5 -0
- package/dist/cli/commands.js +12 -3
- package/dist/cli/completion.js +3 -0
- package/dist/cli/context.d.ts +2 -0
- package/dist/cli/context.js +8 -4
- package/dist/cli/do-background.d.ts +2 -0
- package/dist/cli/do-background.js +15 -0
- package/dist/cli/do.d.ts +4 -0
- package/dist/cli/do.js +17 -1
- package/dist/cli/flows.js +3 -0
- package/dist/cli/identity.d.ts +2 -0
- package/dist/cli/identity.js +67 -11
- package/dist/cli/init.d.ts +12 -0
- package/dist/cli/init.js +219 -0
- package/dist/cli/login-summary.d.ts +6 -0
- package/dist/cli/login-summary.js +11 -0
- package/dist/cli/main.js +3 -0
- package/dist/cli/setup.d.ts +32 -0
- package/dist/cli/setup.js +32 -23
- package/dist/doctor.js +18 -2
- package/dist/engine-camoufox.js +1 -0
- package/dist/engine-lightpanda.js +1 -0
- package/dist/host.d.ts +7 -0
- package/dist/host.js +330 -68
- package/dist/human-input.d.ts +5 -0
- package/dist/human-input.js +1 -1
- package/dist/human-step-overlay.d.ts +73 -0
- package/dist/human-step-overlay.js +285 -0
- package/dist/human-step.d.ts +128 -0
- package/dist/human-step.js +521 -0
- package/dist/index.d.ts +2 -2
- package/dist/index.js +2 -2
- package/dist/live-server.d.ts +7 -3
- package/dist/live-server.js +13 -15
- package/dist/live.d.ts +20 -1
- package/dist/live.js +52 -10
- package/dist/login.d.ts +22 -2
- package/dist/login.js +95 -7
- package/dist/network-redact.d.ts +28 -0
- package/dist/network-redact.js +202 -0
- package/dist/recorder.d.ts +0 -21
- package/dist/recorder.js +1 -41
- package/dist/recordings.d.ts +5 -0
- package/dist/recordings.js +4 -2
- package/dist/redact.d.ts +7 -0
- package/dist/redact.js +32 -1
- package/dist/replay.d.ts +2 -0
- package/dist/replay.js +33 -14
- package/dist/reputation.d.ts +5 -0
- package/dist/reputation.js +12 -7
- package/dist/sensitive.d.ts +79 -0
- package/dist/sensitive.js +275 -0
- package/dist/session.d.ts +12 -0
- package/dist/session.js +24 -0
- package/dist/settings.d.ts +12 -2
- package/dist/settings.js +14 -2
- package/dist/tools.d.ts +5 -0
- package/dist/tools.js +9 -1
- package/dist/tuning.d.ts +4 -1
- package/dist/tuning.js +1 -0
- package/dist/upstream.d.ts +5 -0
- package/dist/watch.d.ts +60 -0
- package/dist/watch.js +114 -0
- package/npm-shrinkwrap.json +2 -2
- package/package.json +1 -1
package/dist/host.js
CHANGED
|
@@ -10,17 +10,22 @@ import { coveredBeforeCall, explainFailure } from "./diagnose.js";
|
|
|
10
10
|
import { createEngine, engineChoice, } from "./engine.js";
|
|
11
11
|
import { TabGovernor } from "./governor.js";
|
|
12
12
|
import { NEEDS_AUTH_HOLD_MS, healthDue, needsAuthMessage, readHealth, runHealthCheck, writeHealth, } from "./health.js";
|
|
13
|
+
import { AgentCursor, CursorInput, cursorTools, resolveCursor } from "./agent-cursor.js";
|
|
13
14
|
import { HumanInput, adaptiveSites, humanizeTools } from "./human-input.js";
|
|
14
15
|
import { identityStateDir } from "./ids.js";
|
|
15
16
|
import { idleMs } from "./idle-timer.js";
|
|
16
|
-
import {
|
|
17
|
+
import { HUMAN_STEP_TOOL, parseStepParams, runHumanStep } from "./human-step.js";
|
|
18
|
+
import { describeChanges, pausedWhileRunningNote, resolveLive, resumeNote, timeoutMessage, viewRefusal, } from "./live.js";
|
|
17
19
|
import { LiveServer } from "./live-server.js";
|
|
18
|
-
import {
|
|
20
|
+
import { names } from "./names.js";
|
|
21
|
+
import { NETWORK_TOOLS, redactNetworkText } from "./network-redact.js";
|
|
22
|
+
import { FlowRecorder } from "./recorder.js";
|
|
19
23
|
import { Recording, tracesTmpDir } from "./recordings.js";
|
|
20
24
|
import { Redactor, SCRIPT_TOOLS, VaultGuard } from "./redact.js";
|
|
21
25
|
import { recordChallenge, watchChallenges } from "./reputation.js";
|
|
22
26
|
import { installResources } from "./resources.js";
|
|
23
|
-
import {
|
|
27
|
+
import { SENSITIVE_MARK, TYPING_TOOLS, maskSnapshots, secretValues, sensitiveFields, sensitiveRefusal, typesIntoSensitive, } from "./sensitive.js";
|
|
28
|
+
import { SESSION_CAPABILITY, SESSION_ENDED_METHOD, SESSION_META_KEY, clientFromMeta, sessionFromMeta, } from "./session.js";
|
|
24
29
|
import { TabSharing, viewConfig } from "./sharing.js";
|
|
25
30
|
import { DEFAULT_TAB_LOCKS, DIRECT_SESSION } from "./tab-locks.js";
|
|
26
31
|
import { readIdentitySettings } from "./settings.js";
|
|
@@ -28,6 +33,7 @@ import { TURNO_TOOLS, checkTurnoTools, optInTools, parseArguments, } from "./too
|
|
|
28
33
|
import { resolveTuning } from "./tuning.js";
|
|
29
34
|
import { vaultHasSites } from "./vault.js";
|
|
30
35
|
import { backendContext, toMcpTool, upstream, } from "./upstream.js";
|
|
36
|
+
import { PlainWatch } from "./watch.js";
|
|
31
37
|
/**
|
|
32
38
|
* turno-browser as the MCP server host.
|
|
33
39
|
*
|
|
@@ -54,7 +60,13 @@ import { backendContext, toMcpTool, upstream, } from "./upstream.js";
|
|
|
54
60
|
* identity.json turns them on when the server starts, and hands those tools MCP sampling,
|
|
55
61
|
* routed to the calling session, when the client offers it;
|
|
56
62
|
* - with the live view on (identity.json `live`, `--live`), holds every browser call while the
|
|
57
|
-
* user has paused the agent, and tells the held call what changed (`live.ts`, `live-server.ts`)
|
|
63
|
+
* user has paused the agent, and tells the held call what changed (`live.ts`, `live-server.ts`);
|
|
64
|
+
* a person step (`browser_request_human`, `human-step.ts`) holds them the same way, with the live
|
|
65
|
+
* view or without it (`watch.ts`);
|
|
66
|
+
* - masks password and one-time code fields in every page snapshot and hides their values from
|
|
67
|
+
* every result, whoever typed them, refuses the agent's typing into them (`sensitive.ts`), drops
|
|
68
|
+
* credential values from network output (`network-redact.ts`), and says when a result ends on
|
|
69
|
+
* a challenge page (`reputation.ts`).
|
|
58
70
|
*
|
|
59
71
|
* stdio, the transport mcpd and every agent use, is served by turno with the MCP SDK.
|
|
60
72
|
* `--port` keeps upstream's HTTP server (Host check, streamable HTTP and SSE) serving the
|
|
@@ -142,6 +154,45 @@ function errorResult(message, json) {
|
|
|
142
154
|
isError: true,
|
|
143
155
|
};
|
|
144
156
|
}
|
|
157
|
+
/** How long a one-time code read from a page stays redacted, as a vault one-time code. */
|
|
158
|
+
const FIELD_CODE_MS = 10 * 60_000;
|
|
159
|
+
/** Longest a result waits for the challenge check of the page it ended on. */
|
|
160
|
+
const CHALLENGE_WAIT_MS = 2_000;
|
|
161
|
+
/** A note after the rest of a result (under `turno.<key>` in JSON). */
|
|
162
|
+
function withTrailer(result, title, note, key, json) {
|
|
163
|
+
const first = result.content[0];
|
|
164
|
+
if (!first || first.type !== "text")
|
|
165
|
+
return { ...result, content: [...result.content, { type: "text", text: `### ${title}\n${note}` }] };
|
|
166
|
+
const text = first.text;
|
|
167
|
+
if (json) {
|
|
168
|
+
try {
|
|
169
|
+
const payload = JSON.parse(text);
|
|
170
|
+
const turno = (payload.turno ?? {});
|
|
171
|
+
const out = JSON.stringify({ ...payload, turno: { ...turno, [key]: note } }, null, 2);
|
|
172
|
+
return { ...result, content: [{ ...first, text: out }, ...result.content.slice(1)] };
|
|
173
|
+
}
|
|
174
|
+
catch {
|
|
175
|
+
/* not JSON after all: add a section */
|
|
176
|
+
}
|
|
177
|
+
}
|
|
178
|
+
return { ...result, content: [{ ...first, text: `${text}\n### ${title}\n${note}` }, ...result.content.slice(1)] };
|
|
179
|
+
}
|
|
180
|
+
/** What the agent reads when the page it ended on is a challenge (`reputation.ts`). */
|
|
181
|
+
export function challengeNote(event) {
|
|
182
|
+
const what = event.kind === "captcha"
|
|
183
|
+
? "a captcha"
|
|
184
|
+
: event.kind === "verify"
|
|
185
|
+
? "a check that asks whether a person is there"
|
|
186
|
+
: "a page that blocks this browser";
|
|
187
|
+
return `This page (${event.site}) shows ${what}. A person should handle it: ask them to (with browser_request_human, if this server offers it), and go on once they are done.`;
|
|
188
|
+
}
|
|
189
|
+
/** A result's text parts with credentials replaced (`network-redact.ts`). */
|
|
190
|
+
function redactNetworkResult(result) {
|
|
191
|
+
return {
|
|
192
|
+
...result,
|
|
193
|
+
content: result.content.map((part) => part.type === "text" ? { ...part, text: redactNetworkText(part.text) } : part),
|
|
194
|
+
};
|
|
195
|
+
}
|
|
145
196
|
/**
|
|
146
197
|
* One MCP session. Implements upstream's server backend contract, so upstream's HTTP server
|
|
147
198
|
* can serve it as well as turno's stdio server.
|
|
@@ -198,10 +249,19 @@ class SessionBackend extends EventEmitter {
|
|
|
198
249
|
recordSettings;
|
|
199
250
|
/** Each client session's flow recorder: a recording belongs to the session that started it. */
|
|
200
251
|
recorders = new Map();
|
|
252
|
+
/** The agent name each client session's calls came with (mcpd, a shared host), for the cursor. */
|
|
253
|
+
clients = new Map();
|
|
254
|
+
/** When the sensitive fields of a page were last read (a snapshot reads them too). */
|
|
255
|
+
fieldsRead = new WeakMap();
|
|
256
|
+
/** The challenge check of each page's last load, until a result reports it. */
|
|
257
|
+
challenges = new WeakMap();
|
|
258
|
+
/** The pause gate and the tabs of this browser: the live server, or a plain watch without it. */
|
|
259
|
+
watch;
|
|
201
260
|
constructor(host, title) {
|
|
202
261
|
super();
|
|
203
262
|
this.host = host;
|
|
204
263
|
this.title = title;
|
|
264
|
+
this.watch = host.live ?? new PlainWatch();
|
|
205
265
|
}
|
|
206
266
|
async initialize(clientInfo) {
|
|
207
267
|
this.clientInfo = clientInfo;
|
|
@@ -288,7 +348,7 @@ class SessionBackend extends EventEmitter {
|
|
|
288
348
|
const profile = launched.engine !== "lightpanda" && launched.engine !== "attach";
|
|
289
349
|
if (identity && profile) {
|
|
290
350
|
try {
|
|
291
|
-
await afterIdentityLaunch(identity, launched, (event) => input?.noteChallenge(event.site));
|
|
351
|
+
await afterIdentityLaunch(identity, launched, (event) => input?.noteChallenge(event.site), (page, found) => this.challenges.set(page, found));
|
|
292
352
|
}
|
|
293
353
|
catch (error) {
|
|
294
354
|
await launched.close().catch(() => { });
|
|
@@ -313,8 +373,19 @@ class SessionBackend extends EventEmitter {
|
|
|
313
373
|
};
|
|
314
374
|
const sessionConfig = configForSession(config, cliOptions, tuning);
|
|
315
375
|
const idle = launched.idleTimer ? new CallIdleTimer(launched.idleTimer, () => launched.isOpen()) : undefined;
|
|
376
|
+
// The agent cursor (on when someone can see this browser): the input tools move it. It wraps
|
|
377
|
+
// the humanized tools, so it is the layer next to the page: humanize's pointer path runs
|
|
378
|
+
// through the cursor's mouse and the arrow follows it step by step.
|
|
379
|
+
const cursor = resolveCursor(tuning.cursor, {
|
|
380
|
+
headless: config.browser?.launchOptions?.headless === true && !launched.userBrowser,
|
|
381
|
+
live: this.host.live !== undefined,
|
|
382
|
+
engine: launched.engine,
|
|
383
|
+
})
|
|
384
|
+
? new AgentCursor({ title, ...(identity ? { identity: identity.id } : {}) })
|
|
385
|
+
: undefined;
|
|
386
|
+
const humanized = input ? humanizeTools(stock, input) : stock;
|
|
316
387
|
// Every backend of this browser (each session's too, with "sharing": "tabs") runs these tools.
|
|
317
|
-
const tools =
|
|
388
|
+
const tools = cursor ? cursorTools(humanized, new CursorInput(cursor, input !== undefined)) : humanized;
|
|
318
389
|
const backend = new BrowserBackend(sessionConfig, launched.context, tools, {
|
|
319
390
|
...(idle ? { idleTimer: idle } : {}),
|
|
320
391
|
dispose: async () => {
|
|
@@ -323,8 +394,20 @@ class SessionBackend extends EventEmitter {
|
|
|
323
394
|
await launched.close();
|
|
324
395
|
},
|
|
325
396
|
});
|
|
397
|
+
// Every snapshot of every page masks its password and one-time code fields.
|
|
398
|
+
const masked = new WeakSet();
|
|
399
|
+
const mask = (page) => {
|
|
400
|
+
if (masked.has(page))
|
|
401
|
+
return;
|
|
402
|
+
masked.add(page);
|
|
403
|
+
maskSnapshots(page, (fields) => this.noteFields(page, fields));
|
|
404
|
+
};
|
|
405
|
+
for (const page of launched.context.pages())
|
|
406
|
+
mask(page);
|
|
407
|
+
launched.context.on("page", mask);
|
|
326
408
|
try {
|
|
327
409
|
await backend.initialize(clientInfo);
|
|
410
|
+
this.hookUpstream(backend);
|
|
328
411
|
// After upstream's own routes (--allowed-origins, --blocked-origins), so this one runs
|
|
329
412
|
// first and hands every request it does not block or serve on to them.
|
|
330
413
|
if (identity && profile)
|
|
@@ -341,6 +424,9 @@ class SessionBackend extends EventEmitter {
|
|
|
341
424
|
title,
|
|
342
425
|
...(noTraces ? { noTraces } : {}),
|
|
343
426
|
...(idleLimit ? { idleMs: idleLimit } : {}),
|
|
427
|
+
beforeSave: async () => {
|
|
428
|
+
await Promise.all(launched.context.pages().map((page) => this.readFields(page)));
|
|
429
|
+
},
|
|
344
430
|
});
|
|
345
431
|
}
|
|
346
432
|
catch (error) {
|
|
@@ -357,6 +443,7 @@ class SessionBackend extends EventEmitter {
|
|
|
357
443
|
const config = viewConfig(forViews);
|
|
358
444
|
const view = new BrowserBackend(config, launched.context, tools, {});
|
|
359
445
|
await view.initialize(clientInfo);
|
|
446
|
+
this.hookUpstream(view);
|
|
360
447
|
return { backend: view, config };
|
|
361
448
|
},
|
|
362
449
|
});
|
|
@@ -391,10 +478,12 @@ class SessionBackend extends EventEmitter {
|
|
|
391
478
|
throw error;
|
|
392
479
|
}
|
|
393
480
|
}
|
|
394
|
-
|
|
481
|
+
// The watch (the live server, or a plain one without the live view) holds this browser
|
|
482
|
+
// during a pause: the user's own, or a person step's.
|
|
483
|
+
{
|
|
395
484
|
const tabs = governor;
|
|
396
485
|
let endIdle;
|
|
397
|
-
unbindLive = this.
|
|
486
|
+
unbindLive = this.watch.bind({
|
|
398
487
|
context: launched.context,
|
|
399
488
|
engine: launched.engine,
|
|
400
489
|
userBrowser: launched.userBrowser === true,
|
|
@@ -405,12 +494,14 @@ class SessionBackend extends EventEmitter {
|
|
|
405
494
|
onPause: async () => {
|
|
406
495
|
tabs.beginCall();
|
|
407
496
|
endIdle = idle?.begin();
|
|
497
|
+
await cursor?.setPaused(launched.context.pages(), true);
|
|
408
498
|
await recording?.pauseForUser();
|
|
409
499
|
},
|
|
410
500
|
onResume: () => {
|
|
411
501
|
endIdle?.();
|
|
412
502
|
endIdle = undefined;
|
|
413
503
|
tabs.endCall();
|
|
504
|
+
void cursor?.setPaused(launched.context.pages(), false);
|
|
414
505
|
},
|
|
415
506
|
});
|
|
416
507
|
}
|
|
@@ -434,6 +525,7 @@ class SessionBackend extends EventEmitter {
|
|
|
434
525
|
...(sharing ? { sharing } : {}),
|
|
435
526
|
...(recording ? { recording } : {}),
|
|
436
527
|
...(input ? { input } : {}),
|
|
528
|
+
...(cursor ? { cursor } : {}),
|
|
437
529
|
};
|
|
438
530
|
})();
|
|
439
531
|
promise.catch(() => {
|
|
@@ -447,9 +539,89 @@ class SessionBackend extends EventEmitter {
|
|
|
447
539
|
* `session` is the client session mcpd put in the call's `_meta`. Only turno tools
|
|
448
540
|
* see it; stock tools run exactly as before. Upstream's HTTP server (`--port`) never passes it.
|
|
449
541
|
*/
|
|
450
|
-
async callTool(name, args, signal, session, sample) {
|
|
542
|
+
async callTool(name, args, signal, session, sample, client, progress) {
|
|
543
|
+
if (client && session)
|
|
544
|
+
this.clients.set(session, client);
|
|
545
|
+
const started = performance.now();
|
|
546
|
+
let result = await this.callLive(name, args, signal, session, sample, progress);
|
|
547
|
+
if (this.touchesBrowser(name))
|
|
548
|
+
result = await this.afterCall(name, args, result, session, started);
|
|
451
549
|
// Every result and error of this server leaves through the redactor.
|
|
452
|
-
return this.host.redactor.result(
|
|
550
|
+
return this.host.redactor.result(result);
|
|
551
|
+
}
|
|
552
|
+
/**
|
|
553
|
+
* After a browser call, before the redactor: the sensitive fields of the tab it ended on are
|
|
554
|
+
* read unless a snapshot of the call read them (the values join the redactor, whoever typed
|
|
555
|
+
* them), credentials leave a network tool's output, and a challenge page is pointed out.
|
|
556
|
+
*/
|
|
557
|
+
async afterCall(name, args, result, session, started) {
|
|
558
|
+
const json = !!args._meta?.json;
|
|
559
|
+
const page = await this.currentPageOf(session ?? DIRECT_SESSION);
|
|
560
|
+
if (NETWORK_TOOLS.has(name))
|
|
561
|
+
result = redactNetworkResult(result);
|
|
562
|
+
if (!page)
|
|
563
|
+
return result;
|
|
564
|
+
if ((this.fieldsRead.get(page) ?? -1) < started)
|
|
565
|
+
await this.readFields(page);
|
|
566
|
+
const check = this.challenges.get(page);
|
|
567
|
+
if (!check)
|
|
568
|
+
return result;
|
|
569
|
+
this.challenges.delete(page);
|
|
570
|
+
let timer;
|
|
571
|
+
const event = await Promise.race([
|
|
572
|
+
check,
|
|
573
|
+
new Promise((r) => {
|
|
574
|
+
timer = setTimeout(() => r(undefined), CHALLENGE_WAIT_MS);
|
|
575
|
+
}),
|
|
576
|
+
]).finally(() => clearTimeout(timer));
|
|
577
|
+
return event ? withTrailer(result, "Challenge", challengeNote(event), "challenge", json) : result;
|
|
578
|
+
}
|
|
579
|
+
/** Values of a page's sensitive fields join the redactor, shown as `[sensitive field]`. */
|
|
580
|
+
noteFields(page, fields) {
|
|
581
|
+
this.fieldsRead.set(page, performance.now());
|
|
582
|
+
for (const f of fields)
|
|
583
|
+
this.host.redactor.addField(f.value, SENSITIVE_MARK, f.kind === "one-time code" ? FIELD_CODE_MS : undefined);
|
|
584
|
+
}
|
|
585
|
+
async readFields(page) {
|
|
586
|
+
if (page.isClosed())
|
|
587
|
+
return;
|
|
588
|
+
this.noteFields(page, await sensitiveFields(page).catch(() => []));
|
|
589
|
+
}
|
|
590
|
+
/**
|
|
591
|
+
* Chains the redactor into upstream's own (`redactSecrets`), which every text result, written
|
|
592
|
+
* file (a snapshot or a network request saved with `filename`) and session log line passes
|
|
593
|
+
* through, and, while a network tool runs, the credential rules of `network-redact.ts`.
|
|
594
|
+
*/
|
|
595
|
+
hookUpstream(backend) {
|
|
596
|
+
const context = backendContext(backend);
|
|
597
|
+
if (!context || typeof context.redactSecrets !== "function")
|
|
598
|
+
return;
|
|
599
|
+
const own = context.redactSecrets.bind(context);
|
|
600
|
+
let network = 0;
|
|
601
|
+
context.redactSecrets = (text) => this.host.redactor.text(network ? redactNetworkText(own(text)) : own(text));
|
|
602
|
+
const call = backend.callTool.bind(backend);
|
|
603
|
+
backend.callTool = async (name, args, signal) => {
|
|
604
|
+
const counted = NETWORK_TOOLS.has(name);
|
|
605
|
+
if (counted)
|
|
606
|
+
network++;
|
|
607
|
+
try {
|
|
608
|
+
return await call(name, args, signal);
|
|
609
|
+
}
|
|
610
|
+
finally {
|
|
611
|
+
if (counted)
|
|
612
|
+
network--;
|
|
613
|
+
}
|
|
614
|
+
};
|
|
615
|
+
}
|
|
616
|
+
/**
|
|
617
|
+
* Typing into a password or one-time code field is the person's or the vault's: the call is
|
|
618
|
+
* refused, unless the text was read from the secret store (a flow replay's `keychain:` value).
|
|
619
|
+
*/
|
|
620
|
+
async refuseSensitive(name, args, page, options) {
|
|
621
|
+
if (!page || options.fromSecretStore || !TYPING_TOOLS.has(name))
|
|
622
|
+
return undefined;
|
|
623
|
+
const kind = await typesIntoSensitive(page, name, args).catch(() => undefined);
|
|
624
|
+
return kind ? sensitiveRefusal(name, kind) : undefined;
|
|
453
625
|
}
|
|
454
626
|
/** True when a call needs the browser, so a pause holds it. */
|
|
455
627
|
touchesBrowser(name) {
|
|
@@ -457,47 +629,51 @@ class SessionBackend extends EventEmitter {
|
|
|
457
629
|
return tool ? tool.browser !== false : true;
|
|
458
630
|
}
|
|
459
631
|
/**
|
|
460
|
-
* A call through the
|
|
461
|
-
*
|
|
462
|
-
* snapshot when the browser changed and the call does more than
|
|
632
|
+
* A call through the pause gate (`live.ts`): it runs at once unless the user paused the agent
|
|
633
|
+
* (the live view) or a person step holds the browser; then it waits, and on resume it runs, or
|
|
634
|
+
* returns an error with a fresh snapshot when the browser changed and the call does more than
|
|
635
|
+
* read. A person step itself is not counted as a call in flight: it waits for those.
|
|
463
636
|
*/
|
|
464
|
-
async callLive(name, args, signal, session, sample) {
|
|
465
|
-
|
|
466
|
-
|
|
467
|
-
|
|
637
|
+
async callLive(name, args, signal, session, sample, progress) {
|
|
638
|
+
if (!this.touchesBrowser(name))
|
|
639
|
+
return await this.callAny(name, args, signal, session, sample, progress);
|
|
640
|
+
const watch = this.watch;
|
|
641
|
+
const step = name === HUMAN_STEP_TOOL;
|
|
468
642
|
const json = !!args._meta?.json;
|
|
469
643
|
const key = session ?? DIRECT_SESSION;
|
|
470
|
-
const outcome = await
|
|
644
|
+
const outcome = await watch.gate.wait(watch.waitMs, signal);
|
|
471
645
|
if (outcome.kind === "timeout")
|
|
472
|
-
return errorResult(timeoutMessage(this.host.identity?.id ?? "", name, outcome.pausedMs, outcome.by,
|
|
646
|
+
return errorResult(timeoutMessage(this.host.identity?.id ?? "", name, outcome.pausedMs, outcome.by, watch.waitMs), json);
|
|
473
647
|
if (outcome.kind === "aborted")
|
|
474
648
|
return errorResult(`${name} was cancelled while the user had the browser`, json);
|
|
475
|
-
const end =
|
|
649
|
+
const end = step ? () => { } : watch.begin();
|
|
476
650
|
try {
|
|
477
651
|
if (outcome.kind === "open") {
|
|
478
|
-
const result = await this.callAny(name, args, signal, session, sample);
|
|
479
|
-
|
|
652
|
+
const result = await this.callAny(name, args, signal, session, sample, progress);
|
|
653
|
+
const paused = watch.gate.paused;
|
|
654
|
+
if (!paused || step)
|
|
480
655
|
return result;
|
|
481
|
-
return withNote(result,
|
|
656
|
+
return withNote(result, pausedWhileRunningNote(paused.step), json);
|
|
482
657
|
}
|
|
483
|
-
const beforePage =
|
|
658
|
+
const beforePage = watch.lastPage(key);
|
|
484
659
|
const afterPage = beforePage ? await this.currentPageOf(key) : undefined;
|
|
485
|
-
const after = await
|
|
486
|
-
const { changed, lines } = describeChanges(outcome.before, after, beforePage ?
|
|
660
|
+
const after = await watch.tabStates();
|
|
661
|
+
const { changed, lines } = describeChanges(outcome.before, after, beforePage ? watch.idOf(beforePage) : undefined, afterPage ? watch.idOf(afterPage) : undefined, outcome.inputs);
|
|
487
662
|
const note = {
|
|
488
663
|
pausedMs: outcome.pausedMs,
|
|
489
664
|
by: outcome.by,
|
|
665
|
+
step: outcome.step,
|
|
490
666
|
lines,
|
|
491
667
|
tool: name,
|
|
492
668
|
recordingFlow: this.recorderFor(session)?.recording !== undefined,
|
|
493
669
|
};
|
|
494
670
|
if (!changed || this.readsOnly(name)) {
|
|
495
|
-
const result = await this.callAny(name, args, signal, session, sample);
|
|
671
|
+
const result = await this.callAny(name, args, signal, session, sample, progress);
|
|
496
672
|
return withNote(result, resumeNote({ ...note, ran: true }), json);
|
|
497
673
|
}
|
|
498
674
|
// A call that was not run is an error, so a client that reads only the outcome (a script,
|
|
499
675
|
// turno browser do) does not take it for done.
|
|
500
|
-
const notRun = `### Error\n${name} was not run: the user changed the browser while you waited (below).`;
|
|
676
|
+
const notRun = `### Error\n${name} was not run: ${outcome.step ? "the person" : "the user"} changed the browser while you waited (below).`;
|
|
501
677
|
if (!this.launched?.isOpen()) {
|
|
502
678
|
const text = `${notRun}\n\n${resumeNote({ ...note, ran: false, noBrowser: true })}`;
|
|
503
679
|
return errorResultOf(json ? JSON.stringify({ isError: true, error: text }, null, 2) : text);
|
|
@@ -524,7 +700,7 @@ class SessionBackend extends EventEmitter {
|
|
|
524
700
|
const view = await running.sharing.view(key).catch(() => undefined);
|
|
525
701
|
return view ? running.sharing.tabView(view).current() : undefined;
|
|
526
702
|
}
|
|
527
|
-
async callAny(name, args, signal, session, sample) {
|
|
703
|
+
async callAny(name, args, signal, session, sample, progress) {
|
|
528
704
|
const tool = this.host.turno.get(name);
|
|
529
705
|
if (!tool)
|
|
530
706
|
return await this.callStock(name, args, signal, {}, session);
|
|
@@ -547,6 +723,7 @@ class SessionBackend extends EventEmitter {
|
|
|
547
723
|
let recording;
|
|
548
724
|
let engine;
|
|
549
725
|
let input;
|
|
726
|
+
let human;
|
|
550
727
|
if (tool.browser !== false) {
|
|
551
728
|
const running = await this.browser();
|
|
552
729
|
engine = running.launched.engine;
|
|
@@ -576,6 +753,10 @@ class SessionBackend extends EventEmitter {
|
|
|
576
753
|
}
|
|
577
754
|
stock = this.stockAccess(view, signal, session);
|
|
578
755
|
recording = running.recording;
|
|
756
|
+
if (name === HUMAN_STEP_TOOL) {
|
|
757
|
+
const access = stock;
|
|
758
|
+
human = (params) => this.requestHuman(running, access, params, json, session, signal, progress);
|
|
759
|
+
}
|
|
579
760
|
}
|
|
580
761
|
const recorder = this.recorderFor(session);
|
|
581
762
|
const secretHook = this.secretHook(recording);
|
|
@@ -593,6 +774,7 @@ class SessionBackend extends EventEmitter {
|
|
|
593
774
|
...(sample ? { sample } : {}),
|
|
594
775
|
...(engine ? { engine } : {}),
|
|
595
776
|
...(this.host.providers ? { providers: this.host.providers } : {}),
|
|
777
|
+
...(human ? { human } : {}),
|
|
596
778
|
...(this.host.identity
|
|
597
779
|
? {
|
|
598
780
|
vault: {
|
|
@@ -608,7 +790,7 @@ class SessionBackend extends EventEmitter {
|
|
|
608
790
|
};
|
|
609
791
|
const result = await tool.handle(context, parsed.params);
|
|
610
792
|
if (stock)
|
|
611
|
-
this.
|
|
793
|
+
this.watch.follow(session ?? DIRECT_SESSION, stock.currentPage());
|
|
612
794
|
if (name === "browser_login") {
|
|
613
795
|
if (!result.isError)
|
|
614
796
|
recorder?.record("browser_login", { site: parsed.params.site });
|
|
@@ -658,22 +840,31 @@ class SessionBackend extends EventEmitter {
|
|
|
658
840
|
let result;
|
|
659
841
|
try {
|
|
660
842
|
await governor.quiet();
|
|
661
|
-
|
|
843
|
+
// A parked tab loads again first, so the field a call would type into is the one it reaches.
|
|
662
844
|
await governor.prepare(name, args);
|
|
663
|
-
|
|
664
|
-
|
|
665
|
-
|
|
666
|
-
|
|
667
|
-
|
|
668
|
-
|
|
669
|
-
|
|
670
|
-
|
|
671
|
-
|
|
672
|
-
|
|
673
|
-
|
|
674
|
-
|
|
675
|
-
|
|
676
|
-
|
|
845
|
+
const refusedField = await this.refuseSensitive(name, args, backendContext(backend)?.currentTab()?.page, options);
|
|
846
|
+
if (refusedField)
|
|
847
|
+
result = errorResult(refusedField, json);
|
|
848
|
+
else {
|
|
849
|
+
const pending = await this.beforeStock(running, session, name, args, backendContext(backend)?.currentTab()?.page);
|
|
850
|
+
if (tuning.backgroundThrottling && !launched.userBrowser && name !== "browser_tabs")
|
|
851
|
+
await backendContext(backend)
|
|
852
|
+
?.currentTab()
|
|
853
|
+
?.page.bringToFront()
|
|
854
|
+
.catch(() => { });
|
|
855
|
+
const page = backendContext(backend)?.currentTab()?.page;
|
|
856
|
+
const key = session ?? DIRECT_SESSION;
|
|
857
|
+
await running.cursor?.begin(page, key, this.clients.get(key));
|
|
858
|
+
result =
|
|
859
|
+
(launched.engine === "lightpanda" ? undefined : await coveredBeforeCall(page, name, args, json)) ??
|
|
860
|
+
(await this.withCursor(running, name, page, () => backend.callTool(name, args, signal)));
|
|
861
|
+
void running.cursor?.end(backendContext(backend)?.currentTab()?.page, key);
|
|
862
|
+
this.watch.follow(key, backendContext(backend)?.currentTab()?.page);
|
|
863
|
+
if (result.isError)
|
|
864
|
+
result = await explainFailure(backendContext(backend)?.currentTab()?.page, name, args, result, json);
|
|
865
|
+
await this.afterStock(running, session, name, args, pending, result, backendContext(backend)?.currentTab()?.page);
|
|
866
|
+
await governor.afterCall();
|
|
867
|
+
}
|
|
677
868
|
}
|
|
678
869
|
finally {
|
|
679
870
|
running.recording?.callEnded();
|
|
@@ -721,25 +912,32 @@ class SessionBackend extends EventEmitter {
|
|
|
721
912
|
};
|
|
722
913
|
}
|
|
723
914
|
const tabs = sharing.tabView(view);
|
|
724
|
-
const pending = await this.beforeStock(running, key, name, args, tabs.current());
|
|
725
915
|
await governor.prepare(name, args, tabs);
|
|
726
|
-
|
|
727
|
-
|
|
728
|
-
|
|
729
|
-
|
|
730
|
-
|
|
731
|
-
|
|
732
|
-
|
|
733
|
-
|
|
734
|
-
|
|
735
|
-
|
|
736
|
-
|
|
737
|
-
|
|
738
|
-
|
|
739
|
-
|
|
740
|
-
|
|
741
|
-
|
|
742
|
-
|
|
916
|
+
const refusedField = await this.refuseSensitive(name, args, tabs.current(), options);
|
|
917
|
+
if (refusedField)
|
|
918
|
+
result = errorResult(refusedField, json);
|
|
919
|
+
else {
|
|
920
|
+
const pending = await this.beforeStock(running, key, name, args, tabs.current());
|
|
921
|
+
if (tuning.backgroundThrottling && name !== "browser_tabs")
|
|
922
|
+
await tabs
|
|
923
|
+
.current()
|
|
924
|
+
?.bringToFront()
|
|
925
|
+
.catch(() => { });
|
|
926
|
+
await running.cursor?.begin(tabs.current(), key, this.clients.get(key));
|
|
927
|
+
result =
|
|
928
|
+
(running.launched.engine === "lightpanda"
|
|
929
|
+
? undefined
|
|
930
|
+
: await coveredBeforeCall(tabs.current(), name, args, json)) ??
|
|
931
|
+
(await this.withCursor(running, name, tabs.current(), () => backend.callTool(name, args, signal)));
|
|
932
|
+
void running.cursor?.end(tabs.current(), key);
|
|
933
|
+
this.watch.follow(key, tabs.current());
|
|
934
|
+
if (result.isError)
|
|
935
|
+
result = await explainFailure(tabs.current(), name, args, result, json);
|
|
936
|
+
await this.afterStock(running, key, name, args, pending, result, tabs.current());
|
|
937
|
+
// The tab the call left current is the session's before the cap counts its tabs.
|
|
938
|
+
sharing.after(key, view);
|
|
939
|
+
await governor.afterCall();
|
|
940
|
+
}
|
|
743
941
|
}
|
|
744
942
|
}
|
|
745
943
|
}
|
|
@@ -762,6 +960,12 @@ class SessionBackend extends EventEmitter {
|
|
|
762
960
|
const text = sharing.annotate(first.text, key, view);
|
|
763
961
|
return { ...annotated, content: [{ ...first, text }, ...annotated.content.slice(1)] };
|
|
764
962
|
}
|
|
963
|
+
/** A stock call, with the agent cursor hidden while it takes the model's screenshot. */
|
|
964
|
+
async withCursor(running, name, page, call) {
|
|
965
|
+
if (!running.cursor || name !== "browser_take_screenshot")
|
|
966
|
+
return await call();
|
|
967
|
+
return await running.cursor.hiddenDuring(page, call);
|
|
968
|
+
}
|
|
765
969
|
/**
|
|
766
970
|
* Before a stock call: the flow step it may become (`recorder.ts`), and a trace stopped and
|
|
767
971
|
* saved before a value goes into a secret field, as before a vault fill (`recordings.ts`).
|
|
@@ -769,6 +973,9 @@ class SessionBackend extends EventEmitter {
|
|
|
769
973
|
async beforeStock(running, session, name, args, page) {
|
|
770
974
|
const { recording } = running;
|
|
771
975
|
await recording?.beforeCall();
|
|
976
|
+
// While a trace records, a value the person typed since the last call is known before it is traced.
|
|
977
|
+
if (recording && page)
|
|
978
|
+
await this.readFields(page);
|
|
772
979
|
const recorder = this.recorderFor(session);
|
|
773
980
|
const pending = recorder?.active ? await recorder.capture(name, args, page).catch(() => undefined) : undefined;
|
|
774
981
|
const secretHook = this.secretHook(recording);
|
|
@@ -814,6 +1021,33 @@ class SessionBackend extends EventEmitter {
|
|
|
814
1021
|
screenshot: true,
|
|
815
1022
|
});
|
|
816
1023
|
}
|
|
1024
|
+
/**
|
|
1025
|
+
* A person step on the session's current tab (`human-step.ts`). The person must be able to see
|
|
1026
|
+
* the browser: a Chrome window (headed, or attach mode), or the live view open on a headless one.
|
|
1027
|
+
*/
|
|
1028
|
+
async requestHuman(running, stock, raw, json, session, signal, progress) {
|
|
1029
|
+
const params = parseStepParams(raw);
|
|
1030
|
+
if ("error" in params)
|
|
1031
|
+
return errorResult(params.error, json);
|
|
1032
|
+
const page = stock.currentPage();
|
|
1033
|
+
if (!page)
|
|
1034
|
+
return errorResult("no page is open; navigate to the page of the step first", json);
|
|
1035
|
+
const { launched } = running;
|
|
1036
|
+
const headless = this.host.config.browser?.launchOptions?.headless === true && !launched.userBrowser;
|
|
1037
|
+
const unseen = headless ? unseenReason(this.host.live, this.host.identity?.id) : undefined;
|
|
1038
|
+
return await runHumanStep(params, {
|
|
1039
|
+
page,
|
|
1040
|
+
watch: this.watch,
|
|
1041
|
+
guard: this.host.guard,
|
|
1042
|
+
session: session ?? DIRECT_SESSION,
|
|
1043
|
+
...(unseen ? { unseen } : {}),
|
|
1044
|
+
window: !headless && !launched.userBrowser,
|
|
1045
|
+
json,
|
|
1046
|
+
...(signal ? { signal } : {}),
|
|
1047
|
+
...(progress ? { progress } : {}),
|
|
1048
|
+
currentPage: () => stock.currentPage(),
|
|
1049
|
+
});
|
|
1050
|
+
}
|
|
817
1051
|
/**
|
|
818
1052
|
* The script tools near the vault: `browser_evaluate` and `browser_run_code_unsafe`
|
|
819
1053
|
* are refused on a page browser_login filled, until it navigates; `browser_run_code_unsafe`
|
|
@@ -838,10 +1072,10 @@ class SessionBackend extends EventEmitter {
|
|
|
838
1072
|
schemas: this.host.stockSchemas,
|
|
839
1073
|
formatError: (error) => upstream().utils.z.prettifyError(error),
|
|
840
1074
|
// A batch, a replay or an act run stops at its next step once the user pauses the agent.
|
|
841
|
-
call: (name, args, options) => this.
|
|
1075
|
+
call: (name, args, options) => this.watch.paused
|
|
842
1076
|
? Promise.resolve(errorResult(`${PAUSED_STEP}: this step and the ones after it were not run. Your next call waits until they resume.`, !!args._meta?.json))
|
|
843
1077
|
: this.callStock(name, args, signal, options, session),
|
|
844
|
-
held: () => (this.
|
|
1078
|
+
held: () => (this.watch.paused ? PAUSED_STEP : undefined),
|
|
845
1079
|
page: async () => {
|
|
846
1080
|
const page = backendContext(view.backend)?.currentTab()?.page;
|
|
847
1081
|
if (!page)
|
|
@@ -856,6 +1090,7 @@ class SessionBackend extends EventEmitter {
|
|
|
856
1090
|
// A flow the session was recording and never stopped is dropped with it.
|
|
857
1091
|
this.recorders.get(session)?.cancel();
|
|
858
1092
|
this.recorders.delete(session);
|
|
1093
|
+
this.clients.delete(session);
|
|
859
1094
|
const current = this.current;
|
|
860
1095
|
if (!current)
|
|
861
1096
|
return;
|
|
@@ -915,7 +1150,7 @@ function refuseWhileNeedsAuth(identity) {
|
|
|
915
1150
|
* log) and, when identity.json has a health check that is due, run it; a page without the
|
|
916
1151
|
* proof of a signed-in session throws NEEDS_AUTH, and the call fails with that message.
|
|
917
1152
|
*/
|
|
918
|
-
async function afterIdentityLaunch(identity, launched, onChallenge) {
|
|
1153
|
+
async function afterIdentityLaunch(identity, launched, onChallenge, onCheck) {
|
|
919
1154
|
const { id, env } = identity;
|
|
920
1155
|
watchChallenges(launched.context, (event) => {
|
|
921
1156
|
onChallenge?.(event);
|
|
@@ -925,7 +1160,7 @@ async function afterIdentityLaunch(identity, launched, onChallenge) {
|
|
|
925
1160
|
catch {
|
|
926
1161
|
/* the log is best effort; a full disk must not break a tool call */
|
|
927
1162
|
}
|
|
928
|
-
});
|
|
1163
|
+
}, onCheck ? { onCheck: (page, found) => onCheck(page, found) } : {});
|
|
929
1164
|
const check = readIdentitySettings(id, env)?.health;
|
|
930
1165
|
if (!check || !healthDue(readHealth(id, env), check))
|
|
931
1166
|
return;
|
|
@@ -1086,6 +1321,17 @@ function samplerFor(server, requestId) {
|
|
|
1086
1321
|
};
|
|
1087
1322
|
};
|
|
1088
1323
|
}
|
|
1324
|
+
/** Progress notifications for a call whose client gave a progress token. */
|
|
1325
|
+
function progressFor(token, send) {
|
|
1326
|
+
if (token === undefined)
|
|
1327
|
+
return undefined;
|
|
1328
|
+
return (elapsedMs, message) => {
|
|
1329
|
+
void send({
|
|
1330
|
+
method: "notifications/progress",
|
|
1331
|
+
params: { progressToken: token, progress: Math.round(elapsedMs / 1000), message },
|
|
1332
|
+
}).catch(() => { });
|
|
1333
|
+
};
|
|
1334
|
+
}
|
|
1089
1335
|
/**
|
|
1090
1336
|
* An MCP server for one connection of this host, on any transport: with its own session
|
|
1091
1337
|
* backend (stdio, one browser per connection), or on the backend of a shared host.
|
|
@@ -1121,7 +1367,11 @@ function createServer(host, shared) {
|
|
|
1121
1367
|
try {
|
|
1122
1368
|
const b = await session();
|
|
1123
1369
|
const args = (request.params.arguments ?? {});
|
|
1124
|
-
const
|
|
1370
|
+
const meta = request.params._meta;
|
|
1371
|
+
const sessionId = sessionFromMeta(meta);
|
|
1372
|
+
// The agent behind the call: mcpd says it; on a shared host each connection is one agent.
|
|
1373
|
+
const client = clientFromMeta(meta) ?? (shared && sessionId ? server.getClientVersion()?.name : undefined);
|
|
1374
|
+
const result = await b.callTool(request.params.name, args, extra.signal, sessionId, samplerFor(server, extra.requestId), client, progressFor(meta?.progressToken, extra.sendNotification));
|
|
1125
1375
|
return mergeTextParts(result);
|
|
1126
1376
|
}
|
|
1127
1377
|
catch (error) {
|
|
@@ -1238,4 +1488,16 @@ export async function runHost(args, options = {}) {
|
|
|
1238
1488
|
core.utils.gracefullyProcessExitDoNotHang(1);
|
|
1239
1489
|
}
|
|
1240
1490
|
}
|
|
1491
|
+
/** Why the person cannot see a headless Chrome now, or undefined when the live view shows it. */
|
|
1492
|
+
export function unseenReason(live, id) {
|
|
1493
|
+
const b = names().browser;
|
|
1494
|
+
if (live && live.viewerCount > 0)
|
|
1495
|
+
return undefined;
|
|
1496
|
+
if (live && id)
|
|
1497
|
+
return `Chrome runs without a window here, so the person sees the page only in the live view, and it is not open. Ask the person to run \`${b} live ${id}\` (it opens the view in their browser), then call ${HUMAN_STEP_TOOL} again.`;
|
|
1498
|
+
const start = id
|
|
1499
|
+
? `\`${b} set ${id} --live true\` and a new session, then \`${b} live ${id}\``
|
|
1500
|
+
: "an identity with the live view on";
|
|
1501
|
+
return `Chrome runs without a window here (--headless) and the live view is off, so the person cannot see the page. Ask the person to run the browser with a window (start turno-browser-mcp without --headless), or to turn the live view on (${start}), then call ${HUMAN_STEP_TOOL} again.`;
|
|
1502
|
+
}
|
|
1241
1503
|
//# sourceMappingURL=host.js.map
|