turno-browser 0.7.0 → 0.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +13 -0
- package/README.md +5 -1
- package/dist/act.js +4 -29
- package/dist/agent-cursor-overlay.d.ts +10 -0
- package/dist/agent-cursor-overlay.js +50 -5
- package/dist/agent-cursor.d.ts +39 -7
- package/dist/agent-cursor.js +106 -20
- package/dist/cli/flows.js +3 -0
- package/dist/cli/identity.js +3 -4
- package/dist/cli/init.js +4 -2
- package/dist/engine-camoufox.js +1 -0
- package/dist/engine-lightpanda.js +1 -0
- package/dist/host.d.ts +7 -0
- package/dist/host.js +313 -76
- package/dist/human-step-overlay.d.ts +73 -0
- package/dist/human-step-overlay.js +285 -0
- package/dist/human-step.d.ts +128 -0
- package/dist/human-step.js +521 -0
- package/dist/index.d.ts +1 -1
- package/dist/index.js +1 -1
- package/dist/live-server.d.ts +7 -3
- package/dist/live-server.js +13 -15
- package/dist/live.d.ts +20 -1
- package/dist/live.js +52 -10
- package/dist/network-redact.d.ts +28 -0
- package/dist/network-redact.js +202 -0
- package/dist/recorder.d.ts +0 -21
- package/dist/recorder.js +1 -41
- package/dist/recordings.d.ts +5 -0
- package/dist/recordings.js +4 -2
- package/dist/redact.d.ts +7 -0
- package/dist/redact.js +32 -1
- package/dist/replay.d.ts +2 -0
- package/dist/replay.js +33 -14
- package/dist/reputation.d.ts +5 -0
- package/dist/reputation.js +12 -7
- package/dist/sensitive.d.ts +79 -0
- package/dist/sensitive.js +275 -0
- package/dist/session.d.ts +12 -0
- package/dist/session.js +24 -0
- package/dist/tools.d.ts +5 -0
- package/dist/tools.js +9 -1
- package/dist/upstream.d.ts +5 -0
- package/dist/watch.d.ts +60 -0
- package/dist/watch.js +114 -0
- package/npm-shrinkwrap.json +2 -2
- package/package.json +1 -1
package/CHANGELOG.md
CHANGED
|
@@ -1,5 +1,18 @@
|
|
|
1
1
|
# turno-browser
|
|
2
2
|
|
|
3
|
+
## 0.8.0
|
|
4
|
+
|
|
5
|
+
### Minor Changes
|
|
6
|
+
|
|
7
|
+
- b94a272: New tool `browser_request_human`: the agent hands the browser to the person for a step only they may do (a password, a captcha, an emailed code, a second factor, accepting terms). The tab comes to the front with a banner that shows the instruction and a Done button, and a ring around the target; every call of every session waits until the step ends on a URL, a selector, Done or the timeout (5 minutes by default, 10 at most). The result says what changed and how many clicks and fields the person made, never what they typed. It works with a Chrome window, or with the live view open on a headless Chrome.
|
|
8
|
+
|
|
9
|
+
### Patch Changes
|
|
10
|
+
|
|
11
|
+
- 298c744: The agent cursor follows the pointer path with humanize on, counts the other busy sessions on the badge with shared tabs, keeps the typing bubble up long enough to see, and keeps its label inside the page. Each session's cursor is named after its own agent ("Claude Code", "Codex", "Cursor", "Copilot", "Gemini CLI") instead of the raw client name or the name of the first one: mcpd sends the client's name with each call (`io.turno/client` in `_meta`), and a shared host uses each connection's own.
|
|
12
|
+
- ce785fd: A password or one-time code never reaches the agent, whoever typed it: every page snapshot (also the snapshot files an action writes to the workspace) shows its value as `[sensitive field]`, and the value is hidden from every later result. `browser_type`, `browser_fill_form` and key presses into a password or one-time code field are refused with an error that says to ask the person (`browser_request_human`) or to use `browser_login`; a flow replay types one only from the secret store (`keychain:<item>`) or from the hidden prompt of `turno browser replay`, which names those parameters for the server it starts in `TURNO_BROWSER_PROMPTED_PARAMS` (listed in `turno help environment`). `browser_network_requests` and `browser_network_request` keep the names of credentials and show `[redacted]` for their values (Authorization, Cookie and Set-Cookie headers, token, code, session, key, secret and password fields), also in a file saved with `filename`. A tool result that ends on a captcha, "verify it's you" or block page says so and suggests that the person handles it.
|
|
13
|
+
- 3a8e4b3: `turno-browser set --help` describes `--engine camoufox` by what it needs and what it lacks, without a "not recommended" label, like the other opt-in settings.
|
|
14
|
+
- 70e2adc: `turno-browser init --agent json` ends with "start a new session of your MCP client" instead of "start a new any MCP client session".
|
|
15
|
+
|
|
3
16
|
## 0.7.0
|
|
4
17
|
|
|
5
18
|
### Minor Changes
|
package/README.md
CHANGED
|
@@ -47,11 +47,12 @@ On from the start:
|
|
|
47
47
|
|---|---|
|
|
48
48
|
| [Identities](https://turno.wimo.dev/docs/browser/identities) | one Chrome profile per identity, with its time zone, languages and locale fixed |
|
|
49
49
|
| [Tab limits](https://turno.wimo.dev/docs/browser/tabs) | at most 8 pages, parked after 5 minutes unused, closed after 15; Chrome closes after 15 minutes without a call |
|
|
50
|
-
| [Credential vault](https://turno.wimo.dev/docs/browser/vault) | `browser_login` fills username, password and one-time code; the agent never sees them |
|
|
50
|
+
| [Credential vault](https://turno.wimo.dev/docs/browser/vault) | `browser_login` fills username, password and one-time code; the agent never sees them, and a password field never comes back to it, whoever typed it |
|
|
51
51
|
| [Session export and import](https://turno.wimo.dev/docs/browser/moving-a-session) | an identity's session as one encrypted file, for another machine |
|
|
52
52
|
| [`browser_batch`](https://turno.wimo.dev/docs/browser/actions) | up to 20 actions in one call, and errors that say what is in the way |
|
|
53
53
|
| [Flows](https://turno.wimo.dev/docs/browser/flows) | save what an agent did on a site once, and replay it with parameters |
|
|
54
54
|
| [From the shell](https://turno.wimo.dev/docs/browser/actions#from-the-shell) | `turno-browser do <id> open`, `snapshot`, `click`, `type`, `read`: the same browser and rules, as shell commands |
|
|
55
|
+
| [Steps only you may do](https://turno.wimo.dev/docs/browser/live-view#when-the-agent-asks-for-you) | `browser_request_human` brings the tab to you with a banner for a password, a captcha, a code or consent; the agent waits, and never gets what you typed |
|
|
55
56
|
| [The agent cursor](https://turno.wimo.dev/docs/browser/agent-cursor) | when you can see the browser, a named cursor and a badge show where the agent is working; the model never sees it |
|
|
56
57
|
| [Health, reputation and doctor](https://turno.wimo.dev/docs/browser/identities#session-health) | a signed-out identity says so, each site's challenges are counted, and `doctor` checks Chrome and every identity |
|
|
57
58
|
|
|
@@ -123,6 +124,9 @@ turno-browser vault list school
|
|
|
123
124
|
and only into frames of those origins.
|
|
124
125
|
- **Never in a result.** Once a value was filled, every result and error shows `[vault: ...]`
|
|
125
126
|
in its place: snapshots, network requests and console messages.
|
|
127
|
+
- **Password fields too.** A password or a one-time code never comes back to the agent,
|
|
128
|
+
whoever typed it: a snapshot shows `[sensitive field]`, and `browser_type` refuses those
|
|
129
|
+
fields.
|
|
126
130
|
- **It does not submit.** The agent clicks the sign-in button, unless you run
|
|
127
131
|
`turno-browser vault submit school auto`.
|
|
128
132
|
|
package/dist/act.js
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { ModelError, estimateTokens, resolveModel } from "./act-model.js";
|
|
2
2
|
import { clip, sections, textOf } from "./batch.js";
|
|
3
3
|
import { actBudget } from "./opt-in-settings.js";
|
|
4
|
-
import {
|
|
4
|
+
import { pageFields, secretValues } from "./sensitive.js";
|
|
5
5
|
import { readIdentitySettings } from "./settings.js";
|
|
6
6
|
import { upstream } from "./upstream.js";
|
|
7
7
|
import { names } from "./names.js";
|
|
@@ -140,38 +140,13 @@ export function checkCall(stock, tool, args) {
|
|
|
140
140
|
return { error: "natural language actions do not write files" };
|
|
141
141
|
return { tool, arguments: a };
|
|
142
142
|
}
|
|
143
|
-
/** The values of password, one-time code and card fields in every frame of the page. */
|
|
144
|
-
async function sensitiveValues(page) {
|
|
145
|
-
const out = [];
|
|
146
|
-
for (const frame of page.frames()) {
|
|
147
|
-
const values = await frame
|
|
148
|
-
.evaluate(({ auto, name }) => {
|
|
149
|
-
const a = new RegExp(auto, "i");
|
|
150
|
-
const n = new RegExp(name, "i");
|
|
151
|
-
const found = [];
|
|
152
|
-
for (const el of Array.from(document.querySelectorAll("input, textarea"))) {
|
|
153
|
-
const field = el;
|
|
154
|
-
if (!field.value)
|
|
155
|
-
continue;
|
|
156
|
-
const type = (field.type || "").toLowerCase();
|
|
157
|
-
if (type === "password" ||
|
|
158
|
-
a.test(field.getAttribute("autocomplete") ?? "") ||
|
|
159
|
-
n.test(`${field.getAttribute("name") ?? ""} ${field.id}`))
|
|
160
|
-
found.push(field.value);
|
|
161
|
-
}
|
|
162
|
-
return found;
|
|
163
|
-
}, { auto: SECRET_AUTOCOMPLETE.source, name: SECRET_NAME.source })
|
|
164
|
-
.catch(() => []);
|
|
165
|
-
out.push(...values);
|
|
166
|
-
}
|
|
167
|
-
return out;
|
|
168
|
-
}
|
|
169
143
|
/** The current tab as the model reads it: page, dialog and snapshot, sensitive values hidden. */
|
|
170
144
|
async function pageForModel(run, reserve) {
|
|
171
145
|
const page = run.stock.currentPage();
|
|
172
146
|
if (page)
|
|
173
|
-
for (const
|
|
174
|
-
|
|
147
|
+
for (const f of await pageFields(page))
|
|
148
|
+
if (f.kind)
|
|
149
|
+
run.redactor.add(f.value, "sensitive value");
|
|
175
150
|
const snap = await run.stock.call("browser_snapshot", {}, { snapshot: true });
|
|
176
151
|
if (snap.isError)
|
|
177
152
|
throw new ActStop(`the page snapshot failed: ${clip(textOf(snap))}`);
|
|
@@ -49,6 +49,11 @@ export type OverlayCommand = {
|
|
|
49
49
|
op: "busy";
|
|
50
50
|
key: string;
|
|
51
51
|
on: boolean;
|
|
52
|
+
}
|
|
53
|
+
/** How many other sessions have a call in flight on this browser, on any page: the badge's "+N". */
|
|
54
|
+
| {
|
|
55
|
+
op: "others";
|
|
56
|
+
n: number;
|
|
52
57
|
} | {
|
|
53
58
|
op: "paused";
|
|
54
59
|
on: boolean;
|
|
@@ -70,12 +75,17 @@ export interface OverlayReport {
|
|
|
70
75
|
busy: boolean;
|
|
71
76
|
paused: boolean;
|
|
72
77
|
badge: string;
|
|
78
|
+
/** `moves`: how many times the arrow was sent somewhere; `pill`: the side its label is on. */
|
|
73
79
|
cursors: {
|
|
74
80
|
key: string;
|
|
75
81
|
label: string;
|
|
76
82
|
x: number;
|
|
77
83
|
y: number;
|
|
84
|
+
moves: number;
|
|
85
|
+
pill: "right" | "left";
|
|
78
86
|
}[];
|
|
87
|
+
/** The sessions whose typing bubble shows. */
|
|
88
|
+
typing: string[];
|
|
79
89
|
};
|
|
80
90
|
}
|
|
81
91
|
/**
|
|
@@ -37,6 +37,9 @@ export function cursorOverlay(commands) {
|
|
|
37
37
|
.cursor{position:fixed;left:0;top:0;transform:translate(var(--x),var(--y));transition:transform var(--ms) cubic-bezier(.22,.61,.36,1),opacity .4s ease;will-change:transform}
|
|
38
38
|
.cursor svg{display:block;overflow:visible;filter:drop-shadow(0 2px 3px rgba(0,0,0,.3))}
|
|
39
39
|
.pill{position:absolute;left:15px;top:19px;padding:3px 8px;border-radius:3px 10px 10px 10px;background:var(--c);color:#fff;white-space:nowrap;box-shadow:0 2px 8px rgba(0,0,0,.2)}
|
|
40
|
+
.fx .pill{left:auto;right:calc(100% - 4px);border-radius:10px 3px 10px 10px}
|
|
41
|
+
.fy .pill{top:auto;bottom:calc(100% - 6px);border-radius:10px 10px 10px 3px}
|
|
42
|
+
.fx.fy .pill{border-radius:10px 10px 3px 10px}
|
|
40
43
|
.idle .cursor{opacity:.6}
|
|
41
44
|
.ring{position:fixed;border:2px solid var(--c);border-radius:7px;box-shadow:0 0 0 4px color-mix(in srgb,var(--c) 20%,transparent);opacity:0;transition:opacity .25s ease,left .2s ease,top .2s ease,width .2s ease,height .2s ease}
|
|
42
45
|
.ring.show{opacity:1}
|
|
@@ -114,6 +117,7 @@ export function cursorOverlay(commands) {
|
|
|
114
117
|
typing: new Map(),
|
|
115
118
|
timers: new Map(),
|
|
116
119
|
busy: new Set(),
|
|
120
|
+
others: 0,
|
|
117
121
|
lastBusy: "",
|
|
118
122
|
paused: false,
|
|
119
123
|
visible: true,
|
|
@@ -165,7 +169,7 @@ export function cursorOverlay(commands) {
|
|
|
165
169
|
pill.className = "pill";
|
|
166
170
|
el.append(svg, pill);
|
|
167
171
|
parts.layer.append(el);
|
|
168
|
-
c = { el, pill, label: "", name: "", color: "#2b59ff", x: -40, y: -40 };
|
|
172
|
+
c = { el, pill, label: "", name: "", color: "#2b59ff", x: -40, y: -40, moves: 0 };
|
|
169
173
|
parts.cursors.set(key, c);
|
|
170
174
|
return c;
|
|
171
175
|
};
|
|
@@ -180,7 +184,10 @@ export function cursorOverlay(commands) {
|
|
|
180
184
|
parts.frame.style.setProperty("--c", color);
|
|
181
185
|
parts.badge.style.setProperty("--c", color);
|
|
182
186
|
const name = parts.cursors.get(key)?.name;
|
|
183
|
-
|
|
187
|
+
// The other sessions at work: those on this page, or on the browser's other pages (shared
|
|
188
|
+
// tabs), whichever says more.
|
|
189
|
+
const n = Math.max(parts.busy.size - 1, parts.others);
|
|
190
|
+
const others = n > 0 ? ` +${n}` : "";
|
|
184
191
|
parts.badgeText.textContent = parts.paused
|
|
185
192
|
? "paused · you have the controls"
|
|
186
193
|
: `turno · ${name ? `${name} at work` : "agent at work"}${others}`;
|
|
@@ -204,6 +211,13 @@ export function cursorOverlay(commands) {
|
|
|
204
211
|
// The arrow's tip is 2px into its box.
|
|
205
212
|
c.el.style.setProperty("--x", `${Math.round(cmd.x - 2)}px`);
|
|
206
213
|
c.el.style.setProperty("--y", `${Math.round(cmd.y - 2)}px`);
|
|
214
|
+
c.moves++;
|
|
215
|
+
// The label goes left of the arrow near the right edge, and above it near the bottom, so
|
|
216
|
+
// it stays inside the page.
|
|
217
|
+
const pw = c.pill.offsetWidth || 8 * c.label.length + 16;
|
|
218
|
+
const ph = c.pill.offsetHeight || 22;
|
|
219
|
+
c.el.classList.toggle("fx", cmd.x + 13 + pw + 4 > window.innerWidth);
|
|
220
|
+
c.el.classList.toggle("fy", cmd.y + 17 + ph + 4 > window.innerHeight);
|
|
207
221
|
break;
|
|
208
222
|
}
|
|
209
223
|
case "ring": {
|
|
@@ -242,12 +256,29 @@ export function cursorOverlay(commands) {
|
|
|
242
256
|
case "typing": {
|
|
243
257
|
const old = parts.typing.get(cmd.key);
|
|
244
258
|
if (!cmd.box) {
|
|
245
|
-
old
|
|
246
|
-
|
|
259
|
+
if (!old)
|
|
260
|
+
break;
|
|
261
|
+
// A fill ends within a frame: the bubble stays up a moment so a person sees it at all.
|
|
262
|
+
const drop = () => {
|
|
263
|
+
old.remove();
|
|
264
|
+
if (parts.typing.get(cmd.key) === old)
|
|
265
|
+
parts.typing.delete(cmd.key);
|
|
266
|
+
};
|
|
267
|
+
const wait = 900 - (Date.now() - Number(old.dataset.since ?? 0));
|
|
268
|
+
if (wait > 0)
|
|
269
|
+
later(`typing:${cmd.key}`, wait, drop);
|
|
270
|
+
else
|
|
271
|
+
drop();
|
|
247
272
|
break;
|
|
248
273
|
}
|
|
274
|
+
const pending = parts.timers.get(`typing:${cmd.key}`);
|
|
275
|
+
if (pending !== undefined) {
|
|
276
|
+
clearTimeout(pending);
|
|
277
|
+
parts.timers.delete(`typing:${cmd.key}`);
|
|
278
|
+
}
|
|
249
279
|
const bubble = old ?? document.createElement("div");
|
|
250
280
|
if (!old) {
|
|
281
|
+
bubble.dataset.since = String(Date.now());
|
|
251
282
|
bubble.className = "typing";
|
|
252
283
|
bubble.append(document.createElement("i"), document.createElement("i"), document.createElement("i"));
|
|
253
284
|
parts.layer.append(bubble);
|
|
@@ -283,6 +314,12 @@ export function cursorOverlay(commands) {
|
|
|
283
314
|
parts.paused = cmd.on;
|
|
284
315
|
paintBadge();
|
|
285
316
|
break;
|
|
317
|
+
case "others":
|
|
318
|
+
if (parts.others !== cmd.n) {
|
|
319
|
+
parts.others = cmd.n;
|
|
320
|
+
paintBadge();
|
|
321
|
+
}
|
|
322
|
+
break;
|
|
286
323
|
case "visible":
|
|
287
324
|
parts.visible = cmd.on;
|
|
288
325
|
parts.host.style.setProperty("visibility", cmd.on ? "visible" : "hidden", "important");
|
|
@@ -306,7 +343,15 @@ export function cursorOverlay(commands) {
|
|
|
306
343
|
busy: parts.busy.size > 0,
|
|
307
344
|
paused: parts.paused,
|
|
308
345
|
badge: parts.badgeText.textContent ?? "",
|
|
309
|
-
cursors: [...parts.cursors.entries()].map(([key, c]) => ({
|
|
346
|
+
cursors: [...parts.cursors.entries()].map(([key, c]) => ({
|
|
347
|
+
key,
|
|
348
|
+
label: c.label,
|
|
349
|
+
x: c.x,
|
|
350
|
+
y: c.y,
|
|
351
|
+
moves: c.moves,
|
|
352
|
+
pill: c.el.classList.contains("fx") ? "left" : "right",
|
|
353
|
+
})),
|
|
354
|
+
typing: [...parts.typing.keys()],
|
|
310
355
|
},
|
|
311
356
|
};
|
|
312
357
|
}
|
package/dist/agent-cursor.d.ts
CHANGED
|
@@ -52,10 +52,18 @@ export declare function inView(box: OverlayBox, view: {
|
|
|
52
52
|
width: number;
|
|
53
53
|
height: number;
|
|
54
54
|
}): boolean;
|
|
55
|
+
/**
|
|
56
|
+
* The agent's name as people know it, from the client name it sends in `initialize`
|
|
57
|
+
* (`clientInfo.name`): "claude-code" reads "Claude Code", "codex-mcp-client" reads "Codex". A
|
|
58
|
+
* name turno does not know is shown as sent; a "(2)" turno added to tell sessions apart stays.
|
|
59
|
+
*/
|
|
60
|
+
export declare function agentName(client: string): string;
|
|
55
61
|
/** The pill of a session's cursor: the client's name, and the identity when there is one. */
|
|
56
62
|
export declare function cursorLabel(client: string, identity?: string): string;
|
|
57
63
|
interface Actor {
|
|
58
64
|
key: string;
|
|
65
|
+
/** The agent's name without the "(2)" that tells sessions apart. */
|
|
66
|
+
base: string;
|
|
59
67
|
/** The client's name: the badge says it is at work. */
|
|
60
68
|
name: string;
|
|
61
69
|
/** The cursor's pill: the name and the identity. */
|
|
@@ -83,9 +91,17 @@ export declare class AgentCursor {
|
|
|
83
91
|
private readonly maxGlideMs;
|
|
84
92
|
private lastKey;
|
|
85
93
|
private paused;
|
|
94
|
+
/** The sessions with a call in flight on this browser, whatever their page. */
|
|
95
|
+
private readonly busy;
|
|
96
|
+
/** The pages the overlay has been on, so a call starting elsewhere updates their badge. */
|
|
97
|
+
private readonly pages;
|
|
86
98
|
constructor(options: AgentCursorOptions);
|
|
87
|
-
/**
|
|
88
|
-
|
|
99
|
+
/**
|
|
100
|
+
* The actor of a session, with its label and color, made on first use. `client` is the name
|
|
101
|
+
* of the session's own agent when the call said it (mcpd, or a shared host's connection);
|
|
102
|
+
* without it the session takes the launching client's name, with "(2)" and up after the first.
|
|
103
|
+
*/
|
|
104
|
+
actor(key: string, client?: string): Actor;
|
|
89
105
|
/** The session acting on this page now, or the last one that acted anywhere. */
|
|
90
106
|
ownerOf(page: Page): string;
|
|
91
107
|
/** Where the session's arrow is on this page, if it has been there. */
|
|
@@ -99,8 +115,15 @@ export declare class AgentCursor {
|
|
|
99
115
|
*/
|
|
100
116
|
send(page: Page, commands: OverlayCommand[], key?: string): Promise<OverlayReport | undefined>;
|
|
101
117
|
private remember;
|
|
102
|
-
/**
|
|
103
|
-
|
|
118
|
+
/** How many sessions other than `key` have a call in flight on this browser: the badge's "+N". */
|
|
119
|
+
private othersThan;
|
|
120
|
+
/** Tells the other pages the overlay is on how many sessions are at work now. */
|
|
121
|
+
private recount;
|
|
122
|
+
/**
|
|
123
|
+
* A call of session `key` begins on this page: the frame and badge come on. `client` names the
|
|
124
|
+
* session's own agent when the call said it.
|
|
125
|
+
*/
|
|
126
|
+
begin(page: Page | undefined, key: string, client?: string): Promise<void>;
|
|
104
127
|
/** The call ended (on this page, which may be a new one): the frame fades after a moment. */
|
|
105
128
|
end(page: Page | undefined, key: string): Promise<void>;
|
|
106
129
|
/** Glides the arrow to `to` and waits for it to arrive. */
|
|
@@ -131,8 +154,16 @@ export declare class CursorInput {
|
|
|
131
154
|
/** The last element marked on each page: where keys typed through the keyboard go. */
|
|
132
155
|
private readonly marked;
|
|
133
156
|
private readonly typingTimers;
|
|
157
|
+
/** Pages where humanize moved the pointer since the last action: the arrow is already there. */
|
|
158
|
+
private readonly moved;
|
|
134
159
|
constructor(cursor: AgentCursor, humanized: boolean);
|
|
135
160
|
private boxOf;
|
|
161
|
+
/**
|
|
162
|
+
* Whether humanize just played a pointer path on this page (and forgets it): then the arrow
|
|
163
|
+
* followed it there, and the action gets no glide of its own. When humanize went straight to
|
|
164
|
+
* the action (no box to aim at), the arrow glides as without humanize.
|
|
165
|
+
*/
|
|
166
|
+
private followed;
|
|
136
167
|
/** Marks the target of an action, runs it, and leaves the mark of its kind. */
|
|
137
168
|
private act;
|
|
138
169
|
/** The cursor view of one of the page's locators. */
|
|
@@ -151,9 +182,10 @@ export declare class CursorInput {
|
|
|
151
182
|
wrapContext<T extends object>(context: T): T;
|
|
152
183
|
}
|
|
153
184
|
/**
|
|
154
|
-
* The stock tools of one browser, with the input tools running on cursor tabs. It goes
|
|
155
|
-
* `humanizeTools` (applied to its result)
|
|
156
|
-
* mouse. Schemas and every other tool are
|
|
185
|
+
* The stock tools of one browser, with the input tools running on cursor tabs. It goes over
|
|
186
|
+
* `humanizeTools` (applied to its result): the outer wrapper's view is the one next to the page,
|
|
187
|
+
* so humanize's pointer path runs through the cursor's mouse. Schemas and every other tool are
|
|
188
|
+
* upstream's own objects.
|
|
157
189
|
*/
|
|
158
190
|
export declare function cursorTools<T extends UpstreamTool>(tools: readonly T[], input: CursorInput): T[];
|
|
159
191
|
export {};
|
package/dist/agent-cursor.js
CHANGED
|
@@ -37,6 +37,29 @@ export function centerOf(box) {
|
|
|
37
37
|
export function inView(box, view) {
|
|
38
38
|
return box.x + box.width > 0 && box.y + box.height > 0 && box.x < view.width && box.y < view.height;
|
|
39
39
|
}
|
|
40
|
+
/**
|
|
41
|
+
* The agent's name as people know it, from the client name it sends in `initialize`
|
|
42
|
+
* (`clientInfo.name`): "claude-code" reads "Claude Code", "codex-mcp-client" reads "Codex". A
|
|
43
|
+
* name turno does not know is shown as sent; a "(2)" turno added to tell sessions apart stays.
|
|
44
|
+
*/
|
|
45
|
+
export function agentName(client) {
|
|
46
|
+
const raw = client.trim();
|
|
47
|
+
const m = /^(.*?)(\s\(\d+\))?$/.exec(raw);
|
|
48
|
+
const base = (m?.[1] ?? raw).trim();
|
|
49
|
+
const suffix = m?.[2] ?? "";
|
|
50
|
+
const known = [
|
|
51
|
+
[/^claude[-_ ]?code$/i, "Claude Code"],
|
|
52
|
+
[/^claude[-_ ]?ai$/i, "Claude"],
|
|
53
|
+
[/^codex([-_ ]mcp[-_ ]client)?$/i, "Codex"],
|
|
54
|
+
[/^cursor([-_ ]vscode)?$/i, "Cursor"],
|
|
55
|
+
[/^(github[-_ ])?copilot([-_ ]cli)?$/i, "Copilot"],
|
|
56
|
+
[/^gemini([-_ ]cli)?([-_ ]mcp[-_ ]client)?$/i, "Gemini CLI"],
|
|
57
|
+
[/^windsurf([-_ ]client)?$/i, "Windsurf"],
|
|
58
|
+
[/^(visual studio code|vscode)$/i, "VS Code"],
|
|
59
|
+
];
|
|
60
|
+
const pretty = known.find(([re]) => re.test(base))?.[1];
|
|
61
|
+
return `${pretty ?? base}${suffix}`;
|
|
62
|
+
}
|
|
40
63
|
/** The pill of a session's cursor: the client's name, and the identity when there is one. */
|
|
41
64
|
export function cursorLabel(client, identity) {
|
|
42
65
|
const name = client.trim() || "Agent";
|
|
@@ -56,22 +79,39 @@ export class AgentCursor {
|
|
|
56
79
|
maxGlideMs;
|
|
57
80
|
lastKey = "";
|
|
58
81
|
paused = false;
|
|
82
|
+
/** The sessions with a call in flight on this browser, whatever their page. */
|
|
83
|
+
busy = new Set();
|
|
84
|
+
/** The pages the overlay has been on, so a call starting elsewhere updates their badge. */
|
|
85
|
+
pages = new Set();
|
|
59
86
|
constructor(options) {
|
|
60
87
|
this.options = options;
|
|
61
88
|
this.sleep = options.sleep ?? defaultSleep;
|
|
62
89
|
this.maxGlideMs = options.maxGlideMs ?? MAX_GLIDE_MS;
|
|
63
90
|
}
|
|
64
|
-
/**
|
|
65
|
-
|
|
91
|
+
/**
|
|
92
|
+
* The actor of a session, with its label and color, made on first use. `client` is the name
|
|
93
|
+
* of the session's own agent when the call said it (mcpd, or a shared host's connection);
|
|
94
|
+
* without it the session takes the launching client's name, with "(2)" and up after the first.
|
|
95
|
+
*/
|
|
96
|
+
actor(key, client) {
|
|
66
97
|
let a = this.actors.get(key);
|
|
67
98
|
if (a)
|
|
68
99
|
return a;
|
|
69
100
|
const n = this.actors.size;
|
|
70
|
-
|
|
101
|
+
let name;
|
|
102
|
+
if (client) {
|
|
103
|
+
const base = agentName(client);
|
|
104
|
+
const same = [...this.actors.values()].filter((o) => o.base === base).length;
|
|
105
|
+
name = same === 0 ? base : `${base} (${same + 1})`;
|
|
106
|
+
}
|
|
107
|
+
else
|
|
108
|
+
name = agentName(n === 0 ? this.options.title : `${this.options.title} (${n + 1})`);
|
|
109
|
+
name = name.trim() || "Agent";
|
|
71
110
|
a = {
|
|
72
111
|
key,
|
|
73
|
-
|
|
74
|
-
|
|
112
|
+
base: name.replace(/\s\(\d+\)$/, ""),
|
|
113
|
+
name,
|
|
114
|
+
label: cursorLabel(name, this.options.identity),
|
|
75
115
|
color: CURSOR_COLORS[n % CURSOR_COLORS.length],
|
|
76
116
|
};
|
|
77
117
|
this.actors.set(key, a);
|
|
@@ -103,10 +143,11 @@ export class AgentCursor {
|
|
|
103
143
|
const at = this.pointOf(page, key);
|
|
104
144
|
if (at)
|
|
105
145
|
prelude.push({ op: "move", key, x: at.x, y: at.y, ms: 0 });
|
|
106
|
-
prelude.push({ op: "paused", on: this.paused });
|
|
146
|
+
prelude.push({ op: "paused", on: this.paused }, { op: "others", n: this.othersThan(key) });
|
|
107
147
|
try {
|
|
108
148
|
const report = await page.evaluate(cursorOverlay, [...prelude, ...commands]);
|
|
109
149
|
this.reports.set(page, report);
|
|
150
|
+
this.pages.add(page);
|
|
110
151
|
return report;
|
|
111
152
|
}
|
|
112
153
|
catch {
|
|
@@ -121,8 +162,31 @@ export class AgentCursor {
|
|
|
121
162
|
}
|
|
122
163
|
m.set(key, point);
|
|
123
164
|
}
|
|
124
|
-
/**
|
|
125
|
-
|
|
165
|
+
/** How many sessions other than `key` have a call in flight on this browser: the badge's "+N". */
|
|
166
|
+
othersThan(key) {
|
|
167
|
+
return this.busy.size - (this.busy.has(key) ? 1 : 0);
|
|
168
|
+
}
|
|
169
|
+
/** Tells the other pages the overlay is on how many sessions are at work now. */
|
|
170
|
+
recount(except) {
|
|
171
|
+
for (const page of this.pages) {
|
|
172
|
+
if (page.isClosed()) {
|
|
173
|
+
this.pages.delete(page);
|
|
174
|
+
continue;
|
|
175
|
+
}
|
|
176
|
+
if (page === except)
|
|
177
|
+
continue;
|
|
178
|
+
const owner = this.ownerOf(page);
|
|
179
|
+
void this.send(page, [], owner);
|
|
180
|
+
}
|
|
181
|
+
}
|
|
182
|
+
/**
|
|
183
|
+
* A call of session `key` begins on this page: the frame and badge come on. `client` names the
|
|
184
|
+
* session's own agent when the call said it.
|
|
185
|
+
*/
|
|
186
|
+
async begin(page, key, client) {
|
|
187
|
+
this.actor(key, client);
|
|
188
|
+
this.busy.add(key);
|
|
189
|
+
this.recount(page);
|
|
126
190
|
if (!page)
|
|
127
191
|
return;
|
|
128
192
|
this.owners.set(page, key);
|
|
@@ -131,6 +195,8 @@ export class AgentCursor {
|
|
|
131
195
|
}
|
|
132
196
|
/** The call ended (on this page, which may be a new one): the frame fades after a moment. */
|
|
133
197
|
async end(page, key) {
|
|
198
|
+
this.busy.delete(key);
|
|
199
|
+
this.recount(page);
|
|
134
200
|
if (!page)
|
|
135
201
|
return;
|
|
136
202
|
this.owners.set(page, key);
|
|
@@ -150,8 +216,10 @@ export class AgentCursor {
|
|
|
150
216
|
/** Moves the arrow along with a pointer path someone else plays (humanize): a short ease per step. */
|
|
151
217
|
async track(page, to) {
|
|
152
218
|
const key = this.ownerOf(page);
|
|
153
|
-
|
|
219
|
+
// Remembered first: the steps are sent without waiting, so the next step's batch must not
|
|
220
|
+
// put the arrow back where this one started.
|
|
154
221
|
this.remember(page, key, to);
|
|
222
|
+
await this.send(page, [{ op: "move", key, x: to.x, y: to.y, ms: 40 }], key);
|
|
155
223
|
}
|
|
156
224
|
/** Puts the arrow at `to` without a glide. */
|
|
157
225
|
async place(page, to) {
|
|
@@ -220,6 +288,8 @@ export class CursorInput {
|
|
|
220
288
|
/** The last element marked on each page: where keys typed through the keyboard go. */
|
|
221
289
|
marked = new WeakMap();
|
|
222
290
|
typingTimers = new WeakMap();
|
|
291
|
+
/** Pages where humanize moved the pointer since the last action: the arrow is already there. */
|
|
292
|
+
moved = new WeakSet();
|
|
223
293
|
constructor(cursor, humanized) {
|
|
224
294
|
this.cursor = cursor;
|
|
225
295
|
this.humanized = humanized;
|
|
@@ -233,9 +303,21 @@ export class CursorInput {
|
|
|
233
303
|
return null;
|
|
234
304
|
}
|
|
235
305
|
}
|
|
306
|
+
/**
|
|
307
|
+
* Whether humanize just played a pointer path on this page (and forgets it): then the arrow
|
|
308
|
+
* followed it there, and the action gets no glide of its own. When humanize went straight to
|
|
309
|
+
* the action (no box to aim at), the arrow glides as without humanize.
|
|
310
|
+
*/
|
|
311
|
+
followed(page) {
|
|
312
|
+
if (!this.humanized || !this.moved.has(page))
|
|
313
|
+
return false;
|
|
314
|
+
this.moved.delete(page);
|
|
315
|
+
return true;
|
|
316
|
+
}
|
|
236
317
|
/** Marks the target of an action, runs it, and leaves the mark of its kind. */
|
|
237
318
|
async act(page, locator, kind, run, to) {
|
|
238
319
|
const c = this.cursor;
|
|
320
|
+
const followed = this.followed(page);
|
|
239
321
|
let box = await this.boxOf(locator);
|
|
240
322
|
const view = c.reportOf(page) ?? (await c.send(page, []));
|
|
241
323
|
if (box && view && !inView(box, view))
|
|
@@ -243,11 +325,11 @@ export class CursorInput {
|
|
|
243
325
|
if (box) {
|
|
244
326
|
this.marked.set(page, box);
|
|
245
327
|
void c.ring(page, box);
|
|
246
|
-
if (!
|
|
328
|
+
if (!followed)
|
|
247
329
|
await c.glideTo(page, centerOf(box));
|
|
248
330
|
if (kind === "type")
|
|
249
331
|
void c.typing(page, box);
|
|
250
|
-
if (kind === "click" && !
|
|
332
|
+
if (kind === "click" && !followed)
|
|
251
333
|
void c.ripple(page, centerOf(box));
|
|
252
334
|
}
|
|
253
335
|
try {
|
|
@@ -259,17 +341,17 @@ export class CursorInput {
|
|
|
259
341
|
await c.place(page, centerOf(end));
|
|
260
342
|
}
|
|
261
343
|
}
|
|
262
|
-
else if (!box || (kind === "click" &&
|
|
344
|
+
else if (!box || (kind === "click" && followed)) {
|
|
263
345
|
// Marked after: the element came into view during the action, or humanize moved there.
|
|
264
346
|
const after = box ?? (await this.boxOf(locator, 200));
|
|
265
347
|
const center = after ? centerOf(after) : undefined;
|
|
266
|
-
const at =
|
|
348
|
+
const at = followed ? (c.pointOf(page) ?? center) : center;
|
|
267
349
|
if (after && !box) {
|
|
268
350
|
this.marked.set(page, after);
|
|
269
351
|
void c.ring(page, after);
|
|
270
352
|
}
|
|
271
353
|
if (at) {
|
|
272
|
-
if (!
|
|
354
|
+
if (!followed)
|
|
273
355
|
await c.place(page, at);
|
|
274
356
|
if (kind === "click")
|
|
275
357
|
void c.ripple(page, at);
|
|
@@ -300,20 +382,23 @@ export class CursorInput {
|
|
|
300
382
|
const c = this.cursor;
|
|
301
383
|
return override(mouse, {
|
|
302
384
|
move: async (x, y, options) => {
|
|
303
|
-
if (this.humanized)
|
|
385
|
+
if (this.humanized) {
|
|
386
|
+
// A step of humanize's pointer path: the arrow follows it.
|
|
387
|
+
this.moved.add(page);
|
|
304
388
|
void c.track(page, { x, y });
|
|
389
|
+
}
|
|
305
390
|
else
|
|
306
391
|
await c.glideTo(page, { x, y });
|
|
307
392
|
await mouse.move(x, y, options);
|
|
308
393
|
},
|
|
309
394
|
click: async (x, y, options) => {
|
|
310
|
-
if (!this.
|
|
395
|
+
if (!this.followed(page))
|
|
311
396
|
await c.glideTo(page, { x, y });
|
|
312
397
|
void c.ripple(page, { x, y });
|
|
313
398
|
await mouse.click(x, y, options);
|
|
314
399
|
},
|
|
315
400
|
dblclick: async (x, y, options) => {
|
|
316
|
-
if (!this.
|
|
401
|
+
if (!this.followed(page))
|
|
317
402
|
await c.glideTo(page, { x, y });
|
|
318
403
|
void c.ripple(page, { x, y });
|
|
319
404
|
await mouse.dblclick(x, y, options);
|
|
@@ -392,9 +477,10 @@ export class CursorInput {
|
|
|
392
477
|
}
|
|
393
478
|
}
|
|
394
479
|
/**
|
|
395
|
-
* The stock tools of one browser, with the input tools running on cursor tabs. It goes
|
|
396
|
-
* `humanizeTools` (applied to its result)
|
|
397
|
-
* mouse. Schemas and every other tool are
|
|
480
|
+
* The stock tools of one browser, with the input tools running on cursor tabs. It goes over
|
|
481
|
+
* `humanizeTools` (applied to its result): the outer wrapper's view is the one next to the page,
|
|
482
|
+
* so humanize's pointer path runs through the cursor's mouse. Schemas and every other tool are
|
|
483
|
+
* upstream's own objects.
|
|
398
484
|
*/
|
|
399
485
|
export function cursorTools(tools, input) {
|
|
400
486
|
return tools.map((tool) => {
|
package/dist/cli/flows.js
CHANGED
|
@@ -135,6 +135,7 @@ export async function replay(c, args, io) {
|
|
|
135
135
|
return fail(`usage: ${c.names.browser} replay <id> <flow> [--param name=value]... [--from <step>] [--headless]`);
|
|
136
136
|
const flow = mod.readFlow(id, name, io.env);
|
|
137
137
|
const params = {};
|
|
138
|
+
const prompted = [];
|
|
138
139
|
for (const pair of p.values.param ?? []) {
|
|
139
140
|
const eq = pair.indexOf("=");
|
|
140
141
|
if (eq <= 0)
|
|
@@ -149,6 +150,7 @@ export async function replay(c, args, io) {
|
|
|
149
150
|
if (!input.isTty() || noInputReason(io.env))
|
|
150
151
|
needsAnswer(`browser replay (parameter ${param.name})`, `--param ${param.name}=keychain:<item>`);
|
|
151
152
|
params[param.name] = await input.hidden(`${param.name} (input hidden): `);
|
|
153
|
+
prompted.push(param.name);
|
|
152
154
|
}
|
|
153
155
|
}
|
|
154
156
|
const from = str(p, "from");
|
|
@@ -159,6 +161,7 @@ export async function replay(c, args, io) {
|
|
|
159
161
|
return fail(holder);
|
|
160
162
|
const result = await mod.replayFlow(id, name, {
|
|
161
163
|
params,
|
|
164
|
+
...(prompted.length ? { prompted } : {}),
|
|
162
165
|
...(from ? { from: Number(from) } : {}),
|
|
163
166
|
headless: bool(p, "headless"),
|
|
164
167
|
env: io.env,
|
package/dist/cli/identity.js
CHANGED
|
@@ -675,10 +675,9 @@ Input:
|
|
|
675
675
|
Which engine runs this identity (all off by default):
|
|
676
676
|
--engine lightpanda read mode: Lightpanda instead of Chrome, for reading pages;
|
|
677
677
|
needs lightpanda on PATH (${b} doctor says how)
|
|
678
|
-
--engine camoufox Camoufox (a Firefox build) with its own profile;
|
|
679
|
-
|
|
680
|
-
|
|
681
|
-
solving, and no picture in the live view
|
|
678
|
+
--engine camoufox Camoufox (a Firefox build) with its own profile; needs
|
|
679
|
+
Camoufox and playwright-core (${b} doctor says how); no
|
|
680
|
+
captcha solving, and no picture in the live view
|
|
682
681
|
--attach auto|<dir>|<url> use the Chrome you have open instead of the identity's own:
|
|
683
682
|
"auto" is your default Chrome profile, after you turn on
|
|
684
683
|
chrome://inspect/#remote-debugging; Chrome asks you to allow
|
package/dist/cli/init.js
CHANGED
|
@@ -208,8 +208,10 @@ export async function init(c, args, io, run = runCommand) {
|
|
|
208
208
|
// What is left.
|
|
209
209
|
io.out(`\n${paint(io, "Next", "1")}`);
|
|
210
210
|
if (agent !== "none") {
|
|
211
|
-
|
|
212
|
-
|
|
211
|
+
// "json" names no agent ("any MCP client"), so it reads as "your MCP client".
|
|
212
|
+
const who = agent === undefined ? "your agent" : agent === "json" ? "your MCP client" : AGENT_NAMES[agent];
|
|
213
|
+
const session = agent === "json" ? `a new session of ${who}` : `a new ${who} session`;
|
|
214
|
+
io.out(`${connected ? "Start" : "Once it is added, start"} ${session} and ask it to open ${firstSite ?? "a site you signed in to"}.`);
|
|
213
215
|
}
|
|
214
216
|
io.out(`${b} doctor checks Chrome and the identity; ${b} show ${id} shows its settings.`);
|
|
215
217
|
return 0;
|
package/dist/engine-camoufox.js
CHANGED
|
@@ -41,6 +41,7 @@ import { names } from "./names.js";
|
|
|
41
41
|
const NOT_IN_CAMOUFOX = {
|
|
42
42
|
browser_run_code_unsafe: "it runs Playwright code against the patchright-core API, and Camoufox runs on stock Playwright",
|
|
43
43
|
browser_solve_captcha: "it sets the token from Patchright's isolated script world and calls the widget's callback in the page's own, and Camoufox runs on stock Playwright",
|
|
44
|
+
browser_request_human: "its banner runs in Patchright's isolated script world, out of the page scripts' reach, and Camoufox runs on stock Playwright",
|
|
44
45
|
};
|
|
45
46
|
/** Why the Camoufox engine refuses this tool, or undefined when it runs it. */
|
|
46
47
|
export function camoufoxRefusal(tool) {
|
|
@@ -70,6 +70,7 @@ const NOT_IN_READ_MODE = {
|
|
|
70
70
|
browser_run_code_unsafe: "read mode runs only the reading tools",
|
|
71
71
|
browser_login: "read mode keeps no vault sign-ins; it is meant for public pages",
|
|
72
72
|
browser_solve_captcha: "read mode runs no captcha widget to solve",
|
|
73
|
+
browser_request_human: "Lightpanda shows no window and no picture, so the person has no page to act on",
|
|
73
74
|
};
|
|
74
75
|
/** Why read mode refuses this tool, or undefined when it runs it. */
|
|
75
76
|
export function readModeRefusal(tool) {
|