turno-browser 0.6.0 → 0.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +40 -0
- package/README.md +13 -5
- package/dist/act.js +4 -29
- package/dist/agent-cursor-overlay.d.ts +95 -0
- package/dist/agent-cursor-overlay.js +358 -0
- package/dist/agent-cursor.d.ts +191 -0
- package/dist/agent-cursor.js +494 -0
- package/dist/api.d.ts +8 -3
- package/dist/api.js +6 -3
- package/dist/argv.d.ts +2 -1
- package/dist/argv.js +7 -1
- package/dist/bin/mcp.js +5 -0
- package/dist/cli/commands.js +12 -3
- package/dist/cli/completion.js +3 -0
- package/dist/cli/context.d.ts +2 -0
- package/dist/cli/context.js +8 -4
- package/dist/cli/do-background.d.ts +2 -0
- package/dist/cli/do-background.js +15 -0
- package/dist/cli/do.d.ts +4 -0
- package/dist/cli/do.js +17 -1
- package/dist/cli/flows.js +3 -0
- package/dist/cli/identity.d.ts +2 -0
- package/dist/cli/identity.js +67 -11
- package/dist/cli/init.d.ts +12 -0
- package/dist/cli/init.js +219 -0
- package/dist/cli/login-summary.d.ts +6 -0
- package/dist/cli/login-summary.js +11 -0
- package/dist/cli/main.js +3 -0
- package/dist/cli/setup.d.ts +32 -0
- package/dist/cli/setup.js +32 -23
- package/dist/doctor.js +18 -2
- package/dist/engine-camoufox.js +1 -0
- package/dist/engine-lightpanda.js +1 -0
- package/dist/host.d.ts +7 -0
- package/dist/host.js +330 -68
- package/dist/human-input.d.ts +5 -0
- package/dist/human-input.js +1 -1
- package/dist/human-step-overlay.d.ts +73 -0
- package/dist/human-step-overlay.js +285 -0
- package/dist/human-step.d.ts +128 -0
- package/dist/human-step.js +521 -0
- package/dist/index.d.ts +2 -2
- package/dist/index.js +2 -2
- package/dist/live-server.d.ts +7 -3
- package/dist/live-server.js +13 -15
- package/dist/live.d.ts +20 -1
- package/dist/live.js +52 -10
- package/dist/login.d.ts +22 -2
- package/dist/login.js +95 -7
- package/dist/network-redact.d.ts +28 -0
- package/dist/network-redact.js +202 -0
- package/dist/recorder.d.ts +0 -21
- package/dist/recorder.js +1 -41
- package/dist/recordings.d.ts +5 -0
- package/dist/recordings.js +4 -2
- package/dist/redact.d.ts +7 -0
- package/dist/redact.js +32 -1
- package/dist/replay.d.ts +2 -0
- package/dist/replay.js +33 -14
- package/dist/reputation.d.ts +5 -0
- package/dist/reputation.js +12 -7
- package/dist/sensitive.d.ts +79 -0
- package/dist/sensitive.js +275 -0
- package/dist/session.d.ts +12 -0
- package/dist/session.js +24 -0
- package/dist/settings.d.ts +12 -2
- package/dist/settings.js +14 -2
- package/dist/tools.d.ts +5 -0
- package/dist/tools.js +9 -1
- package/dist/tuning.d.ts +4 -1
- package/dist/tuning.js +1 -0
- package/dist/upstream.d.ts +5 -0
- package/dist/watch.d.ts +60 -0
- package/dist/watch.js +114 -0
- package/npm-shrinkwrap.json +2 -2
- package/package.json +1 -1
|
@@ -0,0 +1,191 @@
|
|
|
1
|
+
import type { Locator, Page } from "patchright-core";
|
|
2
|
+
import { type OverlayBox, type OverlayCommand, type OverlayReport } from "./agent-cursor-overlay.js";
|
|
3
|
+
import type { CursorMode } from "./settings.js";
|
|
4
|
+
import type { UpstreamTool } from "./upstream.js";
|
|
5
|
+
/**
|
|
6
|
+
* The agent cursor: while an agent drives a browser someone can see (a headed Chrome window, or
|
|
7
|
+
* any browser with the live view on), each page shows who is at work and where. A Figma-like
|
|
8
|
+
* arrow with the agent's name glides to each element it acts on, a ring marks the element, a
|
|
9
|
+
* click leaves a ripple, a field being typed into gets a typing bubble, and a thin frame with a
|
|
10
|
+
* badge ("turno · Claude Code at work", "paused · you have the controls") goes around the page
|
|
11
|
+
* while calls run. Several agent sessions on one identity's Chrome (`"sharing": "tabs"`) each
|
|
12
|
+
* get a cursor and a color of their own.
|
|
13
|
+
*
|
|
14
|
+
* The coordinates are the agent's own: the input tools run on a view of the tab (`CursorInput`)
|
|
15
|
+
* that reads the target's box before each action, as `human-input.ts` does for human-like input.
|
|
16
|
+
* Nothing follows the real mouse, so the user's own pointer never moves the agent's cursor.
|
|
17
|
+
* With humanize on, its pointer path goes through the same view, so the arrow follows the curve.
|
|
18
|
+
*
|
|
19
|
+
* What the page draws is the overlay of `agent-cursor-overlay.ts`, built on first use in each
|
|
20
|
+
* document and kept out of the model's view: out of the accessibility tree (snapshots), out of
|
|
21
|
+
* `document.body` (read), never a hit target, and hidden while the model's screenshot is taken
|
|
22
|
+
* (`hiddenDuring`). Recordings and the live view keep it: they are for the person.
|
|
23
|
+
*/
|
|
24
|
+
/** What the policy needs to know about the browser being launched. */
|
|
25
|
+
export interface CursorContext {
|
|
26
|
+
/** Chrome runs without a window. */
|
|
27
|
+
headless: boolean;
|
|
28
|
+
/** The live view is on for the identity. */
|
|
29
|
+
live: boolean;
|
|
30
|
+
/** The engine that runs it; read mode lays out no page to draw on. */
|
|
31
|
+
engine: string;
|
|
32
|
+
}
|
|
33
|
+
/**
|
|
34
|
+
* Whether the agent cursor is drawn: never in read mode, always with `on`, and with `auto` (the
|
|
35
|
+
* default) when someone can see the browser: a Chrome window, or the live view.
|
|
36
|
+
*/
|
|
37
|
+
export declare function resolveCursor(mode: CursorMode | undefined, ctx: CursorContext): boolean;
|
|
38
|
+
/** One color per agent session, in order: turno's blue first. */
|
|
39
|
+
export declare const CURSOR_COLORS: readonly ["#2b59ff", "#e8590c", "#2f9e44", "#ae3ec9", "#d6336c", "#0c8599"];
|
|
40
|
+
/** The longest glide to a target: long enough to follow, short enough to cost little. */
|
|
41
|
+
export declare const MAX_GLIDE_MS = 420;
|
|
42
|
+
export interface Point {
|
|
43
|
+
x: number;
|
|
44
|
+
y: number;
|
|
45
|
+
}
|
|
46
|
+
/** How long the arrow takes from `from` to `to`: a short base plus the distance, capped. */
|
|
47
|
+
export declare function glideMs(from: Point | undefined, to: Point, max?: number): number;
|
|
48
|
+
/** The center of a box. */
|
|
49
|
+
export declare function centerOf(box: OverlayBox): Point;
|
|
50
|
+
/** True when some of the box is inside a viewport of this size. */
|
|
51
|
+
export declare function inView(box: OverlayBox, view: {
|
|
52
|
+
width: number;
|
|
53
|
+
height: number;
|
|
54
|
+
}): boolean;
|
|
55
|
+
/**
|
|
56
|
+
* The agent's name as people know it, from the client name it sends in `initialize`
|
|
57
|
+
* (`clientInfo.name`): "claude-code" reads "Claude Code", "codex-mcp-client" reads "Codex". A
|
|
58
|
+
* name turno does not know is shown as sent; a "(2)" turno added to tell sessions apart stays.
|
|
59
|
+
*/
|
|
60
|
+
export declare function agentName(client: string): string;
|
|
61
|
+
/** The pill of a session's cursor: the client's name, and the identity when there is one. */
|
|
62
|
+
export declare function cursorLabel(client: string, identity?: string): string;
|
|
63
|
+
interface Actor {
|
|
64
|
+
key: string;
|
|
65
|
+
/** The agent's name without the "(2)" that tells sessions apart. */
|
|
66
|
+
base: string;
|
|
67
|
+
/** The client's name: the badge says it is at work. */
|
|
68
|
+
name: string;
|
|
69
|
+
/** The cursor's pill: the name and the identity. */
|
|
70
|
+
label: string;
|
|
71
|
+
color: string;
|
|
72
|
+
}
|
|
73
|
+
export interface AgentCursorOptions {
|
|
74
|
+
/** The client name of the session that launched the browser (its window title). */
|
|
75
|
+
title: string;
|
|
76
|
+
identity?: string;
|
|
77
|
+
sleep?: (ms: number) => Promise<void>;
|
|
78
|
+
/** The longest glide; 0 places the arrow without a glide. */
|
|
79
|
+
maxGlideMs?: number;
|
|
80
|
+
}
|
|
81
|
+
/** The agent cursor of one running browser: its actors and what each page shows. */
|
|
82
|
+
export declare class AgentCursor {
|
|
83
|
+
private readonly options;
|
|
84
|
+
private readonly actors;
|
|
85
|
+
/** The session acting on each page now (set when a call begins on it). */
|
|
86
|
+
private readonly owners;
|
|
87
|
+
/** Where each session's arrow is on each page, so a new document puts it back. */
|
|
88
|
+
private readonly points;
|
|
89
|
+
private readonly reports;
|
|
90
|
+
private readonly sleep;
|
|
91
|
+
private readonly maxGlideMs;
|
|
92
|
+
private lastKey;
|
|
93
|
+
private paused;
|
|
94
|
+
/** The sessions with a call in flight on this browser, whatever their page. */
|
|
95
|
+
private readonly busy;
|
|
96
|
+
/** The pages the overlay has been on, so a call starting elsewhere updates their badge. */
|
|
97
|
+
private readonly pages;
|
|
98
|
+
constructor(options: AgentCursorOptions);
|
|
99
|
+
/**
|
|
100
|
+
* The actor of a session, with its label and color, made on first use. `client` is the name
|
|
101
|
+
* of the session's own agent when the call said it (mcpd, or a shared host's connection);
|
|
102
|
+
* without it the session takes the launching client's name, with "(2)" and up after the first.
|
|
103
|
+
*/
|
|
104
|
+
actor(key: string, client?: string): Actor;
|
|
105
|
+
/** The session acting on this page now, or the last one that acted anywhere. */
|
|
106
|
+
ownerOf(page: Page): string;
|
|
107
|
+
/** Where the session's arrow is on this page, if it has been there. */
|
|
108
|
+
pointOf(page: Page, key?: string): Point | undefined;
|
|
109
|
+
/** The viewport and motion preference the page last reported. */
|
|
110
|
+
reportOf(page: Page): OverlayReport | undefined;
|
|
111
|
+
/**
|
|
112
|
+
* Sends commands to the page's overlay, with the actor and the arrow's last place first, so a
|
|
113
|
+
* document the overlay has not been on yet shows the same thing. Never throws: a page that
|
|
114
|
+
* closed or navigated away just misses one frame.
|
|
115
|
+
*/
|
|
116
|
+
send(page: Page, commands: OverlayCommand[], key?: string): Promise<OverlayReport | undefined>;
|
|
117
|
+
private remember;
|
|
118
|
+
/** How many sessions other than `key` have a call in flight on this browser: the badge's "+N". */
|
|
119
|
+
private othersThan;
|
|
120
|
+
/** Tells the other pages the overlay is on how many sessions are at work now. */
|
|
121
|
+
private recount;
|
|
122
|
+
/**
|
|
123
|
+
* A call of session `key` begins on this page: the frame and badge come on. `client` names the
|
|
124
|
+
* session's own agent when the call said it.
|
|
125
|
+
*/
|
|
126
|
+
begin(page: Page | undefined, key: string, client?: string): Promise<void>;
|
|
127
|
+
/** The call ended (on this page, which may be a new one): the frame fades after a moment. */
|
|
128
|
+
end(page: Page | undefined, key: string): Promise<void>;
|
|
129
|
+
/** Glides the arrow to `to` and waits for it to arrive. */
|
|
130
|
+
glideTo(page: Page, to: Point): Promise<void>;
|
|
131
|
+
/** Moves the arrow along with a pointer path someone else plays (humanize): a short ease per step. */
|
|
132
|
+
track(page: Page, to: Point): Promise<void>;
|
|
133
|
+
/** Puts the arrow at `to` without a glide. */
|
|
134
|
+
place(page: Page, to: Point): Promise<void>;
|
|
135
|
+
ring(page: Page, box: OverlayBox | null): Promise<void>;
|
|
136
|
+
ripple(page: Page, at: Point): Promise<void>;
|
|
137
|
+
typing(page: Page, box: OverlayBox | null): Promise<void>;
|
|
138
|
+
/** The live view's pause: every page says the user has the controls. */
|
|
139
|
+
setPaused(pages: readonly Page[], on: boolean): Promise<void>;
|
|
140
|
+
/** Runs `fn` with the page's overlay hidden: the model's screenshot shows the page alone. */
|
|
141
|
+
hiddenDuring<T>(page: Page | undefined, fn: () => Promise<T>): Promise<T>;
|
|
142
|
+
}
|
|
143
|
+
/**
|
|
144
|
+
* The input tools' view of a tab that moves the agent cursor: each locator action glides the
|
|
145
|
+
* arrow to its target first, and the coordinate tools' mouse does the same. With humanize on
|
|
146
|
+
* (`humanized`), the pointer path humanize plays goes through this view's mouse, so the arrow
|
|
147
|
+
* follows it step by step instead of gliding on its own.
|
|
148
|
+
*/
|
|
149
|
+
export declare class CursorInput {
|
|
150
|
+
readonly cursor: AgentCursor;
|
|
151
|
+
private readonly humanized;
|
|
152
|
+
/** One view per page, so humanize, which keys its pointer state by page, sees the same one. */
|
|
153
|
+
private readonly views;
|
|
154
|
+
/** The last element marked on each page: where keys typed through the keyboard go. */
|
|
155
|
+
private readonly marked;
|
|
156
|
+
private readonly typingTimers;
|
|
157
|
+
/** Pages where humanize moved the pointer since the last action: the arrow is already there. */
|
|
158
|
+
private readonly moved;
|
|
159
|
+
constructor(cursor: AgentCursor, humanized: boolean);
|
|
160
|
+
private boxOf;
|
|
161
|
+
/**
|
|
162
|
+
* Whether humanize just played a pointer path on this page (and forgets it): then the arrow
|
|
163
|
+
* followed it there, and the action gets no glide of its own. When humanize went straight to
|
|
164
|
+
* the action (no box to aim at), the arrow glides as without humanize.
|
|
165
|
+
*/
|
|
166
|
+
private followed;
|
|
167
|
+
/** Marks the target of an action, runs it, and leaves the mark of its kind. */
|
|
168
|
+
private act;
|
|
169
|
+
/** The cursor view of one of the page's locators. */
|
|
170
|
+
wrapLocator(locator: Locator, page: Page): Locator;
|
|
171
|
+
/** The cursor view of the page's mouse (the coordinate tools, and humanize's pointer path). */
|
|
172
|
+
private wrapMouse;
|
|
173
|
+
/** Keys typed through the keyboard (humanize's typing, browser_press_key): a typing bubble a moment. */
|
|
174
|
+
private keys;
|
|
175
|
+
/** The cursor view of the page's keyboard. */
|
|
176
|
+
private wrapKeyboard;
|
|
177
|
+
/** The cursor view of the page itself: the same object for every call on that page. */
|
|
178
|
+
private viewOf;
|
|
179
|
+
/** A view of one of upstream's tabs whose locators and mouse move the agent cursor. */
|
|
180
|
+
wrapTab<T extends object>(tab: T): T;
|
|
181
|
+
/** A view of upstream's per-session `Context` whose `ensureTab` hands out cursor tabs. */
|
|
182
|
+
wrapContext<T extends object>(context: T): T;
|
|
183
|
+
}
|
|
184
|
+
/**
|
|
185
|
+
* The stock tools of one browser, with the input tools running on cursor tabs. It goes over
|
|
186
|
+
* `humanizeTools` (applied to its result): the outer wrapper's view is the one next to the page,
|
|
187
|
+
* so humanize's pointer path runs through the cursor's mouse. Schemas and every other tool are
|
|
188
|
+
* upstream's own objects.
|
|
189
|
+
*/
|
|
190
|
+
export declare function cursorTools<T extends UpstreamTool>(tools: readonly T[], input: CursorInput): T[];
|
|
191
|
+
export {};
|
|
@@ -0,0 +1,494 @@
|
|
|
1
|
+
import { cursorOverlay } from "./agent-cursor-overlay.js";
|
|
2
|
+
import { HUMANIZED_TOOLS, overlay as override } from "./human-input.js";
|
|
3
|
+
/**
|
|
4
|
+
* Whether the agent cursor is drawn: never in read mode, always with `on`, and with `auto` (the
|
|
5
|
+
* default) when someone can see the browser: a Chrome window, or the live view.
|
|
6
|
+
*/
|
|
7
|
+
export function resolveCursor(mode, ctx) {
|
|
8
|
+
if (ctx.engine === "lightpanda")
|
|
9
|
+
return false;
|
|
10
|
+
const m = mode ?? "auto";
|
|
11
|
+
if (m === "off")
|
|
12
|
+
return false;
|
|
13
|
+
if (m === "on")
|
|
14
|
+
return true;
|
|
15
|
+
return !ctx.headless || ctx.live;
|
|
16
|
+
}
|
|
17
|
+
/** One color per agent session, in order: turno's blue first. */
|
|
18
|
+
export const CURSOR_COLORS = ["#2b59ff", "#e8590c", "#2f9e44", "#ae3ec9", "#d6336c", "#0c8599"];
|
|
19
|
+
/** The longest glide to a target: long enough to follow, short enough to cost little. */
|
|
20
|
+
export const MAX_GLIDE_MS = 420;
|
|
21
|
+
/** How long the arrow takes from `from` to `to`: a short base plus the distance, capped. */
|
|
22
|
+
export function glideMs(from, to, max = MAX_GLIDE_MS) {
|
|
23
|
+
if (max <= 0)
|
|
24
|
+
return 0;
|
|
25
|
+
if (!from)
|
|
26
|
+
return Math.min(max, 220);
|
|
27
|
+
const d = Math.hypot(to.x - from.x, to.y - from.y);
|
|
28
|
+
if (d < 2)
|
|
29
|
+
return 0;
|
|
30
|
+
return Math.round(Math.min(max, 140 + d * 0.35));
|
|
31
|
+
}
|
|
32
|
+
/** The center of a box. */
|
|
33
|
+
export function centerOf(box) {
|
|
34
|
+
return { x: box.x + box.width / 2, y: box.y + box.height / 2 };
|
|
35
|
+
}
|
|
36
|
+
/** True when some of the box is inside a viewport of this size. */
|
|
37
|
+
export function inView(box, view) {
|
|
38
|
+
return box.x + box.width > 0 && box.y + box.height > 0 && box.x < view.width && box.y < view.height;
|
|
39
|
+
}
|
|
40
|
+
/**
|
|
41
|
+
* The agent's name as people know it, from the client name it sends in `initialize`
|
|
42
|
+
* (`clientInfo.name`): "claude-code" reads "Claude Code", "codex-mcp-client" reads "Codex". A
|
|
43
|
+
* name turno does not know is shown as sent; a "(2)" turno added to tell sessions apart stays.
|
|
44
|
+
*/
|
|
45
|
+
export function agentName(client) {
|
|
46
|
+
const raw = client.trim();
|
|
47
|
+
const m = /^(.*?)(\s\(\d+\))?$/.exec(raw);
|
|
48
|
+
const base = (m?.[1] ?? raw).trim();
|
|
49
|
+
const suffix = m?.[2] ?? "";
|
|
50
|
+
const known = [
|
|
51
|
+
[/^claude[-_ ]?code$/i, "Claude Code"],
|
|
52
|
+
[/^claude[-_ ]?ai$/i, "Claude"],
|
|
53
|
+
[/^codex([-_ ]mcp[-_ ]client)?$/i, "Codex"],
|
|
54
|
+
[/^cursor([-_ ]vscode)?$/i, "Cursor"],
|
|
55
|
+
[/^(github[-_ ])?copilot([-_ ]cli)?$/i, "Copilot"],
|
|
56
|
+
[/^gemini([-_ ]cli)?([-_ ]mcp[-_ ]client)?$/i, "Gemini CLI"],
|
|
57
|
+
[/^windsurf([-_ ]client)?$/i, "Windsurf"],
|
|
58
|
+
[/^(visual studio code|vscode)$/i, "VS Code"],
|
|
59
|
+
];
|
|
60
|
+
const pretty = known.find(([re]) => re.test(base))?.[1];
|
|
61
|
+
return `${pretty ?? base}${suffix}`;
|
|
62
|
+
}
|
|
63
|
+
/** The pill of a session's cursor: the client's name, and the identity when there is one. */
|
|
64
|
+
export function cursorLabel(client, identity) {
|
|
65
|
+
const name = client.trim() || "Agent";
|
|
66
|
+
return identity ? `${name} · ${identity}` : name;
|
|
67
|
+
}
|
|
68
|
+
const defaultSleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
|
|
69
|
+
/** The agent cursor of one running browser: its actors and what each page shows. */
|
|
70
|
+
export class AgentCursor {
|
|
71
|
+
options;
|
|
72
|
+
actors = new Map();
|
|
73
|
+
/** The session acting on each page now (set when a call begins on it). */
|
|
74
|
+
owners = new WeakMap();
|
|
75
|
+
/** Where each session's arrow is on each page, so a new document puts it back. */
|
|
76
|
+
points = new WeakMap();
|
|
77
|
+
reports = new WeakMap();
|
|
78
|
+
sleep;
|
|
79
|
+
maxGlideMs;
|
|
80
|
+
lastKey = "";
|
|
81
|
+
paused = false;
|
|
82
|
+
/** The sessions with a call in flight on this browser, whatever their page. */
|
|
83
|
+
busy = new Set();
|
|
84
|
+
/** The pages the overlay has been on, so a call starting elsewhere updates their badge. */
|
|
85
|
+
pages = new Set();
|
|
86
|
+
constructor(options) {
|
|
87
|
+
this.options = options;
|
|
88
|
+
this.sleep = options.sleep ?? defaultSleep;
|
|
89
|
+
this.maxGlideMs = options.maxGlideMs ?? MAX_GLIDE_MS;
|
|
90
|
+
}
|
|
91
|
+
/**
|
|
92
|
+
* The actor of a session, with its label and color, made on first use. `client` is the name
|
|
93
|
+
* of the session's own agent when the call said it (mcpd, or a shared host's connection);
|
|
94
|
+
* without it the session takes the launching client's name, with "(2)" and up after the first.
|
|
95
|
+
*/
|
|
96
|
+
actor(key, client) {
|
|
97
|
+
let a = this.actors.get(key);
|
|
98
|
+
if (a)
|
|
99
|
+
return a;
|
|
100
|
+
const n = this.actors.size;
|
|
101
|
+
let name;
|
|
102
|
+
if (client) {
|
|
103
|
+
const base = agentName(client);
|
|
104
|
+
const same = [...this.actors.values()].filter((o) => o.base === base).length;
|
|
105
|
+
name = same === 0 ? base : `${base} (${same + 1})`;
|
|
106
|
+
}
|
|
107
|
+
else
|
|
108
|
+
name = agentName(n === 0 ? this.options.title : `${this.options.title} (${n + 1})`);
|
|
109
|
+
name = name.trim() || "Agent";
|
|
110
|
+
a = {
|
|
111
|
+
key,
|
|
112
|
+
base: name.replace(/\s\(\d+\)$/, ""),
|
|
113
|
+
name,
|
|
114
|
+
label: cursorLabel(name, this.options.identity),
|
|
115
|
+
color: CURSOR_COLORS[n % CURSOR_COLORS.length],
|
|
116
|
+
};
|
|
117
|
+
this.actors.set(key, a);
|
|
118
|
+
return a;
|
|
119
|
+
}
|
|
120
|
+
/** The session acting on this page now, or the last one that acted anywhere. */
|
|
121
|
+
ownerOf(page) {
|
|
122
|
+
return this.owners.get(page) ?? this.lastKey;
|
|
123
|
+
}
|
|
124
|
+
/** Where the session's arrow is on this page, if it has been there. */
|
|
125
|
+
pointOf(page, key = this.ownerOf(page)) {
|
|
126
|
+
return this.points.get(page)?.get(key);
|
|
127
|
+
}
|
|
128
|
+
/** The viewport and motion preference the page last reported. */
|
|
129
|
+
reportOf(page) {
|
|
130
|
+
return this.reports.get(page);
|
|
131
|
+
}
|
|
132
|
+
/**
|
|
133
|
+
* Sends commands to the page's overlay, with the actor and the arrow's last place first, so a
|
|
134
|
+
* document the overlay has not been on yet shows the same thing. Never throws: a page that
|
|
135
|
+
* closed or navigated away just misses one frame.
|
|
136
|
+
*/
|
|
137
|
+
async send(page, commands, key = this.ownerOf(page)) {
|
|
138
|
+
if (page.isClosed())
|
|
139
|
+
return undefined;
|
|
140
|
+
// The direct session's key is "" (`DIRECT_SESSION`): every key, that one too, is an actor.
|
|
141
|
+
const a = this.actor(key);
|
|
142
|
+
const prelude = [{ op: "actor", key, label: a.label, name: a.name, color: a.color }];
|
|
143
|
+
const at = this.pointOf(page, key);
|
|
144
|
+
if (at)
|
|
145
|
+
prelude.push({ op: "move", key, x: at.x, y: at.y, ms: 0 });
|
|
146
|
+
prelude.push({ op: "paused", on: this.paused }, { op: "others", n: this.othersThan(key) });
|
|
147
|
+
try {
|
|
148
|
+
const report = await page.evaluate(cursorOverlay, [...prelude, ...commands]);
|
|
149
|
+
this.reports.set(page, report);
|
|
150
|
+
this.pages.add(page);
|
|
151
|
+
return report;
|
|
152
|
+
}
|
|
153
|
+
catch {
|
|
154
|
+
return undefined;
|
|
155
|
+
}
|
|
156
|
+
}
|
|
157
|
+
remember(page, key, point) {
|
|
158
|
+
let m = this.points.get(page);
|
|
159
|
+
if (!m) {
|
|
160
|
+
m = new Map();
|
|
161
|
+
this.points.set(page, m);
|
|
162
|
+
}
|
|
163
|
+
m.set(key, point);
|
|
164
|
+
}
|
|
165
|
+
/** How many sessions other than `key` have a call in flight on this browser: the badge's "+N". */
|
|
166
|
+
othersThan(key) {
|
|
167
|
+
return this.busy.size - (this.busy.has(key) ? 1 : 0);
|
|
168
|
+
}
|
|
169
|
+
/** Tells the other pages the overlay is on how many sessions are at work now. */
|
|
170
|
+
recount(except) {
|
|
171
|
+
for (const page of this.pages) {
|
|
172
|
+
if (page.isClosed()) {
|
|
173
|
+
this.pages.delete(page);
|
|
174
|
+
continue;
|
|
175
|
+
}
|
|
176
|
+
if (page === except)
|
|
177
|
+
continue;
|
|
178
|
+
const owner = this.ownerOf(page);
|
|
179
|
+
void this.send(page, [], owner);
|
|
180
|
+
}
|
|
181
|
+
}
|
|
182
|
+
/**
|
|
183
|
+
* A call of session `key` begins on this page: the frame and badge come on. `client` names the
|
|
184
|
+
* session's own agent when the call said it.
|
|
185
|
+
*/
|
|
186
|
+
async begin(page, key, client) {
|
|
187
|
+
this.actor(key, client);
|
|
188
|
+
this.busy.add(key);
|
|
189
|
+
this.recount(page);
|
|
190
|
+
if (!page)
|
|
191
|
+
return;
|
|
192
|
+
this.owners.set(page, key);
|
|
193
|
+
this.lastKey = key;
|
|
194
|
+
await this.send(page, [{ op: "busy", key, on: true }], key);
|
|
195
|
+
}
|
|
196
|
+
/** The call ended (on this page, which may be a new one): the frame fades after a moment. */
|
|
197
|
+
async end(page, key) {
|
|
198
|
+
this.busy.delete(key);
|
|
199
|
+
this.recount(page);
|
|
200
|
+
if (!page)
|
|
201
|
+
return;
|
|
202
|
+
this.owners.set(page, key);
|
|
203
|
+
await this.send(page, [{ op: "busy", key, on: false }], key);
|
|
204
|
+
}
|
|
205
|
+
/** Glides the arrow to `to` and waits for it to arrive. */
|
|
206
|
+
async glideTo(page, to) {
|
|
207
|
+
const key = this.ownerOf(page);
|
|
208
|
+
const report = this.reports.get(page);
|
|
209
|
+
const ms = report?.reducedMotion ? 0 : glideMs(this.pointOf(page, key), to, this.maxGlideMs);
|
|
210
|
+
// Sent from the last place (the batch's prelude puts the arrow there), then remembered.
|
|
211
|
+
await this.send(page, [{ op: "move", key, x: to.x, y: to.y, ms }], key);
|
|
212
|
+
this.remember(page, key, to);
|
|
213
|
+
if (ms > 0)
|
|
214
|
+
await this.sleep(ms);
|
|
215
|
+
}
|
|
216
|
+
/** Moves the arrow along with a pointer path someone else plays (humanize): a short ease per step. */
|
|
217
|
+
async track(page, to) {
|
|
218
|
+
const key = this.ownerOf(page);
|
|
219
|
+
// Remembered first: the steps are sent without waiting, so the next step's batch must not
|
|
220
|
+
// put the arrow back where this one started.
|
|
221
|
+
this.remember(page, key, to);
|
|
222
|
+
await this.send(page, [{ op: "move", key, x: to.x, y: to.y, ms: 40 }], key);
|
|
223
|
+
}
|
|
224
|
+
/** Puts the arrow at `to` without a glide. */
|
|
225
|
+
async place(page, to) {
|
|
226
|
+
const key = this.ownerOf(page);
|
|
227
|
+
await this.send(page, [{ op: "move", key, x: to.x, y: to.y, ms: 0 }], key);
|
|
228
|
+
this.remember(page, key, to);
|
|
229
|
+
}
|
|
230
|
+
async ring(page, box) {
|
|
231
|
+
const key = this.ownerOf(page);
|
|
232
|
+
await this.send(page, [{ op: "ring", key, box }], key);
|
|
233
|
+
}
|
|
234
|
+
async ripple(page, at) {
|
|
235
|
+
const key = this.ownerOf(page);
|
|
236
|
+
await this.send(page, [{ op: "ripple", key, x: at.x, y: at.y }], key);
|
|
237
|
+
}
|
|
238
|
+
async typing(page, box) {
|
|
239
|
+
const key = this.ownerOf(page);
|
|
240
|
+
await this.send(page, [{ op: "typing", key, box }], key);
|
|
241
|
+
}
|
|
242
|
+
/** The live view's pause: every page says the user has the controls. */
|
|
243
|
+
async setPaused(pages, on) {
|
|
244
|
+
this.paused = on;
|
|
245
|
+
await Promise.all(pages.map((page) => this.send(page, [])));
|
|
246
|
+
}
|
|
247
|
+
/** Runs `fn` with the page's overlay hidden: the model's screenshot shows the page alone. */
|
|
248
|
+
async hiddenDuring(page, fn) {
|
|
249
|
+
if (!page || !this.reports.has(page))
|
|
250
|
+
return await fn();
|
|
251
|
+
await this.send(page, [{ op: "visible", on: false }]);
|
|
252
|
+
try {
|
|
253
|
+
return await fn();
|
|
254
|
+
}
|
|
255
|
+
finally {
|
|
256
|
+
await this.send(page, [{ op: "visible", on: true }]);
|
|
257
|
+
}
|
|
258
|
+
}
|
|
259
|
+
}
|
|
260
|
+
/** How long the cursor waits for an element's box before it lets the action run without it. */
|
|
261
|
+
const BOX_TIMEOUT_MS = 500;
|
|
262
|
+
/** The locator methods the input tools call, and the mark each leaves. */
|
|
263
|
+
const LOCATOR_KINDS = {
|
|
264
|
+
click: "click",
|
|
265
|
+
dblclick: "click",
|
|
266
|
+
tap: "click",
|
|
267
|
+
check: "click",
|
|
268
|
+
uncheck: "click",
|
|
269
|
+
setChecked: "click",
|
|
270
|
+
selectOption: "click",
|
|
271
|
+
hover: "hover",
|
|
272
|
+
fill: "type",
|
|
273
|
+
pressSequentially: "type",
|
|
274
|
+
press: "type",
|
|
275
|
+
dragTo: "drag",
|
|
276
|
+
};
|
|
277
|
+
/**
|
|
278
|
+
* The input tools' view of a tab that moves the agent cursor: each locator action glides the
|
|
279
|
+
* arrow to its target first, and the coordinate tools' mouse does the same. With humanize on
|
|
280
|
+
* (`humanized`), the pointer path humanize plays goes through this view's mouse, so the arrow
|
|
281
|
+
* follows it step by step instead of gliding on its own.
|
|
282
|
+
*/
|
|
283
|
+
export class CursorInput {
|
|
284
|
+
cursor;
|
|
285
|
+
humanized;
|
|
286
|
+
/** One view per page, so humanize, which keys its pointer state by page, sees the same one. */
|
|
287
|
+
views = new WeakMap();
|
|
288
|
+
/** The last element marked on each page: where keys typed through the keyboard go. */
|
|
289
|
+
marked = new WeakMap();
|
|
290
|
+
typingTimers = new WeakMap();
|
|
291
|
+
/** Pages where humanize moved the pointer since the last action: the arrow is already there. */
|
|
292
|
+
moved = new WeakSet();
|
|
293
|
+
constructor(cursor, humanized) {
|
|
294
|
+
this.cursor = cursor;
|
|
295
|
+
this.humanized = humanized;
|
|
296
|
+
}
|
|
297
|
+
async boxOf(locator, timeout = BOX_TIMEOUT_MS) {
|
|
298
|
+
try {
|
|
299
|
+
const box = await locator.boundingBox({ timeout });
|
|
300
|
+
return box && box.width > 0 && box.height > 0 ? box : null;
|
|
301
|
+
}
|
|
302
|
+
catch {
|
|
303
|
+
return null;
|
|
304
|
+
}
|
|
305
|
+
}
|
|
306
|
+
/**
|
|
307
|
+
* Whether humanize just played a pointer path on this page (and forgets it): then the arrow
|
|
308
|
+
* followed it there, and the action gets no glide of its own. When humanize went straight to
|
|
309
|
+
* the action (no box to aim at), the arrow glides as without humanize.
|
|
310
|
+
*/
|
|
311
|
+
followed(page) {
|
|
312
|
+
if (!this.humanized || !this.moved.has(page))
|
|
313
|
+
return false;
|
|
314
|
+
this.moved.delete(page);
|
|
315
|
+
return true;
|
|
316
|
+
}
|
|
317
|
+
/** Marks the target of an action, runs it, and leaves the mark of its kind. */
|
|
318
|
+
async act(page, locator, kind, run, to) {
|
|
319
|
+
const c = this.cursor;
|
|
320
|
+
const followed = this.followed(page);
|
|
321
|
+
let box = await this.boxOf(locator);
|
|
322
|
+
const view = c.reportOf(page) ?? (await c.send(page, []));
|
|
323
|
+
if (box && view && !inView(box, view))
|
|
324
|
+
box = null; // Playwright scrolls it in: mark it after
|
|
325
|
+
if (box) {
|
|
326
|
+
this.marked.set(page, box);
|
|
327
|
+
void c.ring(page, box);
|
|
328
|
+
if (!followed)
|
|
329
|
+
await c.glideTo(page, centerOf(box));
|
|
330
|
+
if (kind === "type")
|
|
331
|
+
void c.typing(page, box);
|
|
332
|
+
if (kind === "click" && !followed)
|
|
333
|
+
void c.ripple(page, centerOf(box));
|
|
334
|
+
}
|
|
335
|
+
try {
|
|
336
|
+
const result = await run();
|
|
337
|
+
if (kind === "drag" && to) {
|
|
338
|
+
const end = await this.boxOf(to, 200);
|
|
339
|
+
if (end) {
|
|
340
|
+
void c.ring(page, end);
|
|
341
|
+
await c.place(page, centerOf(end));
|
|
342
|
+
}
|
|
343
|
+
}
|
|
344
|
+
else if (!box || (kind === "click" && followed)) {
|
|
345
|
+
// Marked after: the element came into view during the action, or humanize moved there.
|
|
346
|
+
const after = box ?? (await this.boxOf(locator, 200));
|
|
347
|
+
const center = after ? centerOf(after) : undefined;
|
|
348
|
+
const at = followed ? (c.pointOf(page) ?? center) : center;
|
|
349
|
+
if (after && !box) {
|
|
350
|
+
this.marked.set(page, after);
|
|
351
|
+
void c.ring(page, after);
|
|
352
|
+
}
|
|
353
|
+
if (at) {
|
|
354
|
+
if (!followed)
|
|
355
|
+
await c.place(page, at);
|
|
356
|
+
if (kind === "click")
|
|
357
|
+
void c.ripple(page, at);
|
|
358
|
+
}
|
|
359
|
+
}
|
|
360
|
+
return result;
|
|
361
|
+
}
|
|
362
|
+
finally {
|
|
363
|
+
if (kind === "type")
|
|
364
|
+
void c.typing(page, null);
|
|
365
|
+
}
|
|
366
|
+
}
|
|
367
|
+
/** The cursor view of one of the page's locators. */
|
|
368
|
+
wrapLocator(locator, page) {
|
|
369
|
+
const original = locator;
|
|
370
|
+
const overrides = {};
|
|
371
|
+
for (const [method, kind] of Object.entries(LOCATOR_KINDS)) {
|
|
372
|
+
overrides[method] = (...args) => {
|
|
373
|
+
const to = method === "dragTo" ? args[0] : undefined;
|
|
374
|
+
return this.act(page, locator, kind, () => original[method].apply(locator, args), to);
|
|
375
|
+
};
|
|
376
|
+
}
|
|
377
|
+
return override(locator, overrides);
|
|
378
|
+
}
|
|
379
|
+
/** The cursor view of the page's mouse (the coordinate tools, and humanize's pointer path). */
|
|
380
|
+
wrapMouse(page) {
|
|
381
|
+
const mouse = page.mouse;
|
|
382
|
+
const c = this.cursor;
|
|
383
|
+
return override(mouse, {
|
|
384
|
+
move: async (x, y, options) => {
|
|
385
|
+
if (this.humanized) {
|
|
386
|
+
// A step of humanize's pointer path: the arrow follows it.
|
|
387
|
+
this.moved.add(page);
|
|
388
|
+
void c.track(page, { x, y });
|
|
389
|
+
}
|
|
390
|
+
else
|
|
391
|
+
await c.glideTo(page, { x, y });
|
|
392
|
+
await mouse.move(x, y, options);
|
|
393
|
+
},
|
|
394
|
+
click: async (x, y, options) => {
|
|
395
|
+
if (!this.followed(page))
|
|
396
|
+
await c.glideTo(page, { x, y });
|
|
397
|
+
void c.ripple(page, { x, y });
|
|
398
|
+
await mouse.click(x, y, options);
|
|
399
|
+
},
|
|
400
|
+
dblclick: async (x, y, options) => {
|
|
401
|
+
if (!this.followed(page))
|
|
402
|
+
await c.glideTo(page, { x, y });
|
|
403
|
+
void c.ripple(page, { x, y });
|
|
404
|
+
await mouse.dblclick(x, y, options);
|
|
405
|
+
},
|
|
406
|
+
down: async (options) => {
|
|
407
|
+
const at = c.pointOf(page);
|
|
408
|
+
if (at)
|
|
409
|
+
void c.ripple(page, at);
|
|
410
|
+
await mouse.down(options);
|
|
411
|
+
},
|
|
412
|
+
});
|
|
413
|
+
}
|
|
414
|
+
/** Keys typed through the keyboard (humanize's typing, browser_press_key): a typing bubble a moment. */
|
|
415
|
+
keys(page) {
|
|
416
|
+
const box = this.marked.get(page);
|
|
417
|
+
if (!box)
|
|
418
|
+
return;
|
|
419
|
+
const old = this.typingTimers.get(page);
|
|
420
|
+
if (old)
|
|
421
|
+
clearTimeout(old);
|
|
422
|
+
else
|
|
423
|
+
void this.cursor.typing(page, box);
|
|
424
|
+
const timer = setTimeout(() => {
|
|
425
|
+
this.typingTimers.delete(page);
|
|
426
|
+
void this.cursor.typing(page, null);
|
|
427
|
+
}, 900);
|
|
428
|
+
timer.unref?.();
|
|
429
|
+
this.typingTimers.set(page, timer);
|
|
430
|
+
}
|
|
431
|
+
/** The cursor view of the page's keyboard. */
|
|
432
|
+
wrapKeyboard(page) {
|
|
433
|
+
const keyboard = page.keyboard;
|
|
434
|
+
return override(keyboard, {
|
|
435
|
+
press: async (key, options) => {
|
|
436
|
+
this.keys(page);
|
|
437
|
+
await keyboard.press(key, options);
|
|
438
|
+
},
|
|
439
|
+
type: async (text, options) => {
|
|
440
|
+
this.keys(page);
|
|
441
|
+
await keyboard.type(text, options);
|
|
442
|
+
},
|
|
443
|
+
insertText: async (text) => {
|
|
444
|
+
this.keys(page);
|
|
445
|
+
await keyboard.insertText(text);
|
|
446
|
+
},
|
|
447
|
+
});
|
|
448
|
+
}
|
|
449
|
+
/** The cursor view of the page itself: the same object for every call on that page. */
|
|
450
|
+
viewOf(page) {
|
|
451
|
+
let v = this.views.get(page);
|
|
452
|
+
if (!v) {
|
|
453
|
+
v = override(page, { mouse: this.wrapMouse(page), keyboard: this.wrapKeyboard(page) });
|
|
454
|
+
this.views.set(page, v);
|
|
455
|
+
}
|
|
456
|
+
return v;
|
|
457
|
+
}
|
|
458
|
+
/** A view of one of upstream's tabs whose locators and mouse move the agent cursor. */
|
|
459
|
+
wrapTab(tab) {
|
|
460
|
+
const page = tab.page;
|
|
461
|
+
const t = tab;
|
|
462
|
+
return override(tab, {
|
|
463
|
+
page: this.viewOf(page),
|
|
464
|
+
targetLocator: async (params) => {
|
|
465
|
+
const r = await t.targetLocator(params);
|
|
466
|
+
return { ...r, locator: this.wrapLocator(r.locator, page) };
|
|
467
|
+
},
|
|
468
|
+
targetLocators: async (params) => (await t.targetLocators(params)).map((r) => ({ ...r, locator: this.wrapLocator(r.locator, page) })),
|
|
469
|
+
});
|
|
470
|
+
}
|
|
471
|
+
/** A view of upstream's per-session `Context` whose `ensureTab` hands out cursor tabs. */
|
|
472
|
+
wrapContext(context) {
|
|
473
|
+
const c = context;
|
|
474
|
+
return override(context, {
|
|
475
|
+
ensureTab: async (...args) => this.wrapTab(await c.ensureTab(...args)),
|
|
476
|
+
});
|
|
477
|
+
}
|
|
478
|
+
}
|
|
479
|
+
/**
|
|
480
|
+
* The stock tools of one browser, with the input tools running on cursor tabs. It goes over
|
|
481
|
+
* `humanizeTools` (applied to its result): the outer wrapper's view is the one next to the page,
|
|
482
|
+
* so humanize's pointer path runs through the cursor's mouse. Schemas and every other tool are
|
|
483
|
+
* upstream's own objects.
|
|
484
|
+
*/
|
|
485
|
+
export function cursorTools(tools, input) {
|
|
486
|
+
return tools.map((tool) => {
|
|
487
|
+
if (!HUMANIZED_TOOLS.has(tool.schema.name))
|
|
488
|
+
return tool;
|
|
489
|
+
const handled = tool;
|
|
490
|
+
const handle = (context, params, response, signal) => handled.handle(input.wrapContext(context), params, response, signal);
|
|
491
|
+
return { ...tool, handle };
|
|
492
|
+
});
|
|
493
|
+
}
|
|
494
|
+
//# sourceMappingURL=agent-cursor.js.map
|