scenescout 1.1.0 → 1.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +29 -0
- package/README.md +126 -27
- package/dist/browsers.js +196 -0
- package/dist/cli.js +308 -70
- package/dist/clients.js +213 -0
- package/dist/engine/browser.js +148 -20
- package/dist/engine/hover.js +42 -0
- package/dist/engine/launch.js +12 -5
- package/dist/engine/live-page.js +644 -0
- package/dist/engine/live.js +549 -0
- package/dist/engine/memory.js +3 -0
- package/dist/engine/probes.js +4 -1
- package/dist/engine/report.js +8 -2
- package/dist/installer.js +106 -12
- package/dist/mcp-server.js +268 -28
- package/dist/playbook.js +83 -0
- package/package.json +17 -5
- package/skills/scenescout/SKILL.md +5 -4
package/dist/clients.js
ADDED
|
@@ -0,0 +1,213 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Registering the MCP server with clients other than Claude Code.
|
|
3
|
+
*
|
|
4
|
+
* Each client keeps its server list somewhere different. Where a client has a
|
|
5
|
+
* command for adding a server, that command is used: it knows its own config
|
|
6
|
+
* format and location. Where it has none, the JSON file it reads is edited in
|
|
7
|
+
* place, keeping every other entry.
|
|
8
|
+
*
|
|
9
|
+
* Nothing here launches a process or touches the home directory on its own:
|
|
10
|
+
* the runner, the home directory and the platform are passed in, so every rule
|
|
11
|
+
* can be table-tested.
|
|
12
|
+
*/
|
|
13
|
+
import fs from "node:fs";
|
|
14
|
+
import path from "node:path";
|
|
15
|
+
import { MCP_NAME } from "./installer.js";
|
|
16
|
+
export const OTHER_CLIENTS = ["cursor", "vscode", "codex", "gemini", "copilot", "windsurf"];
|
|
17
|
+
export const CLIENTS = ["claude-code", ...OTHER_CLIENTS];
|
|
18
|
+
export const CLIENT_LABELS = {
|
|
19
|
+
"claude-code": "Claude Code",
|
|
20
|
+
cursor: "Cursor",
|
|
21
|
+
vscode: "VS Code (GitHub Copilot agent mode)",
|
|
22
|
+
codex: "Codex CLI",
|
|
23
|
+
gemini: "Gemini CLI",
|
|
24
|
+
copilot: "GitHub Copilot CLI",
|
|
25
|
+
windsurf: "Windsurf",
|
|
26
|
+
};
|
|
27
|
+
/** Read the value of `--client`. Absent means Claude Code, which is what install has always set up. */
|
|
28
|
+
export function parseClients(value) {
|
|
29
|
+
if (value === undefined)
|
|
30
|
+
return { clients: ["claude-code"] };
|
|
31
|
+
const names = value
|
|
32
|
+
.split(",")
|
|
33
|
+
.map((s) => s.trim().toLowerCase())
|
|
34
|
+
.filter(Boolean);
|
|
35
|
+
const choices = CLIENTS.join(", ");
|
|
36
|
+
if (names.length === 0)
|
|
37
|
+
return { error: `--client needs a value. Choose from: ${choices}.` };
|
|
38
|
+
const picked = new Set();
|
|
39
|
+
for (const name of names) {
|
|
40
|
+
if (!CLIENTS.includes(name))
|
|
41
|
+
return { error: `"${name}" is not a client install knows how to set up. Choose from: ${choices}.` };
|
|
42
|
+
picked.add(name);
|
|
43
|
+
}
|
|
44
|
+
return { clients: CLIENTS.filter((c) => picked.has(c)) };
|
|
45
|
+
}
|
|
46
|
+
const STRATEGIES = {
|
|
47
|
+
cursor: { kind: "file", file: (home) => path.join(home, ".cursor", "mcp.json"), key: "mcpServers" },
|
|
48
|
+
windsurf: { kind: "file", file: (home) => path.join(home, ".codeium", "windsurf", "mcp_config.json"), key: "mcpServers" },
|
|
49
|
+
// `add` replaces an entry of the same name.
|
|
50
|
+
codex: { kind: "command", binary: "codex", add: (launch) => ["mcp", "add", MCP_NAME, "--", ...launch] },
|
|
51
|
+
// `add` updates an entry of the same name. The default scope is the project; a tool like this belongs to the user.
|
|
52
|
+
gemini: { kind: "command", binary: "gemini", add: (launch) => ["mcp", "add", "--scope", "user", MCP_NAME, ...launch] },
|
|
53
|
+
// `add` refuses a name that already exists, so the old entry is removed first.
|
|
54
|
+
copilot: { kind: "command", binary: "copilot", add: (launch) => ["mcp", "add", MCP_NAME, "--", ...launch], removeFirst: ["mcp", "remove", MCP_NAME] },
|
|
55
|
+
};
|
|
56
|
+
const quote = (s) => (/^[\w@%+=:,./-]+$/.test(s) ? s : `'${s.replace(/'/g, `'\\''`)}'`);
|
|
57
|
+
/** The entry every `mcpServers`-style file takes. */
|
|
58
|
+
export function serverEntry(launch) {
|
|
59
|
+
return { command: launch[0], args: launch.slice(1) };
|
|
60
|
+
}
|
|
61
|
+
/**
|
|
62
|
+
* Put the server into a client's JSON server list, keeping everything else in
|
|
63
|
+
* the file. A file that is not valid JSON is left exactly as it is: rewriting
|
|
64
|
+
* it would discard whatever the person had in it.
|
|
65
|
+
*/
|
|
66
|
+
export function registerInFile(file, key, launch) {
|
|
67
|
+
const entry = serverEntry(launch);
|
|
68
|
+
const manual = `add this under "${key}" in ${file}:\n ${JSON.stringify({ [MCP_NAME]: entry })}`;
|
|
69
|
+
let config = {};
|
|
70
|
+
// A config kept in a dotfiles repository is a link. Renaming over the link
|
|
71
|
+
// would replace it with a plain file and leave the real one unchanged, so
|
|
72
|
+
// the write goes to whatever the link points at.
|
|
73
|
+
let target = file;
|
|
74
|
+
let mode = 0o600;
|
|
75
|
+
try {
|
|
76
|
+
if (fs.existsSync(file)) {
|
|
77
|
+
target = fs.realpathSync(file);
|
|
78
|
+
mode = fs.statSync(target).mode & 0o777;
|
|
79
|
+
// Some editors save with a byte-order mark, which JSON.parse rejects.
|
|
80
|
+
const raw = fs.readFileSync(target, "utf8").replace(/^\uFEFF/, "");
|
|
81
|
+
if (raw.trim().length > 0) {
|
|
82
|
+
let parsed;
|
|
83
|
+
try {
|
|
84
|
+
parsed = JSON.parse(raw);
|
|
85
|
+
}
|
|
86
|
+
catch (err) {
|
|
87
|
+
return {
|
|
88
|
+
status: "failed",
|
|
89
|
+
detail: `${file} is not valid JSON (${err instanceof Error ? err.message : String(err)}), so it was left untouched`,
|
|
90
|
+
manual,
|
|
91
|
+
};
|
|
92
|
+
}
|
|
93
|
+
if (parsed === null || typeof parsed !== "object" || Array.isArray(parsed)) {
|
|
94
|
+
return { status: "failed", detail: `${file} does not hold a JSON object, so it was left untouched`, manual };
|
|
95
|
+
}
|
|
96
|
+
config = parsed;
|
|
97
|
+
}
|
|
98
|
+
}
|
|
99
|
+
}
|
|
100
|
+
catch (err) {
|
|
101
|
+
return { status: "failed", detail: `${file} could not be read (${err instanceof Error ? err.message : String(err)})`, manual };
|
|
102
|
+
}
|
|
103
|
+
const existing = config[key];
|
|
104
|
+
if (existing !== undefined && (existing === null || typeof existing !== "object" || Array.isArray(existing))) {
|
|
105
|
+
return { status: "failed", detail: `"${key}" in ${file} is not an object, so the file was left untouched`, manual };
|
|
106
|
+
}
|
|
107
|
+
const servers = (existing ?? {});
|
|
108
|
+
const before = servers[MCP_NAME];
|
|
109
|
+
const notes = [];
|
|
110
|
+
if (before !== undefined && JSON.stringify(before) !== JSON.stringify(entry)) {
|
|
111
|
+
notes.push(`the previous "${MCP_NAME}" entry was replaced; it ran: ${JSON.stringify(before)}`);
|
|
112
|
+
}
|
|
113
|
+
config[key] = { ...servers, [MCP_NAME]: entry };
|
|
114
|
+
// Written beside the target and renamed over it, so a crash cannot leave half a file.
|
|
115
|
+
const tmp = `${target}.scenescout-${process.pid}.tmp`;
|
|
116
|
+
try {
|
|
117
|
+
fs.mkdirSync(path.dirname(target), { recursive: true });
|
|
118
|
+
fs.writeFileSync(tmp, `${JSON.stringify(config, null, 2)}\n`, { mode });
|
|
119
|
+
fs.renameSync(tmp, target);
|
|
120
|
+
}
|
|
121
|
+
catch (err) {
|
|
122
|
+
// The temporary file is a full copy of the config, which can hold secrets in `env`; it must not stay behind.
|
|
123
|
+
fs.rmSync(tmp, { force: true });
|
|
124
|
+
return { status: "failed", detail: `${file} could not be written (${err instanceof Error ? err.message : String(err)}), so it was left untouched`, manual };
|
|
125
|
+
}
|
|
126
|
+
return { status: "registered", where: file, replaced: before !== undefined, notes };
|
|
127
|
+
}
|
|
128
|
+
function registerWithCommand(client, launch, run) {
|
|
129
|
+
const manual = [client.binary, ...client.add(launch)].map(quote).join(" ");
|
|
130
|
+
let removed = false;
|
|
131
|
+
if (client.removeFirst) {
|
|
132
|
+
const removal = run(client.binary, client.removeFirst);
|
|
133
|
+
if (removal.missing)
|
|
134
|
+
return { status: "client-missing", manual };
|
|
135
|
+
// A non-zero exit here only means there was nothing of that name to remove.
|
|
136
|
+
removed = removal.status === 0;
|
|
137
|
+
}
|
|
138
|
+
const added = run(client.binary, client.add(launch));
|
|
139
|
+
if (added.missing)
|
|
140
|
+
return { status: "client-missing", manual };
|
|
141
|
+
if (added.status !== 0) {
|
|
142
|
+
const reason = (added.stdout + added.stderr).trim().split("\n")[0] || `exit code ${added.status}`;
|
|
143
|
+
// Having removed the old entry to make room, say so: the person now has no registration at all.
|
|
144
|
+
const lost = removed ? ` The previous "${MCP_NAME}" entry had already been removed to make room, so ${client.binary} now has none.` : "";
|
|
145
|
+
return { status: "failed", detail: reason + lost, manual };
|
|
146
|
+
}
|
|
147
|
+
return { status: "registered", where: `${client.binary} mcp`, replaced: removed, notes: [] };
|
|
148
|
+
}
|
|
149
|
+
/**
|
|
150
|
+
* The VS Code command line, or null when only a fork is installed. Cursor and
|
|
151
|
+
* Windsurf both install a `code` command of their own, and running that one
|
|
152
|
+
* registers the server in the wrong editor: it reports success and VS Code
|
|
153
|
+
* never sees the entry. A `code` that resolves into another editor's files is
|
|
154
|
+
* therefore not VS Code.
|
|
155
|
+
*
|
|
156
|
+
* The real path is only inspected. What gets run is the command as found on
|
|
157
|
+
* PATH: some installs (snap) link `code` to a launcher that decides what to
|
|
158
|
+
* start from the name it was called by, and running the link's target directly
|
|
159
|
+
* starts the wrong thing.
|
|
160
|
+
*/
|
|
161
|
+
export function vscodeBinary(opts) {
|
|
162
|
+
if (opts.platform === "darwin") {
|
|
163
|
+
for (const root of ["/Applications", path.posix.join(opts.home, "Applications")]) {
|
|
164
|
+
const bundled = path.posix.join(root, "Visual Studio Code.app", "Contents", "Resources", "app", "bin", "code");
|
|
165
|
+
if (opts.exists(bundled))
|
|
166
|
+
return bundled;
|
|
167
|
+
}
|
|
168
|
+
}
|
|
169
|
+
if (!opts.codeOnPath)
|
|
170
|
+
return null;
|
|
171
|
+
// Looked for below the home directory's own name, so an account called "cursor" does not disqualify every install under it.
|
|
172
|
+
const real = opts.codeOnPath.realPath;
|
|
173
|
+
const belowHome = real.toLowerCase().startsWith(opts.home.toLowerCase()) ? real.slice(opts.home.length) : real;
|
|
174
|
+
return /cursor|windsurf|codeium|vscodium/i.test(belowHome) ? null : opts.codeOnPath.command;
|
|
175
|
+
}
|
|
176
|
+
/** What `code --add-mcp` takes: the entry plus its name. */
|
|
177
|
+
export function vscodeAddArgs(launch) {
|
|
178
|
+
return ["--add-mcp", JSON.stringify({ name: MCP_NAME, ...serverEntry(launch) })];
|
|
179
|
+
}
|
|
180
|
+
export function registerWithClient(client, opts) {
|
|
181
|
+
if (client === "vscode") {
|
|
182
|
+
const manual = `in VS Code run "MCP: Add Server…" and choose a command (stdio) server, or run:\n code ${vscodeAddArgs(opts.launch).map(quote).join(" ")}`;
|
|
183
|
+
if (!opts.vscode)
|
|
184
|
+
return { status: "client-missing", manual };
|
|
185
|
+
const added = opts.run(opts.vscode, vscodeAddArgs(opts.launch));
|
|
186
|
+
if (added.missing)
|
|
187
|
+
return { status: "client-missing", manual };
|
|
188
|
+
const output = (added.stdout + added.stderr).trim();
|
|
189
|
+
if (added.status !== 0)
|
|
190
|
+
return { status: "failed", detail: output.split("\n")[0] || `exit code ${added.status}`, manual };
|
|
191
|
+
// This command replaces an entry of the same name and does not say whether there was one.
|
|
192
|
+
return { status: "registered", where: "VS Code's user profile", replaced: false, notes: [] };
|
|
193
|
+
}
|
|
194
|
+
const strategy = STRATEGIES[client];
|
|
195
|
+
return strategy.kind === "file" ? registerInFile(strategy.file(opts.home), strategy.key, opts.launch) : registerWithCommand(strategy, opts.launch, opts.run);
|
|
196
|
+
}
|
|
197
|
+
/** How to register with a client by hand, without running anything. */
|
|
198
|
+
export function manualFor(client, launch, home) {
|
|
199
|
+
if (client === "vscode")
|
|
200
|
+
return `code ${vscodeAddArgs(launch).map(quote).join(" ")}`;
|
|
201
|
+
const strategy = STRATEGIES[client];
|
|
202
|
+
if (strategy.kind === "command")
|
|
203
|
+
return [strategy.binary, ...strategy.add(launch)].map(quote).join(" ");
|
|
204
|
+
return `add under "${strategy.key}" in ${strategy.file(home)}: ${JSON.stringify({ [MCP_NAME]: serverEntry(launch) })}`;
|
|
205
|
+
}
|
|
206
|
+
/** What to tell the person once their clients are set up: how the method reaches an agent that has no skill. */
|
|
207
|
+
export function firstMessageHint(clients) {
|
|
208
|
+
const names = clients.map((c) => CLIENT_LABELS[c]);
|
|
209
|
+
const list = names.length > 1 ? `${names.slice(0, -1).join(", ")} and ${names[names.length - 1]}` : names[0];
|
|
210
|
+
return (`Restart ${list} (or reload the MCP servers there), then ask the agent:\n` +
|
|
211
|
+
` Use SceneScout to test http://localhost:3000\n` +
|
|
212
|
+
`The server hands the agent the testing method through its scout_playbook tool.`);
|
|
213
|
+
}
|
package/dist/engine/browser.js
CHANGED
|
@@ -1,13 +1,15 @@
|
|
|
1
|
-
import { chromium } from "playwright";
|
|
1
|
+
import { chromium, firefox, webkit } from "playwright";
|
|
2
2
|
import fs from "node:fs";
|
|
3
3
|
import path from "node:path";
|
|
4
4
|
import { elementKey, fingerprintState, isNonPageRoute, normalizePath } from "./fingerprint.js";
|
|
5
|
-
import { AUTH_LOSS_PREFIX, MemoryStore } from "./memory.js";
|
|
5
|
+
import { AUTH_LOSS_PREFIX, JOURNEY_END, JOURNEY_START, MemoryStore } from "./memory.js";
|
|
6
6
|
import { AuthLossTracker } from "./authloss.js";
|
|
7
7
|
import { COLLECT_INTERACTABLES_SCRIPT, VISIBLE_SRC, geometryIssues, BROKEN_IMAGES_SCRIPT, brokenImageIssues, } from "./collector.js";
|
|
8
8
|
import { OracleMonitor, formatViolations } from "./oracles.js";
|
|
9
9
|
import { extractCreatedIds, isOwnedResource, normalizeId } from "./ownership.js";
|
|
10
10
|
import { formatJourney, measureJourney } from "./journey.js";
|
|
11
|
+
import { defaultEngine, focusAdvanceKey, REMOVE_SHARED_WORKER_SCRIPT, screencastSupport, serviceWorkerPolicy, sharedWorkersAllowed, } from "../browsers.js";
|
|
12
|
+
import { revealedLines } from "./hover.js";
|
|
11
13
|
import { explainLaunchFailure, isMissingBrowser } from "./launch.js";
|
|
12
14
|
import { ACTION_TIMEOUT_MS, performScroll, probeFocusIndicators, probeOverlays, scrollContainer } from "./probes.js";
|
|
13
15
|
import { BROWSER_MARKER, reapOrphanBrowsers } from "./reaper.js";
|
|
@@ -96,6 +98,8 @@ function actionabilityDiagnostic(message) {
|
|
|
96
98
|
* actions by ref, runs oracles after every action, and records everything in
|
|
97
99
|
* the persistent memory store. Contains no LLM calls — the MCP client is the brain.
|
|
98
100
|
*/
|
|
101
|
+
/** A screencast whose page has been gone for this many 500 ms ticks ends and says so; a re-attach takes fewer. */
|
|
102
|
+
const SCREENCAST_PAGELESS_TICKS = 20;
|
|
99
103
|
export class BrowserEngine {
|
|
100
104
|
browser = null;
|
|
101
105
|
context = null;
|
|
@@ -184,6 +188,8 @@ export class BrowserEngine {
|
|
|
184
188
|
designAuditCount = 0;
|
|
185
189
|
/** Active task-efficiency measurement (scout_journey), if any. */
|
|
186
190
|
journey = null;
|
|
191
|
+
/** The session's task, from scout_attach. Empty when the agent gave none. */
|
|
192
|
+
task = "";
|
|
187
193
|
/**
|
|
188
194
|
* Begin measuring a user JOURNEY — the interaction cost of completing one
|
|
189
195
|
* real task ("create an order", "approve a document"). E2E suites assert
|
|
@@ -198,7 +204,7 @@ export class BrowserEngine {
|
|
|
198
204
|
fromLog: this.memory?.actionLog.length ?? 0,
|
|
199
205
|
startUrl: page.url(),
|
|
200
206
|
};
|
|
201
|
-
this.logAction({ action:
|
|
207
|
+
this.logAction({ action: JOURNEY_START, target: goal, url: page.url() });
|
|
202
208
|
return `JOURNEY STARTED — "${goal}"\nFrom: ${page.url()}\nNow perform the task the way a first-time user would (click through the UI; don't jump straight to a known deep URL, or the measurement is meaningless). Call scout_journey {action:"end"} when the task is complete or you conclude it can't be.`;
|
|
203
209
|
}
|
|
204
210
|
/** Close the journey and report its interaction cost + friction signals. */
|
|
@@ -223,11 +229,13 @@ export class BrowserEngine {
|
|
|
223
229
|
catch {
|
|
224
230
|
/* fact recording is best-effort */
|
|
225
231
|
}
|
|
226
|
-
this.logAction({ action:
|
|
232
|
+
this.logAction({ action: JOURNEY_END, target: j.goal, url: page.url(), result: completed ? "completed" : "abandoned" });
|
|
227
233
|
return formatJourney({ goal: j.goal, completed, seconds, note }, measured);
|
|
228
234
|
}
|
|
229
235
|
/** Whether the browser window is visible — headed hover results carry a physical-cursor caveat. */
|
|
230
236
|
headed = false;
|
|
237
|
+
/** The browser this engine launched; reported by attach so a finding can say where it was seen. */
|
|
238
|
+
engineName = "chromium";
|
|
231
239
|
/** Non-GET requests fired since the last action — surfaces silent state mutation in read-only runs (timestamped for attribution). */
|
|
232
240
|
mutationRequests = [];
|
|
233
241
|
/**
|
|
@@ -254,6 +262,7 @@ export class BrowserEngine {
|
|
|
254
262
|
throw new Error(`storageStatePath does not exist: ${opts.storageStatePath}`);
|
|
255
263
|
}
|
|
256
264
|
this.mode = opts.mode ?? "read-only";
|
|
265
|
+
this.task = (opts.task ?? "").trim().replace(/\s+/g, " ").slice(0, 300);
|
|
257
266
|
this.headed = opts.headed ?? false;
|
|
258
267
|
this.blockedRequests = [];
|
|
259
268
|
this.pendingCreations = new Set();
|
|
@@ -293,11 +302,15 @@ export class BrowserEngine {
|
|
|
293
302
|
this.knownRoutes = [];
|
|
294
303
|
}
|
|
295
304
|
try {
|
|
296
|
-
this.
|
|
305
|
+
this.engineName = opts.browser ?? defaultEngine(process.env);
|
|
306
|
+
this.browser = await this.launchWithRecovery(this.engineName, opts.headed ?? false);
|
|
297
307
|
this.context = await this.browser.newContext({
|
|
298
308
|
storageState: opts.storageStatePath,
|
|
299
309
|
viewport: opts.viewport ?? { width: 1280, height: 900 },
|
|
310
|
+
serviceWorkers: serviceWorkerPolicy(this.engineName),
|
|
300
311
|
});
|
|
312
|
+
if (!sharedWorkersAllowed(this.mode))
|
|
313
|
+
await this.context.addInitScript(REMOVE_SHARED_WORKER_SCRIPT);
|
|
301
314
|
this.page = await this.context.newPage();
|
|
302
315
|
}
|
|
303
316
|
catch (err) {
|
|
@@ -461,6 +474,8 @@ export class BrowserEngine {
|
|
|
461
474
|
`Continuing now tests a logged-out app.`
|
|
462
475
|
: "";
|
|
463
476
|
return (`Attached to ${this.page.url()} (mode=${this.mode}` +
|
|
477
|
+
`${this.engineName === "chromium" ? "" : `, browser=${this.engineName}, service workers blocked because their requests cannot be intercepted here`}` +
|
|
478
|
+
`${focusAdvanceKey(this.engineName, process.platform) === "Tab" ? "" : `, keyboard: Tab stops only at text fields in this browser — press Alt+Tab to reach buttons and links`}` +
|
|
464
479
|
`${opts.storageStatePath ? `, auth=${opts.storageStatePath}` : ""}). ` +
|
|
465
480
|
`Memory: ${this.memory.dir}.${this.memory.loadWarning ? ` WARNING: ${this.memory.loadWarning}` : ""}` +
|
|
466
481
|
`${this.memory.legacyDirNote ? ` ${this.memory.legacyDirNote}` : ""}` +
|
|
@@ -1278,16 +1293,7 @@ export class BrowserEngine {
|
|
|
1278
1293
|
const bodyAfter = (await page.evaluate(`document.body ? document.body.innerText : ""`).catch(() => null));
|
|
1279
1294
|
if (bodyAfter === null)
|
|
1280
1295
|
return { revealed: [], fallbackUsed: false };
|
|
1281
|
-
|
|
1282
|
-
.split("\n")
|
|
1283
|
-
.map((l) => l.trim())
|
|
1284
|
-
.filter(Boolean));
|
|
1285
|
-
revealed = bodyAfter
|
|
1286
|
-
.split("\n")
|
|
1287
|
-
.map((l) => l.trim())
|
|
1288
|
-
.filter((l) => l && !beforeLines.has(l))
|
|
1289
|
-
.slice(0, 5)
|
|
1290
|
-
.map((l) => l.slice(0, 300));
|
|
1296
|
+
revealed = revealedLines(bodyBefore, bodyAfter);
|
|
1291
1297
|
return { revealed, fallbackUsed: revealed.length > 0 };
|
|
1292
1298
|
}
|
|
1293
1299
|
/** Pre-hover baselines: overlay texts, body text, and whether the page is already churning on its own. */
|
|
@@ -1908,6 +1914,125 @@ export class BrowserEngine {
|
|
|
1908
1914
|
this.logAction({ action: "screenshot", url: page.url() });
|
|
1909
1915
|
return { base64: buf.toString("base64"), mimeType: "image/jpeg" };
|
|
1910
1916
|
}
|
|
1917
|
+
/**
|
|
1918
|
+
* What the live view shows next to a session's name. The task is what the
|
|
1919
|
+
* agent said the session is for; the objective is the goal of the journey it
|
|
1920
|
+
* is on right now (scout_journey). Both are the agent's own words — the
|
|
1921
|
+
* engine sees tool calls, never the reasoning behind them. Neither is
|
|
1922
|
+
* redacted here: the caller that writes them anywhere does that.
|
|
1923
|
+
*/
|
|
1924
|
+
get liveDescription() {
|
|
1925
|
+
return {
|
|
1926
|
+
mode: this.mode,
|
|
1927
|
+
browser: this.engineName,
|
|
1928
|
+
headed: this.headed,
|
|
1929
|
+
...(this.task ? { task: this.task } : {}),
|
|
1930
|
+
...(this.journey ? { objective: this.journey.goal, objectiveSince: new Date(this.journey.startedAt).toISOString() } : {}),
|
|
1931
|
+
};
|
|
1932
|
+
}
|
|
1933
|
+
/**
|
|
1934
|
+
* A frame for somebody WATCHING the run, as opposed to scout_screenshot,
|
|
1935
|
+
* which is the agent looking. It is not logged: the action log is the repro
|
|
1936
|
+
* trace attached to findings, and a person glancing at the dashboard is not
|
|
1937
|
+
* a step anyone should replay. It is also bounded, because the moment a
|
|
1938
|
+
* viewer most wants a picture is when the renderer has wedged.
|
|
1939
|
+
*/
|
|
1940
|
+
async liveShot(timeoutMs = 3000) {
|
|
1941
|
+
const page = this.page;
|
|
1942
|
+
if (!page || page.isClosed())
|
|
1943
|
+
return null;
|
|
1944
|
+
let timer;
|
|
1945
|
+
// The driver's own timeout covers a slow capture; the race covers a
|
|
1946
|
+
// renderer that never answers the protocol at all.
|
|
1947
|
+
return Promise.race([
|
|
1948
|
+
page.screenshot({ type: "jpeg", quality: 55, fullPage: false, timeout: timeoutMs }).catch(() => null),
|
|
1949
|
+
new Promise((resolve) => {
|
|
1950
|
+
timer = setTimeout(() => resolve(null), timeoutMs + 500);
|
|
1951
|
+
}),
|
|
1952
|
+
]).finally(() => clearTimeout(timer));
|
|
1953
|
+
}
|
|
1954
|
+
/**
|
|
1955
|
+
* Push frames of this session's page until the returned function is called.
|
|
1956
|
+
* The stream follows the session rather than one tab: adopting a popup
|
|
1957
|
+
* replaces `this.page`, and a stream left on the old tab would show a page
|
|
1958
|
+
* the session is no longer driving.
|
|
1959
|
+
*/
|
|
1960
|
+
async startScreencast(onFrame, onEnd = () => { }) {
|
|
1961
|
+
if (!this.page || this.page.isClosed())
|
|
1962
|
+
return null;
|
|
1963
|
+
let stopped = false;
|
|
1964
|
+
let bound = null;
|
|
1965
|
+
let release = null;
|
|
1966
|
+
// Ticks in a row with no page to take frames from: a re-attach passes
|
|
1967
|
+
// through a few, a closed session never comes back.
|
|
1968
|
+
let pageless = 0;
|
|
1969
|
+
const bind = async (page) => {
|
|
1970
|
+
if (screencastSupport(this.engineName) === "cdp") {
|
|
1971
|
+
const cdp = await page.context().newCDPSession(page);
|
|
1972
|
+
try {
|
|
1973
|
+
cdp.on("Page.screencastFrame", (frame) => {
|
|
1974
|
+
if (!stopped)
|
|
1975
|
+
onFrame(Buffer.from(frame.data, "base64"));
|
|
1976
|
+
void cdp.send("Page.screencastFrameAck", { sessionId: frame.sessionId }).catch(() => { });
|
|
1977
|
+
});
|
|
1978
|
+
await cdp.send("Page.startScreencast", { format: "jpeg", quality: 55, maxWidth: 1280, maxHeight: 900, everyNthFrame: 2 });
|
|
1979
|
+
}
|
|
1980
|
+
catch (err) {
|
|
1981
|
+
// Not bound: the timer tries this page again on its next tick.
|
|
1982
|
+
await cdp.detach().catch(() => { });
|
|
1983
|
+
throw err;
|
|
1984
|
+
}
|
|
1985
|
+
release = async () => {
|
|
1986
|
+
await cdp.send("Page.stopScreencast").catch(() => { });
|
|
1987
|
+
await cdp.detach().catch(() => { });
|
|
1988
|
+
};
|
|
1989
|
+
}
|
|
1990
|
+
else {
|
|
1991
|
+
release = null;
|
|
1992
|
+
}
|
|
1993
|
+
bound = page;
|
|
1994
|
+
};
|
|
1995
|
+
const stop = async () => {
|
|
1996
|
+
stopped = true;
|
|
1997
|
+
clearInterval(timer);
|
|
1998
|
+
const current = release;
|
|
1999
|
+
release = null;
|
|
2000
|
+
await current?.();
|
|
2001
|
+
};
|
|
2002
|
+
await bind(this.page).catch(() => { });
|
|
2003
|
+
// One timer does both jobs: it notices a replaced tab, and where the
|
|
2004
|
+
// browser cannot push frames it is also what takes them.
|
|
2005
|
+
const timer = setInterval(() => {
|
|
2006
|
+
if (stopped)
|
|
2007
|
+
return;
|
|
2008
|
+
const page = this.page;
|
|
2009
|
+
if (!page || page.isClosed()) {
|
|
2010
|
+
pageless += 1;
|
|
2011
|
+
if (pageless >= SCREENCAST_PAGELESS_TICKS)
|
|
2012
|
+
void stop().finally(onEnd);
|
|
2013
|
+
return;
|
|
2014
|
+
}
|
|
2015
|
+
pageless = 0;
|
|
2016
|
+
if (page !== bound) {
|
|
2017
|
+
const previous = release;
|
|
2018
|
+
release = null;
|
|
2019
|
+
void (async () => {
|
|
2020
|
+
await previous?.();
|
|
2021
|
+
if (!stopped)
|
|
2022
|
+
await bind(page).catch(() => { });
|
|
2023
|
+
})();
|
|
2024
|
+
return;
|
|
2025
|
+
}
|
|
2026
|
+
if (screencastSupport(this.engineName) === "poll") {
|
|
2027
|
+
void this.liveShot(1500).then((jpeg) => {
|
|
2028
|
+
if (jpeg && !stopped)
|
|
2029
|
+
onFrame(jpeg);
|
|
2030
|
+
});
|
|
2031
|
+
}
|
|
2032
|
+
}, 500);
|
|
2033
|
+
timer.unref();
|
|
2034
|
+
return stop;
|
|
2035
|
+
}
|
|
1911
2036
|
get currentState() {
|
|
1912
2037
|
return this.currentFingerprint;
|
|
1913
2038
|
}
|
|
@@ -1924,15 +2049,18 @@ export class BrowserEngine {
|
|
|
1924
2049
|
* On the first failure or timeout, reap orphaned Playwright processes and
|
|
1925
2050
|
* try once more before giving up with a diagnosable error.
|
|
1926
2051
|
*/
|
|
1927
|
-
async launchWithRecovery(headed) {
|
|
2052
|
+
async launchWithRecovery(engine, headed) {
|
|
2053
|
+
const types = { chromium, firefox, webkit };
|
|
1928
2054
|
const attempt = async () => {
|
|
1929
2055
|
// The marker is what makes reapOrphanBrowsers safe to run at startup:
|
|
1930
2056
|
// it appears in the child's command line, so the sweep can tell a browser
|
|
1931
2057
|
// WE leaked from one belonging to somebody else's Playwright run.
|
|
1932
2058
|
// `--enable-features` takes arbitrary names and ignores unknown ones.
|
|
1933
|
-
|
|
2059
|
+
// It is a Chromium switch: Firefox and WebKit are launched without it,
|
|
2060
|
+
// so a leaked one of those is not reaped and has to be closed by hand.
|
|
2061
|
+
const launch = types[engine].launch({
|
|
1934
2062
|
headless: !headed,
|
|
1935
|
-
args: [`--enable-features=${BROWSER_MARKER}`],
|
|
2063
|
+
args: engine === "chromium" ? [`--enable-features=${BROWSER_MARKER}`] : [],
|
|
1936
2064
|
});
|
|
1937
2065
|
let timer;
|
|
1938
2066
|
try {
|
|
@@ -1959,13 +2087,13 @@ export class BrowserEngine {
|
|
|
1959
2087
|
const firstMessage = firstErr instanceof Error ? firstErr.message : String(firstErr);
|
|
1960
2088
|
// A browser that was never downloaded will not appear on a second try.
|
|
1961
2089
|
if (isMissingBrowser(firstMessage))
|
|
1962
|
-
throw new Error(explainLaunchFailure(firstMessage, 0));
|
|
2090
|
+
throw new Error(explainLaunchFailure(firstMessage, 0, { engine, headed }));
|
|
1963
2091
|
const reaped = reapOrphanBrowsers();
|
|
1964
2092
|
try {
|
|
1965
2093
|
return await attempt();
|
|
1966
2094
|
}
|
|
1967
2095
|
catch {
|
|
1968
|
-
throw new Error(explainLaunchFailure(firstMessage, reaped));
|
|
2096
|
+
throw new Error(explainLaunchFailure(firstMessage, reaped, { engine, headed }));
|
|
1969
2097
|
}
|
|
1970
2098
|
}
|
|
1971
2099
|
}
|
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The page-text fallback for hover: which text is new after the pointer moved.
|
|
3
|
+
*
|
|
4
|
+
* Pure, so the rule can be table-tested. The browser side only supplies the
|
|
5
|
+
* two `innerText` readings.
|
|
6
|
+
*/
|
|
7
|
+
const squash = (s) => s.replace(/\s+/g, " ").trim();
|
|
8
|
+
/** How many times `needle` occurs in `haystack`, overlaps not counted. */
|
|
9
|
+
function occurrences(haystack, needle) {
|
|
10
|
+
let n = 0;
|
|
11
|
+
for (let at = haystack.indexOf(needle); at >= 0; at = haystack.indexOf(needle, at + needle.length))
|
|
12
|
+
n++;
|
|
13
|
+
return n;
|
|
14
|
+
}
|
|
15
|
+
/**
|
|
16
|
+
* Lines present after the hover whose text was not on the page before it.
|
|
17
|
+
*
|
|
18
|
+
* Comparing line by line is not enough. When something that was showing goes
|
|
19
|
+
* away — the previous hover's tooltip closing as the pointer leaves it — the
|
|
20
|
+
* text around it can re-flow onto a line of its own, and that line is "new"
|
|
21
|
+
* only as a line: every word of it was already visible. How text re-flows
|
|
22
|
+
* differs between browsers. So a line counts as revealed when its text occurs
|
|
23
|
+
* MORE often after the hover than before: text that only moved occurs as often
|
|
24
|
+
* as it did, while a tooltip reading "Delete" on a page that already says
|
|
25
|
+
* "Delete account" occurs once more and is still reported.
|
|
26
|
+
*/
|
|
27
|
+
export function revealedLines(bodyBefore, bodyAfter, limit = 5) {
|
|
28
|
+
const beforeText = squash(bodyBefore);
|
|
29
|
+
const afterText = squash(bodyAfter);
|
|
30
|
+
const seen = new Set();
|
|
31
|
+
return bodyAfter
|
|
32
|
+
.split("\n")
|
|
33
|
+
.map(squash)
|
|
34
|
+
.filter((l) => {
|
|
35
|
+
if (!l || seen.has(l))
|
|
36
|
+
return false;
|
|
37
|
+
seen.add(l);
|
|
38
|
+
return occurrences(afterText, l) > occurrences(beforeText, l);
|
|
39
|
+
})
|
|
40
|
+
.slice(0, limit)
|
|
41
|
+
.map((l) => l.slice(0, 300));
|
|
42
|
+
}
|
package/dist/engine/launch.js
CHANGED
|
@@ -2,21 +2,28 @@
|
|
|
2
2
|
* Turning a failed browser launch into something the person can act on.
|
|
3
3
|
*
|
|
4
4
|
* For anyone who installed from npm and never ran the setup step, the first
|
|
5
|
-
* attach fails because
|
|
5
|
+
* attach fails because the browser was never downloaded — and Playwright reports
|
|
6
6
|
* that as a multi-line box of text with a path in it. That is the single most
|
|
7
7
|
* likely first-run failure, so it gets one plain instruction instead.
|
|
8
8
|
*/
|
|
9
|
+
import { APPROX_DISK_MB, launchTarget } from "../browsers.js";
|
|
9
10
|
/** Playwright's wording when the browser binary is not on disk. */
|
|
10
11
|
const MISSING_BROWSER_RE = /Executable doesn't exist|playwright install|browserType\.launch:.*(not found|ENOENT)/i;
|
|
11
12
|
export function isMissingBrowser(message) {
|
|
12
13
|
return MISSING_BROWSER_RE.test(message);
|
|
13
14
|
}
|
|
14
15
|
/** The error text for a launch that failed. `reaped` is how many orphaned browsers were cleaned up between attempts. */
|
|
15
|
-
export function explainLaunchFailure(message, reaped) {
|
|
16
|
+
export function explainLaunchFailure(message, reaped, need = { engine: "chromium", headed: false }) {
|
|
16
17
|
if (isMissingBrowser(message)) {
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
18
|
+
const target = launchTarget(need.engine, need.headed);
|
|
19
|
+
// Someone who installed only the headless shell did run install, so say what is different about a headed run.
|
|
20
|
+
const why = target === "chromium" && need.headed
|
|
21
|
+
? "A headed run needs the full Chromium browser, which has not been downloaded (the headless shell alone cannot open a window)"
|
|
22
|
+
: `The ${target} build has not been downloaded yet`;
|
|
23
|
+
return (`${why} (one-time, about ${APPROX_DISK_MB[target]} MB on disk). Run this once, then attach again:\n` +
|
|
24
|
+
` npx -y scenescout install --browser-only --browsers ${target}\n` +
|
|
25
|
+
`(from a clone: node dist/cli.js install --browser-only --browsers ${target}). ` +
|
|
26
|
+
`On Linux, if system libraries are missing: npx playwright install --with-deps ${target}`);
|
|
20
27
|
}
|
|
21
28
|
const firstLine = message.split("\n")[0];
|
|
22
29
|
return (`browser launch failed twice (${firstLine})` +
|