pi-chrome-search 0.1.0 → 2.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +37 -76
- package/assets/anchor.html +2 -2
- package/package.json +4 -1
- package/src/cdp/bytes.ts +14 -14
- package/src/cdp/client.ts +687 -0
- package/src/cdp/page.ts +716 -0
- package/src/cdp/readiness.ts +119 -0
- package/src/cdp/shared.ts +70 -0
- package/src/cdp/types.ts +30 -1
- package/src/cli/doctor.ts +14 -17
- package/src/commands/focus.ts +2 -2
- package/src/config/defaults.ts +158 -0
- package/src/config/doc.ts +180 -0
- package/src/config/env.ts +58 -0
- package/src/config/index.ts +10 -0
- package/src/config/load.ts +40 -0
- package/src/config/schema.ts +264 -0
- package/src/engines/duckduckgo.ts +7 -10
- package/src/engines/google.ts +11 -21
- package/src/engines/images.ts +1 -1
- package/src/engines/index.ts +5 -13
- package/src/engines/types.ts +29 -6
- package/src/engines/yandex.ts +11 -18
- package/src/extract/claims.ts +1 -1
- package/src/extract/compact.ts +29 -3
- package/src/extract/goto.ts +124 -2
- package/src/extract/text.ts +43 -0
- package/src/index.ts +52 -80
- package/src/runtime/artifacts.ts +6 -11
- package/src/runtime/attention.ts +2 -2
- package/src/runtime/budget.ts +142 -0
- package/src/runtime/cache.ts +1 -3
- package/src/runtime/doctor.ts +17 -43
- package/src/runtime/error-codes.ts +19 -8
- package/src/runtime/errors-doc.ts +45 -0
- package/src/runtime/limiter.ts +33 -221
- package/src/runtime/log.ts +2 -2
- package/src/runtime/panel.ts +30 -4
- package/src/runtime/types.ts +2 -17
- package/src/{cdp/discovery.ts → session/discover.ts} +3 -3
- package/src/session/index.ts +34 -0
- package/src/session/lock.ts +346 -0
- package/src/{cdp → session}/page-lock.ts +12 -13
- package/src/session/session.ts +684 -0
- package/src/session/tabs.ts +277 -0
- package/src/tools/image_search.ts +21 -7
- package/src/tools/index.ts +18 -7
- package/src/tools/surface.ts +97 -0
- package/src/tools/types.ts +2 -5
- package/src/tools/web_doctor.ts +2 -2
- package/src/tools/web_read.ts +159 -114
- package/src/tools/web_research.ts +33 -207
- package/src/tools/web_search.ts +208 -419
- package/src/version.ts +12 -3
- package/src/cdp/browser.ts +0 -1286
- package/src/cdp/session.ts +0 -1446
- package/src/config.ts +0 -651
- package/src/reflex/classifier.ts +0 -1578
- package/src/reflex/index.ts +0 -172
- package/src/reflex/policy.ts +0 -792
- package/src/reflex/rules.ts +0 -557
- package/src/reflex/types.ts +0 -117
package/src/cdp/page.ts
ADDED
|
@@ -0,0 +1,716 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Document access over a `CdpClient` (SPEC S5.1, S5.4; PLAN P3).
|
|
3
|
+
*
|
|
4
|
+
* This is the only module that turns a target id into a page: navigation commit
|
|
5
|
+
* semantics (`frameNavigated` with a fresh loaderId), readiness, `Runtime.evaluate`
|
|
6
|
+
* and the human input events adapters see. The raw socket, reconnect and target
|
|
7
|
+
* attach live in `client.ts`; the owned-tab lifecycle lives in `src/session/`.
|
|
8
|
+
*
|
|
9
|
+
* Two behaviours are load-bearing and were hard-won on v1:
|
|
10
|
+
*
|
|
11
|
+
* - **A commit, not a load event.** A real SERP never fires a correlated load
|
|
12
|
+
* event. `Page.navigate` resolving means "committed"; the page waits for a
|
|
13
|
+
* new loaderId on `Page.frameNavigated`, then for `readyState interactive`.
|
|
14
|
+
* - **The document is the URL authority.** `url()` is a target-event cache and
|
|
15
|
+
* can be stale after a typed search or a JS redirect; `currentUrl()` asks
|
|
16
|
+
* `location.href` and is what the read proof compares before and after.
|
|
17
|
+
*/
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
import { setTimeout as sleepFor } from "node:timers/promises";
|
|
21
|
+
|
|
22
|
+
import type { Stage } from "../runtime/error-codes.ts";
|
|
23
|
+
import type { WebSearchError } from "../runtime/error-codes.ts";
|
|
24
|
+
import { fail } from "../runtime/errors.ts";
|
|
25
|
+
import type { Clock } from "../runtime/types.ts";
|
|
26
|
+
import { asString, asWebSearchError, finiteNumber, isAborted, isRecord, mainFrameOf, nonNegativeDelay, positiveTimeout, realClock, sameNavigatedUrl } from "./shared.ts";
|
|
27
|
+
import type { CdpClient, CdpPage, CookieInfo, EvaluateOptions, NavigateOptions, TypeOptions } from "./types.ts";
|
|
28
|
+
|
|
29
|
+
export const DEFAULT_NAVIGATE_TIMEOUT_MS = 20_000;
|
|
30
|
+
export const DEFAULT_TYPE_MIN_MS = 30;
|
|
31
|
+
export const DEFAULT_TYPE_MAX_MS = 80;
|
|
32
|
+
|
|
33
|
+
/**
|
|
34
|
+
* `readyState` polling cadence and the budget for accepting `interactive`.
|
|
35
|
+
* A SERP keeps loading ads/trackers long after it is usable; waiting for
|
|
36
|
+
* `complete` would time out on Google, so `interactive` plus a short grace is
|
|
37
|
+
* the readiness bar. The adapter's marker settle waits for real content.
|
|
38
|
+
*/
|
|
39
|
+
const READY_STATE_POLL_MS = 120;
|
|
40
|
+
const READY_STATE_INTERACTIVE_GRACE_MS = 200;
|
|
41
|
+
/**
|
|
42
|
+
* Some navigations return from `Page.navigate` without any commit event and
|
|
43
|
+
* with the target URL already updated (Chrome sometimes skips the event for a
|
|
44
|
+
* fast same-URL reload). One look after this grace avoids a false timeout.
|
|
45
|
+
*/
|
|
46
|
+
const NAVIGATE_URL_FALLBACK_MS = 250;
|
|
47
|
+
const READY_STATE_EXPRESSION = "document.readyState";
|
|
48
|
+
|
|
49
|
+
/** `typeText` accepts an abort signal in addition to the frozen `TypeOptions`. */
|
|
50
|
+
export interface TypeTextOptions extends TypeOptions {
|
|
51
|
+
signal?: AbortSignal;
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
type RemoteObject = {
|
|
55
|
+
value?: unknown;
|
|
56
|
+
subtype?: string;
|
|
57
|
+
unserializableValue?: string;
|
|
58
|
+
};
|
|
59
|
+
|
|
60
|
+
type EvaluateResponse = {
|
|
61
|
+
result?: RemoteObject;
|
|
62
|
+
exceptionDetails?: Record<string, unknown>;
|
|
63
|
+
};
|
|
64
|
+
|
|
65
|
+
type NavigateResponse = {
|
|
66
|
+
frameId?: unknown;
|
|
67
|
+
loaderId?: unknown;
|
|
68
|
+
};
|
|
69
|
+
|
|
70
|
+
function exceptionMessage(details: Record<string, unknown>): string {
|
|
71
|
+
const exception = isRecord(details.exception) ? details.exception : undefined;
|
|
72
|
+
const description = exception === undefined ? undefined : asString(exception.description);
|
|
73
|
+
const text = asString(details.text);
|
|
74
|
+
const raw = description ?? text ?? "evaluation threw";
|
|
75
|
+
// Chrome's `description` includes a stack trace after the first line; the
|
|
76
|
+
// agent needs the error, not local file paths.
|
|
77
|
+
const firstLine = raw.split("\n")[0] ?? raw;
|
|
78
|
+
return firstLine.trim() || "evaluation threw";
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
function clickExpression(selector: string): string {
|
|
82
|
+
return (
|
|
83
|
+
"(() => {" +
|
|
84
|
+
` const el = document.querySelector(${JSON.stringify(selector)});` +
|
|
85
|
+
" if (!el) return null;" +
|
|
86
|
+
' el.scrollIntoView({ block: "center", inline: "center" });' +
|
|
87
|
+
" const rect = el.getBoundingClientRect();" +
|
|
88
|
+
" return { x: rect.left + rect.width / 2, y: rect.top + rect.height / 2 };" +
|
|
89
|
+
" })()"
|
|
90
|
+
);
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
function charKeyCode(text: string): number {
|
|
94
|
+
const upper = text.toUpperCase();
|
|
95
|
+
if (upper.length === 1) {
|
|
96
|
+
const code = upper.charCodeAt(0);
|
|
97
|
+
if (code >= 32 && code <= 126) return code;
|
|
98
|
+
}
|
|
99
|
+
return 0;
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
const KEY_INFO: Record<string, { code: string; vk: number }> = {
|
|
103
|
+
Enter: { code: "Enter", vk: 13 },
|
|
104
|
+
Tab: { code: "Tab", vk: 9 },
|
|
105
|
+
Escape: { code: "Escape", vk: 27 },
|
|
106
|
+
Backspace: { code: "Backspace", vk: 8 },
|
|
107
|
+
Delete: { code: "Delete", vk: 46 },
|
|
108
|
+
Space: { code: "Space", vk: 32 },
|
|
109
|
+
ArrowDown: { code: "ArrowDown", vk: 40 },
|
|
110
|
+
ArrowUp: { code: "ArrowUp", vk: 38 },
|
|
111
|
+
ArrowLeft: { code: "ArrowLeft", vk: 37 },
|
|
112
|
+
ArrowRight: { code: "ArrowRight", vk: 39 },
|
|
113
|
+
Home: { code: "Home", vk: 36 },
|
|
114
|
+
End: { code: "End", vk: 35 },
|
|
115
|
+
PageDown: { code: "PageDown", vk: 34 },
|
|
116
|
+
PageUp: { code: "PageUp", vk: 33 },
|
|
117
|
+
};
|
|
118
|
+
|
|
119
|
+
export interface CdpPageOptions {
|
|
120
|
+
clock?: Clock;
|
|
121
|
+
/** Last known URL when the page is created for an existing target. */
|
|
122
|
+
initialUrl?: string;
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
/**
|
|
126
|
+
* `CdpPage` plus the abort-aware typing signature. The frozen `CdpPage`
|
|
127
|
+
* contract stays the surface adapters see; callers that drive typing directly
|
|
128
|
+
* get the optional signal without a cast.
|
|
129
|
+
*/
|
|
130
|
+
export type CdpPageHandle = CdpPage & {
|
|
131
|
+
typeText(text: string, o?: TypeTextOptions): Promise<void>;
|
|
132
|
+
};
|
|
133
|
+
|
|
134
|
+
/**
|
|
135
|
+
* A `CdpPage` bound to one target. The closure resolves its flattened session id
|
|
136
|
+
* on every call, so a reconnect (which allocates a new session id) is invisible
|
|
137
|
+
* to adapters.
|
|
138
|
+
*/
|
|
139
|
+
export function createCdpPage(
|
|
140
|
+
client: CdpClient,
|
|
141
|
+
targetId: string,
|
|
142
|
+
opts: CdpPageOptions = {},
|
|
143
|
+
): CdpPageHandle {
|
|
144
|
+
const clock = opts.clock ?? client.clock ?? realClock;
|
|
145
|
+
// Seed from the target so a page created right after `createTarget`/`attach`
|
|
146
|
+
// reports the URL Chrome knows instead of an empty string.
|
|
147
|
+
let lastUrl = opts.initialUrl ?? client.targetUrlFor(targetId) ?? "";
|
|
148
|
+
let enabledSessionId: string | undefined;
|
|
149
|
+
let knownSessionId: string | undefined;
|
|
150
|
+
let lost: WebSearchError | undefined;
|
|
151
|
+
let activeNavigation: { reject: (error: WebSearchError) => void } | undefined;
|
|
152
|
+
|
|
153
|
+
const markLost = (message: string): void => {
|
|
154
|
+
const error = fail("E_TARGET_LOST", "navigate", { message });
|
|
155
|
+
lost = error;
|
|
156
|
+
activeNavigation?.reject(error);
|
|
157
|
+
};
|
|
158
|
+
|
|
159
|
+
// Chrome reports a dead target either as a rejected command (-32000/-32001)
|
|
160
|
+
// or as one of these events. Without the event listeners a navigation that is
|
|
161
|
+
// waiting for a commit would sit until its deadline instead of failing as
|
|
162
|
+
// E_TARGET_LOST.
|
|
163
|
+
client.on("Target.detachedFromTarget", (params, sessionId) => {
|
|
164
|
+
const payload = isRecord(params) ? params : {};
|
|
165
|
+
const detachedTarget = asString(payload.targetId);
|
|
166
|
+
const detachedSession = asString(payload.sessionId) ?? sessionId;
|
|
167
|
+
if (detachedTarget === targetId || (detachedSession !== undefined && detachedSession === knownSessionId)) {
|
|
168
|
+
markLost(`target ${targetId} was detached`);
|
|
169
|
+
}
|
|
170
|
+
});
|
|
171
|
+
client.on("Inspector.detached", (_params, sessionId) => {
|
|
172
|
+
if (sessionId !== undefined && sessionId === knownSessionId) {
|
|
173
|
+
markLost(`target ${targetId} was detached`);
|
|
174
|
+
}
|
|
175
|
+
});
|
|
176
|
+
|
|
177
|
+
const sessionIdOrThrow = (stage: Stage): string => {
|
|
178
|
+
if (lost !== undefined) throw lost;
|
|
179
|
+
const sessionId = client.sessionIdFor(targetId);
|
|
180
|
+
if (sessionId === undefined) {
|
|
181
|
+
throw fail("E_TARGET_LOST", stage, {
|
|
182
|
+
message: `target ${targetId} is not attached`,
|
|
183
|
+
hint: "the owned tab was closed; retry so a fresh tab is opened",
|
|
184
|
+
});
|
|
185
|
+
}
|
|
186
|
+
knownSessionId = sessionId;
|
|
187
|
+
return sessionId;
|
|
188
|
+
};
|
|
189
|
+
|
|
190
|
+
/**
|
|
191
|
+
* `Page.enable` makes Chrome emit `Page.*` lifecycle events. It must happen
|
|
192
|
+
* before navigation and exactly once per (re)attached session: after a
|
|
193
|
+
* reconnect Chrome has forgotten the enable, and without it no
|
|
194
|
+
* `frameNavigated`/`navigatedWithinDocument` commit ever arrives (the exact
|
|
195
|
+
* reason the session times out instead of reporting a false success).
|
|
196
|
+
*/
|
|
197
|
+
const ensureEnabled = async (stage: Stage): Promise<string> => {
|
|
198
|
+
const sessionId = sessionIdOrThrow(stage);
|
|
199
|
+
if (enabledSessionId !== sessionId) {
|
|
200
|
+
await client.send("Page.enable", {}, sessionId);
|
|
201
|
+
enabledSessionId = sessionId;
|
|
202
|
+
}
|
|
203
|
+
return sessionId;
|
|
204
|
+
};
|
|
205
|
+
|
|
206
|
+
const evaluateRaw = async <T>(expression: string, o: EvaluateOptions = {}): Promise<T> => {
|
|
207
|
+
const sessionId = sessionIdOrThrow("extract");
|
|
208
|
+
const response = await client.send<EvaluateResponse>(
|
|
209
|
+
"Runtime.evaluate",
|
|
210
|
+
{
|
|
211
|
+
expression,
|
|
212
|
+
returnByValue: true,
|
|
213
|
+
awaitPromise: o.awaitPromise === true,
|
|
214
|
+
},
|
|
215
|
+
sessionId,
|
|
216
|
+
);
|
|
217
|
+
if (response.exceptionDetails !== undefined) {
|
|
218
|
+
throw fail("E_PARSE", "extract", {
|
|
219
|
+
message: exceptionMessage(response.exceptionDetails),
|
|
220
|
+
hint: "the page expression threw; inspect the artifact and the selector/canary",
|
|
221
|
+
});
|
|
222
|
+
}
|
|
223
|
+
const result = response.result;
|
|
224
|
+
if (result === undefined) return undefined as T;
|
|
225
|
+
if (result.unserializableValue !== undefined) return result.unserializableValue as T;
|
|
226
|
+
return result.value as T;
|
|
227
|
+
};
|
|
228
|
+
|
|
229
|
+
const page: CdpPageHandle = {
|
|
230
|
+
url: () => lastUrl,
|
|
231
|
+
|
|
232
|
+
/**
|
|
233
|
+
* Post-navigate proof (P9-B): `url()` is a cache fed by commit events, which
|
|
234
|
+
* a typed search (press Enter inside the page) never produces. Asking the
|
|
235
|
+
* document is the only way to know where the tab really is.
|
|
236
|
+
*/
|
|
237
|
+
async currentUrl(): Promise<string> {
|
|
238
|
+
try {
|
|
239
|
+
const href = await evaluateRaw<unknown>("location.href");
|
|
240
|
+
if (typeof href === "string" && href.trim() !== "") {
|
|
241
|
+
lastUrl = href;
|
|
242
|
+
return href;
|
|
243
|
+
}
|
|
244
|
+
} catch {
|
|
245
|
+
// A detached or navigated-away document has no answer; the last known
|
|
246
|
+
// target URL is still the honest best guess for the caller's proof.
|
|
247
|
+
}
|
|
248
|
+
return lastUrl;
|
|
249
|
+
},
|
|
250
|
+
|
|
251
|
+
async title(): Promise<string> {
|
|
252
|
+
const value = await evaluateRaw<unknown>("document.title");
|
|
253
|
+
return typeof value === "string" ? value : "";
|
|
254
|
+
},
|
|
255
|
+
|
|
256
|
+
async navigate(url: string, o: NavigateOptions = {}): Promise<void> {
|
|
257
|
+
if (lost !== undefined) throw lost;
|
|
258
|
+
const cancelled = (): WebSearchError =>
|
|
259
|
+
fail("E_CANCELLED", "navigate", { message: `navigation to ${url} was aborted` });
|
|
260
|
+
if (isAborted(o.signal)) throw cancelled();
|
|
261
|
+
const sessionId = await ensureEnabled("navigate");
|
|
262
|
+
// `ensureEnabled` awaits; the signal may have been aborted meanwhile and
|
|
263
|
+
// an already-aborted signal never fires the listener below.
|
|
264
|
+
if (lost !== undefined) throw lost;
|
|
265
|
+
if (isAborted(o.signal)) throw cancelled();
|
|
266
|
+
|
|
267
|
+
const timeoutMs = positiveTimeout(o.timeoutMs, DEFAULT_NAVIGATE_TIMEOUT_MS);
|
|
268
|
+
// One deadline for the whole operation: a redirect chain that never
|
|
269
|
+
// commits must not buy extra time by restarting a timer per event.
|
|
270
|
+
const deadline = Date.now() + timeoutMs;
|
|
271
|
+
|
|
272
|
+
await new Promise<void>((resolve, reject) => {
|
|
273
|
+
let settled = false;
|
|
274
|
+
let committed = false;
|
|
275
|
+
let responseReceived = false;
|
|
276
|
+
/** Main-frame loaderId before the navigation: the stale-event discriminator. */
|
|
277
|
+
let preLoaderId: string | undefined;
|
|
278
|
+
/** Main-frame id, refined by `Page.navigate`'s response frameId. */
|
|
279
|
+
let frameFilter: string | undefined;
|
|
280
|
+
let interactiveSince: number | undefined;
|
|
281
|
+
let fallbackTimer: ReturnType<typeof setTimeout> | undefined;
|
|
282
|
+
const cleanups: Array<() => void> = [];
|
|
283
|
+
/** Signals that arrived before the navigate response (ordering barrier). */
|
|
284
|
+
const deferred: Array<() => void> = [];
|
|
285
|
+
|
|
286
|
+
const cleanup = (): void => {
|
|
287
|
+
for (const off of cleanups.splice(0)) off();
|
|
288
|
+
clearTimeout(deadlineTimer);
|
|
289
|
+
if (fallbackTimer !== undefined) {
|
|
290
|
+
clearTimeout(fallbackTimer);
|
|
291
|
+
fallbackTimer = undefined;
|
|
292
|
+
}
|
|
293
|
+
o.signal?.removeEventListener("abort", onAbort);
|
|
294
|
+
if (activeNavigation?.reject === abort) activeNavigation = undefined;
|
|
295
|
+
};
|
|
296
|
+
const succeed = (): void => {
|
|
297
|
+
if (settled) return;
|
|
298
|
+
settled = true;
|
|
299
|
+
cleanup();
|
|
300
|
+
resolve();
|
|
301
|
+
};
|
|
302
|
+
const abort = (error: WebSearchError): void => {
|
|
303
|
+
if (settled) return;
|
|
304
|
+
settled = true;
|
|
305
|
+
cleanup();
|
|
306
|
+
reject(error);
|
|
307
|
+
};
|
|
308
|
+
|
|
309
|
+
const timeoutError = (): WebSearchError =>
|
|
310
|
+
fail("E_TIMEOUT", "navigate", {
|
|
311
|
+
message: `navigate to ${url} did not finish loading within ${timeoutMs}ms`,
|
|
312
|
+
hint:
|
|
313
|
+
"the page never committed a new document or never reached readyState interactive; " +
|
|
314
|
+
"retry, or read a lighter page",
|
|
315
|
+
});
|
|
316
|
+
|
|
317
|
+
const deadlineTimer = setTimeout(() => abort(timeoutError()), timeoutMs);
|
|
318
|
+
deadlineTimer.unref();
|
|
319
|
+
|
|
320
|
+
const onAbort = (): void => abort(cancelled());
|
|
321
|
+
o.signal?.addEventListener("abort", onAbort, { once: true });
|
|
322
|
+
|
|
323
|
+
const forThisSession = (eventSessionId: string | undefined): boolean =>
|
|
324
|
+
eventSessionId !== undefined && eventSessionId === knownSessionId && lost === undefined;
|
|
325
|
+
|
|
326
|
+
const readReadyState = async (): Promise<unknown> => {
|
|
327
|
+
const response = await client.send<EvaluateResponse>(
|
|
328
|
+
"Runtime.evaluate",
|
|
329
|
+
{ expression: READY_STATE_EXPRESSION, returnByValue: true },
|
|
330
|
+
sessionId,
|
|
331
|
+
);
|
|
332
|
+
const result = response.result;
|
|
333
|
+
if (result === undefined || !isRecord(result)) return undefined;
|
|
334
|
+
return result.value;
|
|
335
|
+
};
|
|
336
|
+
|
|
337
|
+
/**
|
|
338
|
+
* Readiness phase: `Page.navigate` resolving means *committed*, not
|
|
339
|
+
* *loaded*. A SERP that waits for `load` (consent hops, long-tail ads)
|
|
340
|
+
* would burn the deadline on a page that is already readable, so the
|
|
341
|
+
* bar is `interactive` (with a short grace for `complete`).
|
|
342
|
+
*/
|
|
343
|
+
const pollReadyState = async (): Promise<void> => {
|
|
344
|
+
while (!settled) {
|
|
345
|
+
const now = Date.now();
|
|
346
|
+
if (now >= deadline) {
|
|
347
|
+
abort(timeoutError());
|
|
348
|
+
return;
|
|
349
|
+
}
|
|
350
|
+
if (
|
|
351
|
+
interactiveSince !== undefined &&
|
|
352
|
+
now - interactiveSince >= READY_STATE_INTERACTIVE_GRACE_MS
|
|
353
|
+
) {
|
|
354
|
+
succeed();
|
|
355
|
+
return;
|
|
356
|
+
}
|
|
357
|
+
|
|
358
|
+
let state: unknown;
|
|
359
|
+
let probeFailed = false;
|
|
360
|
+
try {
|
|
361
|
+
state = await readReadyState();
|
|
362
|
+
} catch (error) {
|
|
363
|
+
if (settled) return;
|
|
364
|
+
const failure = asWebSearchError(error, "E_CDP_DISCONNECTED", "navigate");
|
|
365
|
+
// A destroyed execution context right after the commit is normal
|
|
366
|
+
// ("Cannot find context with specified id"); only a dead target or
|
|
367
|
+
// a dead socket is fatal. Anything else is retried until the
|
|
368
|
+
// deadline, so a transient renderer error cannot fail a live page.
|
|
369
|
+
if (
|
|
370
|
+
failure.details.code === "E_TARGET_LOST" ||
|
|
371
|
+
failure.details.code === "E_CDP_DISCONNECTED"
|
|
372
|
+
) {
|
|
373
|
+
abort(failure);
|
|
374
|
+
return;
|
|
375
|
+
}
|
|
376
|
+
probeFailed = true;
|
|
377
|
+
}
|
|
378
|
+
if (settled) return;
|
|
379
|
+
|
|
380
|
+
if (!probeFailed) {
|
|
381
|
+
if (state === "complete") {
|
|
382
|
+
succeed();
|
|
383
|
+
return;
|
|
384
|
+
}
|
|
385
|
+
if (state === "interactive") interactiveSince ??= Date.now();
|
|
386
|
+
else interactiveSince = undefined;
|
|
387
|
+
}
|
|
388
|
+
|
|
389
|
+
const remaining = deadline - Date.now();
|
|
390
|
+
if (remaining <= 0) continue; // the loop head turns this into E_TIMEOUT
|
|
391
|
+
await sleepFor(Math.min(READY_STATE_POLL_MS, remaining), undefined, { ref: false });
|
|
392
|
+
}
|
|
393
|
+
};
|
|
394
|
+
|
|
395
|
+
const commit = (committedUrl: string): void => {
|
|
396
|
+
if (settled || committed) return;
|
|
397
|
+
committed = true;
|
|
398
|
+
lastUrl = committedUrl;
|
|
399
|
+
if (fallbackTimer !== undefined) {
|
|
400
|
+
clearTimeout(fallbackTimer);
|
|
401
|
+
fallbackTimer = undefined;
|
|
402
|
+
}
|
|
403
|
+
void pollReadyState();
|
|
404
|
+
};
|
|
405
|
+
|
|
406
|
+
const frameMatches = (frameId: string | undefined): boolean =>
|
|
407
|
+
frameFilter === undefined || frameId === undefined || frameId === frameFilter;
|
|
408
|
+
|
|
409
|
+
/**
|
|
410
|
+
* A commit is only the *new* document when it carries a loaderId
|
|
411
|
+
* different from the pre-navigate one. A redirect chain emits several
|
|
412
|
+
* (each hop commits); the first new one is enough. This is also what
|
|
413
|
+
* makes an event replayed from the previous page inert.
|
|
414
|
+
*/
|
|
415
|
+
const applyFrameNavigated = (params: Record<string, unknown>): void => {
|
|
416
|
+
if (settled) return;
|
|
417
|
+
const frame = isRecord(params.frame) ? params.frame : undefined;
|
|
418
|
+
if (frame === undefined) return;
|
|
419
|
+
if (!frameMatches(asString(frame.id))) return;
|
|
420
|
+
const loaderId = asString(frame.loaderId);
|
|
421
|
+
if (preLoaderId !== undefined && (loaderId === undefined || loaderId === preLoaderId)) return;
|
|
422
|
+
const frameUrl = asString(frame.url);
|
|
423
|
+
if (frameUrl !== undefined) lastUrl = frameUrl;
|
|
424
|
+
commit(lastUrl === "" ? url : lastUrl);
|
|
425
|
+
};
|
|
426
|
+
|
|
427
|
+
const applyNavigatedWithinDocument = (params: Record<string, unknown>): void => {
|
|
428
|
+
if (settled) return;
|
|
429
|
+
if (!frameMatches(asString(params.frameId))) return;
|
|
430
|
+
// Same-document navigation: the loaderId intentionally does not
|
|
431
|
+
// change, so the loader gate above must not be applied here.
|
|
432
|
+
const frameUrl = asString(params.url);
|
|
433
|
+
lastUrl = frameUrl ?? url;
|
|
434
|
+
commit(lastUrl);
|
|
435
|
+
};
|
|
436
|
+
|
|
437
|
+
const runOrDefer = (action: () => void): void => {
|
|
438
|
+
if (!responseReceived) {
|
|
439
|
+
deferred.push(action);
|
|
440
|
+
return;
|
|
441
|
+
}
|
|
442
|
+
action();
|
|
443
|
+
};
|
|
444
|
+
|
|
445
|
+
// Both listeners are installed *before* `Page.navigate`, so a commit
|
|
446
|
+
// event cannot be missed even when it arrives in the same socket read
|
|
447
|
+
// as the response.
|
|
448
|
+
cleanups.push(
|
|
449
|
+
client.on("Page.frameNavigated", (params, eventSessionId) => {
|
|
450
|
+
if (!forThisSession(eventSessionId)) return;
|
|
451
|
+
runOrDefer(() => applyFrameNavigated(params));
|
|
452
|
+
}),
|
|
453
|
+
);
|
|
454
|
+
cleanups.push(
|
|
455
|
+
client.on("Page.navigatedWithinDocument", (params, eventSessionId) => {
|
|
456
|
+
if (!forThisSession(eventSessionId)) return;
|
|
457
|
+
runOrDefer(() => applyNavigatedWithinDocument(params));
|
|
458
|
+
}),
|
|
459
|
+
);
|
|
460
|
+
|
|
461
|
+
/**
|
|
462
|
+
* Some navigations return without any commit event and the target URL
|
|
463
|
+
* is already the requested one. One probe after a short grace keeps
|
|
464
|
+
* that case from timing out, without trusting the URL earlier (the tab
|
|
465
|
+
* still shows the previous document until the commit happens).
|
|
466
|
+
*/
|
|
467
|
+
const checkUrlFallback = async (): Promise<void> => {
|
|
468
|
+
fallbackTimer = undefined;
|
|
469
|
+
if (settled || committed) return;
|
|
470
|
+
let current: unknown;
|
|
471
|
+
try {
|
|
472
|
+
current = await evaluateRaw<unknown>("location.href");
|
|
473
|
+
} catch {
|
|
474
|
+
return; // best effort; commit events can still arrive before the deadline
|
|
475
|
+
}
|
|
476
|
+
if (settled || committed) return;
|
|
477
|
+
if (typeof current === "string" && sameNavigatedUrl(current, url)) commit(current);
|
|
478
|
+
};
|
|
479
|
+
|
|
480
|
+
const begin = async (): Promise<void> => {
|
|
481
|
+
const tree = await client.send<Record<string, unknown>>("Page.getFrameTree", {}, sessionId);
|
|
482
|
+
if (settled) return;
|
|
483
|
+
const main = mainFrameOf(tree);
|
|
484
|
+
preLoaderId = main.loaderId;
|
|
485
|
+
frameFilter = main.frameId;
|
|
486
|
+
|
|
487
|
+
const response = await client.send<NavigateResponse>("Page.navigate", { url }, sessionId);
|
|
488
|
+
if (settled) return;
|
|
489
|
+
|
|
490
|
+
responseReceived = true;
|
|
491
|
+
// The response frameId is authoritative when the frame tree did not
|
|
492
|
+
// carry one; `??=` keeps the pre-navigate id otherwise.
|
|
493
|
+
frameFilter ??= asString(response.frameId);
|
|
494
|
+
|
|
495
|
+
while (!settled) {
|
|
496
|
+
const next = deferred.shift();
|
|
497
|
+
if (next === undefined) break;
|
|
498
|
+
next();
|
|
499
|
+
}
|
|
500
|
+
if (settled || committed) return;
|
|
501
|
+
|
|
502
|
+
fallbackTimer = setTimeout(() => {
|
|
503
|
+
void checkUrlFallback();
|
|
504
|
+
}, NAVIGATE_URL_FALLBACK_MS);
|
|
505
|
+
fallbackTimer.unref();
|
|
506
|
+
};
|
|
507
|
+
|
|
508
|
+
activeNavigation = { reject: abort };
|
|
509
|
+
void begin().catch((error: unknown) => abort(asWebSearchError(error, "E_CDP_DISCONNECTED", "navigate")));
|
|
510
|
+
});
|
|
511
|
+
},
|
|
512
|
+
|
|
513
|
+
async evaluate<T>(expression: string, o: EvaluateOptions = {}): Promise<T> {
|
|
514
|
+
return await evaluateRaw<T>(expression, o);
|
|
515
|
+
},
|
|
516
|
+
|
|
517
|
+
async innerText(selector?: string): Promise<string> {
|
|
518
|
+
const expression =
|
|
519
|
+
selector === undefined
|
|
520
|
+
? '(() => (document.body ? document.body.innerText : ""))()'
|
|
521
|
+
: `(() => { const el = document.querySelector(${JSON.stringify(selector)}); return el ? el.innerText : ""; })()`;
|
|
522
|
+
const value = await evaluateRaw<unknown>(expression);
|
|
523
|
+
return typeof value === "string" ? value : "";
|
|
524
|
+
},
|
|
525
|
+
|
|
526
|
+
async html(): Promise<string> {
|
|
527
|
+
const value = await evaluateRaw<unknown>(
|
|
528
|
+
'(() => (document.documentElement ? document.documentElement.outerHTML : ""))()',
|
|
529
|
+
);
|
|
530
|
+
return typeof value === "string" ? value : "";
|
|
531
|
+
},
|
|
532
|
+
|
|
533
|
+
async click(selectorOrPoint: string | { x: number; y: number }): Promise<void> {
|
|
534
|
+
const sessionId = await ensureEnabled("resolve");
|
|
535
|
+
let x: number;
|
|
536
|
+
let y: number;
|
|
537
|
+
|
|
538
|
+
if (typeof selectorOrPoint === "string") {
|
|
539
|
+
const rect = await evaluateRaw<unknown>(clickExpression(selectorOrPoint));
|
|
540
|
+
if (!isRecord(rect)) {
|
|
541
|
+
throw fail("E_PARSE", "resolve", {
|
|
542
|
+
message: `selector not found: ${selectorOrPoint}`,
|
|
543
|
+
hint: "the click target is missing; check the selector canary and the artifact",
|
|
544
|
+
});
|
|
545
|
+
}
|
|
546
|
+
const rectX = finiteNumber(rect.x);
|
|
547
|
+
const rectY = finiteNumber(rect.y);
|
|
548
|
+
if (rectX === undefined || rectY === undefined) {
|
|
549
|
+
throw fail("E_PARSE", "resolve", {
|
|
550
|
+
message: `selector has no clickable box: ${selectorOrPoint}`,
|
|
551
|
+
});
|
|
552
|
+
}
|
|
553
|
+
x = rectX;
|
|
554
|
+
y = rectY;
|
|
555
|
+
} else {
|
|
556
|
+
const pointX = finiteNumber(selectorOrPoint.x);
|
|
557
|
+
const pointY = finiteNumber(selectorOrPoint.y);
|
|
558
|
+
if (pointX === undefined || pointY === undefined) {
|
|
559
|
+
throw fail("E_PARSE", "resolve", { message: "click point must have finite x and y" });
|
|
560
|
+
}
|
|
561
|
+
x = pointX;
|
|
562
|
+
y = pointY;
|
|
563
|
+
}
|
|
564
|
+
|
|
565
|
+
// Chrome expects integer CSS pixels; a fractional coordinate is rounded
|
|
566
|
+
// rather than truncated so the click lands on the visual center.
|
|
567
|
+
const px = Math.round(x);
|
|
568
|
+
const py = Math.round(y);
|
|
569
|
+
const mouse: Record<string, unknown> = {
|
|
570
|
+
x: px,
|
|
571
|
+
y: py,
|
|
572
|
+
button: "left",
|
|
573
|
+
clickCount: 1,
|
|
574
|
+
};
|
|
575
|
+
await client.send("Input.dispatchMouseEvent", { ...mouse, type: "mousePressed" }, sessionId);
|
|
576
|
+
await client.send("Input.dispatchMouseEvent", { ...mouse, type: "mouseReleased" }, sessionId);
|
|
577
|
+
},
|
|
578
|
+
|
|
579
|
+
async typeText(text: string, o: TypeTextOptions = {}): Promise<void> {
|
|
580
|
+
const sessionId = await ensureEnabled("resolve");
|
|
581
|
+
const minMs = nonNegativeDelay(o.minMs, DEFAULT_TYPE_MIN_MS);
|
|
582
|
+
const maxMs = Math.max(minMs, nonNegativeDelay(o.maxMs, DEFAULT_TYPE_MAX_MS));
|
|
583
|
+
const signal = o.signal;
|
|
584
|
+
|
|
585
|
+
// No `Input.insertText`: it is one atomic event and skips the key events
|
|
586
|
+
// a real keyboard produces, which is exactly the pattern a bot detector
|
|
587
|
+
// looks for. Each character is rawKeyDown/char/keyUp with a human delay.
|
|
588
|
+
//
|
|
589
|
+
// The key-down half must be `rawKeyDown`: Chrome inserts `text` on a
|
|
590
|
+
// `keyDown` event, so pairing a text-carrying keyDown with `char` typed
|
|
591
|
+
// every character twice on a real page. `rawKeyDown` carries no text and
|
|
592
|
+
// is what a physical keyboard emits before the character event.
|
|
593
|
+
for (const char of text) {
|
|
594
|
+
if (isAborted(signal)) {
|
|
595
|
+
throw fail("E_CANCELLED", "resolve", { message: "typing was aborted" });
|
|
596
|
+
}
|
|
597
|
+
const waitMs = minMs + Math.floor(Math.random() * (maxMs - minMs + 1));
|
|
598
|
+
try {
|
|
599
|
+
await clock.sleep(waitMs, signal);
|
|
600
|
+
} catch (error) {
|
|
601
|
+
throw asWebSearchError(error, "E_CANCELLED", "resolve");
|
|
602
|
+
}
|
|
603
|
+
if (isAborted(signal)) {
|
|
604
|
+
throw fail("E_CANCELLED", "resolve", { message: "typing was aborted" });
|
|
605
|
+
}
|
|
606
|
+
|
|
607
|
+
const vk = charKeyCode(char);
|
|
608
|
+
await client.send(
|
|
609
|
+
"Input.dispatchKeyEvent",
|
|
610
|
+
{
|
|
611
|
+
type: "rawKeyDown",
|
|
612
|
+
key: char,
|
|
613
|
+
windowsVirtualKeyCode: vk,
|
|
614
|
+
nativeVirtualKeyCode: vk,
|
|
615
|
+
},
|
|
616
|
+
sessionId,
|
|
617
|
+
);
|
|
618
|
+
await client.send(
|
|
619
|
+
"Input.dispatchKeyEvent",
|
|
620
|
+
{ type: "char", key: char, text: char, unmodifiedText: char },
|
|
621
|
+
sessionId,
|
|
622
|
+
);
|
|
623
|
+
await client.send(
|
|
624
|
+
"Input.dispatchKeyEvent",
|
|
625
|
+
{
|
|
626
|
+
type: "keyUp",
|
|
627
|
+
key: char,
|
|
628
|
+
code: "",
|
|
629
|
+
windowsVirtualKeyCode: vk,
|
|
630
|
+
nativeVirtualKeyCode: vk,
|
|
631
|
+
},
|
|
632
|
+
sessionId,
|
|
633
|
+
);
|
|
634
|
+
}
|
|
635
|
+
},
|
|
636
|
+
|
|
637
|
+
async press(key: string): Promise<void> {
|
|
638
|
+
const sessionId = await ensureEnabled("resolve");
|
|
639
|
+
const info = KEY_INFO[key];
|
|
640
|
+
if (info === undefined && key.length !== 1) {
|
|
641
|
+
throw fail("E_UNSUPPORTED", "resolve", {
|
|
642
|
+
message: `unknown key: ${key}`,
|
|
643
|
+
hint: "use Enter, Tab, Escape, Arrow*, Backspace, Delete, Space, Home, End, Page* or one character",
|
|
644
|
+
});
|
|
645
|
+
}
|
|
646
|
+
const vk = info?.vk ?? charKeyCode(key);
|
|
647
|
+
const params: Record<string, unknown> = {
|
|
648
|
+
key,
|
|
649
|
+
code: info?.code ?? "",
|
|
650
|
+
windowsVirtualKeyCode: vk,
|
|
651
|
+
nativeVirtualKeyCode: vk,
|
|
652
|
+
};
|
|
653
|
+
// Enter needs `text: "\r"` to activate a focused control in Chrome; the
|
|
654
|
+
// other mapped keys are navigation/edit keys and must not insert text.
|
|
655
|
+
if (key === "Enter") {
|
|
656
|
+
params.text = "\r";
|
|
657
|
+
params.unmodifiedText = "\r";
|
|
658
|
+
}
|
|
659
|
+
await client.send("Input.dispatchKeyEvent", { ...params, type: "keyDown" }, sessionId);
|
|
660
|
+
await client.send("Input.dispatchKeyEvent", { ...params, type: "keyUp" }, sessionId);
|
|
661
|
+
},
|
|
662
|
+
|
|
663
|
+
async scroll(dy: number): Promise<void> {
|
|
664
|
+
const delta = finiteNumber(dy);
|
|
665
|
+
if (delta === undefined) {
|
|
666
|
+
throw fail("E_PARSE", "navigate", { message: "scroll delta must be a finite number" });
|
|
667
|
+
}
|
|
668
|
+
await evaluateRaw<unknown>(`window.scrollBy(0, ${delta})`);
|
|
669
|
+
},
|
|
670
|
+
|
|
671
|
+
async screenshot(): Promise<Buffer> {
|
|
672
|
+
const sessionId = sessionIdOrThrow("render");
|
|
673
|
+
const response = await client.send<Record<string, unknown>>(
|
|
674
|
+
"Page.captureScreenshot",
|
|
675
|
+
{ format: "jpeg", quality: 70 },
|
|
676
|
+
sessionId,
|
|
677
|
+
);
|
|
678
|
+
const data = asString(response.data);
|
|
679
|
+
if (data === undefined || data === "") {
|
|
680
|
+
throw fail("E_PARSE", "render", {
|
|
681
|
+
message: "Page.captureScreenshot returned no data",
|
|
682
|
+
hint: "the compositor did not produce a frame; retry or run /web-doctor",
|
|
683
|
+
});
|
|
684
|
+
}
|
|
685
|
+
return Buffer.from(data, "base64");
|
|
686
|
+
},
|
|
687
|
+
|
|
688
|
+
async cookiesFor(urls: string[]): Promise<CookieInfo[]> {
|
|
689
|
+
const sessionId = sessionIdOrThrow("read");
|
|
690
|
+
const requested = Array.isArray(urls) ? urls.filter((url) => typeof url === "string") : [];
|
|
691
|
+
const response = await client.send<Record<string, unknown>>(
|
|
692
|
+
"Network.getCookies",
|
|
693
|
+
{ urls: requested },
|
|
694
|
+
sessionId,
|
|
695
|
+
);
|
|
696
|
+
const cookies = Array.isArray(response.cookies) ? response.cookies : [];
|
|
697
|
+
const result: CookieInfo[] = [];
|
|
698
|
+
for (const cookie of cookies) {
|
|
699
|
+
if (!isRecord(cookie)) continue;
|
|
700
|
+
const name = asString(cookie.name);
|
|
701
|
+
const value = asString(cookie.value);
|
|
702
|
+
const domain = asString(cookie.domain);
|
|
703
|
+
if (name === undefined || value === undefined || domain === undefined) continue;
|
|
704
|
+
result.push({ name, value, domain });
|
|
705
|
+
}
|
|
706
|
+
return result;
|
|
707
|
+
},
|
|
708
|
+
|
|
709
|
+
async bringToFront(): Promise<void> {
|
|
710
|
+
const sessionId = sessionIdOrThrow("render");
|
|
711
|
+
await client.send("Page.bringToFront", {}, sessionId);
|
|
712
|
+
},
|
|
713
|
+
};
|
|
714
|
+
|
|
715
|
+
return page;
|
|
716
|
+
}
|