pi-browser-harness 0.10.2 → 0.12.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +54 -0
- package/README.md +21 -10
- package/package.json +6 -3
- package/skills/pi-browser-harness/SKILL.md +30 -1
- package/src/cdp/commands.ts +260 -0
- package/src/cdp/console-buffer.ts +10 -85
- package/src/cdp/discovery.ts +31 -71
- package/src/cdp/errors.ts +4 -0
- package/src/cdp/events.ts +170 -0
- package/src/cdp/network-buffer.ts +42 -109
- package/src/cdp/ownership.ts +0 -17
- package/src/cdp/record-store.ts +61 -0
- package/src/cdp/session.ts +205 -193
- package/src/cdp/target-factory.ts +10 -80
- package/src/cdp/types.ts +21 -12
- package/src/client.ts +39 -95
- package/src/daemon/bridge.ts +375 -265
- package/src/daemon/index.ts +0 -28
- package/src/daemon/protocol.ts +6 -68
- package/src/daemon/queue.ts +85 -0
- package/src/daemon/server.ts +24 -99
- package/src/daemon/spawn.ts +90 -82
- package/src/{cdp/daemon-transport.ts → daemon/transport.ts} +24 -52
- package/src/domains/ax-tree.ts +179 -0
- package/src/domains/box.ts +51 -0
- package/src/domains/cdp-call.ts +55 -0
- package/src/domains/click.ts +19 -37
- package/src/domains/console.ts +48 -4
- package/src/domains/dialog.ts +8 -5
- package/src/domains/drag.ts +8 -7
- package/src/domains/element-call.ts +117 -0
- package/src/domains/files.ts +75 -30
- package/src/domains/fill-engine.ts +80 -0
- package/src/domains/form-batch.ts +64 -0
- package/src/domains/form.ts +115 -112
- package/src/domains/headers.ts +89 -0
- package/src/domains/history.ts +16 -12
- package/src/domains/isolated-tab.ts +26 -55
- package/src/domains/js.ts +32 -51
- package/src/domains/keyboard.ts +38 -34
- package/src/domains/navigate.ts +16 -26
- package/src/domains/network.ts +61 -13
- package/src/domains/page.ts +25 -10
- package/src/domains/readpage/read-page.ts +74 -20
- package/src/domains/readpage/readability.ts +0 -24
- package/src/domains/ref-resolve.ts +24 -58
- package/src/domains/render.ts +6 -27
- package/src/domains/screenshot-capture.ts +80 -0
- package/src/domains/screenshot.ts +9 -90
- package/src/domains/scroll.ts +10 -12
- package/src/domains/search/google-serp.ts +0 -29
- package/src/domains/search/web-search.ts +48 -21
- package/src/domains/setup.ts +1 -13
- package/src/domains/snapshot.ts +36 -280
- package/src/domains/tabs.ts +15 -16
- package/src/domains/target.ts +35 -0
- package/src/domains/viewport.ts +5 -4
- package/src/index.ts +23 -69
- package/src/profile/bind.ts +8 -57
- package/src/profile/browser-process.ts +75 -126
- package/src/profile/command.ts +50 -29
- package/src/profile/launch.ts +22 -40
- package/src/profile/list.ts +14 -54
- package/src/profile/paths.ts +2 -65
- package/src/profile/store.ts +35 -56
- package/src/prompt.ts +22 -122
- package/src/registry.ts +33 -33
- package/src/schemas/parse.ts +23 -0
- package/src/setup.ts +0 -25
- package/src/state.ts +48 -18
- package/src/util/debug.ts +8 -0
- package/src/util/guards.ts +30 -0
- package/src/util/js-template.ts +1 -16
- package/src/util/mutex.ts +0 -5
- package/src/util/paths.ts +11 -1
- package/src/util/result.ts +0 -6
- package/src/util/sharp-shim.ts +12 -19
- package/src/util/tool.ts +17 -42
- package/src/util/truncate.ts +6 -0
- package/tsconfig.json +1 -16
- package/skills/deep-research/SKILL.md +0 -66
- package/src/cdp/event-queue.ts +0 -128
- package/src/cdp/transport.ts +0 -156
- package/src/cdp/window-binding.ts +0 -15
- package/src/deep-research.ts +0 -41
- package/src/domains/forms.ts +0 -355
- package/src/domains/readpage/capture.ts +0 -46
- package/src/domains/search/extract.ts +0 -44
- package/src/profile/picker.ts +0 -73
- package/src/util/keycodes.ts +0 -10
- package/src/util/time.ts +0 -29
|
@@ -0,0 +1,80 @@
|
|
|
1
|
+
import { rename, writeFile } from "node:fs/promises";
|
|
2
|
+
import type { BrowserClient } from "../client";
|
|
3
|
+
import { type Result, ok } from "../util/result";
|
|
4
|
+
import type { ToolErr } from "../util/tool";
|
|
5
|
+
import { screenshotPath } from "../util/paths";
|
|
6
|
+
import { loadSharp } from "../util/sharp-shim";
|
|
7
|
+
import { safeJs } from "../util/js-template";
|
|
8
|
+
import { cdpCall, evalJs } from "./cdp-call";
|
|
9
|
+
|
|
10
|
+
export type CaptureArgs = {
|
|
11
|
+
readonly fullPage?: boolean;
|
|
12
|
+
readonly format?: "png" | "jpeg";
|
|
13
|
+
readonly quality?: number;
|
|
14
|
+
};
|
|
15
|
+
|
|
16
|
+
export const captureBase = async (
|
|
17
|
+
client: BrowserClient,
|
|
18
|
+
args: CaptureArgs,
|
|
19
|
+
): Promise<Result<{ readonly path: string; readonly format: "png" | "jpeg" }, ToolErr>> => {
|
|
20
|
+
const format = args.format ?? "png";
|
|
21
|
+
const quality = args.quality ?? 80;
|
|
22
|
+
const path = screenshotPath(client.namespace, format);
|
|
23
|
+
const r = await cdpCall(client, "Page.captureScreenshot", {
|
|
24
|
+
format,
|
|
25
|
+
captureBeyondViewport: args.fullPage ?? false,
|
|
26
|
+
...(format === "jpeg" ? { quality } : {}),
|
|
27
|
+
});
|
|
28
|
+
if (!r.success) return r;
|
|
29
|
+
await writeFile(path, Buffer.from(r.data.data, "base64"));
|
|
30
|
+
return ok({ path, format });
|
|
31
|
+
};
|
|
32
|
+
|
|
33
|
+
export const resizeIfNeeded = async (
|
|
34
|
+
path: string,
|
|
35
|
+
maxDim: number,
|
|
36
|
+
): Promise<{ readonly note: string }> => {
|
|
37
|
+
const load = await loadSharp();
|
|
38
|
+
if (load.kind === "missing") return { note: " (maxDim ignored: install sharp for auto-resize)" };
|
|
39
|
+
if (load.kind === "error") return { note: ` (maxDim ignored: sharp failed to load: ${load.message})` };
|
|
40
|
+
try {
|
|
41
|
+
const meta = await load.sharp(path).metadata();
|
|
42
|
+
const w = meta.width ?? 0;
|
|
43
|
+
const h = meta.height ?? 0;
|
|
44
|
+
if (Math.max(w, h) <= maxDim) return { note: "" };
|
|
45
|
+
const tmp = `${path}.resized`;
|
|
46
|
+
await load.sharp(path).resize(maxDim, maxDim, { fit: "inside" }).toFile(tmp);
|
|
47
|
+
await rename(tmp, path);
|
|
48
|
+
return { note: ` (resized to fit ${maxDim}px)` };
|
|
49
|
+
} catch (e) {
|
|
50
|
+
return { note: ` (maxDim ignored: sharp threw: ${e instanceof Error ? e.message : String(e)})` };
|
|
51
|
+
}
|
|
52
|
+
};
|
|
53
|
+
|
|
54
|
+
export const captureWithCrosshair = async (
|
|
55
|
+
client: BrowserClient,
|
|
56
|
+
args: { readonly x: number; readonly y: number; readonly format?: "png" | "jpeg"; readonly quality?: number },
|
|
57
|
+
): Promise<Result<{ readonly path: string }, ToolErr>> => {
|
|
58
|
+
const cap = await captureBase(client, args);
|
|
59
|
+
if (!cap.success) return cap;
|
|
60
|
+
const load = await loadSharp();
|
|
61
|
+
if (load.kind !== "ok") return ok({ path: cap.data.path });
|
|
62
|
+
try {
|
|
63
|
+
const dprR = await evalJs(client, safeJs`window.devicePixelRatio`);
|
|
64
|
+
const dpr = dprR.success && typeof dprR.data === "number" ? dprR.data : 1;
|
|
65
|
+
const meta = await load.sharp(cap.data.path).metadata();
|
|
66
|
+
const w = meta.width ?? 0;
|
|
67
|
+
const h = meta.height ?? 0;
|
|
68
|
+
const px = Math.round(args.x * dpr);
|
|
69
|
+
const py = Math.round(args.y * dpr);
|
|
70
|
+
const r = Math.round(15 * dpr);
|
|
71
|
+
const stroke = Math.max(2, Math.round(3 * dpr));
|
|
72
|
+
const svg = `<svg width="${w}" height="${h}"><circle cx="${px}" cy="${py}" r="${r}" fill="none" stroke="red" stroke-width="${stroke}" opacity="0.8"/><line x1="${px - r - 5}" y1="${py}" x2="${px + r + 5}" y2="${py}" stroke="red" stroke-width="${Math.max(1, stroke - 1)}" opacity="0.8"/><line x1="${px}" y1="${py - r - 5}" x2="${px}" y2="${py + r + 5}" stroke="red" stroke-width="${Math.max(1, stroke - 1)}" opacity="0.8"/></svg>`;
|
|
73
|
+
const tmp = `${cap.data.path}.debug`;
|
|
74
|
+
await load.sharp(cap.data.path).composite([{ input: Buffer.from(svg), top: 0, left: 0 }]).toFile(tmp);
|
|
75
|
+
await rename(tmp, cap.data.path);
|
|
76
|
+
return ok({ path: cap.data.path });
|
|
77
|
+
} catch {
|
|
78
|
+
return ok({ path: cap.data.path });
|
|
79
|
+
}
|
|
80
|
+
};
|
|
@@ -1,13 +1,10 @@
|
|
|
1
1
|
import { readFileSync } from "node:fs";
|
|
2
|
-
import { writeFile, rename } from "node:fs/promises";
|
|
3
2
|
import { Type } from "typebox";
|
|
4
3
|
import { Image, type ImageTheme, Text, truncateToWidth, visibleWidth } from "@mariozechner/pi-tui";
|
|
5
|
-
import type
|
|
6
|
-
import { type Result, err, ok } from "../util/result";
|
|
4
|
+
import { type Result, ok } from "../util/result";
|
|
7
5
|
import { defineBrowserTool, type ToolErr, type ToolOk } from "../util/tool";
|
|
8
|
-
import {
|
|
9
|
-
import {
|
|
10
|
-
import { safeJs } from "../util/js-template";
|
|
6
|
+
import { asRecord, asString } from "../util/guards";
|
|
7
|
+
import { captureBase, resizeIfNeeded } from "./screenshot-capture";
|
|
11
8
|
|
|
12
9
|
const ScreenshotArgs = Type.Object({
|
|
13
10
|
fullPage: Type.Optional(Type.Boolean({ default: false, description: "Capture beyond viewport" })),
|
|
@@ -16,50 +13,6 @@ const ScreenshotArgs = Type.Object({
|
|
|
16
13
|
maxDim: Type.Optional(Type.Integer({ minimum: 100, maximum: 8000, description: "If max(w,h) exceeds this, resize via sharp." })),
|
|
17
14
|
});
|
|
18
15
|
|
|
19
|
-
type CaptureArgs = {
|
|
20
|
-
readonly fullPage?: boolean;
|
|
21
|
-
readonly format?: "png" | "jpeg";
|
|
22
|
-
readonly quality?: number;
|
|
23
|
-
};
|
|
24
|
-
|
|
25
|
-
const captureBase = async (
|
|
26
|
-
client: BrowserClient,
|
|
27
|
-
args: CaptureArgs,
|
|
28
|
-
): Promise<Result<{ readonly path: string; readonly format: "png" | "jpeg" }, ToolErr>> => {
|
|
29
|
-
const format = args.format ?? "png";
|
|
30
|
-
const quality = args.quality ?? 80;
|
|
31
|
-
const path = screenshotPath(client.namespace, format);
|
|
32
|
-
const params: Record<string, unknown> = { format, captureBeyondViewport: args.fullPage ?? false };
|
|
33
|
-
if (format === "jpeg") params["quality"] = quality;
|
|
34
|
-
const r = await client.session().call("Page.captureScreenshot", params);
|
|
35
|
-
if (!r.success) return err({ kind: "cdp_error", message: r.error.message });
|
|
36
|
-
// Page.captureScreenshot returns { data: base64 }
|
|
37
|
-
const data = (r.data as { data: string }).data;
|
|
38
|
-
await writeFile(path, Buffer.from(data, "base64"));
|
|
39
|
-
return ok({ path, format });
|
|
40
|
-
};
|
|
41
|
-
|
|
42
|
-
const resizeIfNeeded = async (
|
|
43
|
-
path: string,
|
|
44
|
-
maxDim: number,
|
|
45
|
-
): Promise<{ readonly note: string }> => {
|
|
46
|
-
const load = await loadSharp();
|
|
47
|
-
if (load.kind === "missing") return { note: " (maxDim ignored: install sharp for auto-resize)" };
|
|
48
|
-
if (load.kind === "error") return { note: ` (maxDim ignored: sharp failed to load: ${load.message})` };
|
|
49
|
-
try {
|
|
50
|
-
const meta = await load.sharp(path).metadata();
|
|
51
|
-
const w = meta.width ?? 0;
|
|
52
|
-
const h = meta.height ?? 0;
|
|
53
|
-
if (Math.max(w, h) <= maxDim) return { note: "" };
|
|
54
|
-
const tmp = `${path}.resized`;
|
|
55
|
-
await load.sharp(path).resize(maxDim, maxDim, { fit: "inside" }).toFile(tmp);
|
|
56
|
-
await rename(tmp, path);
|
|
57
|
-
return { note: ` (resized to fit ${maxDim}px)` };
|
|
58
|
-
} catch (e) {
|
|
59
|
-
return { note: ` (maxDim ignored: sharp threw: ${e instanceof Error ? e.message : String(e)})` };
|
|
60
|
-
}
|
|
61
|
-
};
|
|
62
|
-
|
|
63
16
|
export const screenshotTool = defineBrowserTool({
|
|
64
17
|
name: "browser_screenshot",
|
|
65
18
|
label: "Browser Screenshot",
|
|
@@ -67,13 +20,10 @@ export const screenshotTool = defineBrowserTool({
|
|
|
67
20
|
"Capture the page as a JPEG/PNG image. NOT a default exploration tool. Use ONLY when (a) you need to verify visual properties (layout, color, spacing, rendered text legibility), or (b) browser_snapshot and browser_execute_js cannot answer your question. For understanding what's on the page use browser_snapshot. For specific element values use browser_execute_js. For click coordinates use the @(x,y) hints in browser_snapshot's outline.",
|
|
68
21
|
promptSnippet: "Capture a screenshot — visual verification only",
|
|
69
22
|
promptGuidelines: [
|
|
70
|
-
"DO NOT use as a default exploration tool. browser_snapshot is the default for understanding page structure — far cheaper and more reliable.",
|
|
71
|
-
"DO NOT use to find click coordinates. browser_snapshot already includes @(x,y) for every interactive element. If a target isn't there, use browser_execute_js with element.getBoundingClientRect().",
|
|
72
|
-
"DO use to verify visual rendering after an action: did the modal animate in correctly, did the chart render, are colors right, did the layout reflow as expected.",
|
|
73
|
-
"DO use as a debugging fallback when browser_execute_js or browser_snapshot return surprising results and you suspect the rendered page differs from the DOM.",
|
|
74
23
|
"format='jpeg' with quality 60-90 keeps screenshots small for photo-heavy pages. Use maxDim if the page is huge.",
|
|
75
24
|
],
|
|
76
25
|
parameters: ScreenshotArgs,
|
|
26
|
+
concurrency: "parallel",
|
|
77
27
|
async handler(args, { client }): Promise<Result<ToolOk, ToolErr>> {
|
|
78
28
|
const cap = await captureBase(client, args);
|
|
79
29
|
if (!cap.success) return cap;
|
|
@@ -89,15 +39,16 @@ export const screenshotTool = defineBrowserTool({
|
|
|
89
39
|
},
|
|
90
40
|
|
|
91
41
|
renderResult(result, _expanded, theme) {
|
|
92
|
-
const
|
|
93
|
-
const filePath =
|
|
42
|
+
const raw = asRecord(result.details);
|
|
43
|
+
const filePath = raw === undefined ? undefined : asString(raw["path"]);
|
|
94
44
|
if (!filePath) {
|
|
95
45
|
return new Text(theme.fg("error", "Screenshot path missing"), 0, 0);
|
|
96
46
|
}
|
|
47
|
+
const format = raw === undefined ? undefined : asString(raw["format"]);
|
|
97
48
|
try {
|
|
98
49
|
const buf = readFileSync(filePath);
|
|
99
50
|
const b64 = buf.toString("base64");
|
|
100
|
-
const mimeType =
|
|
51
|
+
const mimeType = format === "jpeg" ? "image/jpeg" : "image/png";
|
|
101
52
|
const imageTheme: ImageTheme = {
|
|
102
53
|
fallbackColor: (str: string) => theme.fg("dim", str),
|
|
103
54
|
};
|
|
@@ -106,10 +57,7 @@ export const screenshotTool = defineBrowserTool({
|
|
|
106
57
|
maxHeightCells: 24,
|
|
107
58
|
filename: filePath,
|
|
108
59
|
});
|
|
109
|
-
// Image's text fallback
|
|
110
|
-
// not respect the rendered width, so a long file path can blow past the
|
|
111
|
-
// terminal width and crash the host TUI. Wrap the component to truncate
|
|
112
|
-
// each rendered line to fit.
|
|
60
|
+
// Image's text fallback does not respect the rendered width, so an unwrapped long path can blow past the terminal width and crash the host TUI.
|
|
113
61
|
return {
|
|
114
62
|
invalidate: () => image.invalidate(),
|
|
115
63
|
render: (width: number) =>
|
|
@@ -122,32 +70,3 @@ export const screenshotTool = defineBrowserTool({
|
|
|
122
70
|
}
|
|
123
71
|
},
|
|
124
72
|
});
|
|
125
|
-
|
|
126
|
-
export const captureWithCrosshair = async (
|
|
127
|
-
client: BrowserClient,
|
|
128
|
-
args: { readonly x: number; readonly y: number; readonly format?: "png" | "jpeg"; readonly quality?: number },
|
|
129
|
-
): Promise<Result<{ readonly path: string }, ToolErr>> => {
|
|
130
|
-
const cap = await captureBase(client, args);
|
|
131
|
-
if (!cap.success) return cap;
|
|
132
|
-
const load = await loadSharp();
|
|
133
|
-
if (load.kind !== "ok") return ok({ path: cap.data.path });
|
|
134
|
-
try {
|
|
135
|
-
const dprR = await client.evaluateJs(safeJs`window.devicePixelRatio`);
|
|
136
|
-
const dpr = dprR.success && typeof dprR.data === "number" ? dprR.data : 1;
|
|
137
|
-
const meta = await load.sharp(cap.data.path).metadata();
|
|
138
|
-
const w = meta.width ?? 0;
|
|
139
|
-
const h = meta.height ?? 0;
|
|
140
|
-
const px = Math.round(args.x * dpr);
|
|
141
|
-
const py = Math.round(args.y * dpr);
|
|
142
|
-
const r = Math.round(15 * dpr);
|
|
143
|
-
const stroke = Math.max(2, Math.round(3 * dpr));
|
|
144
|
-
const svg = `<svg width="${w}" height="${h}"><circle cx="${px}" cy="${py}" r="${r}" fill="none" stroke="red" stroke-width="${stroke}" opacity="0.8"/><line x1="${px - r - 5}" y1="${py}" x2="${px + r + 5}" y2="${py}" stroke="red" stroke-width="${Math.max(1, stroke - 1)}" opacity="0.8"/><line x1="${px}" y1="${py - r - 5}" x2="${px}" y2="${py + r + 5}" stroke="red" stroke-width="${Math.max(1, stroke - 1)}" opacity="0.8"/></svg>`;
|
|
145
|
-
const tmp = `${cap.data.path}.debug`;
|
|
146
|
-
await load.sharp(cap.data.path).composite([{ input: Buffer.from(svg), top: 0, left: 0 }]).toFile(tmp);
|
|
147
|
-
await rename(tmp, cap.data.path);
|
|
148
|
-
return ok({ path: cap.data.path });
|
|
149
|
-
} catch {
|
|
150
|
-
// Crosshair overlay is best-effort; on failure return the plain screenshot.
|
|
151
|
-
return ok({ path: cap.data.path });
|
|
152
|
-
}
|
|
153
|
-
};
|
package/src/domains/scroll.ts
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import { Type } from "typebox";
|
|
2
2
|
import { type Result, err, ok } from "../util/result";
|
|
3
3
|
import { defineBrowserTool, type ToolErr, type ToolOk } from "../util/tool";
|
|
4
|
+
import { cdpCall } from "./cdp-call";
|
|
4
5
|
|
|
5
6
|
const ScrollArgs = Type.Object({
|
|
6
7
|
x: Type.Optional(Type.Number({ description: "X coordinate where to scroll. Default: viewport center" })),
|
|
@@ -19,10 +20,10 @@ export const scrollTool = defineBrowserTool({
|
|
|
19
20
|
"Scroll the page at given coordinates. deltaY follows W3C wheel-event convention: positive = scroll down, negative = scroll up. Default scrolls down (deltaY = 300).",
|
|
20
21
|
promptSnippet: "Scroll the page (deltaY positive=down, negative=up)",
|
|
21
22
|
promptGuidelines: [
|
|
22
|
-
"Default behavior scrolls down 300px (deltaY=300). Pass a negative deltaY to scroll up.",
|
|
23
23
|
"Pass x/y to target a specific scrollable region (e.g., a div with overflow); otherwise scrolls the page at viewport center.",
|
|
24
24
|
],
|
|
25
25
|
parameters: ScrollArgs,
|
|
26
|
+
concurrency: "serialized",
|
|
26
27
|
async handler(args, { client }): Promise<Result<ToolOk, ToolErr>> {
|
|
27
28
|
const info = await client.pageInfo();
|
|
28
29
|
if (!info.success) return err({ kind: "cdp_error", message: info.error.message });
|
|
@@ -36,22 +37,19 @@ export const scrollTool = defineBrowserTool({
|
|
|
36
37
|
const cy = args.y ?? Math.round(info.data.height / 2);
|
|
37
38
|
const dx = args.deltaX ?? 0;
|
|
38
39
|
const dy = args.deltaY ?? 300;
|
|
39
|
-
// The page target must be focused or Input events are dropped silently
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
// Establish compositor mouse position before mouseWheel; without this,
|
|
45
|
-
// CDP sometimes drops the wheel event with a timeout.
|
|
46
|
-
const moved = await client.session().call("Input.dispatchMouseEvent", {
|
|
40
|
+
// The page target must be focused or Input events are dropped silently: mouseWheel hangs and synthesizeScrollGesture returns ok without scrolling.
|
|
41
|
+
const front = await cdpCall(client, "Page.bringToFront", {});
|
|
42
|
+
if (!front.success) return front;
|
|
43
|
+
// Establish compositor mouse position first, or CDP sometimes drops the wheel event with a timeout.
|
|
44
|
+
const moved = await cdpCall(client, "Input.dispatchMouseEvent", {
|
|
47
45
|
type: "mouseMoved", x: cx, y: cy, button: "none", buttons: 0,
|
|
48
46
|
});
|
|
49
|
-
if (!moved.success) return
|
|
50
|
-
const wheel = await client
|
|
47
|
+
if (!moved.success) return moved;
|
|
48
|
+
const wheel = await cdpCall(client, "Input.dispatchMouseEvent", {
|
|
51
49
|
type: "mouseWheel", x: cx, y: cy, deltaX: dx, deltaY: dy,
|
|
52
50
|
button: "none", buttons: 0, pointerType: "mouse",
|
|
53
51
|
});
|
|
54
|
-
if (!wheel.success) return
|
|
52
|
+
if (!wheel.success) return wheel;
|
|
55
53
|
return ok({
|
|
56
54
|
text: `Scrolled at (${cx}, ${cy}) by (${dx}, ${dy})`,
|
|
57
55
|
details: { x: cx, y: cy, deltaX: dx, deltaY: dy },
|
|
@@ -1,27 +1,14 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* Pure Google SERP parsing — no CDP, no browser. Everything here is a pure
|
|
3
|
-
* function over data captured from the page, so it is unit-testable against
|
|
4
|
-
* saved fixtures (see test/deep-research/serp-parser-test.ts).
|
|
5
|
-
*
|
|
6
|
-
* ADR-0001: results are extracted by structure/semantics (anchors carrying an
|
|
7
|
-
* <h3> heading in the main results column), never by brittle Google CSS class
|
|
8
|
-
* names, and Google redirect wrappers are unwound to the real target URL.
|
|
9
|
-
*/
|
|
10
|
-
|
|
11
|
-
/** One raw anchor captured in-page: a link with a heading and nearby snippet. */
|
|
12
1
|
export type SerpAnchor = {
|
|
13
2
|
readonly href: string;
|
|
14
3
|
readonly heading: string;
|
|
15
4
|
readonly snippet: string;
|
|
16
5
|
};
|
|
17
6
|
|
|
18
|
-
/** The JSON payload the in-page extraction expression returns. */
|
|
19
7
|
export type SerpExtraction = {
|
|
20
8
|
readonly anchors: ReadonlyArray<SerpAnchor>;
|
|
21
9
|
readonly pageText: string;
|
|
22
10
|
};
|
|
23
11
|
|
|
24
|
-
/** A parsed, ranked search result — the tool's public output shape. */
|
|
25
12
|
export type SearchResult = {
|
|
26
13
|
readonly title: string;
|
|
27
14
|
readonly url: string;
|
|
@@ -29,7 +16,6 @@ export type SearchResult = {
|
|
|
29
16
|
readonly rank: number;
|
|
30
17
|
};
|
|
31
18
|
|
|
32
|
-
/** Why a SERP yielded no usable results, or that it did. */
|
|
33
19
|
export type SerpVerdict = "ok" | "captcha" | "no_results";
|
|
34
20
|
|
|
35
21
|
const CAPTCHA_MARKERS: ReadonlyArray<string> = [
|
|
@@ -42,10 +28,6 @@ const CAPTCHA_MARKERS: ReadonlyArray<string> = [
|
|
|
42
28
|
"our systems have detected unusual traffic",
|
|
43
29
|
];
|
|
44
30
|
|
|
45
|
-
/**
|
|
46
|
-
* Decode a Google redirect wrapper (`/url?q=<real>&sa=...`) to its target.
|
|
47
|
-
* Returns the input unchanged when it is not a wrapper.
|
|
48
|
-
*/
|
|
49
31
|
const unwrapRedirect = (href: string): string => {
|
|
50
32
|
const wrapper = href.match(/^(?:https?:\/\/[^/]*google\.[^/]*)?\/url\?/i);
|
|
51
33
|
if (!wrapper) return href;
|
|
@@ -66,7 +48,6 @@ const isExternalResult = (url: string): boolean => {
|
|
|
66
48
|
}
|
|
67
49
|
};
|
|
68
50
|
|
|
69
|
-
/** Normalized key for dedupe: origin + pathname, trailing slash and hash removed. */
|
|
70
51
|
const dedupeKey = (url: string): string => {
|
|
71
52
|
try {
|
|
72
53
|
const u = new URL(url);
|
|
@@ -76,11 +57,6 @@ const dedupeKey = (url: string): string => {
|
|
|
76
57
|
}
|
|
77
58
|
};
|
|
78
59
|
|
|
79
|
-
/**
|
|
80
|
-
* Parse captured anchors into ranked results: unwrap redirects, keep only
|
|
81
|
-
* external http(s) links, dedupe by normalized url (first wins), rank by input
|
|
82
|
-
* order, cap to `limit`.
|
|
83
|
-
*/
|
|
84
60
|
export const parseGoogleSerp = (anchors: ReadonlyArray<SerpAnchor>, limit: number): ReadonlyArray<SearchResult> => {
|
|
85
61
|
const seen = new Set<string>();
|
|
86
62
|
const results: SearchResult[] = [];
|
|
@@ -101,11 +77,6 @@ export const parseGoogleSerp = (anchors: ReadonlyArray<SerpAnchor>, limit: numbe
|
|
|
101
77
|
return results;
|
|
102
78
|
};
|
|
103
79
|
|
|
104
|
-
/**
|
|
105
|
-
* Decide whether the SERP is usable, blocked by a CAPTCHA, or genuinely empty.
|
|
106
|
-
* `resultCount` is the number of parsed results; `pageText` is a bounded slice
|
|
107
|
-
* of the page's visible text.
|
|
108
|
-
*/
|
|
109
80
|
export const classifySerp = (pageText: string, resultCount: number): SerpVerdict => {
|
|
110
81
|
if (resultCount > 0) return "ok";
|
|
111
82
|
const lower = pageText.toLowerCase();
|
|
@@ -1,16 +1,18 @@
|
|
|
1
1
|
import { Type } from "typebox";
|
|
2
|
+
import { Compile } from "typebox/compile";
|
|
3
|
+
import { parseJson } from "../../schemas/parse";
|
|
2
4
|
import { type Result, err, ok } from "../../util/result";
|
|
3
5
|
import { applyTruncation } from "../../util/truncate";
|
|
4
6
|
import { defineBrowserTool, type ToolErr, type ToolOk } from "../../util/tool";
|
|
5
7
|
import {
|
|
6
8
|
closeIsolatedTab,
|
|
7
9
|
evalInIsolatedTab,
|
|
10
|
+
isJsonText,
|
|
8
11
|
navigateIsolatedTab,
|
|
9
12
|
openIsolatedTab,
|
|
10
13
|
waitForIsolatedLoad,
|
|
11
14
|
} from "../isolated-tab";
|
|
12
15
|
import { renderExpandableText } from "../render";
|
|
13
|
-
import { buildSerpExtractionExpr } from "./extract";
|
|
14
16
|
import { classifySerp, parseGoogleSerp, type SearchResult, type SerpExtraction } from "./google-serp";
|
|
15
17
|
|
|
16
18
|
const LOAD_TIMEOUT_MS = 15_000;
|
|
@@ -22,14 +24,46 @@ const WebSearchArgs = Type.Object({
|
|
|
22
24
|
),
|
|
23
25
|
});
|
|
24
26
|
|
|
27
|
+
const PAGE_TEXT_LIMIT = 4000;
|
|
28
|
+
|
|
29
|
+
const buildSerpExtractionExpr = (): string => `
|
|
30
|
+
(() => {
|
|
31
|
+
const main = document.querySelector('#search, #rso, #main') || document.body;
|
|
32
|
+
const headings = Array.from(main.querySelectorAll('a h3'));
|
|
33
|
+
const seen = new Set();
|
|
34
|
+
const anchors = [];
|
|
35
|
+
for (const h3 of headings) {
|
|
36
|
+
const a = h3.closest('a[href]');
|
|
37
|
+
if (!a) continue;
|
|
38
|
+
const href = a.href;
|
|
39
|
+
if (!href || seen.has(href)) continue;
|
|
40
|
+
seen.add(href);
|
|
41
|
+
const container =
|
|
42
|
+
a.closest('div.g') ||
|
|
43
|
+
a.closest('div[data-hveid]') ||
|
|
44
|
+
a.parentElement?.parentElement ||
|
|
45
|
+
a.parentElement;
|
|
46
|
+
const containerText = container ? container.innerText || '' : '';
|
|
47
|
+
const heading = h3.innerText || h3.textContent || '';
|
|
48
|
+
const snippet = containerText.replace(heading, '').replace(/\\s+/g, ' ').trim().slice(0, 500);
|
|
49
|
+
anchors.push({ href, heading, snippet });
|
|
50
|
+
}
|
|
51
|
+
const pageText = (document.body.innerText || '').slice(0, ${PAGE_TEXT_LIMIT});
|
|
52
|
+
return JSON.stringify({ anchors, pageText });
|
|
53
|
+
})()
|
|
54
|
+
`;
|
|
55
|
+
|
|
25
56
|
const buildSerpUrl = (query: string, limit: number): string =>
|
|
26
57
|
`https://www.google.com/search?q=${encodeURIComponent(query)}&num=${limit}&hl=en`;
|
|
27
58
|
|
|
28
|
-
const
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
59
|
+
const SerpExtractionSchema = Type.Object({
|
|
60
|
+
anchors: Type.Array(
|
|
61
|
+
Type.Object({ href: Type.String(), heading: Type.String(), snippet: Type.String() }),
|
|
62
|
+
),
|
|
63
|
+
pageText: Type.String(),
|
|
64
|
+
});
|
|
65
|
+
|
|
66
|
+
const serpExtractionValidator = Compile(SerpExtractionSchema);
|
|
33
67
|
|
|
34
68
|
const formatResults = (query: string, results: ReadonlyArray<SearchResult>): string => {
|
|
35
69
|
const lines = results.map((r) => `${r.rank}. ${r.title}\n ${r.url}\n ${r.snippet}`);
|
|
@@ -43,12 +77,12 @@ export const webSearchTool = defineBrowserTool({
|
|
|
43
77
|
"Search the web and return ranked results (title, url, snippet, rank). Scrapes a Google results page in the real Chrome — no API key. Returns links only; use browser_read_page to read a result's content.",
|
|
44
78
|
promptSnippet: "Search the web for a query",
|
|
45
79
|
promptGuidelines: [
|
|
46
|
-
"Returns ranked {title, url, snippet, rank} — links only, no page content. Follow up with browser_read_page.",
|
|
47
80
|
"On a CAPTCHA / bot wall the call fails with kind:'invalid_state' and details.reason:'captcha' (plus serpUrl) — surface it to the user; do not retry in a tight loop.",
|
|
48
81
|
"On zero results it fails with details.reason:'no_results' — rephrase the query and try again.",
|
|
49
82
|
"Runs in its own isolated tab and never disturbs the user's current tab.",
|
|
50
83
|
],
|
|
51
84
|
parameters: WebSearchArgs,
|
|
85
|
+
concurrency: "serialized",
|
|
52
86
|
async handler(args, { client, signal }): Promise<Result<ToolOk, ToolErr>> {
|
|
53
87
|
const limit = args.limit ?? 10;
|
|
54
88
|
const serpUrl = buildSerpUrl(args.query, limit);
|
|
@@ -64,16 +98,17 @@ export const webSearchTool = defineBrowserTool({
|
|
|
64
98
|
const loaded = await waitForIsolatedLoad(client, tab, LOAD_TIMEOUT_MS, signal);
|
|
65
99
|
if (!loaded.success) return err(loaded.error);
|
|
66
100
|
|
|
67
|
-
const extracted = await evalInIsolatedTab(client, tab, buildSerpExtractionExpr());
|
|
101
|
+
const extracted = await evalInIsolatedTab(client, tab, buildSerpExtractionExpr(), isJsonText);
|
|
68
102
|
if (!extracted.success) return err(extracted.error);
|
|
69
103
|
|
|
70
|
-
const parsedJson =
|
|
71
|
-
if (!
|
|
72
|
-
return err({ kind: "internal", message:
|
|
104
|
+
const parsedJson = parseJson(extracted.data, serpExtractionValidator);
|
|
105
|
+
if (!parsedJson.success) {
|
|
106
|
+
return err({ kind: "internal", message: `SERP extraction returned an unexpected shape: ${parsedJson.error}` });
|
|
73
107
|
}
|
|
108
|
+
const extraction: SerpExtraction = parsedJson.data;
|
|
74
109
|
|
|
75
|
-
const results = parseGoogleSerp(
|
|
76
|
-
const verdict = classifySerp(
|
|
110
|
+
const results = parseGoogleSerp(extraction.anchors, limit);
|
|
111
|
+
const verdict = classifySerp(extraction.pageText, results.length);
|
|
77
112
|
if (verdict !== "ok") {
|
|
78
113
|
return err({
|
|
79
114
|
kind: "invalid_state",
|
|
@@ -106,11 +141,3 @@ export const webSearchTool = defineBrowserTool({
|
|
|
106
141
|
return renderExpandableText("web_search", result, expanded, theme);
|
|
107
142
|
},
|
|
108
143
|
});
|
|
109
|
-
|
|
110
|
-
const safeParse = (text: string): unknown => {
|
|
111
|
-
try {
|
|
112
|
-
return JSON.parse(text) as unknown;
|
|
113
|
-
} catch {
|
|
114
|
-
return undefined;
|
|
115
|
-
}
|
|
116
|
-
};
|
package/src/domains/setup.ts
CHANGED
|
@@ -1,11 +1,3 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* browser_setup tool — agent-callable browser initialization.
|
|
3
|
-
*
|
|
4
|
-
* The agent can call this directly when browser tools fail with
|
|
5
|
-
* "not initialized" instead of asking the user. Idempotent: safe
|
|
6
|
-
* to call even when already connected.
|
|
7
|
-
*/
|
|
8
|
-
|
|
9
1
|
import { Type } from "typebox";
|
|
10
2
|
import { Text } from "@mariozechner/pi-tui";
|
|
11
3
|
import type { Result } from "../util/result";
|
|
@@ -23,17 +15,13 @@ export const setupTool = defineBrowserTool({
|
|
|
23
15
|
"Call when browser tools fail with 'Browser harness not initialized'. Idempotent.",
|
|
24
16
|
promptSnippet: "Initialize browser connection",
|
|
25
17
|
promptGuidelines: [
|
|
26
|
-
"Call browser_setup when you get a 'Browser harness not initialized' error from any browser tool.",
|
|
27
|
-
"This tool is idempotent — calling it when already connected is harmless.",
|
|
28
18
|
"After browser_setup succeeds, retry the browser tool that failed.",
|
|
29
19
|
],
|
|
30
20
|
parameters: SetupArgs,
|
|
31
|
-
|
|
21
|
+
concurrency: "parallel",
|
|
32
22
|
ensureAlive: false,
|
|
33
23
|
renderCall: () => new Text("🔧 Initializing browser...", 0, 0),
|
|
34
24
|
async handler(_args, { client, extensionCtx }): Promise<Result<ToolOk, ToolErr>> {
|
|
35
|
-
// extensionCtx carries the UI, so the agent calling browser_setup gets the
|
|
36
|
-
// same first-run profile picker the user gets from /browser-setup.
|
|
37
25
|
const result = await performSetup(client, extensionCtx);
|
|
38
26
|
if (result.success) {
|
|
39
27
|
return ok({ text: result.data });
|