pi-browser-harness 0.10.2 → 0.12.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +54 -0
- package/README.md +21 -10
- package/package.json +6 -3
- package/skills/pi-browser-harness/SKILL.md +30 -1
- package/src/cdp/commands.ts +260 -0
- package/src/cdp/console-buffer.ts +10 -85
- package/src/cdp/discovery.ts +31 -71
- package/src/cdp/errors.ts +4 -0
- package/src/cdp/events.ts +170 -0
- package/src/cdp/network-buffer.ts +42 -109
- package/src/cdp/ownership.ts +0 -17
- package/src/cdp/record-store.ts +61 -0
- package/src/cdp/session.ts +205 -193
- package/src/cdp/target-factory.ts +10 -80
- package/src/cdp/types.ts +21 -12
- package/src/client.ts +39 -95
- package/src/daemon/bridge.ts +375 -265
- package/src/daemon/index.ts +0 -28
- package/src/daemon/protocol.ts +6 -68
- package/src/daemon/queue.ts +85 -0
- package/src/daemon/server.ts +24 -99
- package/src/daemon/spawn.ts +90 -82
- package/src/{cdp/daemon-transport.ts → daemon/transport.ts} +24 -52
- package/src/domains/ax-tree.ts +179 -0
- package/src/domains/box.ts +51 -0
- package/src/domains/cdp-call.ts +55 -0
- package/src/domains/click.ts +19 -37
- package/src/domains/console.ts +48 -4
- package/src/domains/dialog.ts +8 -5
- package/src/domains/drag.ts +8 -7
- package/src/domains/element-call.ts +117 -0
- package/src/domains/files.ts +75 -30
- package/src/domains/fill-engine.ts +80 -0
- package/src/domains/form-batch.ts +64 -0
- package/src/domains/form.ts +115 -112
- package/src/domains/headers.ts +89 -0
- package/src/domains/history.ts +16 -12
- package/src/domains/isolated-tab.ts +26 -55
- package/src/domains/js.ts +32 -51
- package/src/domains/keyboard.ts +38 -34
- package/src/domains/navigate.ts +16 -26
- package/src/domains/network.ts +61 -13
- package/src/domains/page.ts +25 -10
- package/src/domains/readpage/read-page.ts +74 -20
- package/src/domains/readpage/readability.ts +0 -24
- package/src/domains/ref-resolve.ts +24 -58
- package/src/domains/render.ts +6 -27
- package/src/domains/screenshot-capture.ts +80 -0
- package/src/domains/screenshot.ts +9 -90
- package/src/domains/scroll.ts +10 -12
- package/src/domains/search/google-serp.ts +0 -29
- package/src/domains/search/web-search.ts +48 -21
- package/src/domains/setup.ts +1 -13
- package/src/domains/snapshot.ts +36 -280
- package/src/domains/tabs.ts +15 -16
- package/src/domains/target.ts +35 -0
- package/src/domains/viewport.ts +5 -4
- package/src/index.ts +23 -69
- package/src/profile/bind.ts +8 -57
- package/src/profile/browser-process.ts +75 -126
- package/src/profile/command.ts +50 -29
- package/src/profile/launch.ts +22 -40
- package/src/profile/list.ts +14 -54
- package/src/profile/paths.ts +2 -65
- package/src/profile/store.ts +35 -56
- package/src/prompt.ts +22 -122
- package/src/registry.ts +33 -33
- package/src/schemas/parse.ts +23 -0
- package/src/setup.ts +0 -25
- package/src/state.ts +48 -18
- package/src/util/debug.ts +8 -0
- package/src/util/guards.ts +30 -0
- package/src/util/js-template.ts +1 -16
- package/src/util/mutex.ts +0 -5
- package/src/util/paths.ts +11 -1
- package/src/util/result.ts +0 -6
- package/src/util/sharp-shim.ts +12 -19
- package/src/util/tool.ts +17 -42
- package/src/util/truncate.ts +6 -0
- package/tsconfig.json +1 -16
- package/skills/deep-research/SKILL.md +0 -66
- package/src/cdp/event-queue.ts +0 -128
- package/src/cdp/transport.ts +0 -156
- package/src/cdp/window-binding.ts +0 -15
- package/src/deep-research.ts +0 -41
- package/src/domains/forms.ts +0 -355
- package/src/domains/readpage/capture.ts +0 -46
- package/src/domains/search/extract.ts +0 -44
- package/src/profile/picker.ts +0 -73
- package/src/util/keycodes.ts +0 -10
- package/src/util/time.ts +0 -29
|
@@ -0,0 +1,80 @@
|
|
|
1
|
+
export type FillOptions = {
|
|
2
|
+
readonly rejectSelect: boolean;
|
|
3
|
+
readonly focusFirst: boolean;
|
|
4
|
+
};
|
|
5
|
+
|
|
6
|
+
const ENGINE = `
|
|
7
|
+
const el = this;
|
|
8
|
+
if (!el || el.nodeType !== 1) return { ok: false, reason: "ref does not point to an element" };
|
|
9
|
+
const tagName = el.tagName || "";
|
|
10
|
+
const tag = tagName.toLowerCase();
|
|
11
|
+
const type = (el.type || "").toLowerCase();
|
|
12
|
+
|
|
13
|
+
if (__rejectSelect && tag === "select") {
|
|
14
|
+
return { ok: false, reason: "element is a <select>", kind: "select", tag: tagName };
|
|
15
|
+
}
|
|
16
|
+
if (el.disabled) return { ok: false, reason: "element is disabled", tag: tagName };
|
|
17
|
+
|
|
18
|
+
const fire = function (t) { el.dispatchEvent(new Event(t, { bubbles: true })); };
|
|
19
|
+
if (__focusFirst) { try { el.focus(); } catch (e) {} }
|
|
20
|
+
|
|
21
|
+
if (tag === "select") {
|
|
22
|
+
const want = String(value);
|
|
23
|
+
let matched = null;
|
|
24
|
+
for (let i = 0; i < el.options.length; i++) {
|
|
25
|
+
const o = el.options[i];
|
|
26
|
+
if (o.value === want || o.label === want || o.text === want) { matched = o; break; }
|
|
27
|
+
}
|
|
28
|
+
if (!matched) {
|
|
29
|
+
const opts = [];
|
|
30
|
+
for (let i = 0; i < el.options.length; i++) opts.push({ value: el.options[i].value, text: el.options[i].text });
|
|
31
|
+
return { ok: false, reason: "no matching option", kind: "select", options: opts, tag: tagName };
|
|
32
|
+
}
|
|
33
|
+
el.value = matched.value;
|
|
34
|
+
fire("input"); fire("change");
|
|
35
|
+
return { ok: true, kind: "select", value: el.value, text: matched.text, tag: tagName };
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
if (tag === "input" && (type === "checkbox" || type === "radio")) {
|
|
39
|
+
const want = value === true || value === "true" || value === "on" || value === 1;
|
|
40
|
+
let changed = false;
|
|
41
|
+
if (el.checked !== want) {
|
|
42
|
+
try { el.click(); } catch (e) {}
|
|
43
|
+
if (el.checked !== want) { el.checked = want; fire("input"); fire("change"); }
|
|
44
|
+
changed = true;
|
|
45
|
+
}
|
|
46
|
+
return { ok: true, kind: type, checked: el.checked, changed: changed, tag: tagName };
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
if (el.isContentEditable) {
|
|
50
|
+
try { el.focus(); } catch (e) {}
|
|
51
|
+
let done = false;
|
|
52
|
+
try {
|
|
53
|
+
const sel = window.getSelection();
|
|
54
|
+
if (sel) { sel.removeAllRanges(); const rng = document.createRange(); rng.selectNodeContents(el); sel.addRange(rng); }
|
|
55
|
+
done = document.execCommand("insertText", false, String(value));
|
|
56
|
+
} catch (e) { done = false; }
|
|
57
|
+
if (!done) {
|
|
58
|
+
el.textContent = String(value);
|
|
59
|
+
el.dispatchEvent(new InputEvent("input", { bubbles: true }));
|
|
60
|
+
}
|
|
61
|
+
return { ok: true, kind: "contenteditable", value: el.textContent || "", tag: tagName };
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
if (tag === "input" || tag === "textarea") {
|
|
65
|
+
const proto = tag === "textarea" ? window.HTMLTextAreaElement.prototype : window.HTMLInputElement.prototype;
|
|
66
|
+
const desc = Object.getOwnPropertyDescriptor(proto, "value");
|
|
67
|
+
if (desc && desc.set) desc.set.call(el, String(value));
|
|
68
|
+
else el.value = String(value);
|
|
69
|
+
fire("input"); fire("change");
|
|
70
|
+
return { ok: true, kind: el.type || tag, value: el.value, tag: tagName };
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
return { ok: false, reason: "element is not a fillable field (tag=" + tag + ")", tag: tagName };
|
|
74
|
+
`;
|
|
75
|
+
|
|
76
|
+
export const fillBody = (opts: FillOptions): string =>
|
|
77
|
+
`const __rejectSelect = ${opts.rejectSelect}; const __focusFirst = ${opts.focusFirst};\n${ENGINE}`;
|
|
78
|
+
|
|
79
|
+
export const fillDeclaration = (opts: FillOptions): string =>
|
|
80
|
+
`function (value) { ${fillBody(opts)} }`;
|
|
@@ -0,0 +1,64 @@
|
|
|
1
|
+
import { Type } from "typebox";
|
|
2
|
+
import { type Result, err, ok } from "../util/result";
|
|
3
|
+
import { defineBrowserTool, type ToolErr, type ToolOk } from "../util/tool";
|
|
4
|
+
import { fillDeclaration } from "./fill-engine";
|
|
5
|
+
import { detailsOf, resolveAndCall } from "./element-call";
|
|
6
|
+
|
|
7
|
+
const FILL_FN = fillDeclaration({ rejectSelect: false, focusFirst: true });
|
|
8
|
+
|
|
9
|
+
const FieldValue = Type.Union([Type.String(), Type.Boolean()], {
|
|
10
|
+
description: "Value to set. String for text inputs / textareas / selects / contenteditable; boolean for checkboxes & radios.",
|
|
11
|
+
});
|
|
12
|
+
|
|
13
|
+
const FillFormArgs = Type.Object({
|
|
14
|
+
fields: Type.Array(
|
|
15
|
+
Type.Object({
|
|
16
|
+
ref: Type.String({ description: "Stable element ref from browser_snapshot (e.g. 'e7')." }),
|
|
17
|
+
value: FieldValue,
|
|
18
|
+
}),
|
|
19
|
+
{ minItems: 1, description: "Fields to fill in one batch, each by ref." },
|
|
20
|
+
),
|
|
21
|
+
});
|
|
22
|
+
|
|
23
|
+
export const fillFormTool = defineBrowserTool({
|
|
24
|
+
name: "browser_fill_form",
|
|
25
|
+
label: "Browser Fill Form",
|
|
26
|
+
description:
|
|
27
|
+
"Fill multiple form fields in one call — the efficient way to complete a form. Each field is { ref, value }; refs come from browser_snapshot's [eN]. Handles text inputs, textareas, selects, checkboxes, radios, and contenteditable, drives React/Vue controlled inputs correctly, and reports the resulting value of every field.",
|
|
28
|
+
promptSnippet: "Fill many form fields at once by ref — React-safe, self-confirming",
|
|
29
|
+
promptGuidelines: [
|
|
30
|
+
"Preferred for forms: snapshot once, then fill all fields in a single browser_fill_form call.",
|
|
31
|
+
"The result summarizes how many fields filled cleanly and flags any value mismatches or per-field errors.",
|
|
32
|
+
"Re-run browser_snapshot afterwards to verify the form state if needed.",
|
|
33
|
+
],
|
|
34
|
+
parameters: FillFormArgs,
|
|
35
|
+
concurrency: "serialized",
|
|
36
|
+
async handler(args, { client }): Promise<Result<ToolOk, ToolErr>> {
|
|
37
|
+
const results: Array<Readonly<Record<string, unknown>>> = [];
|
|
38
|
+
let okCount = 0;
|
|
39
|
+
const issues: string[] = [];
|
|
40
|
+
for (const f of args.fields) {
|
|
41
|
+
const r = await resolveAndCall(client, f.ref, FILL_FN, [f.value]);
|
|
42
|
+
if (!r.success) {
|
|
43
|
+
results.push({ ref: f.ref, ok: false, error: r.error.message });
|
|
44
|
+
issues.push(`ref ${f.ref} (${r.error.message})`);
|
|
45
|
+
continue;
|
|
46
|
+
}
|
|
47
|
+
const res = r.data;
|
|
48
|
+
const wanted = typeof f.value === "boolean" ? undefined : String(f.value);
|
|
49
|
+
const got = res.value !== undefined ? String(res.value) : undefined;
|
|
50
|
+
const mismatch = wanted !== undefined && got !== undefined && got !== wanted && res.text !== wanted;
|
|
51
|
+
const fieldOk = res.ok === true && !mismatch;
|
|
52
|
+
if (fieldOk) okCount++;
|
|
53
|
+
else issues.push(`ref ${f.ref} (${res.ok === false ? (res.reason ?? "failed") : "value mismatch"})`);
|
|
54
|
+
results.push({ ref: f.ref, ok: fieldOk, mismatch, ...detailsOf(f.ref, res) });
|
|
55
|
+
}
|
|
56
|
+
const text =
|
|
57
|
+
`Filled ${okCount}/${args.fields.length} fields` +
|
|
58
|
+
(issues.length > 0 ? `; issues: ${issues.join(", ")}` : "");
|
|
59
|
+
if (okCount === 0) {
|
|
60
|
+
return err({ kind: "invalid_state", message: text, details: { results } });
|
|
61
|
+
}
|
|
62
|
+
return ok({ text, details: { results } });
|
|
63
|
+
},
|
|
64
|
+
});
|
package/src/domains/form.ts
CHANGED
|
@@ -1,9 +1,9 @@
|
|
|
1
1
|
import { Type } from "typebox";
|
|
2
|
-
import type { BrowserClient } from "../client";
|
|
3
|
-
import { safeJs } from "../util/js-template";
|
|
4
2
|
import { type Result, err, ok } from "../util/result";
|
|
5
3
|
import { defineBrowserTool, type ToolErr, type ToolOk } from "../util/tool";
|
|
6
|
-
import {
|
|
4
|
+
import { detailsOf, isPageResult, type PageResult, resolveAndCall, runOnElement } from "./element-call";
|
|
5
|
+
import { fillBody } from "./fill-engine";
|
|
6
|
+
import { interactiveDiff } from "./ref-resolve";
|
|
7
7
|
|
|
8
8
|
const FillArgs = Type.Object({
|
|
9
9
|
ref: Type.Optional(
|
|
@@ -16,137 +16,71 @@ const FillArgs = Type.Object({
|
|
|
16
16
|
value: Type.String({ description: "Value to set in the field" }),
|
|
17
17
|
});
|
|
18
18
|
|
|
19
|
-
|
|
20
|
-
* Run an element-scoped function either against a CSS selector (querySelector +
|
|
21
|
-
* apply) or a ref (resolve to objectId, callFunctionOn with element as `this`).
|
|
22
|
-
* The shared `fnBody` is a function body where `this` is the target element and
|
|
23
|
-
* the single argument is the value — identical logic on both paths. Returns the
|
|
24
|
-
* function's return value (parsed) or a ToolErr.
|
|
25
|
-
*
|
|
26
|
-
* On the ref path a detached node yields a stale-ref error; on the selector path
|
|
27
|
-
* a missing element returns { status: "not_found" } from the function body.
|
|
28
|
-
*/
|
|
29
|
-
const runOnElement = async (
|
|
30
|
-
client: BrowserClient,
|
|
31
|
-
opts: { ref?: string | undefined; selector?: string | undefined; fnBody: string; arg: unknown },
|
|
32
|
-
): Promise<Result<unknown, ToolErr>> => {
|
|
33
|
-
if (opts.ref !== undefined) {
|
|
34
|
-
const objectId = await resolveRefToObjectId(client, opts.ref);
|
|
35
|
-
if (!objectId.success) return objectId;
|
|
36
|
-
const r = await client.session().call("Runtime.callFunctionOn", {
|
|
37
|
-
objectId: objectId.data,
|
|
38
|
-
functionDeclaration: `function (arg) { ${opts.fnBody} }`,
|
|
39
|
-
arguments: [{ value: opts.arg }],
|
|
40
|
-
returnByValue: true,
|
|
41
|
-
});
|
|
42
|
-
if (!r.success) return err({ kind: "cdp_error", message: r.error.message });
|
|
43
|
-
return ok((r.data as { result?: { value?: unknown } }).result?.value);
|
|
44
|
-
}
|
|
45
|
-
if (opts.selector === undefined) {
|
|
46
|
-
return err({ kind: "invalid_state", message: "Provide either `ref` or `selector`." });
|
|
47
|
-
}
|
|
48
|
-
// Selector path: bind the same trusted function body to the matched element
|
|
49
|
-
// via .call(). fnBody is harness-authored (not user input); only the selector
|
|
50
|
-
// and value are interpolated, and those go through safeJs.
|
|
51
|
-
const prelude = safeJs`
|
|
52
|
-
(() => {
|
|
53
|
-
const el = document.querySelector(${opts.selector});
|
|
54
|
-
if (!el) return { status: "not_found" };
|
|
55
|
-
const __arg = ${opts.arg};
|
|
56
|
-
return (function (arg) {`;
|
|
57
|
-
const expr = `${prelude} ${opts.fnBody} }).call(el, __arg); })()`;
|
|
58
|
-
const r = await client.evaluateJs(expr);
|
|
59
|
-
if (!r.success) return err({ kind: "cdp_error", message: r.error.message });
|
|
60
|
-
return ok(r.data);
|
|
61
|
-
};
|
|
19
|
+
const FILL_BODY = `const value = arg;\n${fillBody({ rejectSelect: true, focusFirst: false })}`;
|
|
62
20
|
|
|
63
|
-
|
|
64
|
-
* Result shape returned by the in-page fill script. Discriminated by `status`
|
|
65
|
-
* so the handler can map page-level outcomes to typed ToolErr kinds.
|
|
66
|
-
*/
|
|
67
|
-
type FillResult =
|
|
68
|
-
| { status: "ok"; tag: string; kind: string; value: string }
|
|
69
|
-
| { status: "not_found" }
|
|
70
|
-
| { status: "not_fillable"; tag: string };
|
|
71
|
-
|
|
72
|
-
// Element-scoped fill logic. `this` is the target element, `arg` is the value.
|
|
73
|
-
// Write via the native prototype value setter so React's _valueTracker
|
|
74
|
-
// registers the change — assigning el.value directly does NOT, which is why
|
|
75
|
-
// controlled inputs revert on re-render. Shared verbatim by the ref path
|
|
76
|
-
// (callFunctionOn) and the selector path (querySelector + .call).
|
|
77
|
-
const FILL_FN = `
|
|
78
|
-
const el = this;
|
|
79
|
-
const tag = el.tagName;
|
|
80
|
-
const proto =
|
|
81
|
-
el instanceof HTMLTextAreaElement ? HTMLTextAreaElement.prototype
|
|
82
|
-
: el instanceof HTMLInputElement ? HTMLInputElement.prototype
|
|
83
|
-
: null;
|
|
84
|
-
if (proto) {
|
|
85
|
-
const desc = Object.getOwnPropertyDescriptor(proto, "value");
|
|
86
|
-
const setter = desc && desc.set;
|
|
87
|
-
if (setter) setter.call(el, arg);
|
|
88
|
-
else el.value = arg;
|
|
89
|
-
el.dispatchEvent(new Event("input", { bubbles: true }));
|
|
90
|
-
el.dispatchEvent(new Event("change", { bubbles: true }));
|
|
91
|
-
return { status: "ok", tag, kind: el.type || tag.toLowerCase(), value: el.value };
|
|
92
|
-
}
|
|
93
|
-
if (el.isContentEditable) {
|
|
94
|
-
// Rich-text editors (Slack/Notion/ProseMirror) listen for beforeinput/input.
|
|
95
|
-
el.focus();
|
|
96
|
-
let done = false;
|
|
97
|
-
try {
|
|
98
|
-
const sel = window.getSelection();
|
|
99
|
-
if (sel) { sel.removeAllRanges(); const rng = document.createRange(); rng.selectNodeContents(el); sel.addRange(rng); }
|
|
100
|
-
done = document.execCommand("insertText", false, arg);
|
|
101
|
-
} catch (e) { done = false; }
|
|
102
|
-
if (!done) {
|
|
103
|
-
el.textContent = arg;
|
|
104
|
-
el.dispatchEvent(new InputEvent("input", { bubbles: true }));
|
|
105
|
-
}
|
|
106
|
-
return { status: "ok", tag, kind: "contenteditable", value: el.textContent || "" };
|
|
107
|
-
}
|
|
108
|
-
return { status: "not_fillable", tag };
|
|
109
|
-
`;
|
|
21
|
+
const NOT_FOUND: PageResult = { ok: false, reason: "not_found" };
|
|
110
22
|
|
|
111
23
|
export const fillTool = defineBrowserTool({
|
|
112
24
|
name: "browser_fill",
|
|
113
25
|
label: "Browser Fill",
|
|
114
26
|
description:
|
|
115
|
-
"Fill a form field (input/textarea/contenteditable). PREFERRED: pass `ref` (e.g. 'e7') from browser_snapshot — survives re-renders. Fallback: a CSS `selector`. Writes through the native value setter and fires bubbling 'input'/'change' events, so React/Vue/Angular controlled components and rich-text editors update correctly — unlike browser_type. Returns the field's value after writing, plus a compact diff of page changes.",
|
|
27
|
+
"Fill a form field (input/textarea/contenteditable), or tick a checkbox/radio by passing 'true'/'false'. PREFERRED: pass `ref` (e.g. 'e7') from browser_snapshot — survives re-renders. Fallback: a CSS `selector`. Writes through the native value setter and fires bubbling 'input'/'change' events, so React/Vue/Angular controlled components and rich-text editors update correctly — unlike browser_type. Refuses a disabled field rather than writing a value the page will ignore. Returns the field's value after writing, plus a compact diff of page changes.",
|
|
116
28
|
promptSnippet: "Fill a form field by ref (preferred) or selector (works with React/Vue controlled inputs)",
|
|
117
29
|
promptGuidelines: [
|
|
118
|
-
"PREFER `ref` from browser_snapshot (the '[eN]' handle) over a guessed CSS selector — refs survive re-renders and don't require you to invent a selector.",
|
|
119
30
|
"Pass the desired value; no browser_click is needed first.",
|
|
120
|
-
"
|
|
31
|
+
"For a checkbox or radio pass 'true' or 'false' — it toggles `checked`, not the submit value. browser_set_checked does the same thing with a boolean.",
|
|
121
32
|
"Use browser_type instead only for keystroke-sensitive widgets (autocomplete, masked/segmented inputs).",
|
|
122
33
|
"Fill sets the value but does not focus or submit. To submit a tag/autocomplete input, follow with browser_dispatch_key({ ref, key: 'Enter' }) — not browser_press_key, which targets the focused element and may miss this field.",
|
|
123
34
|
"For a custom (div-based) dropdown, browser_fill won't validate against its option list — open it, run browser_snapshot, then browser_click the option's ref.",
|
|
124
|
-
"
|
|
35
|
+
"Confirm the field value browser_fill returns matches what you intended.",
|
|
125
36
|
],
|
|
126
37
|
parameters: FillArgs,
|
|
38
|
+
concurrency: "serialized",
|
|
127
39
|
async handler(args, { client }): Promise<Result<ToolOk, ToolErr>> {
|
|
128
|
-
const r = await runOnElement(client, {
|
|
40
|
+
const r = await runOnElement(client, {
|
|
41
|
+
ref: args.ref,
|
|
42
|
+
selector: args.selector,
|
|
43
|
+
fnBody: FILL_BODY,
|
|
44
|
+
arg: args.value,
|
|
45
|
+
notFound: NOT_FOUND,
|
|
46
|
+
});
|
|
129
47
|
if (!r.success) return r;
|
|
130
|
-
const res = r.data
|
|
48
|
+
const res = isPageResult(r.data) ? r.data : undefined;
|
|
131
49
|
const target = args.ref ?? args.selector ?? "";
|
|
132
|
-
if (res === undefined || res.
|
|
50
|
+
if (res === undefined || res.reason === "not_found") {
|
|
133
51
|
return err({
|
|
134
52
|
kind: "invalid_state",
|
|
135
53
|
message: `No element matched: ${target}`,
|
|
136
54
|
details: { ref: args.ref, selector: args.selector },
|
|
137
55
|
});
|
|
138
56
|
}
|
|
139
|
-
if (res.
|
|
57
|
+
if (!res.ok) {
|
|
58
|
+
const details = { ref: args.ref, selector: args.selector, ...(res.tag !== undefined ? { tag: res.tag } : {}) };
|
|
59
|
+
if (res.kind === "select") {
|
|
60
|
+
return err({
|
|
61
|
+
kind: "invalid_state",
|
|
62
|
+
message: `Element <select> is not fillable (not an input, textarea, or contenteditable): ${target}. For <select> use browser_select_option.`,
|
|
63
|
+
details,
|
|
64
|
+
});
|
|
65
|
+
}
|
|
140
66
|
return err({
|
|
141
67
|
kind: "invalid_state",
|
|
142
|
-
message: `
|
|
143
|
-
details
|
|
68
|
+
message: `Could not fill ${target}: ${res.reason ?? "unknown reason"}`,
|
|
69
|
+
details,
|
|
144
70
|
});
|
|
145
71
|
}
|
|
72
|
+
const shown = res.value !== undefined ? JSON.stringify(res.value) : `checked=${res.checked === true}`;
|
|
146
73
|
const diff = await interactiveDiff(client);
|
|
147
74
|
return ok({
|
|
148
|
-
text: `Filled ${target} = ${
|
|
149
|
-
details: {
|
|
75
|
+
text: `Filled ${target} = ${shown}${diff}`,
|
|
76
|
+
details: {
|
|
77
|
+
ref: args.ref,
|
|
78
|
+
selector: args.selector,
|
|
79
|
+
value: args.value,
|
|
80
|
+
verified: res.value ?? res.checked,
|
|
81
|
+
...(res.tag !== undefined ? { tag: res.tag } : {}),
|
|
82
|
+
...(res.kind !== undefined ? { kind: res.kind } : {}),
|
|
83
|
+
},
|
|
150
84
|
});
|
|
151
85
|
},
|
|
152
86
|
});
|
|
@@ -160,6 +94,13 @@ const FocusArgs = Type.Object({
|
|
|
160
94
|
|
|
161
95
|
type FocusResult = { status: "ok"; tag: string } | { status: "not_found" } | { status: "not_focusable"; tag: string };
|
|
162
96
|
|
|
97
|
+
const isFocusResult = (v: unknown): v is FocusResult => {
|
|
98
|
+
if (typeof v !== "object" || v === null || !("status" in v)) return false;
|
|
99
|
+
if (v.status === "not_found") return true;
|
|
100
|
+
if (v.status === "ok" || v.status === "not_focusable") return "tag" in v && typeof v.tag === "string";
|
|
101
|
+
return false;
|
|
102
|
+
};
|
|
103
|
+
|
|
163
104
|
const FOCUS_FN = `
|
|
164
105
|
const el = this;
|
|
165
106
|
if (typeof el.focus !== "function") return { status: "not_focusable", tag: el.tagName };
|
|
@@ -174,15 +115,14 @@ export const focusTool = defineBrowserTool({
|
|
|
174
115
|
"Focus an element via the DOM .focus() method — deterministic, no coordinate accuracy needed. PREFERRED: pass `ref` from browser_snapshot; fallback: a CSS `selector`. Use before browser_type when a click might miss the field.",
|
|
175
116
|
promptSnippet: "Focus an element by ref (preferred) or selector",
|
|
176
117
|
promptGuidelines: [
|
|
177
|
-
"PREFER `ref` from browser_snapshot over a CSS selector — survives re-renders.",
|
|
178
|
-
"Use before browser_type to guarantee the right field is focused without relying on click coordinates.",
|
|
179
118
|
"For simply setting a value, prefer browser_fill (it doesn't need a separate focus step).",
|
|
180
119
|
],
|
|
181
120
|
parameters: FocusArgs,
|
|
121
|
+
concurrency: "serialized",
|
|
182
122
|
async handler(args, { client }): Promise<Result<ToolOk, ToolErr>> {
|
|
183
123
|
const r = await runOnElement(client, { ref: args.ref, selector: args.selector, fnBody: FOCUS_FN, arg: "" });
|
|
184
124
|
if (!r.success) return r;
|
|
185
|
-
const res = r.data
|
|
125
|
+
const res = isFocusResult(r.data) ? r.data : undefined;
|
|
186
126
|
const target = args.ref ?? args.selector ?? "";
|
|
187
127
|
if (res === undefined || res.status === "not_found") {
|
|
188
128
|
return err({ kind: "invalid_state", message: `No element matched: ${target}`, details: { ref: args.ref, selector: args.selector } });
|
|
@@ -210,7 +150,24 @@ type SelectResult =
|
|
|
210
150
|
| { status: "not_select"; tag: string }
|
|
211
151
|
| { status: "no_match"; options: ReadonlyArray<{ value: string; label: string }> };
|
|
212
152
|
|
|
213
|
-
|
|
153
|
+
const isSelectOption = (v: unknown): v is { value: string; label: string } =>
|
|
154
|
+
typeof v === "object" && v !== null
|
|
155
|
+
&& "value" in v && typeof v.value === "string"
|
|
156
|
+
&& "label" in v && typeof v.label === "string";
|
|
157
|
+
|
|
158
|
+
const isSelectResult = (v: unknown): v is SelectResult => {
|
|
159
|
+
if (typeof v !== "object" || v === null || !("status" in v)) return false;
|
|
160
|
+
if (v.status === "not_found") return true;
|
|
161
|
+
if (v.status === "not_select") return "tag" in v && typeof v.tag === "string";
|
|
162
|
+
if (v.status === "ok") {
|
|
163
|
+
return "value" in v && typeof v.value === "string" && "label" in v && typeof v.label === "string";
|
|
164
|
+
}
|
|
165
|
+
if (v.status === "no_match") {
|
|
166
|
+
return "options" in v && Array.isArray(v.options) && v.options.every(isSelectOption);
|
|
167
|
+
}
|
|
168
|
+
return false;
|
|
169
|
+
};
|
|
170
|
+
|
|
214
171
|
const SELECT_FN = `
|
|
215
172
|
const el = this;
|
|
216
173
|
if (el.tagName !== "SELECT") return { status: "not_select", tag: el.tagName };
|
|
@@ -234,12 +191,11 @@ export const selectOptionTool = defineBrowserTool({
|
|
|
234
191
|
"Select an option in a native <select> element by value, visible label, or index. PREFERRED: pass `ref` from browser_snapshot; fallback: a CSS `selector`. Sets the selection and fires bubbling 'input'/'change' events so framework listeners update. On no match, returns the available options.",
|
|
235
192
|
promptSnippet: "Select an option in a native <select> by ref (preferred) or selector",
|
|
236
193
|
promptGuidelines: [
|
|
237
|
-
"PREFER `ref` from browser_snapshot over a CSS selector — survives re-renders.",
|
|
238
194
|
"Provide exactly one of value, label, or index to choose the option.",
|
|
239
195
|
"For custom (non-native) dropdowns built from divs, click to open then browser_click the option instead.",
|
|
240
|
-
"If no option matches, the error details list the available options so you can retry.",
|
|
241
196
|
],
|
|
242
197
|
parameters: SelectOptionArgs,
|
|
198
|
+
concurrency: "serialized",
|
|
243
199
|
async handler(args, { client }): Promise<Result<ToolOk, ToolErr>> {
|
|
244
200
|
if (args.value === undefined && args.label === undefined && args.index === undefined) {
|
|
245
201
|
return err({ kind: "invalid_state", message: "Provide one of value, label, or index." });
|
|
@@ -248,7 +204,7 @@ export const selectOptionTool = defineBrowserTool({
|
|
|
248
204
|
const arg = { wv: args.value ?? null, wl: args.label ?? null, wi: args.index ?? null };
|
|
249
205
|
const r = await runOnElement(client, { ref: args.ref, selector: args.selector, fnBody: SELECT_FN, arg });
|
|
250
206
|
if (!r.success) return r;
|
|
251
|
-
const res = r.data
|
|
207
|
+
const res = isSelectResult(r.data) ? r.data : undefined;
|
|
252
208
|
const target = args.ref ?? args.selector ?? "";
|
|
253
209
|
if (res === undefined || res.status === "not_found") {
|
|
254
210
|
return err({ kind: "invalid_state", message: `No element matched: ${target}`, details: { ref: args.ref, selector: args.selector } });
|
|
@@ -270,3 +226,50 @@ export const selectOptionTool = defineBrowserTool({
|
|
|
270
226
|
});
|
|
271
227
|
},
|
|
272
228
|
});
|
|
229
|
+
|
|
230
|
+
const CHECK_FN = `
|
|
231
|
+
function(checked) {
|
|
232
|
+
const el = this;
|
|
233
|
+
const tag = (el.tagName || "").toLowerCase();
|
|
234
|
+
const type = (el.type || "").toLowerCase();
|
|
235
|
+
if (tag !== "input" || (type !== "checkbox" && type !== "radio")) return { ok: false, reason: "ref is not a checkbox or radio" };
|
|
236
|
+
if (el.disabled) return { ok: false, reason: "element is disabled" };
|
|
237
|
+
const want = checked === true;
|
|
238
|
+
let changed = false;
|
|
239
|
+
if (el.checked !== want) {
|
|
240
|
+
try { el.click(); } catch (e) {}
|
|
241
|
+
if (el.checked !== want) { el.checked = want; el.dispatchEvent(new Event("change", { bubbles: true })); }
|
|
242
|
+
changed = true;
|
|
243
|
+
}
|
|
244
|
+
return { ok: true, checked: el.checked, changed: changed };
|
|
245
|
+
}`;
|
|
246
|
+
|
|
247
|
+
const SetCheckedArgs = Type.Object({
|
|
248
|
+
ref: Type.String({ description: "Stable element ref of a checkbox or radio from browser_snapshot (e.g. 'e7')." }),
|
|
249
|
+
checked: Type.Boolean({ description: "Desired checked state." }),
|
|
250
|
+
});
|
|
251
|
+
|
|
252
|
+
export const setCheckedTool = defineBrowserTool({
|
|
253
|
+
name: "browser_set_checked",
|
|
254
|
+
label: "Browser Set Checked",
|
|
255
|
+
description:
|
|
256
|
+
"Set a checkbox or radio to a desired checked state by ref. Only acts if the current state differs, fires change, and reports the final state. Get the ref from browser_snapshot's [eN].",
|
|
257
|
+
promptSnippet: "Set a checkbox/radio checked state by ref",
|
|
258
|
+
promptGuidelines: [
|
|
259
|
+
"For radios, set checked:true on the option you want; the group deselects the others.",
|
|
260
|
+
],
|
|
261
|
+
parameters: SetCheckedArgs,
|
|
262
|
+
concurrency: "serialized",
|
|
263
|
+
async handler(args, { client }): Promise<Result<ToolOk, ToolErr>> {
|
|
264
|
+
const r = await resolveAndCall(client, args.ref, CHECK_FN, [args.checked]);
|
|
265
|
+
if (!r.success) return r;
|
|
266
|
+
const res = r.data;
|
|
267
|
+
if (!res.ok) {
|
|
268
|
+
return err({ kind: "invalid_state", message: `Could not set checked on ref ${args.ref}: ${res.reason ?? "unknown reason"}`, details: detailsOf(args.ref, res) });
|
|
269
|
+
}
|
|
270
|
+
return ok({
|
|
271
|
+
text: `Set ref ${args.ref} checked=${res.checked}` + (res.changed === false ? " (already)" : ""),
|
|
272
|
+
details: detailsOf(args.ref, res),
|
|
273
|
+
});
|
|
274
|
+
},
|
|
275
|
+
});
|
|
@@ -0,0 +1,89 @@
|
|
|
1
|
+
import { Type } from "typebox";
|
|
2
|
+
import type { BrowserClient } from "../client";
|
|
3
|
+
import { type Result, err, ok } from "../util/result";
|
|
4
|
+
import { defineBrowserTool, type ToolErr, type ToolOk } from "../util/tool";
|
|
5
|
+
import { cdpCall } from "./cdp-call";
|
|
6
|
+
|
|
7
|
+
const HeaderValue = Type.String({
|
|
8
|
+
maxLength: 8192,
|
|
9
|
+
pattern: "^[^\\r\\n]*$",
|
|
10
|
+
});
|
|
11
|
+
|
|
12
|
+
const SetHeadersArgs = Type.Object({
|
|
13
|
+
headers: Type.Record(Type.String(), HeaderValue, {
|
|
14
|
+
minProperties: 1,
|
|
15
|
+
maxProperties: 64,
|
|
16
|
+
description: "Extra HTTP request headers for the current owned tab. Names must be valid HTTP tokens.",
|
|
17
|
+
}),
|
|
18
|
+
});
|
|
19
|
+
|
|
20
|
+
const HEADER_NAME = /^[!#$%&'*+\-.^_`|~0-9A-Za-z]+$/;
|
|
21
|
+
|
|
22
|
+
const currentOwnedTarget = (
|
|
23
|
+
client: BrowserClient,
|
|
24
|
+
): Result<{ readonly targetId: string }, ToolErr> => {
|
|
25
|
+
const current = client.current();
|
|
26
|
+
if (current === undefined || current === null) {
|
|
27
|
+
return err({ kind: "invalid_state", message: "No tab attached." });
|
|
28
|
+
}
|
|
29
|
+
if (!client.owns(current.targetId)) {
|
|
30
|
+
return err({
|
|
31
|
+
kind: "invalid_state",
|
|
32
|
+
message: `Current tab ${current.targetId} is not owned by this harness session.`,
|
|
33
|
+
});
|
|
34
|
+
}
|
|
35
|
+
return ok({ targetId: current.targetId });
|
|
36
|
+
};
|
|
37
|
+
|
|
38
|
+
export const setHeadersTool = defineBrowserTool({
|
|
39
|
+
name: "browser_set_headers",
|
|
40
|
+
label: "Browser Set Headers",
|
|
41
|
+
description:
|
|
42
|
+
"Set extra HTTP request headers on the current owned tab. Headers persist for subsequent document, subresource, fetch, and XHR requests until cleared or the tab closes.",
|
|
43
|
+
promptSnippet: "Set extra request headers on the current owned tab",
|
|
44
|
+
promptGuidelines: [
|
|
45
|
+
"If no tab is attached yet, call browser_new_tab without a URL first.",
|
|
46
|
+
"Call before navigation when a site requires routing or environment headers.",
|
|
47
|
+
"Call browser_clear_headers before switching tabs or navigating this tab to another origin.",
|
|
48
|
+
"Header values are never echoed in tool output.",
|
|
49
|
+
],
|
|
50
|
+
parameters: SetHeadersArgs,
|
|
51
|
+
concurrency: "serialized",
|
|
52
|
+
async handler(args, { client }): Promise<Result<ToolOk, ToolErr>> {
|
|
53
|
+
const target = currentOwnedTarget(client);
|
|
54
|
+
if (!target.success) return target;
|
|
55
|
+
const invalidName = Object.keys(args.headers).find((name) => !HEADER_NAME.test(name));
|
|
56
|
+
if (invalidName !== undefined) {
|
|
57
|
+
return err({ kind: "invalid_state", message: `Invalid HTTP header name: ${JSON.stringify(invalidName)}` });
|
|
58
|
+
}
|
|
59
|
+
const result = await cdpCall(client, "Network.setExtraHTTPHeaders", { headers: args.headers });
|
|
60
|
+
if (!result.success) return result;
|
|
61
|
+
const headerNames = Object.keys(args.headers).sort();
|
|
62
|
+
return ok({
|
|
63
|
+
text: `Set ${headerNames.length} extra HTTP header(s) on tab ${target.data.targetId}: ${headerNames.join(", ")}. Values are hidden. Clear them before switching tabs or navigating to another origin.`,
|
|
64
|
+
details: { targetId: target.data.targetId, headerNames, count: headerNames.length },
|
|
65
|
+
});
|
|
66
|
+
},
|
|
67
|
+
});
|
|
68
|
+
|
|
69
|
+
export const clearHeadersTool = defineBrowserTool({
|
|
70
|
+
name: "browser_clear_headers",
|
|
71
|
+
label: "Browser Clear Headers",
|
|
72
|
+
description: "Clear all extra HTTP request headers previously set on the current owned tab.",
|
|
73
|
+
promptSnippet: "Clear extra request headers from the current owned tab",
|
|
74
|
+
promptGuidelines: [
|
|
75
|
+
"Call after finishing an environment-specific workflow, before switching tabs, or before navigating to another origin.",
|
|
76
|
+
],
|
|
77
|
+
parameters: Type.Object({}),
|
|
78
|
+
concurrency: "serialized",
|
|
79
|
+
async handler(_args, { client }): Promise<Result<ToolOk, ToolErr>> {
|
|
80
|
+
const target = currentOwnedTarget(client);
|
|
81
|
+
if (!target.success) return target;
|
|
82
|
+
const result = await cdpCall(client, "Network.setExtraHTTPHeaders", { headers: {} });
|
|
83
|
+
if (!result.success) return result;
|
|
84
|
+
return ok({
|
|
85
|
+
text: `Cleared extra HTTP headers from tab ${target.data.targetId}.`,
|
|
86
|
+
details: { targetId: target.data.targetId },
|
|
87
|
+
});
|
|
88
|
+
},
|
|
89
|
+
});
|
package/src/domains/history.ts
CHANGED
|
@@ -2,14 +2,15 @@ import { Type } from "typebox";
|
|
|
2
2
|
import type { BrowserClient } from "../client";
|
|
3
3
|
import { type Result, err, ok } from "../util/result";
|
|
4
4
|
import { defineBrowserTool, type ToolErr, type ToolOk } from "../util/tool";
|
|
5
|
+
import { cdpCall } from "./cdp-call";
|
|
5
6
|
|
|
6
7
|
type HistoryEntry = { readonly id: number; readonly url: string; readonly title: string };
|
|
7
8
|
type History = { readonly entries: ReadonlyArray<HistoryEntry>; readonly currentIndex: number };
|
|
8
9
|
|
|
9
10
|
const fetchHistory = async (client: BrowserClient): Promise<Result<History, ToolErr>> => {
|
|
10
|
-
const r = await client
|
|
11
|
-
if (!r.success) return
|
|
12
|
-
return ok(r.data
|
|
11
|
+
const r = await cdpCall(client, "Page.getNavigationHistory", {});
|
|
12
|
+
if (!r.success) return r;
|
|
13
|
+
return ok(r.data);
|
|
13
14
|
};
|
|
14
15
|
|
|
15
16
|
export const goBackTool = defineBrowserTool({
|
|
@@ -17,16 +18,17 @@ export const goBackTool = defineBrowserTool({
|
|
|
17
18
|
label: "Browser Go Back",
|
|
18
19
|
description: "Navigate back one page in history.",
|
|
19
20
|
promptSnippet: "Go back one page",
|
|
20
|
-
promptGuidelines: [
|
|
21
|
+
promptGuidelines: [],
|
|
21
22
|
parameters: Type.Object({}),
|
|
23
|
+
concurrency: "serialized",
|
|
22
24
|
async handler(_a, { client }): Promise<Result<ToolOk, ToolErr>> {
|
|
23
25
|
const h = await fetchHistory(client);
|
|
24
26
|
if (!h.success) return h;
|
|
25
27
|
if (h.data.currentIndex <= 0) return ok({ text: "Already at the beginning of history." });
|
|
26
28
|
const target = h.data.entries[h.data.currentIndex - 1];
|
|
27
29
|
if (!target) return err({ kind: "internal", message: "history entry missing at index" });
|
|
28
|
-
const nav = await client
|
|
29
|
-
if (!nav.success) return
|
|
30
|
+
const nav = await cdpCall(client, "Page.navigateToHistoryEntry", { entryId: target.id });
|
|
31
|
+
if (!nav.success) return nav;
|
|
30
32
|
return ok({ text: `Navigated back to: ${target.url}`, details: { url: target.url } });
|
|
31
33
|
},
|
|
32
34
|
});
|
|
@@ -36,16 +38,17 @@ export const goForwardTool = defineBrowserTool({
|
|
|
36
38
|
label: "Browser Go Forward",
|
|
37
39
|
description: "Navigate forward one page in history.",
|
|
38
40
|
promptSnippet: "Go forward one page",
|
|
39
|
-
promptGuidelines: [
|
|
41
|
+
promptGuidelines: [],
|
|
40
42
|
parameters: Type.Object({}),
|
|
43
|
+
concurrency: "serialized",
|
|
41
44
|
async handler(_a, { client }): Promise<Result<ToolOk, ToolErr>> {
|
|
42
45
|
const h = await fetchHistory(client);
|
|
43
46
|
if (!h.success) return h;
|
|
44
47
|
if (h.data.currentIndex >= h.data.entries.length - 1) return ok({ text: "Already at the end of history." });
|
|
45
48
|
const target = h.data.entries[h.data.currentIndex + 1];
|
|
46
49
|
if (!target) return err({ kind: "internal", message: "history entry missing at index" });
|
|
47
|
-
const nav = await client
|
|
48
|
-
if (!nav.success) return
|
|
50
|
+
const nav = await cdpCall(client, "Page.navigateToHistoryEntry", { entryId: target.id });
|
|
51
|
+
if (!nav.success) return nav;
|
|
49
52
|
return ok({ text: `Navigated forward to: ${target.url}`, details: { url: target.url } });
|
|
50
53
|
},
|
|
51
54
|
});
|
|
@@ -55,11 +58,12 @@ export const reloadTool = defineBrowserTool({
|
|
|
55
58
|
label: "Browser Reload",
|
|
56
59
|
description: "Reload the current page.",
|
|
57
60
|
promptSnippet: "Reload the page",
|
|
58
|
-
promptGuidelines: [
|
|
61
|
+
promptGuidelines: [],
|
|
59
62
|
parameters: Type.Object({}),
|
|
63
|
+
concurrency: "serialized",
|
|
60
64
|
async handler(_a, { client }): Promise<Result<ToolOk, ToolErr>> {
|
|
61
|
-
const r = await client
|
|
62
|
-
if (!r.success) return
|
|
65
|
+
const r = await cdpCall(client, "Page.reload", {});
|
|
66
|
+
if (!r.success) return r;
|
|
63
67
|
const info = await client.pageInfo();
|
|
64
68
|
if (!info.success) return err({ kind: "cdp_error", message: info.error.message });
|
|
65
69
|
if ("dialog" in info.data) {
|