webhand 0.2.0 → 0.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,70 @@
1
+ /**
2
+ * Collect while scrolling: find every item of a list longer than the screen that meets Claude's condition.
3
+ *
4
+ * webhand scrolls, from the top, one screen at a time (the same scan as compare.ts); Jev only judges. After
5
+ * each scroll, the list and table items in view that were not seen before are asked about, one question each:
6
+ * does it meet the condition? The items that do are kept as text, since they may leave the page. Nothing is
7
+ * clicked or typed. Afterwards the page is scrolled back to where it was, so what comes before the list (a
8
+ * search box, a form) is in view again. Claude gets the items and writes the answer from them.
9
+ */
10
+ import { MAX_SCROLLS, scanList, scroll } from "./compare.js";
11
+ /** Items shown to Claude; the count of all matches is kept beyond it. */
12
+ export const MAX_COLLECTED = 100;
13
+ /** Items asked about in one Jev request. */
14
+ const BATCH = 40;
15
+ export const MATCH = "MATCH";
16
+ export const NO = "NO";
17
+ export async function collectScan(browser, collector, plan, criterion, recorder, options = {}) {
18
+ const attempt = options.attempt ?? plan.attempt ?? 1;
19
+ const seen = new Set();
20
+ const items = [];
21
+ let matched = 0;
22
+ const scan = await scanList(browser, options.maxScrolls ?? MAX_SCROLLS, async (page, round) => {
23
+ const texts = await browser.rows();
24
+ const fresh = texts.filter((text) => !seen.has(text) && seen.add(text));
25
+ let found = 0;
26
+ for (let start = 0; start < fresh.length; start += BATCH) {
27
+ const rows = fresh.slice(start, start + BATCH).map((text, i) => ({ key: String(start + i + 1), text }));
28
+ const keys = await collector.collect(page, plan, criterion, rows);
29
+ for (const call of collector.calls)
30
+ recorder.call(call);
31
+ collector.calls.length = 0;
32
+ for (const row of rows) {
33
+ if (!keys.has(row.key))
34
+ continue;
35
+ found += 1;
36
+ if (items.length < MAX_COLLECTED)
37
+ items.push(row.text);
38
+ }
39
+ }
40
+ matched += found;
41
+ recorder.event("collect_round", {
42
+ attempt,
43
+ round,
44
+ y: page.scroll.y,
45
+ height: page.scroll.height,
46
+ rows: texts.length,
47
+ new: fresh.length,
48
+ matched: found,
49
+ });
50
+ }, options.loadMore ?? null);
51
+ let page = scan.page;
52
+ // Back to where the page was; on a later page, to its top, where a page's own form usually is.
53
+ const y = page.url === scan.startUrl ? scan.startY : 0;
54
+ if (page.scroll.y !== y) {
55
+ await browser.act(scroll(y - page.scroll.y), page);
56
+ page = await browser.observe();
57
+ }
58
+ const collected = {
59
+ criterion,
60
+ items,
61
+ matched,
62
+ seen: seen.size,
63
+ rounds: scan.rounds,
64
+ stopped: scan.stopped,
65
+ ...(options.loadMore ? { pages: scan.pages } : {}),
66
+ };
67
+ const { items: _items, ...counts } = collected;
68
+ recorder.event("collect_end", { attempt, ...counts });
69
+ return { collected, page };
70
+ }
@@ -0,0 +1,173 @@
1
+ /**
2
+ * Compare while scrolling: find the item that best meets Claude's criterion in a list longer than the screen,
3
+ * before anything is acted on.
4
+ *
5
+ * webhand scrolls, from the top, one screen at a time; Jev only compares. After each scroll Jev chooses among
6
+ * the items in view and the best one so far (kept as text, since it may have left the page), or keeps it.
7
+ * The scan stops at 20 scrolls, or at the bottom of the page once waiting adds nothing more. Nothing is clicked
8
+ * or typed. Afterwards the page is scrolled back to where the best item was, and the normal loop acts on it.
9
+ */
10
+ import { PolicyError } from "./types.js";
11
+ // Screens a scan goes through. 20 cut long lists short in BU Bench (round gv1: a list of 200 rows, read to
12
+ // about 105, three times); 100 screens still stop an endless feed.
13
+ export const MAX_SCROLLS = 100;
14
+ /** How many times a scan may press Claude's control for more items (--more). */
15
+ export const MAX_PAGES = 10;
16
+ /** How long the bottom of the page is watched for more items to load. */
17
+ const GROW_WAIT_MS = 2000;
18
+ const GROW_POLL_MS = 250;
19
+ /** Each scroll leaves this much of the last screen in view, so an item is never cut in half unseen. */
20
+ const OVERLAP_PX = 120;
21
+ export const KEEP = "KEEP";
22
+ export const NONE = "NONE";
23
+ export function describeBest(best) {
24
+ if (!best)
25
+ return null;
26
+ return best.context && best.context !== best.label ? `${best.label} — ${best.context}` : best.label;
27
+ }
28
+ export async function candidates(browser, page) {
29
+ const seen = new Set();
30
+ const clicks = page.actions.filter((a) => {
31
+ if (a.kind !== "click" || seen.has(a.node))
32
+ return false;
33
+ seen.add(a.node);
34
+ return true;
35
+ });
36
+ const texts = await browser.contexts(clicks.map((a) => a.node));
37
+ return clicks.map((action, i) => ({ key: String(i + 1), action, context: texts[String(action.node)] ?? "" }));
38
+ }
39
+ /** A scan's own scroll. It is not a decision by Jev, so it is not checked for staleness (scan: true). */
40
+ export function scroll(delta) {
41
+ return { id: "scroll", kind: "scroll", label: "Scroll", delta, scan: true };
42
+ }
43
+ const bottom = (page) => page.scroll.y + page.h >= page.scroll.height - 2;
44
+ /** Waits at the bottom for the page to grow. Returns the grown page, or null when nothing more loaded. */
45
+ async function grown(browser, page) {
46
+ for (let waited = 0; waited < GROW_WAIT_MS; waited += GROW_POLL_MS) {
47
+ await new Promise((resolve) => setTimeout(resolve, GROW_POLL_MS));
48
+ const next = await browser.observe();
49
+ if (next.scroll.height > page.scroll.height)
50
+ return next;
51
+ }
52
+ return null;
53
+ }
54
+ /**
55
+ * Scrolls through the page from the top one screen at a time, calling onScreen on each, until the scroll
56
+ * limit, or the bottom once waiting there adds nothing more. Only scrolls; nothing is clicked or typed, except
57
+ * by loadMore: it is asked where the main list ends (when the browser can tell) and at the bottom.
58
+ */
59
+ export async function scanList(browser, maxScrolls, onScreen, loadMore = null) {
60
+ let page = await browser.observe();
61
+ const startY = page.scroll.y;
62
+ const startUrl = page.url;
63
+ let pages = 0;
64
+ if (page.scroll.y > 0) {
65
+ await browser.act(scroll(-page.scroll.y), page);
66
+ page = await browser.observe();
67
+ }
68
+ let rounds = 0;
69
+ let asked = "";
70
+ while (true) {
71
+ await onScreen(page, rounds);
72
+ const end = (stopped) => ({ rounds, stopped, page, startY, startUrl, pages });
73
+ if (rounds >= maxScrolls)
74
+ return end("limit");
75
+ // The control for more items is usually right below the list, which may be far above the bottom of the
76
+ // page. Where the list ends, it is looked for once, before scrolling past it; then again at the bottom.
77
+ const here = `${page.url}|${page.scroll.height}`;
78
+ let listEnd = loadMore && !bottom(page) && asked !== here ? await browser.listEnd?.() : null;
79
+ if (listEnd != null && listEnd > page.h)
80
+ listEnd = null;
81
+ if (bottom(page) || listEnd != null) {
82
+ const next = await grown(browser, page);
83
+ if (!next) {
84
+ if (!loadMore)
85
+ return end("bottom");
86
+ if (pages >= MAX_PAGES)
87
+ return end("pages");
88
+ if (listEnd != null && listEnd > page.h / 2) {
89
+ // Bring what is below the list into view.
90
+ await browser.act(scroll(Math.round(listEnd - page.h / 3)), page);
91
+ page = await browser.observe();
92
+ }
93
+ asked = `${page.url}|${page.scroll.height}`;
94
+ const loaded = await loadMore(page);
95
+ if (loaded) {
96
+ // More items, on this page or the next one: look at the screen as it is now before scrolling on.
97
+ pages += 1;
98
+ rounds += 1;
99
+ page = loaded;
100
+ continue;
101
+ }
102
+ if (bottom(page))
103
+ return end("bottom");
104
+ }
105
+ else {
106
+ page = next;
107
+ }
108
+ }
109
+ await browser.act(scroll(Math.max(100, page.h - OVERLAP_PX)), page);
110
+ rounds += 1;
111
+ page = await browser.observe();
112
+ }
113
+ }
114
+ export async function compareScan(browser, comparer, plan, criterion, recorder, options = {}) {
115
+ const attempt = options.attempt ?? plan.attempt ?? 1;
116
+ const flush = () => {
117
+ for (const call of comparer.calls)
118
+ recorder.call(call);
119
+ comparer.calls.length = 0;
120
+ };
121
+ let best = null;
122
+ const scan = await scanList(browser, options.maxScrolls ?? MAX_SCROLLS, async (page, round) => {
123
+ const items = await candidates(browser, page);
124
+ const choice = await comparer.compare(page, plan, criterion, best, items);
125
+ flush();
126
+ const chosen = items.find((item) => item.key === choice);
127
+ if (chosen) {
128
+ best = { label: chosen.action.label, context: chosen.context, y: page.scroll.y, url: page.url, round };
129
+ }
130
+ else if (choice !== KEEP && choice !== NONE) {
131
+ throw new PolicyError(`Comparison choice ${choice} is not offered; nothing was chosen.`);
132
+ }
133
+ recorder.event("compare_round", {
134
+ attempt,
135
+ round,
136
+ y: page.scroll.y,
137
+ height: page.scroll.height,
138
+ candidates: items.length,
139
+ choice: chosen ? "item" : choice,
140
+ best: best?.label ?? null,
141
+ });
142
+ }, options.loadMore ?? null);
143
+ const { rounds, stopped } = scan;
144
+ let page = scan.page;
145
+ const found = best;
146
+ if (found) {
147
+ if (found.url && found.url !== page.url && browser.goto) {
148
+ // The best item was on an earlier page: open that page again (it was already visited).
149
+ await browser.goto(found.url);
150
+ page = await browser.observe();
151
+ }
152
+ if (found.y !== page.scroll.y) {
153
+ await browser.act(scroll(found.y - page.scroll.y), page);
154
+ page = await browser.observe();
155
+ }
156
+ const label = found.label;
157
+ const inView = page.actions.some((a) => a.kind === "click" && a.label === label);
158
+ const reveal = page.actions.find((a) => a.kind === "reveal" && a.label === `Scroll to ${label}`);
159
+ if (!inView && reveal) {
160
+ await browser.act(reveal, page);
161
+ page = await browser.observe();
162
+ }
163
+ }
164
+ const compared = {
165
+ criterion,
166
+ best: describeBest(found),
167
+ rounds,
168
+ stopped,
169
+ ...(options.loadMore ? { pages: scan.pages } : {}),
170
+ };
171
+ recorder.event("compare_end", { attempt, ...compared, best_label: found?.label ?? null });
172
+ return { compared, page };
173
+ }
package/dist/daemon.js ADDED
@@ -0,0 +1,125 @@
1
+ /**
2
+ * Holds one browser for a session, for engines that lose their pages when a CDP connection ends (Lightpanda, Obscura).
3
+ * Started by session.startChrome as `node daemon.ts SOCKET ENGINE BINARY PORT`: it runs the engine's `serve`, attaches
4
+ * one Browser to it, and answers the CLI's requests on SOCKET (see remote.ts). The socket is owner-only.
5
+ */
6
+ import { spawn } from "node:child_process";
7
+ import { chmodSync, rmSync } from "node:fs";
8
+ import { createServer } from "node:net";
9
+ import { Browser } from "./browser.js";
10
+ import { sleep } from "./format.js";
11
+ import { failure, lines, METHODS } from "./remote.js";
12
+ const STARTUP_MS = 15_000;
13
+ /** Attaches once the engine accepts connections. */
14
+ async function attach(engine, port, child) {
15
+ const deadline = performance.now() + STARTUP_MS;
16
+ for (;;) {
17
+ try {
18
+ // about:blank gives the run its own tab; Lightpanda's placeholder tab is not one.
19
+ return await Browser.open("about:blank", {
20
+ port,
21
+ rememberTab: false,
22
+ trackWindows: true, // Neither engine opens a second window for target=_blank or window.open: the log should say so
23
+ compat: engine === "obscura",
24
+ });
25
+ }
26
+ catch (error) {
27
+ if (child.exitCode !== null || performance.now() > deadline)
28
+ throw error;
29
+ await sleep(50);
30
+ }
31
+ }
32
+ }
33
+ /** The engine's own command line. Obscura refuses loopback and private addresses unless told otherwise (its SSRF guard). */
34
+ function serveArguments(engine, port) {
35
+ if (engine === "obscura") {
36
+ const open = process.env.WEBHAND_OBSCURA_ALLOW_PRIVATE === "1" ? ["--allow-private-network"] : [];
37
+ return ["serve", "--port", String(port), ...open];
38
+ }
39
+ return ["serve", "--host", "127.0.0.1", "--port", String(port)];
40
+ }
41
+ async function run(socketPath, engine, executable, port) {
42
+ const child = spawn(executable, serveArguments(engine, port), { stdio: "ignore" });
43
+ child.on("exit", () => process.exit(1));
44
+ const browser = await attach(engine, port, child);
45
+ const server = createServer();
46
+ const stop = async () => {
47
+ server.close();
48
+ await browser.close();
49
+ child.kill("SIGTERM");
50
+ rmSync(socketPath, { force: true });
51
+ process.exit(0);
52
+ };
53
+ process.on("SIGTERM", stop);
54
+ process.on("SIGINT", stop);
55
+ async function handle(request) {
56
+ try {
57
+ if (!METHODS.includes(request.method))
58
+ throw new Error(`Unknown method ${request.method}`);
59
+ browser.view = request.view;
60
+ const [a, b, c] = request.params;
61
+ switch (request.method) {
62
+ case "version":
63
+ return { id: request.id, result: browser.version };
64
+ case "observe":
65
+ return { id: request.id, result: await browser.observe() };
66
+ case "fresh":
67
+ return { id: request.id, result: await browser.fresh(a, b) };
68
+ case "act":
69
+ await browser.act(a, b, c);
70
+ return { id: request.id, result: null };
71
+ case "contexts":
72
+ return { id: request.id, result: await browser.contexts(a) };
73
+ case "goto":
74
+ await browser.goto(a);
75
+ return { id: request.id, result: null };
76
+ case "rows":
77
+ return { id: request.id, result: await browser.rows() };
78
+ case "listEnd":
79
+ return { id: request.id, result: await browser.listEnd() };
80
+ case "check":
81
+ return { id: request.id, result: await browser.check() };
82
+ case "unseen":
83
+ return { id: request.id, result: await browser.unseen() };
84
+ case "ignoredWindows":
85
+ return { id: request.id, result: await browser.ignoredWindows() };
86
+ case "screenshot":
87
+ return { id: request.id, result: (await browser.screenshot()).toString("base64") };
88
+ case "evaluate":
89
+ return { id: request.id, result: await browser.evaluate(a, b) };
90
+ case "storageState":
91
+ return { id: request.id, result: await browser.storageState() };
92
+ case "restore":
93
+ return { id: request.id, result: await browser.restore(a, b) };
94
+ default:
95
+ setImmediate(stop);
96
+ return { id: request.id, result: null };
97
+ }
98
+ }
99
+ catch (error) {
100
+ return { id: request.id, error: failure(error) };
101
+ }
102
+ }
103
+ // One request at a time: a command is one thing happening to the page.
104
+ let queue = Promise.resolve();
105
+ server.on("connection", (socket) => {
106
+ socket.on("error", () => { });
107
+ socket.on("data", lines((line) => {
108
+ const request = JSON.parse(line);
109
+ queue = queue.then(async () => {
110
+ const reply = await handle(request);
111
+ if (!socket.destroyed)
112
+ socket.write(`${JSON.stringify(reply)}\n`);
113
+ });
114
+ }));
115
+ });
116
+ await new Promise((resolve, reject) => {
117
+ server.once("error", reject);
118
+ server.listen(socketPath, () => {
119
+ chmodSync(socketPath, 0o600);
120
+ resolve();
121
+ });
122
+ });
123
+ }
124
+ const [socketPath, engine, executable, port] = process.argv.slice(2);
125
+ run(socketPath, engine, executable, Number.parseInt(port, 10)).catch(() => process.exit(1));
package/dist/format.js ADDED
@@ -0,0 +1,44 @@
1
+ /** How values are written into what Claude reads, what Jev receives, and the logs. */
2
+ /** A value as JSON, the way the CLI quotes a value inside a message. */
3
+ export function quote(value) {
4
+ return toJson(value);
5
+ }
6
+ /** Strings as they are, anything else as JSON. */
7
+ export function show(value) {
8
+ return typeof value === "string" ? value : quote(value);
9
+ }
10
+ /**
11
+ * Compact JSON. Maps keep their insertion order, which a plain object would not keep for keys like
12
+ * "2" and "1". Undefined object entries are dropped; undefined elsewhere is null.
13
+ */
14
+ export function toJson(value) {
15
+ const encode = (item) => {
16
+ if (item === null || item === undefined)
17
+ return "null";
18
+ if (typeof item === "number")
19
+ return Number.isFinite(item) ? JSON.stringify(item) : "null";
20
+ if (typeof item === "string" || typeof item === "boolean")
21
+ return JSON.stringify(item);
22
+ if (Array.isArray(item))
23
+ return `[${item.map(encode).join(",")}]`;
24
+ const entries = item instanceof Map ? [...item] : Object.entries(item);
25
+ return `{${entries
26
+ .filter(([, v]) => v !== undefined)
27
+ .map(([k, v]) => `${JSON.stringify(String(k))}:${encode(v)}`)
28
+ .join(",")}}`;
29
+ };
30
+ return encode(value);
31
+ }
32
+ /** A run folder name: YYYYMMDDTHHMMSS in local time. */
33
+ export function stamp(date = new Date()) {
34
+ const pad = (n) => String(n).padStart(2, "0");
35
+ return (`${date.getFullYear()}${pad(date.getMonth() + 1)}${pad(date.getDate())}` +
36
+ `T${pad(date.getHours())}${pad(date.getMinutes())}${pad(date.getSeconds())}`);
37
+ }
38
+ /** An error for a log line: its name and message. */
39
+ export function describeError(error) {
40
+ if (error instanceof Error)
41
+ return `${error.name}: ${error.message}`;
42
+ return `Error: ${String(error)}`;
43
+ }
44
+ export const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
@@ -0,0 +1,206 @@
1
+ /**
2
+ * What webhand learns about a site from its own runs (v2, stage 3), kept in the user's folder:
3
+ * $WEBHAND_HOME/knowledge/<host>.json. For each kind of page (the address with ids and query values taken out),
4
+ * the operations tried on it (kind and label) and what each did: how often the page changed, and where it went.
5
+ * Only labels of controls that read as plain names are kept, and none from inside a list item (a result, a row,
6
+ * a card: its label is the page's content, such as a name or a title); typed text, values, query values, and anything that
7
+ * looks like personal data never are. Nothing here is shown to Jev yet: this step only records.
8
+ */
9
+ import { mkdirSync, readdirSync, readFileSync, renameSync, rmSync, writeFileSync } from "node:fs";
10
+ import { join } from "node:path";
11
+ import { home } from "./workspace.js";
12
+ const MAX_PAGES = 200;
13
+ const MAX_TRANSITIONS = 30;
14
+ const MAX_LABEL = 60;
15
+ const RECORDED_KINDS = new Set(["click", "select"]);
16
+ /** The kind of page an address is: numbers and ids in the path become :n and :id, query values are dropped. */
17
+ export function pageShape(url) {
18
+ if (typeof url !== "string")
19
+ return null;
20
+ let parsed;
21
+ try {
22
+ parsed = new URL(url);
23
+ }
24
+ catch {
25
+ return null;
26
+ }
27
+ if (!/^(https?|file):$/.test(parsed.protocol))
28
+ return null;
29
+ const segments = parsed.pathname
30
+ .split("/")
31
+ .filter(Boolean)
32
+ .map((segment) => {
33
+ if (/^\d+$/.test(segment))
34
+ return ":n";
35
+ if (/^[0-9a-f]{8}-[0-9a-f-]{27,}$/i.test(segment) || /^[0-9a-f]{12,}$/i.test(segment))
36
+ return ":id";
37
+ if (segment.length > 40 || (/\d/.test(segment) && /[a-z]/i.test(segment) && segment.length > 16))
38
+ return ":x";
39
+ return segment.toLowerCase();
40
+ });
41
+ const keys = [...new Set([...parsed.searchParams.keys()])].sort();
42
+ const host = parsed.protocol === "file:" ? "local" : parsed.host.toLowerCase();
43
+ return { host, shape: `/${segments.join("/")}${keys.length ? `?${keys.join("&")}` : ""}` };
44
+ }
45
+ /** A control label that is safe to keep: short, plain, with no address, no long number, nothing like a name field. */
46
+ export function plainLabel(label) {
47
+ if (typeof label !== "string")
48
+ return null;
49
+ const text = label.replace(/\s+/g, " ").trim();
50
+ if (!text || text.length > MAX_LABEL)
51
+ return null;
52
+ if (/@|https?:|\d{5,}/.test(text))
53
+ return null;
54
+ return text;
55
+ }
56
+ const MAX_HINTS = 5;
57
+ // Entries in a run's history that are not an operation. After a step, an answer or a NOT_VISIBLE choice means the run was stuck there.
58
+ const DEAD_END = new Set(["answer", "missing"]);
59
+ export class Knowledge {
60
+ dir;
61
+ use;
62
+ /** use: whether hints() offers what is known; recording goes on either way. */
63
+ constructor(dir = join(home(), "knowledge"), use = true) {
64
+ this.dir = dir;
65
+ this.use = use;
66
+ }
67
+ file(host) {
68
+ return join(this.dir, `${host.replace(/[^a-z0-9.-]/g, "_")}.json`);
69
+ }
70
+ load(host) {
71
+ try {
72
+ return JSON.parse(readFileSync(this.file(host), "utf8"));
73
+ }
74
+ catch {
75
+ return { host, pages: {} };
76
+ }
77
+ }
78
+ /** One executed operation: what was tried on the page at `from`, and whether the page at `to` differs. */
79
+ record(step) {
80
+ if (!RECORDED_KINDS.has(step.kind) || step.inItem)
81
+ return false;
82
+ const label = plainLabel(step.label);
83
+ const from = pageShape(step.from);
84
+ if (!label || !from)
85
+ return false;
86
+ const to = pageShape(step.to);
87
+ const knowledge = this.load(from.host);
88
+ const keys = Object.keys(knowledge.pages);
89
+ if (!knowledge.pages[from.shape] && keys.length >= MAX_PAGES)
90
+ return false;
91
+ knowledge.pages[from.shape] ??= { seen: 0, transitions: {} };
92
+ const page = knowledge.pages[from.shape];
93
+ page.seen += 1;
94
+ const key = `${step.kind}|${label}`;
95
+ if (!page.transitions[key] && Object.keys(page.transitions).length >= MAX_TRANSITIONS)
96
+ return false;
97
+ page.transitions[key] ??= { kind: step.kind, label, tried: 0, changed: 0, to: {} };
98
+ const transition = page.transitions[key];
99
+ transition.tried += 1;
100
+ if (step.changed) {
101
+ transition.changed += 1;
102
+ if (to && to.host === from.host && to.shape !== from.shape)
103
+ transition.to[to.shape] = (transition.to[to.shape] ?? 0) + 1;
104
+ }
105
+ mkdirSync(this.dir, { recursive: true });
106
+ const target = this.file(from.host);
107
+ writeFileSync(`${target}.tmp`, JSON.stringify(knowledge, null, 1));
108
+ renameSync(`${target}.tmp`, target);
109
+ return true;
110
+ }
111
+ /**
112
+ * A run finished as done: its click and select steps are marked as the way that worked, except the ones that led
113
+ * nowhere. A step does not count when the run was stuck on the page it led to (the next entry is an answer to
114
+ * Jev's question, or a NOT_VISIBLE choice) or when it changed nothing. `history` is the whole history, so that
115
+ * those entries are there to be seen.
116
+ */
117
+ confirm(history) {
118
+ const seen = new Set();
119
+ history.forEach((step, i) => {
120
+ const label = plainLabel(step.action);
121
+ const from = pageShape(step.url);
122
+ if (!label || !from || !RECORDED_KINDS.has(String(step.kind)) || step.page_changed === false)
123
+ return;
124
+ if (DEAD_END.has(String(history[i + 1]?.kind)))
125
+ return;
126
+ const knowledge = this.load(from.host);
127
+ const transition = knowledge.pages[from.shape]?.transitions[`${step.kind}|${label}`];
128
+ const mark = `${from.host}|${from.shape}|${step.kind}|${label}`;
129
+ if (!transition || seen.has(mark))
130
+ return;
131
+ seen.add(mark);
132
+ transition.ok = (transition.ok ?? 0) + 1;
133
+ const target = this.file(from.host);
134
+ writeFileSync(`${target}.tmp`, JSON.stringify(knowledge, null, 1));
135
+ renameSync(`${target}.tmp`, target);
136
+ });
137
+ }
138
+ /** The sites that have knowledge, with how much. */
139
+ hosts() {
140
+ let files = [];
141
+ try {
142
+ files = readdirSync(this.dir).filter((name) => name.endsWith(".json"));
143
+ }
144
+ catch {
145
+ return [];
146
+ }
147
+ return files.sort().map((name) => {
148
+ const knowledge = JSON.parse(readFileSync(join(this.dir, name), "utf8"));
149
+ const transitions = Object.values(knowledge.pages).flatMap((page) => Object.values(page.transitions));
150
+ return {
151
+ host: knowledge.host,
152
+ pages: Object.keys(knowledge.pages).length,
153
+ operations: transitions.length,
154
+ worked: transitions.filter((t) => (t.ok ?? 0) > 0).length,
155
+ };
156
+ });
157
+ }
158
+ /** What is known about a site, as lines a person can read. */
159
+ describe(host) {
160
+ const knowledge = this.load(host);
161
+ return Object.entries(knowledge.pages).flatMap(([shape, page]) => [
162
+ `${shape} (seen ${page.seen})`,
163
+ ...Object.values(page.transitions).map((t) => {
164
+ const goes = Object.entries(t.to).sort((a, b) => b[1] - a[1])[0]?.[0];
165
+ return ` ${t.kind.toUpperCase()} "${t.label}": tried ${t.tried}, page changed ${t.changed}, worked ${t.ok ?? 0}${goes ? `, goes to ${goes}` : ""}`;
166
+ }),
167
+ ]);
168
+ }
169
+ /** Forgets a site, or every site when host is null. Returns how many files went. */
170
+ forget(host) {
171
+ const names = host === null ? this.hosts().map((h) => h.host) : [host];
172
+ let removed = 0;
173
+ for (const name of names) {
174
+ try {
175
+ rmSync(this.file(name));
176
+ removed += 1;
177
+ }
178
+ catch {
179
+ // Nothing was known about it.
180
+ }
181
+ }
182
+ return removed;
183
+ }
184
+ /** What worked before on this kind of page: operations of runs that finished as done, the most often first. */
185
+ hints(url) {
186
+ const from = pageShape(url);
187
+ if (!this.use || !from)
188
+ return [];
189
+ const page = this.load(from.host).pages[from.shape];
190
+ if (!page)
191
+ return [];
192
+ return Object.values(page.transitions)
193
+ .filter((t) => (t.ok ?? 0) > 0)
194
+ .sort((a, b) => (b.ok ?? 0) - (a.ok ?? 0) || b.changed / b.tried - a.changed / a.tried)
195
+ .slice(0, MAX_HINTS)
196
+ .map((t) => {
197
+ const goes = Object.entries(t.to).sort((a, b) => b[1] - a[1])[0]?.[0];
198
+ return {
199
+ operation: t.kind.toUpperCase(),
200
+ label: t.label,
201
+ worked: `${t.ok} of ${t.tried} tries`,
202
+ ...(goes ? { goes_to: goes } : {}),
203
+ };
204
+ });
205
+ }
206
+ }
package/dist/learn.js CHANGED
@@ -4,11 +4,10 @@
4
4
  * The record is untrusted: pages can forge binding calls, so it only ever becomes a draft for review.
5
5
  */
6
6
  import { asset, READ_STATE } from "./browser.js";
7
- import { pyRound } from "./py.js";
8
7
  // A function replacer: the snapshot holds "$" sequences that a string replacement would expand.
9
8
  const RECORDER = asset("recorder.js").replaceAll("__SNAPSHOT__", () => READ_STATE.trim());
10
- const FIELDS = ["operation", "role", "label", "value", "checked", "reason", "url", "title", "text", "region"];
11
- const KINDS = new Set(["action", "page", "unsupported"]);
9
+ const FIELDS = ["change", "operation", "role", "label", "value", "checked", "reason", "url", "title", "text", "region"];
10
+ const KINDS = new Set(["action", "page", "unsupported", "tab"]);
12
11
  const MAX_REGIONS = 60;
13
12
  export const REGION_FIELDS = {
14
13
  key: "str",
@@ -46,7 +45,7 @@ export async function record(browser, url, recorder, until) {
46
45
  const started = performance.now();
47
46
  const write = (kind, fields) => {
48
47
  counts[kind] = (counts[kind] ?? 0) + 1;
49
- recorder.event(kind, { at_ms: pyRound(performance.now() - started), ...fields });
48
+ recorder.event(kind, { at_ms: Math.round(performance.now() - started), ...fields });
50
49
  };
51
50
  const receive = (_source, event) => {
52
51
  if (event === null || typeof event !== "object" || Array.isArray(event))
@@ -54,6 +53,9 @@ export async function record(browser, url, recorder, until) {
54
53
  const data = event;
55
54
  if (typeof data.kind !== "string" || !KINDS.has(data.kind))
56
55
  return;
56
+ // A page can only say that its tab came to the front; opened and closed tabs are seen from outside.
57
+ if (data.kind === "tab" && data.change !== "switched")
58
+ return;
57
59
  const fields = {};
58
60
  for (const key of FIELDS)
59
61
  if (key in data)
@@ -65,7 +67,11 @@ export async function record(browser, url, recorder, until) {
65
67
  const context = browser.context;
66
68
  await context.exposeBinding("__browserLabRecord", receive);
67
69
  await context.addInitScript({ content: RECORDER });
68
- context.on("page", () => write("unsupported", { reason: "new_tab" }));
70
+ // A tab is a change the agent can follow (TAB_n), so it is recorded as an event, not as unsupported.
71
+ context.on("page", (page) => {
72
+ write("tab", { change: "opened", count: context.pages().length });
73
+ page.on("close", () => write("tab", { change: "closed", count: context.pages().length }));
74
+ });
69
75
  await browser.page.goto(url, { waitUntil: "load" });
70
76
  while (browser.browser.isConnected() && context.pages().length && !(await until())) {
71
77
  try {