webhand 0.2.0 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.ja.md +104 -0
- package/README.md +70 -219
- package/assets/recorder.js +4 -0
- package/assets/snapshot.js +224 -4
- package/dist/agent.js +154 -13
- package/dist/auth.js +62 -0
- package/dist/bin.js +0 -0
- package/dist/browser.js +702 -23
- package/dist/cli.js +684 -86
- package/dist/collect.js +70 -0
- package/dist/compare.js +173 -0
- package/dist/daemon.js +125 -0
- package/dist/format.js +44 -0
- package/dist/knowledge.js +206 -0
- package/dist/learn.js +11 -5
- package/dist/metrics.js +64 -4
- package/dist/more.js +58 -0
- package/dist/policies/base.js +110 -14
- package/dist/policies/jev.js +245 -18
- package/dist/policies/scripted.js +2 -2
- package/dist/remote.js +165 -0
- package/dist/runbook.js +34 -19
- package/dist/session.js +261 -17
- package/dist/types.js +12 -0
- package/package.json +9 -6
- package/dist/py.js +0 -141
- package/dist/run.js +0 -80
- package/dist/tasks.js +0 -37
package/assets/snapshot.js
CHANGED
|
@@ -1,4 +1,6 @@
|
|
|
1
|
-
// view = {focus: [...], skip: [...]}: region patterns Claude chose from what was
|
|
1
|
+
// view = {focus: [...], skip: [...], read: [...], whole: true}: region patterns Claude chose from what was
|
|
2
|
+
// learned, the kinds of content to read beyond the viewport, and (v2) whether elements beyond the viewport are
|
|
3
|
+
// offered with their own operations. Empty: DOM order, viewport only.
|
|
2
4
|
(view => {
|
|
3
5
|
if (!document.body) return null;
|
|
4
6
|
|
|
@@ -6,9 +8,13 @@
|
|
|
6
8
|
|
|
7
9
|
const TEXT_LIMIT = 6000;
|
|
8
10
|
const MAX_ACTIONS = 250;
|
|
11
|
+
const MAX_REVEALS = 60;
|
|
12
|
+
// v2: elements beyond the viewport offered as they are, the nearest first.
|
|
13
|
+
const MAX_BEYOND = 200;
|
|
9
14
|
const SCROLL_STEP = 560;
|
|
10
15
|
const MAX_REGIONS = 40;
|
|
11
16
|
const REGION_NAME_LIMIT = 40;
|
|
17
|
+
const READ_LIMIT = 12000;
|
|
12
18
|
|
|
13
19
|
const INTERACTIVE_ROLES = ['button', 'link', 'checkbox', 'radio', 'switch', 'tab', 'menuitem', 'menuitemradio',
|
|
14
20
|
'option', 'gridcell', 'combobox', 'textbox', 'searchbox', 'spinbutton'];
|
|
@@ -210,6 +216,31 @@
|
|
|
210
216
|
return false;
|
|
211
217
|
};
|
|
212
218
|
|
|
219
|
+
// What lies over (x, y) instead of the element: the nearest dialog, or fixed or sticky layer, above the topmost
|
|
220
|
+
// element there, as its role and a short name. For telling Jev why a click on a covered element did not run.
|
|
221
|
+
state.cover = (x, y) => {
|
|
222
|
+
const top = deepElementFromPoint(x, y);
|
|
223
|
+
let layer = null;
|
|
224
|
+
for (let node = top; node instanceof Element; node = flatParent(node)) {
|
|
225
|
+
const position = getComputedStyle(node).position;
|
|
226
|
+
if (node.tagName === 'DIALOG' || ['dialog', 'alertdialog'].includes(node.getAttribute('role')) ||
|
|
227
|
+
node.getAttribute('aria-modal') === 'true' || position === 'fixed' || position === 'sticky') {
|
|
228
|
+
layer = node;
|
|
229
|
+
break;
|
|
230
|
+
}
|
|
231
|
+
}
|
|
232
|
+
const shown = layer || top;
|
|
233
|
+
if (!shown) return 'something';
|
|
234
|
+
const role = shown.getAttribute('role') || (shown.tagName === 'DIALOG' ? 'dialog' : shown.tagName.toLowerCase());
|
|
235
|
+
const heading = shown.querySelector?.('h1,h2,h3,h4,[role="heading"],legend');
|
|
236
|
+
const name = (shown.getAttribute('aria-label') || heading?.textContent || shown.textContent || '')
|
|
237
|
+
.replace(/\s+/g, ' ').trim().slice(0, 60);
|
|
238
|
+
const modal = layer && (layer.tagName === 'DIALOG' || ['dialog', 'alertdialog'].includes(layer.getAttribute('role')) ||
|
|
239
|
+
layer.getAttribute('aria-modal') === 'true');
|
|
240
|
+
const kind = modal ? 'a dialog' : layer ? 'a fixed ' + role : 'a ' + role;
|
|
241
|
+
return name ? kind + ' "' + name + '"' : kind;
|
|
242
|
+
};
|
|
243
|
+
|
|
213
244
|
state.describe = element => {
|
|
214
245
|
let target = closestDeep(element, INTERACTIVE_SELECTOR);
|
|
215
246
|
for (let current = element; !target && current instanceof Element; current = flatParent(current)) {
|
|
@@ -234,6 +265,8 @@
|
|
|
234
265
|
element.getAttribute('aria-checked'), element.getAttribute('aria-selected'),
|
|
235
266
|
element.getAttribute('href'),
|
|
236
267
|
scope?.innerText?.slice(0, TEXT_LIMIT) || '',
|
|
268
|
+
// Whether the scope is a list item or row (its text says which item the target belongs to).
|
|
269
|
+
scope?.matches?.('article,li,tr,[role="row"]') ? 'item' : 'other',
|
|
237
270
|
];
|
|
238
271
|
};
|
|
239
272
|
|
|
@@ -284,6 +317,13 @@
|
|
|
284
317
|
return actions;
|
|
285
318
|
};
|
|
286
319
|
|
|
320
|
+
// Elements beyond the viewport, in DOM order, offered only to be scrolled into view (read layer only).
|
|
321
|
+
const reveals = [];
|
|
322
|
+
const revealed = new Set();
|
|
323
|
+
// v2 (view.whole): elements beyond the viewport with their own operations; acting scrolls them into view first.
|
|
324
|
+
const whole = Boolean(view.whole);
|
|
325
|
+
const beyond = [];
|
|
326
|
+
|
|
287
327
|
// In DOM order. Each action's region is kept aside so the freshness marker does not depend on it.
|
|
288
328
|
const collectActions = () => {
|
|
289
329
|
const actions = [];
|
|
@@ -295,6 +335,26 @@
|
|
|
295
335
|
const region = regionOf(element);
|
|
296
336
|
if (!centerInViewport(rect)) {
|
|
297
337
|
region.offscreen_actions++;
|
|
338
|
+
if (whole) {
|
|
339
|
+
const base = baseAction(element, role, rect);
|
|
340
|
+
const found = element.tagName === 'SELECT' ? selectActions(element, base) : otherActions(element, role, base);
|
|
341
|
+
const at = where(rect);
|
|
342
|
+
const distance = rect.bottom <= 0 ? -rect.bottom : rect.top - innerHeight;
|
|
343
|
+
for (const action of found) {
|
|
344
|
+
action.at = at;
|
|
345
|
+
action.offscreen = true;
|
|
346
|
+
actionRegions.set(action, region);
|
|
347
|
+
beyond.push({action, distance});
|
|
348
|
+
}
|
|
349
|
+
continue;
|
|
350
|
+
}
|
|
351
|
+
// With a read layer, an element beyond the viewport can be scrolled into view, never acted on as it is.
|
|
352
|
+
if (readKinds.length && readable(element) && !revealed.has(element)) {
|
|
353
|
+
revealed.add(element);
|
|
354
|
+
// Named as a scroll, so the action history never reads as if the element had been acted on.
|
|
355
|
+
reveals.push({node: nodeId(element), role, label: 'Scroll to ' + (accessibleName(element) || role),
|
|
356
|
+
kind: 'reveal', at: where(rect)});
|
|
357
|
+
}
|
|
298
358
|
continue;
|
|
299
359
|
}
|
|
300
360
|
const base = baseAction(element, role, rect);
|
|
@@ -460,7 +520,8 @@
|
|
|
460
520
|
range.selectNodeContents(node);
|
|
461
521
|
const region = regionOf(parent);
|
|
462
522
|
region.text_chars += value.length;
|
|
463
|
-
|
|
523
|
+
const rect = range.getBoundingClientRect();
|
|
524
|
+
lines.push({value, region, parent, rect, inView: intersectsViewport(rect)});
|
|
464
525
|
}
|
|
465
526
|
return lines;
|
|
466
527
|
};
|
|
@@ -496,6 +557,138 @@
|
|
|
496
557
|
return fitText(ordered.flatMap(([region, regionLines]) => [{value: '## ' + region.key}, ...regionLines]));
|
|
497
558
|
};
|
|
498
559
|
|
|
560
|
+
// ---- Read layer ----
|
|
561
|
+
// Chosen kinds of content from the whole page, beyond the viewport, for reading only: nothing in it can be
|
|
562
|
+
// acted on, and the freshness marker never includes it. Each kind is classified by page structure, never by
|
|
563
|
+
// site. The outline always comes first. Field values are never read; controls give a role and a name only.
|
|
564
|
+
// Text, list items, links, and controls in view are left out: the viewport text and elements already hold them.
|
|
565
|
+
// The outline and tables are kept whole, since their structure is the point.
|
|
566
|
+
|
|
567
|
+
const READ_KINDS = ['outline', 'text', 'table', 'list', 'link', 'control', 'media'];
|
|
568
|
+
const TABLE_SELECTOR = 'table,[role="table"],[role="grid"]';
|
|
569
|
+
const LIST_ITEM_SELECTOR = 'li,[role="listitem"]';
|
|
570
|
+
const LINK_SELECTOR = 'a[href],[role="link"]';
|
|
571
|
+
const CONTROL_SELECTOR = 'button,input,select,textarea,summary,label,[contenteditable="true"],' +
|
|
572
|
+
INTERACTIVE_ROLES.filter(role => role !== 'link').map(role => '[role="' + role + '"]').join(',');
|
|
573
|
+
const MEDIA_SELECTOR = 'img[alt],[role="img"][aria-label],figcaption';
|
|
574
|
+
// Text inside these belongs to another kind, so the text kind never repeats it.
|
|
575
|
+
const NOT_TEXT_SELECTOR = [TABLE_SELECTOR, LIST_ITEM_SELECTOR, HEADING_SELECTOR, LINK_SELECTOR, CONTROL_SELECTOR,
|
|
576
|
+
'figcaption', 'option'].join(',');
|
|
577
|
+
const requested = Array.isArray(view?.read) ? view.read : [];
|
|
578
|
+
const readKinds = requested.some(kind => READ_KINDS.includes(kind))
|
|
579
|
+
? READ_KINDS.filter(kind => kind === 'outline' || requested.includes(kind)) : [];
|
|
580
|
+
|
|
581
|
+
const squash = (text, limit) => String(text || '').replace(/\s+/g, ' ').trim().slice(0, limit);
|
|
582
|
+
// Where content is, rounded to 100px: the direction to scroll and how far.
|
|
583
|
+
const where = rect => {
|
|
584
|
+
if (rect.bottom <= 0) return 'above ' + Math.max(100, Math.round(-rect.bottom / 100) * 100) + 'px';
|
|
585
|
+
if (rect.top >= innerHeight) return 'below ' + Math.max(100, Math.round((rect.top - innerHeight) / 100) * 100) + 'px';
|
|
586
|
+
return 'in view';
|
|
587
|
+
};
|
|
588
|
+
// With focus patterns, only focused regions (and open dialogs) are read; skipped regions never are.
|
|
589
|
+
const readable = element => {
|
|
590
|
+
const decision = decide(regionOf(element));
|
|
591
|
+
if (decision.view === 'skip') return false;
|
|
592
|
+
return focusPatterns.length === 0 || decision.view === 'focus' || decision.order === 0;
|
|
593
|
+
};
|
|
594
|
+
const readElements = selector => queryAllDeep(selector)
|
|
595
|
+
.filter(element => !element.closest(NON_CONTENT_SELECTOR) && isVisible(element) && readable(element));
|
|
596
|
+
// Lines with a position; a position line is written only when it changes.
|
|
597
|
+
const placed = items => {
|
|
598
|
+
const out = [];
|
|
599
|
+
let last = null;
|
|
600
|
+
for (const {at, line} of items) {
|
|
601
|
+
if (at !== last) out.push('@ ' + at);
|
|
602
|
+
out.push(line);
|
|
603
|
+
last = at;
|
|
604
|
+
}
|
|
605
|
+
return out;
|
|
606
|
+
};
|
|
607
|
+
|
|
608
|
+
const readers = {
|
|
609
|
+
outline: () => placed(readElements(HEADING_SELECTOR).map(heading => {
|
|
610
|
+
const level = /^H[1-6]$/.test(heading.tagName) ? heading.tagName.toLowerCase()
|
|
611
|
+
: 'h' + (heading.getAttribute('aria-level') || 2);
|
|
612
|
+
return {at: where(heading.getBoundingClientRect()), line: level + ' ' + squash(heading.textContent, 120)};
|
|
613
|
+
}).filter(item => item.line.length > 3)),
|
|
614
|
+
text: () => placed(lines
|
|
615
|
+
.filter(line => !closestDeep(line.parent, NOT_TEXT_SELECTOR) && readable(line.parent))
|
|
616
|
+
.map(line => ({at: where(line.rect), line: squash(line.value, 1000)}))
|
|
617
|
+
.filter(item => item.at !== 'in view')),
|
|
618
|
+
table: () => readElements(TABLE_SELECTOR)
|
|
619
|
+
.filter(table => !table.parentElement?.closest(TABLE_SELECTOR))
|
|
620
|
+
.flatMap((table, index) => {
|
|
621
|
+
const rows = [...table.querySelectorAll('tr,[role="row"]')]
|
|
622
|
+
.filter(row => row.closest(TABLE_SELECTOR) === table && isVisible(row))
|
|
623
|
+
.map(row => [...row.querySelectorAll('th,td,[role="cell"],[role="gridcell"],[role="columnheader"],' +
|
|
624
|
+
'[role="rowheader"]')].filter(cell => cell.closest('tr,[role="row"]') === row)
|
|
625
|
+
.map(cell => squash(cell.innerText, 80)).join(' | '))
|
|
626
|
+
.filter(row => row.replace(/[\s|]/g, ''));
|
|
627
|
+
const caption = squash(table.querySelector('caption')?.textContent || table.getAttribute('aria-label'), 80);
|
|
628
|
+
return ['## table ' + (index + 1) + ' @ ' + where(table.getBoundingClientRect()) +
|
|
629
|
+
(caption ? ': ' + caption : '') + ' (' + rows.length + ' rows)', ...rows];
|
|
630
|
+
}),
|
|
631
|
+
list: () => {
|
|
632
|
+
const items = readElements(LIST_ITEM_SELECTOR)
|
|
633
|
+
.filter(item => !item.parentElement?.closest(LIST_ITEM_SELECTOR) && !item.closest(TABLE_SELECTOR));
|
|
634
|
+
return placed(items.map(item => ({at: where(item.getBoundingClientRect()), line: '- ' + squash(item.innerText, 300)}))
|
|
635
|
+
.filter(item => item.line.length > 2 && item.at !== 'in view'));
|
|
636
|
+
},
|
|
637
|
+
link: () => {
|
|
638
|
+
const seen = new Set();
|
|
639
|
+
return placed(readElements(LINK_SELECTOR).map(link => ({at: where(link.getBoundingClientRect()),
|
|
640
|
+
line: '- ' + squash(accessibleName(link), 80)}))
|
|
641
|
+
.filter(item => item.at !== 'in view' && item.line.length > 2 && !seen.has(item.line) && seen.add(item.line)));
|
|
642
|
+
},
|
|
643
|
+
control: () => placed(readElements(CONTROL_SELECTOR)
|
|
644
|
+
.filter(element => !element.matches(LINK_SELECTOR) && !(element.type === 'hidden'))
|
|
645
|
+
.map(element => {
|
|
646
|
+
const rect = element.getBoundingClientRect();
|
|
647
|
+
return {at: where(rect), line: '- ' + (roleOf(element) || element.tagName.toLowerCase()) + ' ' +
|
|
648
|
+
JSON.stringify(squash(accessibleName(element), 60))};
|
|
649
|
+
})
|
|
650
|
+
.filter(item => item.at !== 'in view')),
|
|
651
|
+
media: () => placed(readElements(MEDIA_SELECTOR).map(element => ({at: where(element.getBoundingClientRect()),
|
|
652
|
+
line: '- ' + squash(element.matches('figcaption') ? element.textContent
|
|
653
|
+
: element.getAttribute('alt') || element.getAttribute('aria-label'), 160)}))
|
|
654
|
+
.filter(item => item.line.length > 2)),
|
|
655
|
+
};
|
|
656
|
+
|
|
657
|
+
// Each kind gets an equal share of the limit; what a small kind leaves goes to the others. Lines are kept
|
|
658
|
+
// whole; only a single line longer than its share is cut. Reports shown and total characters per kind.
|
|
659
|
+
const readLayer = () => {
|
|
660
|
+
if (!readKinds.length) return null;
|
|
661
|
+
const full = readKinds.map(kind => {
|
|
662
|
+
const kindLines = readers[kind]();
|
|
663
|
+
return {kind, lines: kindLines, total: kindLines.reduce((sum, line) => sum + line.length + 1, 0)};
|
|
664
|
+
});
|
|
665
|
+
let remaining = READ_LIMIT;
|
|
666
|
+
const share = new Map();
|
|
667
|
+
[...full].sort((a, b) => a.total - b.total).forEach((entry, index, sorted) => {
|
|
668
|
+
const given = Math.min(entry.total, Math.floor(remaining / (sorted.length - index)));
|
|
669
|
+
share.set(entry.kind, given);
|
|
670
|
+
remaining -= given;
|
|
671
|
+
});
|
|
672
|
+
const chars = {};
|
|
673
|
+
const sections = full.map(({kind, lines: kindLines, total}) => {
|
|
674
|
+
const taken = [];
|
|
675
|
+
let used = 0;
|
|
676
|
+
for (const line of kindLines) {
|
|
677
|
+
if (used + line.length + 1 > share.get(kind)) {
|
|
678
|
+
if (!taken.length && share.get(kind) > 1) taken.push(line.slice(0, share.get(kind) - 1));
|
|
679
|
+
break;
|
|
680
|
+
}
|
|
681
|
+
taken.push(line);
|
|
682
|
+
used += line.length + 1;
|
|
683
|
+
}
|
|
684
|
+
const shown = taken.reduce((sum, line) => sum + line.length + 1, 0);
|
|
685
|
+
chars[kind] = [shown, total];
|
|
686
|
+
const more = total > shown ? ['... ' + (total - shown) + ' more characters'] : [];
|
|
687
|
+
return ['# ' + kind + (kindLines.length ? '' : ' (none)'), ...taken, ...more].join('\n');
|
|
688
|
+
});
|
|
689
|
+
return {kinds: readKinds, text: sections.join('\n'), chars, limit: READ_LIMIT};
|
|
690
|
+
};
|
|
691
|
+
|
|
499
692
|
// ---- Snapshot ----
|
|
500
693
|
|
|
501
694
|
const actions = collectActions();
|
|
@@ -528,7 +721,31 @@
|
|
|
528
721
|
}
|
|
529
722
|
chosen = chosen.slice(0, MAX_ACTIONS);
|
|
530
723
|
const omitted_actions = actions.length - chosen.length;
|
|
531
|
-
|
|
724
|
+
// The region key goes on the action for the run log only; the marker above was taken without it.
|
|
725
|
+
for (const action of chosen) {
|
|
726
|
+
const region = actionRegions.get(action);
|
|
727
|
+
region.shown_actions++;
|
|
728
|
+
action.region = region.key;
|
|
729
|
+
}
|
|
730
|
+
// v2: the nearest elements beyond the viewport, back in DOM order, after those in view.
|
|
731
|
+
const beyondKept = beyond
|
|
732
|
+
.map((entry, index) => ({...entry, index}))
|
|
733
|
+
.filter(({action}) => !viewing || decide(actionRegions.get(action)).view !== 'skip')
|
|
734
|
+
.sort((a, b) => a.distance - b.distance || a.index - b.index)
|
|
735
|
+
.slice(0, MAX_BEYOND)
|
|
736
|
+
.sort((a, b) => a.index - b.index)
|
|
737
|
+
.map(({action}) => action);
|
|
738
|
+
for (const action of beyondKept) {
|
|
739
|
+
if (!(action.node in guards)) guards[action.node] = state.guard(state.nodes.get(action.node));
|
|
740
|
+
action.region = actionRegions.get(action).key;
|
|
741
|
+
}
|
|
742
|
+
// The nearest and the farthest are kept: the end of a list is what loads more of it.
|
|
743
|
+
const kept = reveals.length > MAX_REVEALS
|
|
744
|
+
? [...reveals.slice(0, MAX_REVEALS / 2), ...reveals.slice(-MAX_REVEALS / 2)] : reveals;
|
|
745
|
+
for (const reveal of kept) {
|
|
746
|
+
if (!(reveal.node in guards)) guards[reveal.node] = state.guard(state.nodes.get(reveal.node));
|
|
747
|
+
}
|
|
748
|
+
chosen = [...chosen, ...beyondKept, ...kept];
|
|
532
749
|
chosen.forEach((action, index) => action.id = 'e' + (index + 1));
|
|
533
750
|
if (scrollY + innerHeight < height - 2) {
|
|
534
751
|
chosen.push({id: 'scroll_down', kind: 'scroll', label: 'Scroll down', delta: SCROLL_STEP});
|
|
@@ -549,8 +766,11 @@
|
|
|
549
766
|
});
|
|
550
767
|
|
|
551
768
|
state.region = element => regionOf(element).key;
|
|
769
|
+
const read = readLayer();
|
|
552
770
|
|
|
553
771
|
return {url: location.href, title: document.title, w: innerWidth, h: innerHeight, text: shown.text,
|
|
554
772
|
scroll: {y: scrollY, height}, actions: chosen, marker, page_key, guards, omitted_actions,
|
|
555
|
-
regions: regionSummary, omitted_regions: outline.length - regionSummary.length}
|
|
773
|
+
regions: regionSummary, omitted_regions: outline.length - regionSummary.length, ...(read ? {read} : {}),
|
|
774
|
+
...(reveals.length > kept.length ? {omitted_reveals: reveals.length - kept.length} : {}),
|
|
775
|
+
...(beyond.length > beyondKept.length ? {omitted_beyond: beyond.length - beyondKept.length} : {})};
|
|
556
776
|
})
|
package/dist/agent.js
CHANGED
|
@@ -3,8 +3,9 @@
|
|
|
3
3
|
*
|
|
4
4
|
* The loop returns to Claude only when it cannot continue on its own. The task's verify() decides success.
|
|
5
5
|
*/
|
|
6
|
-
import {
|
|
7
|
-
import {
|
|
6
|
+
import { quote } from "./format.js";
|
|
7
|
+
import { pageEvent, safeUrl } from "./metrics.js";
|
|
8
|
+
import { Decision, MutationUncertain, PolicyError, StalePage, } from "./types.js";
|
|
8
9
|
// Why the loop handed control back to Claude.
|
|
9
10
|
export const GOAL_REPORTED = "GOAL_REPORTED"; // Jev chose DONE. Claude checks the page; finish verifies.
|
|
10
11
|
export const BLOCKED = "BLOCKED";
|
|
@@ -12,6 +13,7 @@ export const STUCK = "STUCK"; // Three actions in a row did not change the page.
|
|
|
12
13
|
export const ATTEMPT_BUDGET = "ATTEMPT_BUDGET";
|
|
13
14
|
export const NEEDS_HUMAN = "NEEDS_HUMAN"; // A person acts in the window: sign-in, codes, CAPTCHA, [person] steps.
|
|
14
15
|
export const NEEDS_VALUE = "NEEDS_VALUE"; // A field needs text and none of Claude's values fits.
|
|
16
|
+
export const ESCALATE = "ESCALATE"; // v2: Jev handed the decision to Claude. `answer` continues the same attempt.
|
|
15
17
|
export const JEV_FAILED = "JEV_FAILED"; // No usable decision. Nothing was executed.
|
|
16
18
|
export const UNCERTAIN = "UNCERTAIN"; // A mutation may have happened. Never retried, never replanned.
|
|
17
19
|
/** Per attempt. A replan starts a new attempt with a fresh budget. */
|
|
@@ -27,15 +29,95 @@ export class Budget {
|
|
|
27
29
|
}
|
|
28
30
|
const MISSING = "missing"; // A history entry for a NOT_VISIBLE choice: nothing ran, but Jev must see it.
|
|
29
31
|
const STALE = "stale"; // A history entry for a choice the browser refused as stale: nothing ran.
|
|
30
|
-
/**
|
|
32
|
+
/** What the observation offered when Jev found the element it needed missing: counts for the run log only. */
|
|
33
|
+
function offered(page) {
|
|
34
|
+
const actions = page.actions ?? [];
|
|
35
|
+
const scroll = page.scroll ?? {};
|
|
36
|
+
return {
|
|
37
|
+
in_view: actions.filter((a) => a.kind !== "reveal").length,
|
|
38
|
+
offscreen: actions.filter((a) => a.kind === "reveal").length,
|
|
39
|
+
omitted_actions: page.omitted_actions ?? 0,
|
|
40
|
+
omitted_reveals: page.omitted_reveals ?? 0,
|
|
41
|
+
regions_with_offscreen: (page.regions ?? []).filter((r) => r.offscreen_actions)
|
|
42
|
+
.length,
|
|
43
|
+
below_px: Math.max(0, Math.round((scroll.height ?? 0) - (scroll.y ?? 0) - (page.h ?? 0))),
|
|
44
|
+
};
|
|
45
|
+
}
|
|
46
|
+
/** Outcomes after which the log keeps a page check, to tell a bot check or an error page from a plan problem. */
|
|
47
|
+
export const CHECKED_OUTCOMES = [BLOCKED, STUCK, ATTEMPT_BUDGET, NEEDS_HUMAN];
|
|
48
|
+
/** Logs a page_check event. Read-only, never sent to Jev; a failed check is logged and never stops the run. */
|
|
49
|
+
export async function checkPage(browser, recorder, when, attempt, url) {
|
|
50
|
+
if (!browser.check)
|
|
51
|
+
return;
|
|
52
|
+
try {
|
|
53
|
+
recorder.event("page_check", { when, attempt, url: safeUrl(url), ...(await browser.check()) });
|
|
54
|
+
}
|
|
55
|
+
catch (error) {
|
|
56
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
57
|
+
recorder.event("page_check_failed", { when, attempt, error: message });
|
|
58
|
+
}
|
|
59
|
+
}
|
|
60
|
+
/**
|
|
61
|
+
* Opt-in with WEBHAND_SCREENSHOTS=1: the harness, not a model, captures the page after each action so an
|
|
62
|
+
* outside judge can check the run. Never sent to Jev. A failed capture is logged and never stops the run.
|
|
63
|
+
*/
|
|
64
|
+
export async function captureScreenshot(browser, recorder, step, attempt, after, url) {
|
|
65
|
+
if (process.env.WEBHAND_SCREENSHOTS !== "1" || !browser.screenshot)
|
|
66
|
+
return;
|
|
67
|
+
try {
|
|
68
|
+
recorder.screenshot(step, attempt, after, await browser.screenshot(), url);
|
|
69
|
+
}
|
|
70
|
+
catch (error) {
|
|
71
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
72
|
+
recorder.event("screenshot_failed", { step, attempt, error: message });
|
|
73
|
+
}
|
|
74
|
+
}
|
|
75
|
+
/** Executed actions only, in every attempt: not a person's step, a NOT_VISIBLE choice, or a refused stale one. */
|
|
76
|
+
export function executed(history) {
|
|
77
|
+
return history.filter((h) => !["human", "answer", MISSING, STALE].includes(h.kind));
|
|
78
|
+
}
|
|
79
|
+
/** Executed actions of one attempt; they count against the budget. */
|
|
31
80
|
export function actionsIn(history, attempt) {
|
|
32
|
-
return history.filter((h) => (h.attempt ?? 1) === attempt
|
|
81
|
+
return executed(history).filter((h) => (h.attempt ?? 1) === attempt);
|
|
33
82
|
}
|
|
34
83
|
/** Three entries without progress: unchanged pages, targets Jev could not see, or refused stale choices. */
|
|
35
84
|
function stuck(history, attempt = 1) {
|
|
36
85
|
const recent = history.filter((h) => (h.attempt ?? 1) === attempt && h.kind !== "human").slice(-3);
|
|
37
86
|
return recent.length === 3 && recent.every((s) => s.page_changed === false && s.kind !== "wait");
|
|
38
87
|
}
|
|
88
|
+
/** Entries of this attempt after Claude's last answer: an answer starts the count again, so Jev is asked before any stop. */
|
|
89
|
+
function sinceAnswer(history, attempt) {
|
|
90
|
+
const own = history.filter((h) => (h.attempt ?? 1) === attempt && h.kind !== "human");
|
|
91
|
+
const last = own.map((h) => h.kind).lastIndexOf("answer");
|
|
92
|
+
return own.slice(last + 1);
|
|
93
|
+
}
|
|
94
|
+
/** v2: back and forth scrolling (three round trips) with nothing else done between. Each scroll changes the page, so stuck() never sees it. */
|
|
95
|
+
export function looping(history, attempt = 1) {
|
|
96
|
+
const recent = sinceAnswer(history, attempt).slice(-6);
|
|
97
|
+
const scrolls = recent.map((h) => h.operation);
|
|
98
|
+
return (recent.length === 6 &&
|
|
99
|
+
scrolls.every((op, i) => (op === "SCROLL_DOWN" || op === "SCROLL_UP") && (i === 0 || op !== scrolls[i - 1])));
|
|
100
|
+
}
|
|
101
|
+
/** How many frames, canvases, and plugin objects the page has: parts webhand cannot see into. */
|
|
102
|
+
export function unseenParts(page) {
|
|
103
|
+
const unseen = (page.status?.unseen ?? {});
|
|
104
|
+
return (unseen.frames ?? 0) + (unseen.canvases ?? 0) + (unseen.objects ?? 0);
|
|
105
|
+
}
|
|
106
|
+
/** v2: the needed element was not offered twice in a row, on a page with parts webhand cannot see into. */
|
|
107
|
+
function missingInUnseen(history, page, attempt = 1) {
|
|
108
|
+
const recent = sinceAnswer(history, attempt).slice(-2);
|
|
109
|
+
return recent.length === 2 && recent.every((h) => h.kind === MISSING) && unseenParts(page) > 0;
|
|
110
|
+
}
|
|
111
|
+
/** A step for the run log: the value's label instead of the typed text, and the URL without query values. */
|
|
112
|
+
function logged(step, region, values) {
|
|
113
|
+
const { text, url, ...rest } = step;
|
|
114
|
+
const entry = { ...rest, region: typeof region === "string" ? region : null };
|
|
115
|
+
entry.url = safeUrl(url);
|
|
116
|
+
entry.value = text === null ? null : (values.find((v) => v.value === text)?.label ?? null);
|
|
117
|
+
if (text !== null)
|
|
118
|
+
entry.text_chars = [...String(text)].length;
|
|
119
|
+
return entry;
|
|
120
|
+
}
|
|
39
121
|
/** Runs once. Logged before the next observation so a navigation cannot erase it. */
|
|
40
122
|
export async function execute(browser, page, decision, history, recorder, options) {
|
|
41
123
|
const onChange = options.onChange ?? (() => { });
|
|
@@ -54,18 +136,55 @@ export async function execute(browser, page, decision, history, recorder, option
|
|
|
54
136
|
page_changed: null,
|
|
55
137
|
};
|
|
56
138
|
history.push(step);
|
|
57
|
-
recorder.event("step", step);
|
|
139
|
+
recorder.event("step", logged(step, action.region, options.values ?? []));
|
|
58
140
|
onChange(page);
|
|
141
|
+
const asked = browser.ignoredWindows ? await browser.ignoredWindows().catch(() => []) : [];
|
|
59
142
|
const newPage = await browser.observe();
|
|
60
|
-
step.page_changed = !
|
|
143
|
+
step.page_changed = !samePage(newPage, page);
|
|
61
144
|
recorder.event("step_result", { step: step.step, page_changed: step.page_changed });
|
|
145
|
+
options.knowledge?.record({
|
|
146
|
+
from: page.url,
|
|
147
|
+
to: newPage.url,
|
|
148
|
+
kind: action.kind,
|
|
149
|
+
label: action.label,
|
|
150
|
+
changed: step.page_changed,
|
|
151
|
+
inItem: page.guards?.[String(action.node)]?.[14] === "item",
|
|
152
|
+
});
|
|
153
|
+
if (asked.length) {
|
|
154
|
+
recorder.event("window_ignored", { step: step.step, attempt: options.attempt, urls: asked.map(safeUrl) });
|
|
155
|
+
}
|
|
156
|
+
if (step.page_changed)
|
|
157
|
+
recorder.event("page", pageEvent(newPage, options.attempt));
|
|
158
|
+
await captureScreenshot(browser, recorder, step.step, options.attempt, "action", newPage.url);
|
|
62
159
|
onChange(newPage);
|
|
63
160
|
return newPage;
|
|
64
161
|
}
|
|
162
|
+
/**
|
|
163
|
+
* v2: Jev chose an element beyond the viewport. It is scrolled into view (a scroll, never an input), the page
|
|
164
|
+
* is observed again, and the same element with the same operation is taken from that observation. When it is
|
|
165
|
+
* no longer offered, nothing runs: StalePage, as for any refused action.
|
|
166
|
+
*/
|
|
167
|
+
async function intoView(browser, page, decision, recorder, attempt) {
|
|
168
|
+
const action = decision.action;
|
|
169
|
+
await browser.act({ kind: "reveal", node: action.node, label: action.label }, page);
|
|
170
|
+
const next = await browser.observe();
|
|
171
|
+
const same = (next.actions ?? []).find((a) => !a.offscreen && a.node === action.node && a.kind === action.kind && (a.value ?? null) === (action.value ?? null));
|
|
172
|
+
recorder.event("into_view", { attempt, at: action.at ?? null, found: Boolean(same) });
|
|
173
|
+
if (!same)
|
|
174
|
+
throw new StalePage("Target was not in view after scrolling to it", ["target_unreachable"]);
|
|
175
|
+
const chosen = new Decision(decision.operation, same, { confidence: decision.confidence, text: decision.text });
|
|
176
|
+
return { page: next, decision: chosen };
|
|
177
|
+
}
|
|
65
178
|
/** Markers are JSON values from the page; equal content means the same page. */
|
|
66
179
|
function sameMarker(a, b) {
|
|
67
180
|
return JSON.stringify(a) === JSON.stringify(b);
|
|
68
181
|
}
|
|
182
|
+
/** The same page in the same tab setup and frames: a new or closed tab, a switch, or a change inside a frame is a change even when the page is not. */
|
|
183
|
+
function samePage(a, b) {
|
|
184
|
+
return (sameMarker(a.marker, b.marker) &&
|
|
185
|
+
sameMarker(a.tabs ?? null, b.tabs ?? null) &&
|
|
186
|
+
sameMarker(a.frame_markers ?? null, b.frame_markers ?? null));
|
|
187
|
+
}
|
|
69
188
|
/** Runs one attempt until an outcome above. history is extended in place. */
|
|
70
189
|
export async function runAgent(browser, policy, plan, recorder, budget = new Budget(), options = {}) {
|
|
71
190
|
const attempt = plan.attempt ?? 1;
|
|
@@ -86,12 +205,18 @@ export async function runAgent(browser, policy, plan, recorder, budget = new Bud
|
|
|
86
205
|
page,
|
|
87
206
|
actions: actionsIn(history, attempt).length,
|
|
88
207
|
decisions,
|
|
89
|
-
elapsed_ms:
|
|
208
|
+
elapsed_ms: Math.round(performance.now() - started),
|
|
90
209
|
...extra,
|
|
91
210
|
});
|
|
92
211
|
while (true) {
|
|
93
212
|
if (stuck(history, attempt))
|
|
94
213
|
return finish(STUCK, "three actions without a page change");
|
|
214
|
+
if (plan.flow === "v2" && looping(history, attempt)) {
|
|
215
|
+
return finish(STUCK, "scrolled up and down three times with nothing else done");
|
|
216
|
+
}
|
|
217
|
+
if (plan.flow === "v2" && missingInUnseen(history, page, attempt)) {
|
|
218
|
+
return finish(STUCK, "the needed element was not offered twice, and the page has parts webhand cannot see");
|
|
219
|
+
}
|
|
95
220
|
if (actionsIn(history, attempt).length >= budget.maxActions || decisions >= budget.maxDecisions) {
|
|
96
221
|
return finish(ATTEMPT_BUDGET, `${actionsIn(history, attempt).length} actions, ${decisions} decisions`);
|
|
97
222
|
}
|
|
@@ -102,7 +227,11 @@ export async function runAgent(browser, policy, plan, recorder, budget = new Bud
|
|
|
102
227
|
page = await browser.observe();
|
|
103
228
|
decisions += 1;
|
|
104
229
|
onDecision(decisions);
|
|
105
|
-
|
|
230
|
+
// v2: what worked before on this kind of page, when the site is known (stage 3).
|
|
231
|
+
const known = options.knowledge?.hints(page.url) ?? [];
|
|
232
|
+
if (known.length)
|
|
233
|
+
recorder.event("known_offered", { attempt, count: known.length });
|
|
234
|
+
decision = await policy.choose(known.length ? { ...page, known } : page, plan, history);
|
|
106
235
|
flushCalls();
|
|
107
236
|
if (decision.operation === "NOT_VISIBLE") {
|
|
108
237
|
const entry = {
|
|
@@ -116,14 +245,21 @@ export async function runAgent(browser, policy, plan, recorder, budget = new Bud
|
|
|
116
245
|
page_changed: false,
|
|
117
246
|
};
|
|
118
247
|
history.push(entry);
|
|
119
|
-
recorder.event("not_visible", {
|
|
248
|
+
recorder.event("not_visible", {
|
|
249
|
+
attempt,
|
|
250
|
+
wanted: decision.wanted,
|
|
251
|
+
confidence: decision.confidence,
|
|
252
|
+
offered: offered(page),
|
|
253
|
+
...(browser.unseen ? { unseen: await browser.unseen().catch(() => null) } : {}),
|
|
254
|
+
...(page.frames_log ? { frames: page.frames_log } : {}),
|
|
255
|
+
});
|
|
120
256
|
onChange(page);
|
|
121
257
|
continue;
|
|
122
258
|
}
|
|
123
259
|
if (decision.action === null) {
|
|
124
260
|
if (!(await browser.fresh(page)))
|
|
125
261
|
continue;
|
|
126
|
-
const outcome = { DONE: GOAL_REPORTED, NEEDS_HUMAN }[decision.operation] ?? BLOCKED;
|
|
262
|
+
const outcome = { DONE: GOAL_REPORTED, NEEDS_HUMAN, ESCALATE }[decision.operation] ?? BLOCKED;
|
|
127
263
|
return finish(outcome, `Jev chose ${decision.operation}`);
|
|
128
264
|
}
|
|
129
265
|
if (decision.action.kind === "fill" && decision.text === null) {
|
|
@@ -132,10 +268,14 @@ export async function runAgent(browser, policy, plan, recorder, budget = new Bud
|
|
|
132
268
|
field[key] = decision.action[key] ?? null;
|
|
133
269
|
return finish(NEEDS_VALUE, "no value from the plan fits this field", { field });
|
|
134
270
|
}
|
|
271
|
+
if (decision.action.offscreen)
|
|
272
|
+
({ page, decision } = await intoView(browser, page, decision, recorder, attempt));
|
|
135
273
|
page = await execute(browser, page, decision, history, recorder, {
|
|
136
274
|
decidedBy: policy.name,
|
|
137
275
|
attempt,
|
|
138
276
|
onChange,
|
|
277
|
+
values: plan.values,
|
|
278
|
+
knowledge: options.knowledge,
|
|
139
279
|
});
|
|
140
280
|
}
|
|
141
281
|
catch (error) {
|
|
@@ -149,11 +289,11 @@ export async function runAgent(browser, policy, plan, recorder, budget = new Bud
|
|
|
149
289
|
attempt,
|
|
150
290
|
operation: "STALE",
|
|
151
291
|
kind: STALE,
|
|
152
|
-
action: `${decision.operation} ${
|
|
292
|
+
action: `${decision.operation} ${quote(decision.action.label)} did not run: ${error.message}`,
|
|
153
293
|
text: null,
|
|
154
294
|
url: observed.url,
|
|
155
295
|
// Unchanged: the same choice would be refused again, so it counts toward STUCK.
|
|
156
|
-
page_changed: !
|
|
296
|
+
page_changed: !samePage(page, observed),
|
|
157
297
|
};
|
|
158
298
|
history.push(entry);
|
|
159
299
|
recorder.event("stale", {
|
|
@@ -161,13 +301,14 @@ export async function runAgent(browser, policy, plan, recorder, budget = new Bud
|
|
|
161
301
|
wanted: decision.operation,
|
|
162
302
|
target: decision.action.label,
|
|
163
303
|
reason: error.message,
|
|
304
|
+
changed: error.changed,
|
|
164
305
|
page_changed: entry.page_changed,
|
|
165
306
|
});
|
|
166
307
|
}
|
|
167
308
|
onChange(page);
|
|
168
309
|
}
|
|
169
310
|
else if (error instanceof PolicyError) {
|
|
170
|
-
return finish(JEV_FAILED, error.message);
|
|
311
|
+
return finish(JEV_FAILED, error.message, error.diagnostic ? { jev: error.diagnostic } : {});
|
|
171
312
|
}
|
|
172
313
|
else if (error instanceof MutationUncertain) {
|
|
173
314
|
return finish(UNCERTAIN, `mutation uncertain: ${error.message}`);
|
package/dist/auth.js
ADDED
|
@@ -0,0 +1,62 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Saved logins: the cookies and localStorage of a session, kept so a later new session can start signed in
|
|
3
|
+
* (`start --auth NAME`). Unlike a saved profile, a login is not tied to Chrome: any engine can take it.
|
|
4
|
+
*
|
|
5
|
+
* A login file is the sign-in itself. It lives only in the user's state folder, readable by the owner alone, and
|
|
6
|
+
* its contents are never printed.
|
|
7
|
+
*/
|
|
8
|
+
import { existsSync, mkdirSync, readdirSync, readFileSync, renameSync, rmSync, writeFileSync } from "node:fs";
|
|
9
|
+
import { join } from "node:path";
|
|
10
|
+
import { quote } from "./format.js";
|
|
11
|
+
import { home, SessionError } from "./session.js";
|
|
12
|
+
const NAME = /^[\p{L}\p{N}_-]{1,40}$/u;
|
|
13
|
+
/** A login name can be a file name. */
|
|
14
|
+
export function checkName(name) {
|
|
15
|
+
if (!NAME.test(name))
|
|
16
|
+
throw new SessionError(`Expected a login name of letters, digits, - or _ (up to 40), got ${quote(name)}`);
|
|
17
|
+
return name;
|
|
18
|
+
}
|
|
19
|
+
function folder() {
|
|
20
|
+
return join(home(), "auth");
|
|
21
|
+
}
|
|
22
|
+
function path(name) {
|
|
23
|
+
return join(folder(), `${checkName(name)}.json`);
|
|
24
|
+
}
|
|
25
|
+
export function exists(name) {
|
|
26
|
+
return existsSync(path(name));
|
|
27
|
+
}
|
|
28
|
+
export function save(name, state) {
|
|
29
|
+
const target = path(name);
|
|
30
|
+
mkdirSync(folder(), { recursive: true, mode: 0o700 });
|
|
31
|
+
const temporary = `${target}.tmp`;
|
|
32
|
+
const login = { saved_at: new Date().toISOString(), state };
|
|
33
|
+
writeFileSync(temporary, JSON.stringify(login), { encoding: "utf8", mode: 0o600 });
|
|
34
|
+
renameSync(temporary, target);
|
|
35
|
+
}
|
|
36
|
+
export function load(name) {
|
|
37
|
+
return JSON.parse(readFileSync(path(name), "utf8"));
|
|
38
|
+
}
|
|
39
|
+
export function names() {
|
|
40
|
+
if (!existsSync(folder()))
|
|
41
|
+
return [];
|
|
42
|
+
return readdirSync(folder())
|
|
43
|
+
.filter((file) => file.endsWith(".json"))
|
|
44
|
+
.map((file) => file.slice(0, -".json".length))
|
|
45
|
+
.sort();
|
|
46
|
+
}
|
|
47
|
+
export function remove(name) {
|
|
48
|
+
if (!exists(name))
|
|
49
|
+
return false;
|
|
50
|
+
rmSync(path(name), { force: true });
|
|
51
|
+
return true;
|
|
52
|
+
}
|
|
53
|
+
/** What a login holds, without its values: how many cookies, and which sites. */
|
|
54
|
+
export function summary(login) {
|
|
55
|
+
const hosts = new Set();
|
|
56
|
+
for (const cookie of login.state.cookies)
|
|
57
|
+
hosts.add(cookie.domain.replace(/^\./, ""));
|
|
58
|
+
for (const origin of login.state.origins)
|
|
59
|
+
hosts.add(new URL(origin.origin).host);
|
|
60
|
+
const stored = login.state.origins.reduce((sum, origin) => sum + origin.localStorage.length, 0);
|
|
61
|
+
return `${login.state.cookies.length} cookies, ${stored} localStorage items; ${[...hosts].sort().join(", ") || "no sites"}`;
|
|
62
|
+
}
|
package/dist/bin.js
CHANGED
|
File without changes
|