webhand 0.0.0-stage → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.ja.md +89 -0
- package/README.md +88 -2
- package/THIRD_PARTY_LICENSE_jev-ultrafast.txt +21 -0
- package/assets/recorder.js +113 -0
- package/assets/snapshot.js +561 -0
- package/dist/agent.js +197 -0
- package/dist/bin.js +22 -0
- package/dist/browser.js +236 -0
- package/dist/cli.js +1136 -0
- package/dist/format.js +44 -0
- package/dist/learn.js +78 -0
- package/dist/metrics.js +107 -0
- package/dist/policies/base.js +178 -0
- package/dist/policies/jev.js +198 -0
- package/dist/policies/scripted.js +48 -0
- package/dist/runbook.js +296 -0
- package/dist/session.js +331 -0
- package/dist/types.js +34 -0
- package/dist/workspace.js +78 -0
- package/package.json +60 -4
|
@@ -0,0 +1,561 @@
|
|
|
1
|
+
// view = {focus: [...], skip: [...]}: region patterns Claude chose from what was learned. Empty: DOM order.
|
|
2
|
+
(view => {
|
|
3
|
+
if (!document.body) return null;
|
|
4
|
+
|
|
5
|
+
// ---- Limits and vocabularies ----
|
|
6
|
+
|
|
7
|
+
const TEXT_LIMIT = 6000;
|
|
8
|
+
const MAX_ACTIONS = 250;
|
|
9
|
+
const SCROLL_STEP = 560;
|
|
10
|
+
const MAX_REGIONS = 40;
|
|
11
|
+
const REGION_NAME_LIMIT = 40;
|
|
12
|
+
|
|
13
|
+
const INTERACTIVE_ROLES = ['button', 'link', 'checkbox', 'radio', 'switch', 'tab', 'menuitem', 'menuitemradio',
|
|
14
|
+
'option', 'gridcell', 'combobox', 'textbox', 'searchbox', 'spinbutton'];
|
|
15
|
+
const INTERACTIVE_SELECTOR = 'a[href],button,input,textarea,select,summary,[contenteditable="true"],' +
|
|
16
|
+
INTERACTIVE_ROLES.map(role => '[role="' + role + '"]').join(',');
|
|
17
|
+
const INPUT_ROLES = {
|
|
18
|
+
checkbox: 'checkbox', radio: 'radio',
|
|
19
|
+
button: 'button', submit: 'button', reset: 'button', image: 'button',
|
|
20
|
+
search: 'searchbox', number: 'spinbutton',
|
|
21
|
+
text: 'textbox', email: 'textbox', url: 'textbox', tel: 'textbox',
|
|
22
|
+
};
|
|
23
|
+
const TEXT_ROLES = ['textbox', 'searchbox', 'spinbutton'];
|
|
24
|
+
const BUTTON_INPUT_TYPES = ['button', 'submit', 'reset'];
|
|
25
|
+
// Values of these inputs are never read.
|
|
26
|
+
const UNOBSERVED_INPUT_TYPES = ['password', 'file', 'hidden'];
|
|
27
|
+
const HIDDEN_SELECTOR = '[aria-hidden="true"],[inert]';
|
|
28
|
+
const FORM_FIELD_SELECTOR = 'input,textarea,select';
|
|
29
|
+
// The surrounding region whose text is part of an element's freshness guard.
|
|
30
|
+
const GUARD_SCOPE_SELECTOR = 'form,dialog,[role="dialog"],article,li,tr,[role="row"]';
|
|
31
|
+
const NON_CONTENT_SELECTOR = 'script,style,noscript,template';
|
|
32
|
+
// Regions the page is divided into. An explicit role decides the kind before the tag does.
|
|
33
|
+
const REGION_ROLES = {
|
|
34
|
+
dialog: 'dialog', alertdialog: 'dialog', banner: 'header', navigation: 'nav', main: 'main',
|
|
35
|
+
complementary: 'aside', contentinfo: 'footer', search: 'search', form: 'form', grid: 'table',
|
|
36
|
+
listbox: 'listbox', menu: 'menu', region: 'section',
|
|
37
|
+
};
|
|
38
|
+
const REGION_TAGS = {DIALOG: 'dialog', HEADER: 'header', NAV: 'nav', MAIN: 'main', ASIDE: 'aside',
|
|
39
|
+
FOOTER: 'footer', FORM: 'form', FIELDSET: 'fieldset', TABLE: 'table', SECTION: 'section'};
|
|
40
|
+
const REGION_SELECTOR = [...Object.keys(REGION_TAGS).map(tag => tag.toLowerCase()),
|
|
41
|
+
...Object.keys(REGION_ROLES).map(role => '[role="' + role + '"]')].join(',');
|
|
42
|
+
const HEADING_SELECTOR = 'h1,h2,h3,h4,h5,h6,[role="heading"]';
|
|
43
|
+
|
|
44
|
+
// ---- Node ids (stable across snapshots of the same document) ----
|
|
45
|
+
|
|
46
|
+
const state = window.__browserLab ||= {ids: new WeakMap(), nodes: new Map(), next: 1};
|
|
47
|
+
|
|
48
|
+
const nodeId = element => {
|
|
49
|
+
if (!state.ids.has(element)) state.ids.set(element, state.next++);
|
|
50
|
+
const id = state.ids.get(element);
|
|
51
|
+
state.nodes.set(id, element);
|
|
52
|
+
return id;
|
|
53
|
+
};
|
|
54
|
+
|
|
55
|
+
const forgetDisconnected = () => {
|
|
56
|
+
for (const [id, element] of state.nodes) {
|
|
57
|
+
if (!element.isConnected) state.nodes.delete(id);
|
|
58
|
+
}
|
|
59
|
+
};
|
|
60
|
+
forgetDisconnected();
|
|
61
|
+
|
|
62
|
+
// ---- Flat-tree traversal ----
|
|
63
|
+
// Open shadow roots are walked as part of the page. Closed roots are not reachable from page script.
|
|
64
|
+
|
|
65
|
+
// NodeFilter.SHOW_* bits are 1 << (nodeType - 1).
|
|
66
|
+
const matchesFilter = (node, whatToShow) => (1 << (node.nodeType - 1)) & whatToShow;
|
|
67
|
+
|
|
68
|
+
const walkDeep = function* (root, whatToShow) {
|
|
69
|
+
// Elements are always visited so that their shadow roots can be entered.
|
|
70
|
+
const walker = document.createTreeWalker(root, whatToShow | NodeFilter.SHOW_ELEMENT);
|
|
71
|
+
for (let node = walker.nextNode(); node; node = walker.nextNode()) {
|
|
72
|
+
if (node.nodeType === Node.ELEMENT_NODE && node.shadowRoot) yield* walkDeep(node.shadowRoot, whatToShow);
|
|
73
|
+
if (matchesFilter(node, whatToShow)) yield node;
|
|
74
|
+
}
|
|
75
|
+
};
|
|
76
|
+
|
|
77
|
+
const queryAllDeep = selector =>
|
|
78
|
+
[...walkDeep(document, NodeFilter.SHOW_ELEMENT)].filter(element => element.matches(selector));
|
|
79
|
+
|
|
80
|
+
// Like Element.closest, but continues from a shadow root to its host.
|
|
81
|
+
const closestDeep = (element, selector) => {
|
|
82
|
+
for (let current = element; current; current = current.getRootNode().host) {
|
|
83
|
+
const found = current.closest(selector);
|
|
84
|
+
if (found) return found;
|
|
85
|
+
}
|
|
86
|
+
return null;
|
|
87
|
+
};
|
|
88
|
+
|
|
89
|
+
// Slotted nodes belong to their slot, shadow roots to their host.
|
|
90
|
+
// A shadow root's host, never an anchor's .host, which is a URL string.
|
|
91
|
+
const flatParent = node => node.assignedSlot ||
|
|
92
|
+
(node.parentNode instanceof ShadowRoot ? node.parentNode.host : node.parentNode);
|
|
93
|
+
|
|
94
|
+
const flatChildren = element => {
|
|
95
|
+
if (element.shadowRoot) return [...element.shadowRoot.childNodes];
|
|
96
|
+
if (element.tagName === 'SLOT' && element.assignedNodes().length) return element.assignedNodes({flatten: true});
|
|
97
|
+
return [...element.childNodes];
|
|
98
|
+
};
|
|
99
|
+
|
|
100
|
+
// Like document.elementFromPoint, but descends into open shadow roots.
|
|
101
|
+
const deepElementFromPoint = (x, y) => {
|
|
102
|
+
let hit = document.elementFromPoint(x, y);
|
|
103
|
+
while (hit?.shadowRoot) {
|
|
104
|
+
const inner = hit.shadowRoot.elementFromPoint(x, y);
|
|
105
|
+
if (!inner || inner === hit) break;
|
|
106
|
+
hit = inner;
|
|
107
|
+
}
|
|
108
|
+
return hit;
|
|
109
|
+
};
|
|
110
|
+
|
|
111
|
+
// ---- Element properties ----
|
|
112
|
+
|
|
113
|
+
const isObservableInput = element => !UNOBSERVED_INPUT_TYPES.includes(element.type);
|
|
114
|
+
|
|
115
|
+
const isVisible = element => !closestDeep(element, HIDDEN_SELECTOR) &&
|
|
116
|
+
element.checkVisibility({checkOpacity: true, checkVisibilityCSS: true});
|
|
117
|
+
|
|
118
|
+
const isDisabled = element => element.matches(':disabled') || closestDeep(element, '[aria-disabled="true"]');
|
|
119
|
+
|
|
120
|
+
// A simplified accessible name computation. `seen` stops reference cycles.
|
|
121
|
+
const accessibleName = (element, seen = new Set()) => {
|
|
122
|
+
if (!element || seen.has(element)) return '';
|
|
123
|
+
seen.add(element);
|
|
124
|
+
const nameOf = node => accessibleName(node, seen);
|
|
125
|
+
|
|
126
|
+
const referenced = (element.getAttribute('aria-labelledby') || '').split(/\s+/)
|
|
127
|
+
.map(id => nameOf(element.getRootNode().getElementById?.(id))).filter(Boolean).join(' ');
|
|
128
|
+
if (referenced) return referenced;
|
|
129
|
+
|
|
130
|
+
const ariaLabel = element.getAttribute('aria-label');
|
|
131
|
+
if (ariaLabel) return ariaLabel;
|
|
132
|
+
|
|
133
|
+
const labels = [...(element.labels || [])].map(nameOf).filter(Boolean).join(' ');
|
|
134
|
+
if (labels) return labels;
|
|
135
|
+
|
|
136
|
+
if (BUTTON_INPUT_TYPES.includes(element.type) && element.value) return element.value;
|
|
137
|
+
|
|
138
|
+
const alt = element.getAttribute('alt');
|
|
139
|
+
if (alt) return alt;
|
|
140
|
+
|
|
141
|
+
if (element.tagName !== 'INPUT') {
|
|
142
|
+
const content = flatChildren(element).map(child => {
|
|
143
|
+
if (child.nodeType === Node.TEXT_NODE) return child.textContent;
|
|
144
|
+
if (child.nodeType === Node.ELEMENT_NODE && child.getAttribute('aria-hidden') !== 'true') return nameOf(child);
|
|
145
|
+
return '';
|
|
146
|
+
}).join(' ').trim();
|
|
147
|
+
if (content) return content;
|
|
148
|
+
}
|
|
149
|
+
|
|
150
|
+
return element.getAttribute('title') || element.getAttribute('placeholder') || '';
|
|
151
|
+
};
|
|
152
|
+
|
|
153
|
+
const roleOf = element => {
|
|
154
|
+
const explicit = element.getAttribute('role');
|
|
155
|
+
if (INTERACTIVE_ROLES.includes(explicit)) return explicit;
|
|
156
|
+
switch (element.tagName) {
|
|
157
|
+
case 'BUTTON':
|
|
158
|
+
case 'SUMMARY': return 'button';
|
|
159
|
+
case 'A': return 'link';
|
|
160
|
+
case 'SELECT': return 'combobox';
|
|
161
|
+
case 'TEXTAREA': return 'textbox';
|
|
162
|
+
}
|
|
163
|
+
if (element.isContentEditable) return 'textbox';
|
|
164
|
+
if (element.tagName === 'INPUT') return INPUT_ROLES[element.type] ?? null;
|
|
165
|
+
return null;
|
|
166
|
+
};
|
|
167
|
+
|
|
168
|
+
// An element with no interactive tag or role that a script makes clickable: an onclick attribute, a tabindex,
|
|
169
|
+
// or the outermost element of a pointer cursor. It must have a name and must not be inside or around a
|
|
170
|
+
// control, which is acted on directly. A label is acted on through its control.
|
|
171
|
+
// Frames are not observed inside, so they are not acted on either.
|
|
172
|
+
const NOT_CLICKABLE_TAGS = ['HTML', 'BODY', 'LABEL', 'OPTION', 'OPTGROUP', 'IFRAME', 'FRAME', 'EMBED', 'OBJECT'];
|
|
173
|
+
const cursorOf = node => node instanceof Element ? getComputedStyle(node).cursor : null;
|
|
174
|
+
const isClickable = element => {
|
|
175
|
+
if (NOT_CLICKABLE_TAGS.includes(element.tagName) || element.matches(INTERACTIVE_SELECTOR)) return false;
|
|
176
|
+
const signal = element.hasAttribute('onclick') || (element.hasAttribute('tabindex') && element.tabIndex >= 0) ||
|
|
177
|
+
(cursorOf(element) === 'pointer' && cursorOf(flatParent(element)) !== 'pointer');
|
|
178
|
+
if (!signal) return false;
|
|
179
|
+
const parent = flatParent(element);
|
|
180
|
+
if (parent instanceof Element && closestDeep(parent, INTERACTIVE_SELECTOR)) return false;
|
|
181
|
+
if (element.querySelector(INTERACTIVE_SELECTOR) || element.shadowRoot?.querySelector(INTERACTIVE_SELECTOR)) {
|
|
182
|
+
return false;
|
|
183
|
+
}
|
|
184
|
+
return Boolean(accessibleName(element));
|
|
185
|
+
};
|
|
186
|
+
|
|
187
|
+
// The role an action on element is observed with, or null when it is not acted on.
|
|
188
|
+
const targetRole = element =>
|
|
189
|
+
element.matches(INTERACTIVE_SELECTOR) ? roleOf(element) : isClickable(element) ? 'clickable' : null;
|
|
190
|
+
|
|
191
|
+
const isEditable = (element, role) => !element.readOnly && element.getAttribute('aria-readonly') !== 'true' &&
|
|
192
|
+
(TEXT_ROLES.includes(role) || (role === 'combobox' && ['INPUT', 'TEXTAREA'].includes(element.tagName)));
|
|
193
|
+
|
|
194
|
+
// ---- Freshness keys ----
|
|
195
|
+
|
|
196
|
+
const viewportKey = () => [performance.timeOrigin, location.href, scrollX, scrollY, innerWidth, innerHeight];
|
|
197
|
+
|
|
198
|
+
const formFieldsKey = () => queryAllDeep(FORM_FIELD_SELECTOR).filter(isObservableInput)
|
|
199
|
+
.map(field => [nodeId(field), field.value, field.checked, field.selectedIndex, field.disabled, field.readOnly]);
|
|
200
|
+
|
|
201
|
+
// ---- API used by browser.py and recorder.js ----
|
|
202
|
+
|
|
203
|
+
state.closest = closestDeep;
|
|
204
|
+
|
|
205
|
+
// Whether a click at (x, y) would land on element or one of its flat-tree descendants.
|
|
206
|
+
state.hit = (element, x, y) => {
|
|
207
|
+
for (let node = deepElementFromPoint(x, y); node; node = flatParent(node)) {
|
|
208
|
+
if (node === element) return true;
|
|
209
|
+
}
|
|
210
|
+
return false;
|
|
211
|
+
};
|
|
212
|
+
|
|
213
|
+
state.describe = element => {
|
|
214
|
+
let target = closestDeep(element, INTERACTIVE_SELECTOR);
|
|
215
|
+
for (let current = element; !target && current instanceof Element; current = flatParent(current)) {
|
|
216
|
+
if (isClickable(current)) target = current;
|
|
217
|
+
}
|
|
218
|
+
if (!target) return null;
|
|
219
|
+
const role = targetRole(target);
|
|
220
|
+
return {role, label: accessibleName(target) || role || '', element: target};
|
|
221
|
+
};
|
|
222
|
+
|
|
223
|
+
state.pageKey = () => [...viewportKey(), formFieldsKey()];
|
|
224
|
+
|
|
225
|
+
// What must still hold for an action on element to mean what it meant when observed.
|
|
226
|
+
state.guard = element => {
|
|
227
|
+
if (!element?.isConnected || !isVisible(element)) return null;
|
|
228
|
+
const scope = closestDeep(element, GUARD_SCOPE_SELECTOR) || element.parentElement || element.getRootNode().host;
|
|
229
|
+
return [
|
|
230
|
+
nodeId(element), targetRole(element), accessibleName(element),
|
|
231
|
+
element.value ?? null, element.checked ?? null, element.selectedIndex ?? null, element.readOnly ?? null,
|
|
232
|
+
element.matches(':disabled'),
|
|
233
|
+
element.getAttribute('aria-disabled'), element.getAttribute('aria-expanded'),
|
|
234
|
+
element.getAttribute('aria-checked'), element.getAttribute('aria-selected'),
|
|
235
|
+
element.getAttribute('href'),
|
|
236
|
+
scope?.innerText?.slice(0, TEXT_LIMIT) || '',
|
|
237
|
+
];
|
|
238
|
+
};
|
|
239
|
+
|
|
240
|
+
// ---- Actions ----
|
|
241
|
+
|
|
242
|
+
const centerOf = rect => ({x: rect.x + rect.width / 2, y: rect.y + rect.height / 2});
|
|
243
|
+
|
|
244
|
+
const centerInViewport = rect => {
|
|
245
|
+
const {x, y} = centerOf(rect);
|
|
246
|
+
return x >= 0 && y >= 0 && x < innerWidth && y < innerHeight;
|
|
247
|
+
};
|
|
248
|
+
|
|
249
|
+
// The element has a role and a nonempty box. Its center must also be inside the viewport to be acted on.
|
|
250
|
+
const isTarget = (element, role, rect) => {
|
|
251
|
+
if (!role || rect.width <= 0 || rect.height <= 0) return false;
|
|
252
|
+
// A grid cell that holds a button is acted on through the button.
|
|
253
|
+
return !(role === 'gridcell' && element.querySelector('button,[role="button"]'));
|
|
254
|
+
};
|
|
255
|
+
|
|
256
|
+
const baseAction = (element, role, rect) => {
|
|
257
|
+
const base = {node: nodeId(element), role, label: accessibleName(element) || role,
|
|
258
|
+
rect: {x: rect.x, y: rect.y, w: rect.width, h: rect.height}};
|
|
259
|
+
for (const key of ['checked', 'selected', 'expanded']) {
|
|
260
|
+
const value = element.getAttribute('aria-' + key);
|
|
261
|
+
if (value !== null) base[key] = value;
|
|
262
|
+
}
|
|
263
|
+
if (['checkbox', 'radio'].includes(element.type)) base.checked = String(element.checked);
|
|
264
|
+
return base;
|
|
265
|
+
};
|
|
266
|
+
|
|
267
|
+
// One action per option that could be chosen.
|
|
268
|
+
const selectActions = (select, base) => {
|
|
269
|
+
const currentValue = [...select.selectedOptions].map(option => option.label).join(', ');
|
|
270
|
+
return [...select.options]
|
|
271
|
+
.filter(option => !option.selected && !option.disabled && !option.closest('optgroup[disabled]'))
|
|
272
|
+
.map(option => ({...base, kind: 'select', value: option.value,
|
|
273
|
+
current_value: currentValue, label: base.label + ' → ' + option.label}));
|
|
274
|
+
};
|
|
275
|
+
|
|
276
|
+
// A fill for editable fields (plus a click to open them), otherwise a click.
|
|
277
|
+
const otherActions = (element, role, base) => {
|
|
278
|
+
const editable = isEditable(element, role);
|
|
279
|
+
let value = '';
|
|
280
|
+
if ('value' in element) value = String(element.value);
|
|
281
|
+
else if (element.isContentEditable || role === 'combobox') value = element.innerText.trim();
|
|
282
|
+
const actions = [{...base, kind: editable ? 'fill' : 'click', value}];
|
|
283
|
+
if (editable) actions.push({...base, kind: 'click', value, label: 'Open ' + base.label});
|
|
284
|
+
return actions;
|
|
285
|
+
};
|
|
286
|
+
|
|
287
|
+
// In DOM order. Each action's region is kept aside so the freshness marker does not depend on it.
|
|
288
|
+
const collectActions = () => {
|
|
289
|
+
const actions = [];
|
|
290
|
+
for (const element of walkDeep(document, NodeFilter.SHOW_ELEMENT)) {
|
|
291
|
+
const role = targetRole(element);
|
|
292
|
+
if (!role || !isObservableInput(element) || !isVisible(element) || isDisabled(element)) continue;
|
|
293
|
+
const rect = element.getBoundingClientRect();
|
|
294
|
+
if (!isTarget(element, role, rect)) continue;
|
|
295
|
+
const region = regionOf(element);
|
|
296
|
+
if (!centerInViewport(rect)) {
|
|
297
|
+
region.offscreen_actions++;
|
|
298
|
+
continue;
|
|
299
|
+
}
|
|
300
|
+
const base = baseAction(element, role, rect);
|
|
301
|
+
const found = element.tagName === 'SELECT' ? selectActions(element, base) : otherActions(element, role, base);
|
|
302
|
+
for (const action of found) actionRegions.set(action, region);
|
|
303
|
+
region.actions += found.length;
|
|
304
|
+
actions.push(...found);
|
|
305
|
+
}
|
|
306
|
+
return actions;
|
|
307
|
+
};
|
|
308
|
+
|
|
309
|
+
// ---- Regions ----
|
|
310
|
+
// A key names a region by the regions around it: main/form:予約/fieldset:追加プラン. A segment is
|
|
311
|
+
// kind:name, or kind#n among unnamed siblings of one kind, or the kind alone when it has no such sibling.
|
|
312
|
+
|
|
313
|
+
const regionKind = element => {
|
|
314
|
+
const role = element.getAttribute('role');
|
|
315
|
+
if (role) return REGION_ROLES[role] ?? null;
|
|
316
|
+
const kind = REGION_TAGS[element.tagName] ?? null;
|
|
317
|
+
if (kind === 'section' && !element.hasAttribute('aria-label') && !element.hasAttribute('aria-labelledby')) return null;
|
|
318
|
+
return kind;
|
|
319
|
+
};
|
|
320
|
+
|
|
321
|
+
const regionElements = new Map();
|
|
322
|
+
for (const element of queryAllDeep(REGION_SELECTOR)) {
|
|
323
|
+
const kind = regionKind(element);
|
|
324
|
+
if (kind) regionElements.set(element, {kind});
|
|
325
|
+
}
|
|
326
|
+
|
|
327
|
+
const regionAncestor = node => {
|
|
328
|
+
for (let current = node; current; current = flatParent(current)) {
|
|
329
|
+
if (regionElements.has(current)) return current;
|
|
330
|
+
}
|
|
331
|
+
return null;
|
|
332
|
+
};
|
|
333
|
+
|
|
334
|
+
// Names come from labels and headings, never from field values. '/' separates segments, so it is dropped.
|
|
335
|
+
const cleanName = text => (text || '').replace(/[\s/]+/g, ' ').trim().slice(0, REGION_NAME_LIMIT).trim();
|
|
336
|
+
|
|
337
|
+
const regionName = element => {
|
|
338
|
+
const referenced = (element.getAttribute('aria-labelledby') || '').split(/\s+/)
|
|
339
|
+
.map(id => accessibleName(element.getRootNode().getElementById?.(id))).filter(Boolean).join(' ');
|
|
340
|
+
if (referenced) return cleanName(referenced);
|
|
341
|
+
const label = element.getAttribute('aria-label');
|
|
342
|
+
if (label) return cleanName(label);
|
|
343
|
+
const caption = element.tagName === 'FIELDSET' ? element.querySelector(':scope > legend') :
|
|
344
|
+
element.tagName === 'TABLE' ? element.caption : null;
|
|
345
|
+
if (caption?.textContent.trim()) return cleanName(caption.textContent);
|
|
346
|
+
// A heading names the region only when it opens it, not when it titles a part further down.
|
|
347
|
+
const heading = element.querySelector(HEADING_SELECTOR);
|
|
348
|
+
if (heading && regionAncestor(heading) === element && heading.contains(firstText(element))) {
|
|
349
|
+
return cleanName(heading.textContent);
|
|
350
|
+
}
|
|
351
|
+
return '';
|
|
352
|
+
};
|
|
353
|
+
|
|
354
|
+
const firstText = element => {
|
|
355
|
+
for (const node of walkDeep(element, NodeFilter.SHOW_TEXT)) {
|
|
356
|
+
if (node.textContent.trim() && !node.parentElement?.closest(NON_CONTENT_SELECTOR)) return node;
|
|
357
|
+
}
|
|
358
|
+
return null;
|
|
359
|
+
};
|
|
360
|
+
|
|
361
|
+
const newRegion = (key, kind, element) => ({key, kind, element, text_chars: 0, shown_chars: 0,
|
|
362
|
+
actions: 0, shown_actions: 0, offscreen_actions: 0});
|
|
363
|
+
|
|
364
|
+
for (const [element, info] of regionElements) {
|
|
365
|
+
info.name = regionName(element);
|
|
366
|
+
info.parent = regionElements.get(regionAncestor(flatParent(element))) ?? null;
|
|
367
|
+
}
|
|
368
|
+
const unnamed = new Map();
|
|
369
|
+
for (const info of regionElements.values()) {
|
|
370
|
+
if (info.name) continue;
|
|
371
|
+
const siblings = unnamed.get(info.parent) ?? {};
|
|
372
|
+
siblings[info.kind] = [...(siblings[info.kind] ?? []), info];
|
|
373
|
+
unnamed.set(info.parent, siblings);
|
|
374
|
+
}
|
|
375
|
+
for (const siblings of unnamed.values()) {
|
|
376
|
+
for (const group of Object.values(siblings)) {
|
|
377
|
+
if (group.length > 1) group.forEach((info, index) => info.ordinal = index + 1);
|
|
378
|
+
}
|
|
379
|
+
}
|
|
380
|
+
const keyOf = info => {
|
|
381
|
+
if (info.key) return info.key;
|
|
382
|
+
const segment = info.name ? info.kind + ':' + info.name : info.kind + (info.ordinal ? '#' + info.ordinal : '');
|
|
383
|
+
return info.key = (info.parent ? keyOf(info.parent) + '/' : '') + segment;
|
|
384
|
+
};
|
|
385
|
+
const regions = [];
|
|
386
|
+
const pageRegion = newRegion('page', 'page', null);
|
|
387
|
+
const regionByElement = new Map();
|
|
388
|
+
for (const [element, info] of regionElements) {
|
|
389
|
+
const region = newRegion(keyOf(info), info.kind, element);
|
|
390
|
+
regions.push(region);
|
|
391
|
+
regionByElement.set(element, region);
|
|
392
|
+
}
|
|
393
|
+
const regionOf = node => regionByElement.get(regionAncestor(node)) ?? pageRegion;
|
|
394
|
+
const actionRegions = new WeakMap();
|
|
395
|
+
|
|
396
|
+
// A pattern matches a run of segments anywhere in a key, so it covers the region and everything in it.
|
|
397
|
+
// A pattern segment with a name or number must be equal; a bare kind matches any segment of that kind.
|
|
398
|
+
const segmentMatches = (pattern, segment) =>
|
|
399
|
+
/[:#]/.test(pattern) ? pattern === segment : segment.split(/[:#]/)[0] === pattern;
|
|
400
|
+
|
|
401
|
+
const matchDepth = (pattern, segments) => {
|
|
402
|
+
const parts = String(pattern).split('/');
|
|
403
|
+
let depth = -1;
|
|
404
|
+
for (let start = 0; start + parts.length <= segments.length; start++) {
|
|
405
|
+
if (parts.every((part, i) => segmentMatches(part, segments[start + i]))) depth = start + parts.length - 1;
|
|
406
|
+
}
|
|
407
|
+
return depth;
|
|
408
|
+
};
|
|
409
|
+
|
|
410
|
+
const focusPatterns = Array.isArray(view?.focus) ? view.focus : [];
|
|
411
|
+
const skipPatterns = Array.isArray(view?.skip) ? view.skip : [];
|
|
412
|
+
const viewing = focusPatterns.length > 0 || skipPatterns.length > 0;
|
|
413
|
+
const NORMAL_ORDER = 1e6;
|
|
414
|
+
|
|
415
|
+
// The deepest match wins, focus on a tie. Open dialogs come first unless skipped.
|
|
416
|
+
const decide = region => {
|
|
417
|
+
if (region.decision) return region.decision;
|
|
418
|
+
const segments = region.key.split('/');
|
|
419
|
+
let focus = {depth: -1, rank: 0};
|
|
420
|
+
focusPatterns.forEach((pattern, rank) => {
|
|
421
|
+
const depth = matchDepth(pattern, segments);
|
|
422
|
+
if (depth > focus.depth) focus = {depth, rank};
|
|
423
|
+
});
|
|
424
|
+
const skip = Math.max(-1, ...skipPatterns.map(pattern => matchDepth(pattern, segments)));
|
|
425
|
+
const inDialog = segments.some(segment => segmentMatches('dialog', segment));
|
|
426
|
+
let decision;
|
|
427
|
+
if (skip > focus.depth) decision = {view: 'skip', order: Infinity};
|
|
428
|
+
else if (inDialog) decision = {view: focus.depth >= 0 ? 'focus' : null, order: 0, whole: focus.depth >= 0};
|
|
429
|
+
else if (focus.depth >= 0) decision = {view: 'focus', order: 1 + focus.rank, whole: true};
|
|
430
|
+
else decision = {view: null, order: NORMAL_ORDER};
|
|
431
|
+
return region.decision = decision;
|
|
432
|
+
};
|
|
433
|
+
|
|
434
|
+
const positionOf = region => {
|
|
435
|
+
if (!region.element) return 'visible';
|
|
436
|
+
const rect = region.element.getBoundingClientRect();
|
|
437
|
+
if (rect.bottom <= 0) return 'above';
|
|
438
|
+
if (rect.top >= innerHeight) return 'below';
|
|
439
|
+
return 'visible';
|
|
440
|
+
};
|
|
441
|
+
|
|
442
|
+
// ---- Text ----
|
|
443
|
+
|
|
444
|
+
const intersectsViewport = rect => rect.width > 0 && rect.height > 0 &&
|
|
445
|
+
rect.bottom > 0 && rect.top < innerHeight && rect.right > 0 && rect.left < innerWidth;
|
|
446
|
+
|
|
447
|
+
// Every visible text node on the page, with its region and whether it is in the viewport.
|
|
448
|
+
const textLines = () => {
|
|
449
|
+
const lines = [];
|
|
450
|
+
const range = document.createRange();
|
|
451
|
+
const visibleParents = new Map();
|
|
452
|
+
for (const node of walkDeep(document.body, NodeFilter.SHOW_TEXT)) {
|
|
453
|
+
const value = node.textContent.trim();
|
|
454
|
+
const parent = node.parentElement;
|
|
455
|
+
if (!value || !parent) continue;
|
|
456
|
+
if (!visibleParents.has(parent)) {
|
|
457
|
+
visibleParents.set(parent, !parent.closest(NON_CONTENT_SELECTOR) && isVisible(parent));
|
|
458
|
+
}
|
|
459
|
+
if (!visibleParents.get(parent)) continue;
|
|
460
|
+
range.selectNodeContents(node);
|
|
461
|
+
const region = regionOf(parent);
|
|
462
|
+
region.text_chars += value.length;
|
|
463
|
+
lines.push({value, region, inView: intersectsViewport(range.getBoundingClientRect())});
|
|
464
|
+
}
|
|
465
|
+
return lines;
|
|
466
|
+
};
|
|
467
|
+
|
|
468
|
+
// Lines are taken until the limit is reached, then joined and cut. Returns the text and each line's share.
|
|
469
|
+
const fitText = lines => {
|
|
470
|
+
const taken = [];
|
|
471
|
+
let length = 0;
|
|
472
|
+
for (const line of lines) {
|
|
473
|
+
if (length >= TEXT_LIMIT) break;
|
|
474
|
+
taken.push(line);
|
|
475
|
+
length += line.value.length;
|
|
476
|
+
}
|
|
477
|
+
let position = 0;
|
|
478
|
+
for (const line of taken) {
|
|
479
|
+
line.shown = Math.max(0, Math.min(line.value.length, TEXT_LIMIT - position));
|
|
480
|
+
position += line.value.length + 1;
|
|
481
|
+
}
|
|
482
|
+
return {text: taken.map(line => line.value).join('\n').slice(0, TEXT_LIMIT), taken};
|
|
483
|
+
};
|
|
484
|
+
|
|
485
|
+
// Without a view: the viewport in DOM order, exactly as before. With one: grouped by region in view order,
|
|
486
|
+
// focused regions from the whole page, the rest from the viewport, skipped regions left out.
|
|
487
|
+
const viewText = lines => {
|
|
488
|
+
const groups = new Map();
|
|
489
|
+
for (const line of lines) {
|
|
490
|
+
const decision = decide(line.region);
|
|
491
|
+
if (decision.view === 'skip' || (!decision.whole && !line.inView)) continue;
|
|
492
|
+
if (!groups.has(line.region)) groups.set(line.region, []);
|
|
493
|
+
groups.get(line.region).push(line);
|
|
494
|
+
}
|
|
495
|
+
const ordered = [...groups].sort(([a], [b]) => decide(a).order - decide(b).order);
|
|
496
|
+
return fitText(ordered.flatMap(([region, regionLines]) => [{value: '## ' + region.key}, ...regionLines]));
|
|
497
|
+
};
|
|
498
|
+
|
|
499
|
+
// ---- Snapshot ----
|
|
500
|
+
|
|
501
|
+
const actions = collectActions();
|
|
502
|
+
const lines = textLines();
|
|
503
|
+
// The freshness marker always uses the viewport text in DOM order, whatever the view.
|
|
504
|
+
const base = fitText(lines.filter(line => line.inView));
|
|
505
|
+
const text = base.text;
|
|
506
|
+
const shown = viewing ? viewText(lines) : base;
|
|
507
|
+
for (const line of shown.taken) if (line.region) line.region.shown_chars += line.shown;
|
|
508
|
+
const height = document.documentElement.scrollHeight;
|
|
509
|
+
const formFields = formFieldsKey();
|
|
510
|
+
const page_key = [...viewportKey(), formFields];
|
|
511
|
+
|
|
512
|
+
const guards = {};
|
|
513
|
+
for (const action of actions) {
|
|
514
|
+
if (!(action.node in guards)) guards[action.node] = state.guard(state.nodes.get(action.node));
|
|
515
|
+
}
|
|
516
|
+
|
|
517
|
+
// Compare meaning and identity. Geometry is always resolved and hit-tested just before input.
|
|
518
|
+
const semantics = actions.map(({rect, ...action}) => action);
|
|
519
|
+
const marker = [...viewportKey(), document.title, text, semantics, formFields];
|
|
520
|
+
|
|
521
|
+
let chosen = actions;
|
|
522
|
+
if (viewing) {
|
|
523
|
+
chosen = actions.filter(action => decide(actionRegions.get(action)).view !== 'skip')
|
|
524
|
+
.map((action, index) => ({action, index}))
|
|
525
|
+
.sort((a, b) => decide(actionRegions.get(a.action)).order - decide(actionRegions.get(b.action)).order ||
|
|
526
|
+
a.index - b.index)
|
|
527
|
+
.map(({action}) => action);
|
|
528
|
+
}
|
|
529
|
+
chosen = chosen.slice(0, MAX_ACTIONS);
|
|
530
|
+
const omitted_actions = actions.length - chosen.length;
|
|
531
|
+
// The region key goes on the action for the run log only; the marker above was taken without it.
|
|
532
|
+
for (const action of chosen) {
|
|
533
|
+
const region = actionRegions.get(action);
|
|
534
|
+
region.shown_actions++;
|
|
535
|
+
action.region = region.key;
|
|
536
|
+
}
|
|
537
|
+
chosen.forEach((action, index) => action.id = 'e' + (index + 1));
|
|
538
|
+
if (scrollY + innerHeight < height - 2) {
|
|
539
|
+
chosen.push({id: 'scroll_down', kind: 'scroll', label: 'Scroll down', delta: SCROLL_STEP});
|
|
540
|
+
}
|
|
541
|
+
if (scrollY > 0) chosen.push({id: 'scroll_up', kind: 'scroll', label: 'Scroll up', delta: -SCROLL_STEP});
|
|
542
|
+
chosen.push({id: 'wait', kind: 'wait', label: 'Wait for the page to update'});
|
|
543
|
+
|
|
544
|
+
// The page's outline: every region with content, how much of it is shown, and where it is.
|
|
545
|
+
const outline = [pageRegion, ...regions]
|
|
546
|
+
.filter(region => region.text_chars || region.actions || region.offscreen_actions)
|
|
547
|
+
.filter(region => !region.element || isVisible(region.element));
|
|
548
|
+
const regionSummary = outline.slice(0, MAX_REGIONS).map(region => {
|
|
549
|
+
const summary = {key: region.key, position: positionOf(region), text_chars: region.text_chars,
|
|
550
|
+
shown_chars: region.shown_chars, actions: region.actions, shown_actions: region.shown_actions,
|
|
551
|
+
offscreen_actions: region.offscreen_actions};
|
|
552
|
+
if (viewing && decide(region).view) summary.view = decide(region).view;
|
|
553
|
+
return summary;
|
|
554
|
+
});
|
|
555
|
+
|
|
556
|
+
state.region = element => regionOf(element).key;
|
|
557
|
+
|
|
558
|
+
return {url: location.href, title: document.title, w: innerWidth, h: innerHeight, text: shown.text,
|
|
559
|
+
scroll: {y: scrollY, height}, actions: chosen, marker, page_key, guards, omitted_actions,
|
|
560
|
+
regions: regionSummary, omitted_regions: outline.length - regionSummary.length};
|
|
561
|
+
})
|