visor-ai 0.2.7 → 0.2.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/adapters.js +224 -22
- package/dist/appMap.js +2486 -0
- package/dist/cli.js +424 -45
- package/dist/daemon.js +87 -11
- package/dist/localRuntime.js +124 -0
- package/dist/runner.js +25 -2
- package/package.json +4 -1
package/dist/appMap.js
ADDED
|
@@ -0,0 +1,2486 @@
|
|
|
1
|
+
import { createHash, randomUUID } from 'node:crypto';
|
|
2
|
+
import fs from 'node:fs';
|
|
3
|
+
import os from 'node:os';
|
|
4
|
+
import path from 'node:path';
|
|
5
|
+
import { resolveTapMode } from './adapters.js';
|
|
6
|
+
import { canonicalJson, ensureDir, signatureFor, utcNowIso } from './utils.js';
|
|
7
|
+
export const APP_MAP_SCHEMA_VERSION = 1;
|
|
8
|
+
const RISKY_WORDS = [
|
|
9
|
+
'delete',
|
|
10
|
+
'remove',
|
|
11
|
+
'logout',
|
|
12
|
+
'log out',
|
|
13
|
+
'sign out',
|
|
14
|
+
'submit',
|
|
15
|
+
'send',
|
|
16
|
+
'pay',
|
|
17
|
+
'purchase',
|
|
18
|
+
'buy',
|
|
19
|
+
'confirm',
|
|
20
|
+
'destroy',
|
|
21
|
+
'archive'
|
|
22
|
+
];
|
|
23
|
+
const INPUT_WORDS = ['password', 'email', 'username', 'search', 'input', 'text field', 'textfield'];
|
|
24
|
+
const AUTH_WORDS = ['login', 'log in', 'sign in', 'password', 'username', 'auth', 'authentication'];
|
|
25
|
+
const DEFAULT_ACTION_SETTLE_MS = 400;
|
|
26
|
+
const DEFAULT_ACTION_SETTLE_POLL_MS = 400;
|
|
27
|
+
const DEFAULT_CRAWL_SETTLE_MS = 1500;
|
|
28
|
+
const DEFAULT_CRAWL_SETTLE_POLL_MS = 300;
|
|
29
|
+
function envNoMap() {
|
|
30
|
+
const raw = process.env.VISOR_NO_MAP ?? process.env.VISOR_DISABLE_MAP;
|
|
31
|
+
return ['1', 'true', 'yes', 'on'].includes(String(raw ?? '').trim().toLowerCase());
|
|
32
|
+
}
|
|
33
|
+
function resolveOptions(platform, options) {
|
|
34
|
+
const enabled = options?.enabled !== false && !envNoMap();
|
|
35
|
+
return {
|
|
36
|
+
enabled,
|
|
37
|
+
rootDir: options?.rootDir ?? process.env.VISOR_APP_MAP_DIR ?? path.join(process.cwd(), '.visor', 'maps'),
|
|
38
|
+
appId: options?.appId ?? `default-${platform}`,
|
|
39
|
+
repairDepth: options?.repairDepth ?? 2,
|
|
40
|
+
repairTimeoutMs: options?.repairTimeoutMs ?? 30000,
|
|
41
|
+
repair: options?.repair === true,
|
|
42
|
+
crawl: options?.crawl === true,
|
|
43
|
+
crawlDepth: options?.crawlDepth ?? 2,
|
|
44
|
+
crawlLimit: options?.crawlLimit ?? 24,
|
|
45
|
+
crawlSettleMs: nonNegativeNumber(options?.crawlSettleMs, DEFAULT_CRAWL_SETTLE_MS),
|
|
46
|
+
crawlSettlePollMs: positiveNumber(options?.crawlSettlePollMs, DEFAULT_CRAWL_SETTLE_POLL_MS)
|
|
47
|
+
};
|
|
48
|
+
}
|
|
49
|
+
function nonNegativeNumber(value, defaultValue) {
|
|
50
|
+
return typeof value === 'number' && Number.isFinite(value) && value >= 0 ? value : defaultValue;
|
|
51
|
+
}
|
|
52
|
+
function positiveNumber(value, defaultValue) {
|
|
53
|
+
return typeof value === 'number' && Number.isFinite(value) && value > 0 ? value : defaultValue;
|
|
54
|
+
}
|
|
55
|
+
function mapIdentity(platform, appId) {
|
|
56
|
+
return `${platform}:${appId}`;
|
|
57
|
+
}
|
|
58
|
+
function mapFilePath(rootDir, identity) {
|
|
59
|
+
const digest = createHash('sha256').update(identity, 'utf8').digest('hex').slice(0, 16);
|
|
60
|
+
return path.join(ensureDir(rootDir), `${digest}.json`);
|
|
61
|
+
}
|
|
62
|
+
function newMap(platform, appId) {
|
|
63
|
+
const now = utcNowIso();
|
|
64
|
+
return {
|
|
65
|
+
schema_version: APP_MAP_SCHEMA_VERSION,
|
|
66
|
+
identity: mapIdentity(platform, appId),
|
|
67
|
+
app_id: appId,
|
|
68
|
+
platform,
|
|
69
|
+
created_at: now,
|
|
70
|
+
updated_at: now,
|
|
71
|
+
screens: [],
|
|
72
|
+
variants: [],
|
|
73
|
+
edges: []
|
|
74
|
+
};
|
|
75
|
+
}
|
|
76
|
+
function readMap(filePath, platform, appId) {
|
|
77
|
+
if (!fs.existsSync(filePath)) {
|
|
78
|
+
return { map: newMap(platform, appId), scrubbed: false };
|
|
79
|
+
}
|
|
80
|
+
try {
|
|
81
|
+
const parsed = JSON.parse(fs.readFileSync(filePath, 'utf8'));
|
|
82
|
+
if (parsed.schema_version !== APP_MAP_SCHEMA_VERSION || parsed.identity !== mapIdentity(platform, appId)) {
|
|
83
|
+
return { map: newMap(platform, appId), scrubbed: false };
|
|
84
|
+
}
|
|
85
|
+
const originalEdgeCount = parsed.edges.length;
|
|
86
|
+
let enrichedActions = false;
|
|
87
|
+
for (const variant of parsed.variants) {
|
|
88
|
+
if (!Array.isArray(variant.actions)) {
|
|
89
|
+
variant.actions = actionAffordancesForElements(variant.elements);
|
|
90
|
+
enrichedActions = true;
|
|
91
|
+
}
|
|
92
|
+
}
|
|
93
|
+
parsed.edges = parsed.edges.filter((edge) => !isValueBearingAction(edge.command, edge.args));
|
|
94
|
+
return { map: parsed, scrubbed: parsed.edges.length !== originalEdgeCount || enrichedActions };
|
|
95
|
+
}
|
|
96
|
+
catch {
|
|
97
|
+
return { map: newMap(platform, appId), scrubbed: false };
|
|
98
|
+
}
|
|
99
|
+
}
|
|
100
|
+
function writeMap(filePath, map) {
|
|
101
|
+
ensureDir(path.dirname(filePath));
|
|
102
|
+
const tempPath = `${filePath}.${process.pid}.${randomUUID()}.tmp`;
|
|
103
|
+
fs.writeFileSync(tempPath, `${JSON.stringify(map, null, 2)}\n`, 'utf8');
|
|
104
|
+
fs.renameSync(tempPath, filePath);
|
|
105
|
+
}
|
|
106
|
+
export function createMapSummary(adapter, options) {
|
|
107
|
+
const platform = adapter.capability().platform;
|
|
108
|
+
const resolved = resolveOptions(platform, options);
|
|
109
|
+
const identity = mapIdentity(platform, resolved.appId);
|
|
110
|
+
return {
|
|
111
|
+
enabled: resolved.enabled,
|
|
112
|
+
used: false,
|
|
113
|
+
updated: false,
|
|
114
|
+
repaired: false,
|
|
115
|
+
repairs: 0,
|
|
116
|
+
schema_version: APP_MAP_SCHEMA_VERSION,
|
|
117
|
+
path: resolved.enabled ? mapFilePath(resolved.rootDir, identity) : undefined,
|
|
118
|
+
identity
|
|
119
|
+
};
|
|
120
|
+
}
|
|
121
|
+
export function createAppMapContext(adapter, options) {
|
|
122
|
+
const platform = adapter.capability().platform;
|
|
123
|
+
const resolved = resolveOptions(platform, options);
|
|
124
|
+
if (!resolved.enabled) {
|
|
125
|
+
return null;
|
|
126
|
+
}
|
|
127
|
+
const identity = mapIdentity(platform, resolved.appId);
|
|
128
|
+
const filePath = mapFilePath(resolved.rootDir, identity);
|
|
129
|
+
const summary = createMapSummary(adapter, options);
|
|
130
|
+
const loaded = readMap(filePath, platform, resolved.appId);
|
|
131
|
+
return {
|
|
132
|
+
adapter,
|
|
133
|
+
map: loaded.map,
|
|
134
|
+
filePath,
|
|
135
|
+
options: resolved,
|
|
136
|
+
summary,
|
|
137
|
+
changed: loaded.scrubbed
|
|
138
|
+
};
|
|
139
|
+
}
|
|
140
|
+
export function persistAppMapContext(context) {
|
|
141
|
+
if (!context) {
|
|
142
|
+
return undefined;
|
|
143
|
+
}
|
|
144
|
+
context.summary.updated = context.summary.updated || context.changed;
|
|
145
|
+
if (context.changed) {
|
|
146
|
+
context.map.updated_at = utcNowIso();
|
|
147
|
+
writeMap(context.filePath, context.map);
|
|
148
|
+
context.changed = false;
|
|
149
|
+
}
|
|
150
|
+
return context.summary;
|
|
151
|
+
}
|
|
152
|
+
function attributeMap(input) {
|
|
153
|
+
const attrs = {};
|
|
154
|
+
const pattern = /([A-Za-z_:][-A-Za-z0-9_:.]*)\s*=\s*("([^"]*)"|'([^']*)')/g;
|
|
155
|
+
for (const match of input.matchAll(pattern)) {
|
|
156
|
+
attrs[match[1]] = decodeXmlEntities(match[3] ?? match[4] ?? '');
|
|
157
|
+
}
|
|
158
|
+
return attrs;
|
|
159
|
+
}
|
|
160
|
+
function decodeXmlEntities(value) {
|
|
161
|
+
return value
|
|
162
|
+
.replace(/&#x([0-9a-f]+);/gi, (_, hex) => String.fromCodePoint(Number.parseInt(hex, 16)))
|
|
163
|
+
.replace(/&#(\d+);/g, (_, decimal) => String.fromCodePoint(Number.parseInt(decimal, 10)))
|
|
164
|
+
.replace(/"/g, '"')
|
|
165
|
+
.replace(/'/g, "'")
|
|
166
|
+
.replace(/</g, '<')
|
|
167
|
+
.replace(/>/g, '>')
|
|
168
|
+
.replace(/&/g, '&');
|
|
169
|
+
}
|
|
170
|
+
function normalizeLabel(value) {
|
|
171
|
+
return value
|
|
172
|
+
.replace(/[0-9a-f]{8,}/gi, '<id>')
|
|
173
|
+
.replace(/\b\d{1,2}:\d{2}(?::\d{2})?\b/g, '<time>')
|
|
174
|
+
.replace(/\b\d+\b/g, '<number>')
|
|
175
|
+
.replace(/\s+/g, ' ')
|
|
176
|
+
.trim()
|
|
177
|
+
.toLowerCase();
|
|
178
|
+
}
|
|
179
|
+
function unique(values) {
|
|
180
|
+
return Array.from(new Set(values.filter((value) => value.trim() !== '')));
|
|
181
|
+
}
|
|
182
|
+
function isScrollableTag(tag) {
|
|
183
|
+
const normalized = tag.toLowerCase();
|
|
184
|
+
return (normalized.includes('scrollview') ||
|
|
185
|
+
normalized.includes('scroll') ||
|
|
186
|
+
normalized.includes('list') ||
|
|
187
|
+
normalized.includes('table'));
|
|
188
|
+
}
|
|
189
|
+
function attrBool(attrs, key, defaultValue) {
|
|
190
|
+
const raw = attrs[key];
|
|
191
|
+
if (raw === undefined) {
|
|
192
|
+
return defaultValue;
|
|
193
|
+
}
|
|
194
|
+
return raw.trim().toLowerCase() === 'true';
|
|
195
|
+
}
|
|
196
|
+
function attrNumber(attrs, key) {
|
|
197
|
+
const parsed = Number(attrs[key]);
|
|
198
|
+
return Number.isFinite(parsed) ? parsed : null;
|
|
199
|
+
}
|
|
200
|
+
function expandSemanticLabels(labels) {
|
|
201
|
+
return labels.flatMap((label) => {
|
|
202
|
+
const lines = unique(label.split(/\n+/).map((line) => line.trim()));
|
|
203
|
+
if (lines.length === 1 && lines[0] !== label) {
|
|
204
|
+
return [label, lines[0]];
|
|
205
|
+
}
|
|
206
|
+
if (lines.length > 1 && label.split(/\n+/).filter((line) => line.trim() !== '').every((line) => line.trim() === lines[0])) {
|
|
207
|
+
return [label, lines[0]];
|
|
208
|
+
}
|
|
209
|
+
const rawLines = label.split(/\n+/).map((line) => line.trim()).filter(Boolean);
|
|
210
|
+
if (rawLines.length >= 3 && rawLines[0] === rawLines[rawLines.length - 1]) {
|
|
211
|
+
return [label, rawLines[0]];
|
|
212
|
+
}
|
|
213
|
+
return [label];
|
|
214
|
+
});
|
|
215
|
+
}
|
|
216
|
+
function isLikelyFormControl(tag, attrs) {
|
|
217
|
+
const haystack = [
|
|
218
|
+
tag,
|
|
219
|
+
attrs.type ?? '',
|
|
220
|
+
attrs.class ?? '',
|
|
221
|
+
attrs.name ?? '',
|
|
222
|
+
attrs.label ?? '',
|
|
223
|
+
attrs.placeholder ?? ''
|
|
224
|
+
]
|
|
225
|
+
.join(' ')
|
|
226
|
+
.toLowerCase();
|
|
227
|
+
return [
|
|
228
|
+
'input',
|
|
229
|
+
'textfield',
|
|
230
|
+
'text field',
|
|
231
|
+
'edittext',
|
|
232
|
+
'secure',
|
|
233
|
+
'password',
|
|
234
|
+
'email',
|
|
235
|
+
'username',
|
|
236
|
+
'search',
|
|
237
|
+
'phone',
|
|
238
|
+
'credit',
|
|
239
|
+
'card'
|
|
240
|
+
].some((word) => haystack.includes(word));
|
|
241
|
+
}
|
|
242
|
+
function structuralElementTarget(tag, attrs) {
|
|
243
|
+
if (!isScrollableTag(tag)) {
|
|
244
|
+
return '';
|
|
245
|
+
}
|
|
246
|
+
return [
|
|
247
|
+
'scroll-container',
|
|
248
|
+
tag,
|
|
249
|
+
attrs.x ?? '',
|
|
250
|
+
attrs.y ?? '',
|
|
251
|
+
attrs.width ?? '',
|
|
252
|
+
attrs.height ?? ''
|
|
253
|
+
].join('=');
|
|
254
|
+
}
|
|
255
|
+
function redactLabel(value) {
|
|
256
|
+
return value
|
|
257
|
+
.replace(/[A-Z0-9._%+-]+@[A-Z0-9.-]+\.[A-Z]{2,}/gi, '<redacted-email>')
|
|
258
|
+
.replace(/\b(?:\+?\d[\d .()-]{7,}\d)\b/g, '<redacted-phone>')
|
|
259
|
+
.replace(/\b(?:\d[ -]*?){13,19}\b/g, '<redacted-card>');
|
|
260
|
+
}
|
|
261
|
+
function classifyControl(tag, labels, attrs) {
|
|
262
|
+
const haystack = [...labels, tag, attrs.type ?? '', attrs.class ?? ''].join(' ').toLowerCase();
|
|
263
|
+
if (RISKY_WORDS.some((word) => haystack.includes(word))) {
|
|
264
|
+
return 'risky';
|
|
265
|
+
}
|
|
266
|
+
if (INPUT_WORDS.some((word) => haystack.includes(word))) {
|
|
267
|
+
return 'needs-input';
|
|
268
|
+
}
|
|
269
|
+
if (tag.toLowerCase().includes('button') ||
|
|
270
|
+
attrs.clickable === 'true' ||
|
|
271
|
+
attrs.enabled === 'true' ||
|
|
272
|
+
attrs.accessible === 'true') {
|
|
273
|
+
return 'safe';
|
|
274
|
+
}
|
|
275
|
+
return 'unknown';
|
|
276
|
+
}
|
|
277
|
+
function extractElements(source) {
|
|
278
|
+
const elements = [];
|
|
279
|
+
const tagPattern = /<([A-Za-z][A-Za-z0-9_.:-]*)([^<>]*?)(?:\/>|>([^<>]*)<\/\1>)/g;
|
|
280
|
+
for (const match of source.matchAll(tagPattern)) {
|
|
281
|
+
const tag = match[1];
|
|
282
|
+
const attrs = attributeMap(match[2] ?? '');
|
|
283
|
+
const bodyText = (match[3] ?? '').trim();
|
|
284
|
+
const formControl = isLikelyFormControl(tag, attrs);
|
|
285
|
+
const rawLabels = unique(expandSemanticLabels([
|
|
286
|
+
redactLabel(attrs.name ?? ''),
|
|
287
|
+
redactLabel(attrs.label ?? ''),
|
|
288
|
+
formControl ? '' : redactLabel(attrs.text ?? ''),
|
|
289
|
+
formControl ? '' : redactLabel(attrs.value ?? ''),
|
|
290
|
+
redactLabel(attrs['content-desc'] ?? ''),
|
|
291
|
+
redactLabel(attrs['contentDescription'] ?? ''),
|
|
292
|
+
formControl ? '' : redactLabel(bodyText)
|
|
293
|
+
]));
|
|
294
|
+
const id = attrs['resource-id'] ?? attrs.id ?? attrs.identifier ?? '';
|
|
295
|
+
const structuralTarget = structuralElementTarget(tag, attrs);
|
|
296
|
+
const targets = unique([
|
|
297
|
+
...rawLabels,
|
|
298
|
+
...rawLabels.map((label) => `text=${label}`),
|
|
299
|
+
id ? `id=${id}` : '',
|
|
300
|
+
structuralTarget
|
|
301
|
+
]);
|
|
302
|
+
if (targets.length === 0) {
|
|
303
|
+
continue;
|
|
304
|
+
}
|
|
305
|
+
const stable = Boolean(rawLabels.length > 0 || id || structuralTarget);
|
|
306
|
+
const selector = id ? `id=${id}` : rawLabels[0] ?? targets[0];
|
|
307
|
+
elements.push({
|
|
308
|
+
tag,
|
|
309
|
+
selector,
|
|
310
|
+
targets,
|
|
311
|
+
labels: rawLabels,
|
|
312
|
+
stable,
|
|
313
|
+
safety: classifyControl(tag, rawLabels, attrs),
|
|
314
|
+
enabled: attrBool(attrs, 'enabled', true),
|
|
315
|
+
visible: attrBool(attrs, 'visible', true),
|
|
316
|
+
clickable: attrBool(attrs, 'clickable', false),
|
|
317
|
+
x: attrNumber(attrs, 'x'),
|
|
318
|
+
y: attrNumber(attrs, 'y'),
|
|
319
|
+
width: attrNumber(attrs, 'width'),
|
|
320
|
+
height: attrNumber(attrs, 'height')
|
|
321
|
+
});
|
|
322
|
+
}
|
|
323
|
+
return elements;
|
|
324
|
+
}
|
|
325
|
+
function observeSource(source) {
|
|
326
|
+
const elements = extractElements(source);
|
|
327
|
+
const normalizedLabels = elements.flatMap((element) => element.labels.map(normalizeLabel));
|
|
328
|
+
const elementKeys = unique([
|
|
329
|
+
...elements.flatMap((element) => element.targets.map(normalizeLabel)),
|
|
330
|
+
...normalizedLabels
|
|
331
|
+
]).sort();
|
|
332
|
+
const normalizedFingerprint = signatureFor(elementKeys);
|
|
333
|
+
const authRequired = AUTH_WORDS.some((word) => normalizedLabels.some((label) => label.includes(word)));
|
|
334
|
+
return {
|
|
335
|
+
fingerprint: signatureFor(source),
|
|
336
|
+
normalized_fingerprint: normalizedFingerprint,
|
|
337
|
+
elements,
|
|
338
|
+
element_keys: elementKeys,
|
|
339
|
+
auth_required: authRequired
|
|
340
|
+
};
|
|
341
|
+
}
|
|
342
|
+
function similarity(left, right) {
|
|
343
|
+
if (left.length === 0 && right.length === 0) {
|
|
344
|
+
return 1;
|
|
345
|
+
}
|
|
346
|
+
const leftSet = new Set(left);
|
|
347
|
+
const rightSet = new Set(right);
|
|
348
|
+
const intersection = Array.from(leftSet).filter((value) => rightSet.has(value)).length;
|
|
349
|
+
const union = new Set([...left, ...right]).size;
|
|
350
|
+
const jaccard = union === 0 ? 0 : intersection / union;
|
|
351
|
+
const smaller = Math.min(leftSet.size, rightSet.size);
|
|
352
|
+
const containment = smaller === 0 ? 0 : intersection / smaller;
|
|
353
|
+
return containment >= 0.8 ? Math.max(jaccard, containment) : jaccard;
|
|
354
|
+
}
|
|
355
|
+
function identityKeys(elements, includeStatic) {
|
|
356
|
+
const navigationElements = new Set(navigationControlsForElements(elements).map((control) => control.element));
|
|
357
|
+
return unique(elements.flatMap((element) => {
|
|
358
|
+
if (navigationElements.has(element)) {
|
|
359
|
+
return [];
|
|
360
|
+
}
|
|
361
|
+
const tag = element.tag.toLowerCase();
|
|
362
|
+
if (!includeStatic && isStaticIdentityElement(element)) {
|
|
363
|
+
return [];
|
|
364
|
+
}
|
|
365
|
+
const labels = element.labels
|
|
366
|
+
.filter((label) => !isGlobalIdentityLabel(label, element))
|
|
367
|
+
.map(normalizeLabel);
|
|
368
|
+
const ids = element.targets.filter((target) => target.startsWith('id=')).map(normalizeLabel);
|
|
369
|
+
return [...labels, ...ids].map((value) => `${tag}:${element.safety}:${value}`);
|
|
370
|
+
})).sort();
|
|
371
|
+
}
|
|
372
|
+
function isStaticIdentityElement(element) {
|
|
373
|
+
const tag = element.tag.toLowerCase();
|
|
374
|
+
return element.safety === 'unknown' || tag.includes('statictext') || tag.includes('image') || isScrollableTag(tag);
|
|
375
|
+
}
|
|
376
|
+
function screenSimilarity(left, right) {
|
|
377
|
+
const leftControlKeys = identityKeys(left, false);
|
|
378
|
+
const rightControlKeys = identityKeys(right, false);
|
|
379
|
+
const fullScore = similarity(identityKeys(left, true), identityKeys(right, true));
|
|
380
|
+
if (leftControlKeys.length === 0 && rightControlKeys.length === 0) {
|
|
381
|
+
return fullScore;
|
|
382
|
+
}
|
|
383
|
+
return Math.max(fullScore, similarity(leftControlKeys, rightControlKeys));
|
|
384
|
+
}
|
|
385
|
+
function isGlobalIdentityLabel(label, element) {
|
|
386
|
+
const normalized = normalizeControlTarget(label);
|
|
387
|
+
if (isBundleLikeLabel(normalized)) {
|
|
388
|
+
return true;
|
|
389
|
+
}
|
|
390
|
+
if (['settings settings', 'notifications notifications'].includes(normalized)) {
|
|
391
|
+
return true;
|
|
392
|
+
}
|
|
393
|
+
return false;
|
|
394
|
+
}
|
|
395
|
+
function isBundleLikeLabel(normalized) {
|
|
396
|
+
return /^[a-z][a-z0-9_]*(?:\.[a-z][a-z0-9_]*){2,}$/.test(normalized);
|
|
397
|
+
}
|
|
398
|
+
function newVariant(map, observation) {
|
|
399
|
+
const screen = {
|
|
400
|
+
id: `screen_${map.screens.length + 1}`,
|
|
401
|
+
variant_ids: []
|
|
402
|
+
};
|
|
403
|
+
const variant = {
|
|
404
|
+
id: `variant_${map.variants.length + 1}`,
|
|
405
|
+
screen_id: screen.id,
|
|
406
|
+
fingerprint: observation.fingerprint,
|
|
407
|
+
normalized_fingerprint: observation.normalized_fingerprint,
|
|
408
|
+
elements: observation.elements,
|
|
409
|
+
element_keys: observation.element_keys,
|
|
410
|
+
actions: actionAffordancesForElements(observation.elements),
|
|
411
|
+
auth_required: observation.auth_required,
|
|
412
|
+
observations: 1,
|
|
413
|
+
confidence: 0.6,
|
|
414
|
+
last_observed_at: utcNowIso()
|
|
415
|
+
};
|
|
416
|
+
screen.variant_ids.push(variant.id);
|
|
417
|
+
map.screens.push(screen);
|
|
418
|
+
map.variants.push(variant);
|
|
419
|
+
return variant;
|
|
420
|
+
}
|
|
421
|
+
function upsertVariant(map, observation) {
|
|
422
|
+
let best = null;
|
|
423
|
+
let bestScore = 0;
|
|
424
|
+
const observedNavigationTarget = preferredNavigationTargetForElements(observation.elements);
|
|
425
|
+
const observedHasBottomNavigation = navigationControlsForElements(observation.elements).length >= 2;
|
|
426
|
+
const observedHasScrollableSurface = elementsHaveScrollableSurface(observation.elements);
|
|
427
|
+
const observedControlKeys = identityKeys(observation.elements, false);
|
|
428
|
+
for (const variant of map.variants) {
|
|
429
|
+
const variantNavigationTarget = preferredNavigationTarget(variant);
|
|
430
|
+
if (observedNavigationTarget &&
|
|
431
|
+
variantNavigationTarget &&
|
|
432
|
+
observedNavigationTarget !== variantNavigationTarget) {
|
|
433
|
+
continue;
|
|
434
|
+
}
|
|
435
|
+
if (observedHasScrollableSurface || hasScrollableSurface(variant)) {
|
|
436
|
+
const variantControlKeys = identityKeys(variant.elements, false);
|
|
437
|
+
if ((observedControlKeys.length > 0 || variantControlKeys.length > 0) &&
|
|
438
|
+
similarity(observedControlKeys, variantControlKeys) < 0.3) {
|
|
439
|
+
continue;
|
|
440
|
+
}
|
|
441
|
+
}
|
|
442
|
+
const score = variant.normalized_fingerprint === observation.normalized_fingerprint
|
|
443
|
+
? 1
|
|
444
|
+
: screenSimilarity(variant.elements, observation.elements);
|
|
445
|
+
if (score > bestScore) {
|
|
446
|
+
best = variant;
|
|
447
|
+
bestScore = score;
|
|
448
|
+
}
|
|
449
|
+
}
|
|
450
|
+
const requiredScore = observedHasBottomNavigation || (best ? hasBottomNavigation(best) : false)
|
|
451
|
+
? 0.62
|
|
452
|
+
: observedHasScrollableSurface || (best ? hasScrollableSurface(best) : false)
|
|
453
|
+
? 0.62
|
|
454
|
+
: 0.45;
|
|
455
|
+
if (!best || bestScore < requiredScore) {
|
|
456
|
+
return {
|
|
457
|
+
observation,
|
|
458
|
+
variant: newVariant(map, observation),
|
|
459
|
+
created: true
|
|
460
|
+
};
|
|
461
|
+
}
|
|
462
|
+
best.fingerprint = observation.fingerprint;
|
|
463
|
+
best.normalized_fingerprint = observation.normalized_fingerprint;
|
|
464
|
+
best.elements = observation.elements;
|
|
465
|
+
best.element_keys = observation.element_keys;
|
|
466
|
+
best.actions = actionAffordancesForElements(observation.elements);
|
|
467
|
+
best.auth_required = observation.auth_required;
|
|
468
|
+
best.observations += 1;
|
|
469
|
+
best.confidence = Math.min(1, best.confidence + 0.05);
|
|
470
|
+
best.last_observed_at = utcNowIso();
|
|
471
|
+
return {
|
|
472
|
+
observation,
|
|
473
|
+
variant: best,
|
|
474
|
+
created: false
|
|
475
|
+
};
|
|
476
|
+
}
|
|
477
|
+
async function captureCurrentSource(context) {
|
|
478
|
+
const sourceDir = ensureDir(path.join(os.tmpdir(), 'visor-app-map-source'));
|
|
479
|
+
const sourcePath = path.join(sourceDir, `${process.pid}-${Date.now()}-${randomUUID()}.xml`);
|
|
480
|
+
const details = await context.adapter.source({ label: 'app-map-source', path: sourcePath });
|
|
481
|
+
const args = details.args;
|
|
482
|
+
const sourceArgs = args && typeof args === 'object' && !Array.isArray(args)
|
|
483
|
+
? args
|
|
484
|
+
: {};
|
|
485
|
+
const detailPath = typeof sourceArgs.path === 'string'
|
|
486
|
+
? sourceArgs.path
|
|
487
|
+
: sourcePath;
|
|
488
|
+
const source = fs.existsSync(detailPath) ? fs.readFileSync(detailPath, 'utf8') : '';
|
|
489
|
+
try {
|
|
490
|
+
fs.rmSync(detailPath, { force: true });
|
|
491
|
+
}
|
|
492
|
+
catch {
|
|
493
|
+
// Source observations are transient; stale temp files should not fail a run.
|
|
494
|
+
}
|
|
495
|
+
return { source };
|
|
496
|
+
}
|
|
497
|
+
function isTransientObservation(observation) {
|
|
498
|
+
return observation.elements.length === 0 || observation.element_keys.length === 0;
|
|
499
|
+
}
|
|
500
|
+
async function captureCurrentObservation(context) {
|
|
501
|
+
let observation = null;
|
|
502
|
+
for (let attempt = 0; attempt < 5; attempt += 1) {
|
|
503
|
+
const captured = await captureCurrentSource(context);
|
|
504
|
+
observation = observeSource(captured.source);
|
|
505
|
+
if (!isTransientObservation(observation) || attempt === 4) {
|
|
506
|
+
break;
|
|
507
|
+
}
|
|
508
|
+
await delay(500);
|
|
509
|
+
}
|
|
510
|
+
return observation ?? observeSource('');
|
|
511
|
+
}
|
|
512
|
+
function observeScreenObservation(context, observation) {
|
|
513
|
+
const observed = upsertVariant(context.map, observation);
|
|
514
|
+
context.changed = true;
|
|
515
|
+
return observed;
|
|
516
|
+
}
|
|
517
|
+
async function observeCurrentScreen(context) {
|
|
518
|
+
return observeScreenObservation(context, await captureCurrentObservation(context));
|
|
519
|
+
}
|
|
520
|
+
async function observeAfterAction(context, target, options = {}) {
|
|
521
|
+
const observed = await observeCurrentScreen(context);
|
|
522
|
+
if (target && restoreMatch(observed, target)) {
|
|
523
|
+
return observed;
|
|
524
|
+
}
|
|
525
|
+
if (!shouldSettleObservation(observed.observation)) {
|
|
526
|
+
return observed;
|
|
527
|
+
}
|
|
528
|
+
const settleMs = nonNegativeNumber(options.settleMs, DEFAULT_ACTION_SETTLE_MS);
|
|
529
|
+
if (settleMs === 0) {
|
|
530
|
+
return observed;
|
|
531
|
+
}
|
|
532
|
+
const pollMs = positiveNumber(options.pollMs, DEFAULT_ACTION_SETTLE_POLL_MS);
|
|
533
|
+
let best = observed;
|
|
534
|
+
let previous = observed;
|
|
535
|
+
let elapsed = 0;
|
|
536
|
+
while (elapsed < settleMs) {
|
|
537
|
+
const waitMs = Math.min(pollMs, settleMs - elapsed);
|
|
538
|
+
await settleDelay(context, waitMs);
|
|
539
|
+
elapsed += waitMs;
|
|
540
|
+
let next;
|
|
541
|
+
try {
|
|
542
|
+
next = await observeCurrentScreen(context);
|
|
543
|
+
}
|
|
544
|
+
catch {
|
|
545
|
+
return best;
|
|
546
|
+
}
|
|
547
|
+
if (target && restoreMatch(next, target)) {
|
|
548
|
+
return next;
|
|
549
|
+
}
|
|
550
|
+
if (isRicherObservation(next.observation, best.observation)) {
|
|
551
|
+
best = next;
|
|
552
|
+
}
|
|
553
|
+
if (next.observation.fingerprint === previous.observation.fingerprint && !shouldSettleObservation(next.observation)) {
|
|
554
|
+
return isRicherObservation(next.observation, best.observation) ? next : best;
|
|
555
|
+
}
|
|
556
|
+
previous = next;
|
|
557
|
+
}
|
|
558
|
+
return best;
|
|
559
|
+
}
|
|
560
|
+
async function observeAfterCrawlAction(context) {
|
|
561
|
+
const observation = await captureCurrentObservation(context);
|
|
562
|
+
const settled = await settleCrawlObservation(context, observation);
|
|
563
|
+
return observeScreenObservation(context, settled);
|
|
564
|
+
}
|
|
565
|
+
async function settleCrawlObservation(context, observation) {
|
|
566
|
+
const settleMs = context.options.crawlSettleMs;
|
|
567
|
+
if (settleMs === 0) {
|
|
568
|
+
return observation;
|
|
569
|
+
}
|
|
570
|
+
const pollMs = context.options.crawlSettlePollMs;
|
|
571
|
+
let best = observation;
|
|
572
|
+
let previous = observation;
|
|
573
|
+
let elapsed = 0;
|
|
574
|
+
while (elapsed < settleMs) {
|
|
575
|
+
const waitMs = Math.min(pollMs, settleMs - elapsed);
|
|
576
|
+
await settleDelay(context, waitMs);
|
|
577
|
+
elapsed += waitMs;
|
|
578
|
+
let next;
|
|
579
|
+
try {
|
|
580
|
+
next = await captureCurrentObservation(context);
|
|
581
|
+
}
|
|
582
|
+
catch {
|
|
583
|
+
return best;
|
|
584
|
+
}
|
|
585
|
+
if (isRicherObservation(next, best)) {
|
|
586
|
+
best = next;
|
|
587
|
+
}
|
|
588
|
+
if (next.fingerprint === previous.fingerprint && !shouldSettleObservation(next)) {
|
|
589
|
+
return isRicherObservation(next, best) ? next : best;
|
|
590
|
+
}
|
|
591
|
+
previous = next;
|
|
592
|
+
}
|
|
593
|
+
return best;
|
|
594
|
+
}
|
|
595
|
+
async function settleDelay(context, ms) {
|
|
596
|
+
if (ms <= 0) {
|
|
597
|
+
return;
|
|
598
|
+
}
|
|
599
|
+
try {
|
|
600
|
+
await context.adapter.wait({ ms, label: 'app-map-settle' });
|
|
601
|
+
}
|
|
602
|
+
catch {
|
|
603
|
+
await delay(ms);
|
|
604
|
+
}
|
|
605
|
+
}
|
|
606
|
+
function isRicherObservation(candidate, current) {
|
|
607
|
+
if (isTransientObservation(current) && !isTransientObservation(candidate)) {
|
|
608
|
+
return true;
|
|
609
|
+
}
|
|
610
|
+
if (isTransientObservation(candidate)) {
|
|
611
|
+
return false;
|
|
612
|
+
}
|
|
613
|
+
const candidateScore = observationRichness(candidate);
|
|
614
|
+
const currentScore = observationRichness(current);
|
|
615
|
+
return candidateScore > currentScore;
|
|
616
|
+
}
|
|
617
|
+
function observationRichness(observation) {
|
|
618
|
+
const stableControls = observation.elements.filter((element) => element.stable && element.visible !== false && element.safety === 'safe').length;
|
|
619
|
+
return observation.element_keys.length + observation.elements.length * 2 + stableControls * 3;
|
|
620
|
+
}
|
|
621
|
+
function shouldSettleObservation(observation) {
|
|
622
|
+
if (isTransientObservation(observation)) {
|
|
623
|
+
return true;
|
|
624
|
+
}
|
|
625
|
+
const labels = observation.elements.flatMap((element) => element.labels.map(normalizeControlTarget));
|
|
626
|
+
const hasTransitionLabel = labels.some((label) => /\b(loading|syncing|refreshing|please wait|progress)\b/.test(label));
|
|
627
|
+
return hasTransitionLabel && observation.elements.length <= 3;
|
|
628
|
+
}
|
|
629
|
+
function delay(ms) {
|
|
630
|
+
return new Promise((resolve) => setTimeout(resolve, ms));
|
|
631
|
+
}
|
|
632
|
+
function targetDescriptor(command, args) {
|
|
633
|
+
if (isValueBearingAction(command, args)) {
|
|
634
|
+
return { kind: 'unknown', confidence: 0 };
|
|
635
|
+
}
|
|
636
|
+
if (command === 'tap') {
|
|
637
|
+
try {
|
|
638
|
+
if (resolveTapMode(args) === 'coordinates') {
|
|
639
|
+
return { kind: 'coordinate', target: coordinateTarget(args), confidence: 0.5 };
|
|
640
|
+
}
|
|
641
|
+
}
|
|
642
|
+
catch {
|
|
643
|
+
return { kind: 'unknown', confidence: 0 };
|
|
644
|
+
}
|
|
645
|
+
}
|
|
646
|
+
if (command === 'scroll') {
|
|
647
|
+
return { kind: 'stable', target: scrollTarget(args), confidence: 0.4 };
|
|
648
|
+
}
|
|
649
|
+
const target = typeof args.target === 'string' ? args.target : undefined;
|
|
650
|
+
if (!target) {
|
|
651
|
+
return { kind: 'unknown', confidence: 0 };
|
|
652
|
+
}
|
|
653
|
+
if (target.startsWith('first-in-section=')) {
|
|
654
|
+
return { kind: 'section-first', target, value: target.slice(17), confidence: 0.55 };
|
|
655
|
+
}
|
|
656
|
+
if (target.startsWith('text~=')) {
|
|
657
|
+
return { kind: 'text-contains', target, value: target.slice(6), confidence: 0.45 };
|
|
658
|
+
}
|
|
659
|
+
if (target.startsWith('text=')) {
|
|
660
|
+
return { kind: 'text', target, value: target.slice(5), confidence: 0.6 };
|
|
661
|
+
}
|
|
662
|
+
if (target.startsWith('xpath=')) {
|
|
663
|
+
return { kind: 'unknown', target, value: target.slice(6), confidence: 0.2 };
|
|
664
|
+
}
|
|
665
|
+
return { kind: 'stable', target, value: target.replace(/^(id=|accessibility=)/, ''), confidence: 0.9 };
|
|
666
|
+
}
|
|
667
|
+
function coordinateTarget(args) {
|
|
668
|
+
const x = Number(args.x);
|
|
669
|
+
const y = Number(args.y);
|
|
670
|
+
const base = `x=${Number.isFinite(x) ? x : String(args.x)},y=${Number.isFinite(y) ? y : String(args.y)}`;
|
|
671
|
+
return args.normalized === true ? `${base},normalized=true` : base;
|
|
672
|
+
}
|
|
673
|
+
function scrollTarget(args) {
|
|
674
|
+
const direction = String(args.direction ?? 'down').trim().toLowerCase() || 'down';
|
|
675
|
+
const percent = args.percent === undefined ? '' : `,percent=${String(args.percent)}`;
|
|
676
|
+
return `scroll=${direction}${percent}`;
|
|
677
|
+
}
|
|
678
|
+
function isValueBearingAction(command, args) {
|
|
679
|
+
return command === 'act' && args.name === 'type' && args.value !== undefined;
|
|
680
|
+
}
|
|
681
|
+
function replayableEdge(edge) {
|
|
682
|
+
return !isValueBearingAction(edge.command, edge.args);
|
|
683
|
+
}
|
|
684
|
+
function elementContainsTarget(element, descriptor) {
|
|
685
|
+
if (descriptor.kind === 'section-first') {
|
|
686
|
+
return false;
|
|
687
|
+
}
|
|
688
|
+
if (!descriptor.target && !descriptor.value) {
|
|
689
|
+
return false;
|
|
690
|
+
}
|
|
691
|
+
const target = descriptor.target ?? '';
|
|
692
|
+
const value = descriptor.value ?? target;
|
|
693
|
+
if (element.targets.includes(target) || element.targets.includes(value)) {
|
|
694
|
+
return true;
|
|
695
|
+
}
|
|
696
|
+
if (descriptor.kind === 'text-contains') {
|
|
697
|
+
const normalizedValue = normalizeControlTarget(value);
|
|
698
|
+
return element.labels.some((label) => normalizeControlTarget(label).includes(normalizedValue));
|
|
699
|
+
}
|
|
700
|
+
return element.labels.some((label) => label === value);
|
|
701
|
+
}
|
|
702
|
+
function variantContainsTarget(variant, descriptor) {
|
|
703
|
+
if (descriptor.kind === 'section-first') {
|
|
704
|
+
return sectionFirstTapArgs(variant, descriptor) !== null;
|
|
705
|
+
}
|
|
706
|
+
return variant.elements.some((element) => elementContainsTarget(element, descriptor));
|
|
707
|
+
}
|
|
708
|
+
async function liveContainsTarget(context, descriptor) {
|
|
709
|
+
if (descriptor.kind === 'section-first') {
|
|
710
|
+
return false;
|
|
711
|
+
}
|
|
712
|
+
const target = descriptor.target ?? descriptor.value;
|
|
713
|
+
if (!target) {
|
|
714
|
+
return false;
|
|
715
|
+
}
|
|
716
|
+
try {
|
|
717
|
+
return await context.adapter.exists(target);
|
|
718
|
+
}
|
|
719
|
+
catch {
|
|
720
|
+
return false;
|
|
721
|
+
}
|
|
722
|
+
}
|
|
723
|
+
function candidateVariants(map, descriptor) {
|
|
724
|
+
return map.variants.filter((variant) => variantContainsTarget(variant, descriptor) && variantMatchesDescriptorContext(map, variant, descriptor));
|
|
725
|
+
}
|
|
726
|
+
function variantMatchesDescriptorContext(map, variant, descriptor) {
|
|
727
|
+
if (descriptor.kind !== 'section-first') {
|
|
728
|
+
return true;
|
|
729
|
+
}
|
|
730
|
+
const navigationHint = sectionNavigationHint(descriptor.value ?? descriptor.target ?? '', map);
|
|
731
|
+
if (!navigationHint) {
|
|
732
|
+
return true;
|
|
733
|
+
}
|
|
734
|
+
const navigationTarget = preferredNavigationTarget(variant);
|
|
735
|
+
return !navigationTarget || navigationTarget === navigationHint;
|
|
736
|
+
}
|
|
737
|
+
function summarizeEdge(edge) {
|
|
738
|
+
return {
|
|
739
|
+
command: edge.command,
|
|
740
|
+
target: edge.target,
|
|
741
|
+
confidence: Math.round(edge.confidence * 100) / 100
|
|
742
|
+
};
|
|
743
|
+
}
|
|
744
|
+
function summarizeControl(command, target, confidence) {
|
|
745
|
+
return {
|
|
746
|
+
command,
|
|
747
|
+
target,
|
|
748
|
+
confidence: Math.round(confidence * 100) / 100
|
|
749
|
+
};
|
|
750
|
+
}
|
|
751
|
+
function planRoute(map, fromVariantId, descriptor) {
|
|
752
|
+
if (descriptor.kind === 'coordinate' || descriptor.kind === 'unknown' || !routeableDescriptor(descriptor)) {
|
|
753
|
+
return { edges: [], ambiguous: false, authRequired: false };
|
|
754
|
+
}
|
|
755
|
+
const destinations = candidateVariants(map, descriptor);
|
|
756
|
+
if (destinations.length === 0) {
|
|
757
|
+
return { edges: [], ambiguous: false, authRequired: false };
|
|
758
|
+
}
|
|
759
|
+
const destinationIds = new Set(destinations.map((variant) => variant.id));
|
|
760
|
+
const visited = new Set([fromVariantId]);
|
|
761
|
+
const queue = [{ variantId: fromVariantId, edges: [] }];
|
|
762
|
+
const reachableRoutes = [];
|
|
763
|
+
let shortestDestinationLength = null;
|
|
764
|
+
while (queue.length > 0) {
|
|
765
|
+
const current = queue.shift();
|
|
766
|
+
if (!current) {
|
|
767
|
+
break;
|
|
768
|
+
}
|
|
769
|
+
if (destinationIds.has(current.variantId)) {
|
|
770
|
+
if (shortestDestinationLength === null) {
|
|
771
|
+
shortestDestinationLength = current.edges.length;
|
|
772
|
+
}
|
|
773
|
+
if (current.edges.length === shortestDestinationLength) {
|
|
774
|
+
reachableRoutes.push(current);
|
|
775
|
+
}
|
|
776
|
+
continue;
|
|
777
|
+
}
|
|
778
|
+
if (shortestDestinationLength !== null && current.edges.length >= shortestDestinationLength) {
|
|
779
|
+
continue;
|
|
780
|
+
}
|
|
781
|
+
const outgoing = [
|
|
782
|
+
...map.edges,
|
|
783
|
+
...navigationVirtualEdges(map, map.variants.find((variant) => variant.id === current.variantId))
|
|
784
|
+
]
|
|
785
|
+
.filter((edge) => edge.from_variant_id === current.variantId && !edge.stale && edge.confidence >= 0.25 && replayableEdge(edge))
|
|
786
|
+
.sort((left, right) => right.confidence - left.confidence);
|
|
787
|
+
for (const edge of outgoing) {
|
|
788
|
+
if (visited.has(edge.to_variant_id)) {
|
|
789
|
+
continue;
|
|
790
|
+
}
|
|
791
|
+
visited.add(edge.to_variant_id);
|
|
792
|
+
queue.push({ variantId: edge.to_variant_id, edges: [...current.edges, edge] });
|
|
793
|
+
}
|
|
794
|
+
}
|
|
795
|
+
if (reachableRoutes.length > 0) {
|
|
796
|
+
const rankedRoutes = reachableRoutes
|
|
797
|
+
.map((route) => ({
|
|
798
|
+
...route,
|
|
799
|
+
score: destinationTargetScore(map, map.variants.find((variant) => variant.id === route.variantId), descriptor) + routeTargetScore(map, route.edges, descriptor)
|
|
800
|
+
}))
|
|
801
|
+
.sort((left, right) => right.score - left.score);
|
|
802
|
+
const best = rankedRoutes[0];
|
|
803
|
+
if (!best) {
|
|
804
|
+
return { edges: [], ambiguous: false, authRequired: false };
|
|
805
|
+
}
|
|
806
|
+
const tiedBest = rankedRoutes.filter((route) => route.score === best.score);
|
|
807
|
+
const ambiguous = (descriptor.kind === 'text' || descriptor.kind === 'text-contains' || descriptor.kind === 'section-first') &&
|
|
808
|
+
tiedBest.length > 1;
|
|
809
|
+
const diagnostics = routePlanDiagnostics(map, descriptor, rankedRoutes, ambiguous ? undefined : best, ambiguous);
|
|
810
|
+
if (ambiguous) {
|
|
811
|
+
return { edges: [], ambiguous: true, authRequired: false, diagnostics };
|
|
812
|
+
}
|
|
813
|
+
const route = best;
|
|
814
|
+
const destination = map.variants.find((variant) => variant.id === route?.variantId);
|
|
815
|
+
return {
|
|
816
|
+
edges: route?.edges ?? [],
|
|
817
|
+
ambiguous: false,
|
|
818
|
+
authRequired: Boolean(destination?.auth_required),
|
|
819
|
+
diagnostics
|
|
820
|
+
};
|
|
821
|
+
}
|
|
822
|
+
return { edges: [], ambiguous: false, authRequired: destinations.some((variant) => variant.auth_required) };
|
|
823
|
+
}
|
|
824
|
+
function routePlanDiagnostics(map, descriptor, rankedRoutes, selected, ambiguous) {
|
|
825
|
+
const bestScore = rankedRoutes[0]?.score;
|
|
826
|
+
return {
|
|
827
|
+
target: descriptor.target ?? descriptor.value ?? descriptor.kind,
|
|
828
|
+
selected_variant_id: selected?.variantId,
|
|
829
|
+
ambiguous,
|
|
830
|
+
route_candidates: rankedRoutes.map((route) => {
|
|
831
|
+
const variant = map.variants.find((candidate) => candidate.id === route.variantId);
|
|
832
|
+
const isSelected = selected?.variantId === route.variantId;
|
|
833
|
+
return {
|
|
834
|
+
variant_id: route.variantId,
|
|
835
|
+
screen_id: variant?.screen_id,
|
|
836
|
+
score: Math.round(route.score * 100) / 100,
|
|
837
|
+
route: route.edges.map(summarizeEdge),
|
|
838
|
+
selected: isSelected,
|
|
839
|
+
...(isSelected
|
|
840
|
+
? {}
|
|
841
|
+
: { rejected_reason: ambiguous && route.score === bestScore ? 'ambiguous_tie' : 'lower_score' })
|
|
842
|
+
};
|
|
843
|
+
})
|
|
844
|
+
};
|
|
845
|
+
}
|
|
846
|
+
function routeableDescriptor(descriptor) {
|
|
847
|
+
if (descriptor.kind === 'section-first') {
|
|
848
|
+
const value = descriptor.value ?? descriptor.target ?? '';
|
|
849
|
+
return /[a-z]/i.test(value);
|
|
850
|
+
}
|
|
851
|
+
if (descriptor.kind !== 'text-contains') {
|
|
852
|
+
return true;
|
|
853
|
+
}
|
|
854
|
+
const value = descriptor.value ?? descriptor.target ?? '';
|
|
855
|
+
return /[a-z]/i.test(value);
|
|
856
|
+
}
|
|
857
|
+
function navigationVirtualEdges(map, variant) {
|
|
858
|
+
if (!variant) {
|
|
859
|
+
return [];
|
|
860
|
+
}
|
|
861
|
+
const currentNavigationTarget = preferredNavigationTarget(variant);
|
|
862
|
+
const edges = [];
|
|
863
|
+
const destinationsByTarget = new Map();
|
|
864
|
+
for (const destination of map.variants) {
|
|
865
|
+
if (destination.id === variant.id) {
|
|
866
|
+
continue;
|
|
867
|
+
}
|
|
868
|
+
const target = preferredNavigationTarget(destination);
|
|
869
|
+
if (!target || target === currentNavigationTarget || destinationsByTarget.has(target)) {
|
|
870
|
+
continue;
|
|
871
|
+
}
|
|
872
|
+
destinationsByTarget.set(target, destination);
|
|
873
|
+
}
|
|
874
|
+
for (const candidate of safeTapCandidates(variant, { skipCurrentNavigation: true })) {
|
|
875
|
+
const target = normalizeControlTarget(candidate.target);
|
|
876
|
+
const destination = destinationsByTarget.get(target);
|
|
877
|
+
if (!destination) {
|
|
878
|
+
continue;
|
|
879
|
+
}
|
|
880
|
+
if (hasRealEdgeForTap(map, variant.id, candidate)) {
|
|
881
|
+
continue;
|
|
882
|
+
}
|
|
883
|
+
edges.push({
|
|
884
|
+
id: `virtual_nav_${variant.id}_${target}`,
|
|
885
|
+
from_variant_id: variant.id,
|
|
886
|
+
to_variant_id: destination.id,
|
|
887
|
+
command: 'tap',
|
|
888
|
+
args: structuredClone(candidate.args),
|
|
889
|
+
target: targetDescriptor('tap', candidate.args).target,
|
|
890
|
+
confidence: 0.4,
|
|
891
|
+
successes: 0,
|
|
892
|
+
failures: 0,
|
|
893
|
+
stale: false,
|
|
894
|
+
candidate: true,
|
|
895
|
+
destination_contract: {
|
|
896
|
+
variant_id: destination.id,
|
|
897
|
+
required_targets: contractTargets(destination),
|
|
898
|
+
normalized_fingerprint: destination.normalized_fingerprint
|
|
899
|
+
},
|
|
900
|
+
last_observed_at: utcNowIso()
|
|
901
|
+
});
|
|
902
|
+
}
|
|
903
|
+
return edges;
|
|
904
|
+
}
|
|
905
|
+
function hasRealEdgeForTap(map, fromVariantId, candidate) {
|
|
906
|
+
const descriptor = targetDescriptor('tap', candidate.args);
|
|
907
|
+
const candidateTargets = new Set([candidate.key, candidate.target, descriptor.target, descriptor.value].filter(Boolean));
|
|
908
|
+
return map.edges.some((edge) => edge.from_variant_id === fromVariantId &&
|
|
909
|
+
!edge.stale &&
|
|
910
|
+
edge.confidence >= 0.25 &&
|
|
911
|
+
replayableEdge(edge) &&
|
|
912
|
+
candidateTargets.has(edge.target));
|
|
913
|
+
}
|
|
914
|
+
function destinationTargetScore(map, variant, descriptor) {
|
|
915
|
+
if (!variant) {
|
|
916
|
+
return 0;
|
|
917
|
+
}
|
|
918
|
+
if (descriptor.kind === 'section-first') {
|
|
919
|
+
const element = firstItemInSection(variant, descriptor);
|
|
920
|
+
if (!element) {
|
|
921
|
+
return 0;
|
|
922
|
+
}
|
|
923
|
+
let score = isHighValueContentCard(element) ? 10 : 7;
|
|
924
|
+
const navigationHint = sectionNavigationHint(descriptor.value ?? descriptor.target ?? '', map);
|
|
925
|
+
const navigationTarget = preferredNavigationTarget(variant);
|
|
926
|
+
if (navigationHint && navigationTarget === navigationHint) {
|
|
927
|
+
score += 5;
|
|
928
|
+
}
|
|
929
|
+
else if (navigationHint && navigationTarget && navigationTarget !== navigationHint) {
|
|
930
|
+
score -= 5;
|
|
931
|
+
}
|
|
932
|
+
return score;
|
|
933
|
+
}
|
|
934
|
+
let score = 0;
|
|
935
|
+
for (const element of variant.elements) {
|
|
936
|
+
if (!elementContainsTarget(element, descriptor)) {
|
|
937
|
+
continue;
|
|
938
|
+
}
|
|
939
|
+
let elementScore = 1;
|
|
940
|
+
if (isLikelyTapElement(element)) {
|
|
941
|
+
elementScore += 4;
|
|
942
|
+
}
|
|
943
|
+
else if (element.visible === false) {
|
|
944
|
+
elementScore -= 2;
|
|
945
|
+
}
|
|
946
|
+
if (isHighValueContentCard(element)) {
|
|
947
|
+
elementScore += 3;
|
|
948
|
+
}
|
|
949
|
+
const value = normalizeControlTarget(descriptor.value ?? descriptor.target ?? '');
|
|
950
|
+
if (value &&
|
|
951
|
+
element.labels.some((label) => normalizeControlTarget(label) === value ||
|
|
952
|
+
label.split(/\n+/).some((line) => normalizeControlTarget(line) === value))) {
|
|
953
|
+
elementScore += 2;
|
|
954
|
+
}
|
|
955
|
+
score = Math.max(score, elementScore);
|
|
956
|
+
}
|
|
957
|
+
if (preferredNavigationTarget(variant) && score > 0) {
|
|
958
|
+
score += 1;
|
|
959
|
+
}
|
|
960
|
+
if (score > 0 && isFeedLikeVariant(variant) && !descriptorTargetsFeed(descriptor)) {
|
|
961
|
+
score -= 3;
|
|
962
|
+
}
|
|
963
|
+
return score;
|
|
964
|
+
}
|
|
965
|
+
function isFeedLikeVariant(variant) {
|
|
966
|
+
return contentLabels(variant).some((label) => /^(for you|activity|feed|latest|timeline|updates?)$/.test(label));
|
|
967
|
+
}
|
|
968
|
+
function descriptorTargetsFeed(descriptor) {
|
|
969
|
+
const value = normalizeControlTarget(descriptor.value ?? descriptor.target ?? '');
|
|
970
|
+
return /\b(activity|feed|post|comment|update|timeline)\b/.test(value);
|
|
971
|
+
}
|
|
972
|
+
function routeTargetScore(map, route, descriptor) {
|
|
973
|
+
if (descriptor.kind !== 'section-first') {
|
|
974
|
+
return 0;
|
|
975
|
+
}
|
|
976
|
+
const navigationHint = sectionNavigationHint(descriptor.value ?? descriptor.target ?? '', map);
|
|
977
|
+
if (!navigationHint) {
|
|
978
|
+
return 0;
|
|
979
|
+
}
|
|
980
|
+
return route.reduce((score, edge) => {
|
|
981
|
+
const target = edgeNavigationTarget(map, edge);
|
|
982
|
+
if (!target) {
|
|
983
|
+
return score;
|
|
984
|
+
}
|
|
985
|
+
return score + (target === navigationHint ? 3 : -3);
|
|
986
|
+
}, 0);
|
|
987
|
+
}
|
|
988
|
+
function edgeNavigationTarget(map, edge) {
|
|
989
|
+
const normalizedTarget = normalizeControlTarget(edge.target ?? '');
|
|
990
|
+
const from = map.variants.find((variant) => variant.id === edge.from_variant_id);
|
|
991
|
+
if (from && isNavigationTarget(from, normalizedTarget)) {
|
|
992
|
+
return normalizedTarget;
|
|
993
|
+
}
|
|
994
|
+
if (!from || edge.command !== 'tap') {
|
|
995
|
+
return null;
|
|
996
|
+
}
|
|
997
|
+
const edgeCoordinateTarget = coordinateTarget(edge.args);
|
|
998
|
+
for (const control of navigationControls(from)) {
|
|
999
|
+
const elementCoordinateTarget = coordinateArgsForElement(control.element);
|
|
1000
|
+
if (elementCoordinateTarget && coordinateTarget(elementCoordinateTarget) === edgeCoordinateTarget) {
|
|
1001
|
+
return control.target;
|
|
1002
|
+
}
|
|
1003
|
+
}
|
|
1004
|
+
return null;
|
|
1005
|
+
}
|
|
1006
|
+
function contractTargets(variant) {
|
|
1007
|
+
const navigationElements = new Set(navigationControls(variant).map((control) => control.element));
|
|
1008
|
+
const contentTargets = variant.elements
|
|
1009
|
+
.filter((element) => element.stable)
|
|
1010
|
+
.filter((element) => !navigationElements.has(element))
|
|
1011
|
+
.flatMap((element) => element.targets.filter((target) => !isGlobalIdentityLabel(normalizeControlTarget(target), element)));
|
|
1012
|
+
const fallbackTargets = variant.elements
|
|
1013
|
+
.filter((element) => element.stable)
|
|
1014
|
+
.flatMap((element) => element.targets);
|
|
1015
|
+
return unique([...contentTargets, ...fallbackTargets]).slice(0, 8);
|
|
1016
|
+
}
|
|
1017
|
+
function matchingEdge(map, fromVariantId, command, target) {
|
|
1018
|
+
return map.edges.find((edge) => edge.from_variant_id === fromVariantId &&
|
|
1019
|
+
edge.command === command &&
|
|
1020
|
+
String(edge.target ?? '') === String(target ?? ''));
|
|
1021
|
+
}
|
|
1022
|
+
function recordEdgeSuccess(context, from, to, command, args, candidate) {
|
|
1023
|
+
if (from.id === to.id) {
|
|
1024
|
+
return;
|
|
1025
|
+
}
|
|
1026
|
+
const descriptor = targetDescriptor(command, args);
|
|
1027
|
+
if (descriptor.kind === 'unknown') {
|
|
1028
|
+
return;
|
|
1029
|
+
}
|
|
1030
|
+
const edge = matchingEdge(context.map, from.id, command, descriptor.target);
|
|
1031
|
+
if (edge) {
|
|
1032
|
+
edge.to_variant_id = to.id;
|
|
1033
|
+
edge.successes += 1;
|
|
1034
|
+
edge.failures = 0;
|
|
1035
|
+
edge.stale = false;
|
|
1036
|
+
edge.candidate = edge.candidate && edge.successes < 2;
|
|
1037
|
+
edge.confidence = Math.min(1, edge.confidence + 0.15);
|
|
1038
|
+
edge.destination_contract = {
|
|
1039
|
+
variant_id: to.id,
|
|
1040
|
+
required_targets: contractTargets(to),
|
|
1041
|
+
normalized_fingerprint: to.normalized_fingerprint
|
|
1042
|
+
};
|
|
1043
|
+
edge.last_observed_at = utcNowIso();
|
|
1044
|
+
context.changed = true;
|
|
1045
|
+
return;
|
|
1046
|
+
}
|
|
1047
|
+
context.map.edges.push({
|
|
1048
|
+
id: `edge_${context.map.edges.length + 1}`,
|
|
1049
|
+
from_variant_id: from.id,
|
|
1050
|
+
to_variant_id: to.id,
|
|
1051
|
+
command,
|
|
1052
|
+
args: structuredClone(args),
|
|
1053
|
+
target: descriptor.target,
|
|
1054
|
+
confidence: candidate ? 0.45 : descriptor.confidence,
|
|
1055
|
+
successes: 1,
|
|
1056
|
+
failures: 0,
|
|
1057
|
+
stale: false,
|
|
1058
|
+
candidate,
|
|
1059
|
+
destination_contract: {
|
|
1060
|
+
variant_id: to.id,
|
|
1061
|
+
required_targets: contractTargets(to),
|
|
1062
|
+
normalized_fingerprint: to.normalized_fingerprint
|
|
1063
|
+
},
|
|
1064
|
+
last_observed_at: utcNowIso()
|
|
1065
|
+
});
|
|
1066
|
+
context.changed = true;
|
|
1067
|
+
}
|
|
1068
|
+
function demoteEdge(context, edge) {
|
|
1069
|
+
edge.failures += 1;
|
|
1070
|
+
edge.confidence = Math.max(0, edge.confidence - 0.25);
|
|
1071
|
+
if (edge.failures >= 2 || edge.confidence < 0.25) {
|
|
1072
|
+
edge.stale = true;
|
|
1073
|
+
}
|
|
1074
|
+
edge.last_observed_at = utcNowIso();
|
|
1075
|
+
context.changed = true;
|
|
1076
|
+
}
|
|
1077
|
+
function shouldRequireObservedEffect(command, args) {
|
|
1078
|
+
return command === 'tap' || command === 'navigate' || command === 'scroll' || (command === 'act' && args.name === 'back');
|
|
1079
|
+
}
|
|
1080
|
+
function assertObservedEffect(command, args, before, after, descriptor) {
|
|
1081
|
+
if (!shouldRequireObservedEffect(command, args)) {
|
|
1082
|
+
return;
|
|
1083
|
+
}
|
|
1084
|
+
if (before.observation.fingerprint !== after.observation.fingerprint) {
|
|
1085
|
+
return;
|
|
1086
|
+
}
|
|
1087
|
+
const target = descriptor.target ?? descriptor.value ?? command;
|
|
1088
|
+
throw new Error(`Action '${command}' on '${target}' had no effect: screen source did not change after the command`);
|
|
1089
|
+
}
|
|
1090
|
+
function variantSatisfiesContract(variant, contract) {
|
|
1091
|
+
if (variant.id === contract.variant_id) {
|
|
1092
|
+
return true;
|
|
1093
|
+
}
|
|
1094
|
+
const observedTargets = new Set(variant.elements.flatMap((element) => element.targets));
|
|
1095
|
+
const required = contract.required_targets;
|
|
1096
|
+
if (required.length === 0) {
|
|
1097
|
+
return variant.normalized_fingerprint === contract.normalized_fingerprint;
|
|
1098
|
+
}
|
|
1099
|
+
const matched = required.filter((target) => observedTargets.has(target)).length;
|
|
1100
|
+
return matched / required.length >= 0.6;
|
|
1101
|
+
}
|
|
1102
|
+
function restoreContractHasSharedAnchors(observed, target) {
|
|
1103
|
+
const observedTargets = new Set(observed.elements.flatMap((element) => element.targets));
|
|
1104
|
+
const required = contractTargets(target);
|
|
1105
|
+
if (required.length === 0) {
|
|
1106
|
+
return false;
|
|
1107
|
+
}
|
|
1108
|
+
const matched = unique(required.filter((targetValue) => observedTargets.has(targetValue)));
|
|
1109
|
+
return matched.length >= 2 && matched.length / required.length >= 0.25;
|
|
1110
|
+
}
|
|
1111
|
+
function restoreMatch(observed, target) {
|
|
1112
|
+
if (observed.variant.id === target.variant.id) {
|
|
1113
|
+
return { acceptedBy: 'exact' };
|
|
1114
|
+
}
|
|
1115
|
+
const observedNavigationTarget = preferredNavigationTarget(observed.variant);
|
|
1116
|
+
const targetNavigationTarget = preferredNavigationTarget(target.variant);
|
|
1117
|
+
if (observedNavigationTarget && targetNavigationTarget && observedNavigationTarget !== targetNavigationTarget) {
|
|
1118
|
+
return null;
|
|
1119
|
+
}
|
|
1120
|
+
if (hasScrollableSurface(observed.variant) || hasScrollableSurface(target.variant)) {
|
|
1121
|
+
const observedControlKeys = identityKeys(observed.variant.elements, false);
|
|
1122
|
+
const targetControlKeys = identityKeys(target.variant.elements, false);
|
|
1123
|
+
if ((observedControlKeys.length > 0 || targetControlKeys.length > 0) &&
|
|
1124
|
+
similarity(observedControlKeys, targetControlKeys) < 0.3) {
|
|
1125
|
+
return null;
|
|
1126
|
+
}
|
|
1127
|
+
}
|
|
1128
|
+
const contract = {
|
|
1129
|
+
variant_id: target.variant.id,
|
|
1130
|
+
required_targets: contractTargets(target.variant),
|
|
1131
|
+
normalized_fingerprint: target.variant.normalized_fingerprint
|
|
1132
|
+
};
|
|
1133
|
+
if (variantSatisfiesContract(observed.variant, contract)) {
|
|
1134
|
+
return { acceptedBy: 'contract' };
|
|
1135
|
+
}
|
|
1136
|
+
if (observedNavigationTarget &&
|
|
1137
|
+
targetNavigationTarget &&
|
|
1138
|
+
observedNavigationTarget === targetNavigationTarget &&
|
|
1139
|
+
restoreContractHasSharedAnchors(observed.variant, target.variant)) {
|
|
1140
|
+
return { acceptedBy: 'contract' };
|
|
1141
|
+
}
|
|
1142
|
+
const score = screenSimilarity(target.variant.elements, observed.variant.elements);
|
|
1143
|
+
if (score >= 0.72) {
|
|
1144
|
+
return { acceptedBy: 'similarity', score };
|
|
1145
|
+
}
|
|
1146
|
+
return null;
|
|
1147
|
+
}
|
|
1148
|
+
function satisfiesContract(edge, observed) {
|
|
1149
|
+
return variantSatisfiesContract(observed.variant, edge.destination_contract);
|
|
1150
|
+
}
|
|
1151
|
+
async function driveRoute(context, current, route) {
|
|
1152
|
+
for (const edge of route) {
|
|
1153
|
+
const args = structuredClone(edge.args);
|
|
1154
|
+
let observed;
|
|
1155
|
+
try {
|
|
1156
|
+
await context.adapter[edge.command](args);
|
|
1157
|
+
observed = await observeAfterAction(context);
|
|
1158
|
+
}
|
|
1159
|
+
catch {
|
|
1160
|
+
demoteEdge(context, edge);
|
|
1161
|
+
try {
|
|
1162
|
+
current = await observeCurrentScreen(context);
|
|
1163
|
+
}
|
|
1164
|
+
catch {
|
|
1165
|
+
// Keep the last known screen if the app cannot provide source after a failed cached action.
|
|
1166
|
+
}
|
|
1167
|
+
return { current, failedEdge: edge };
|
|
1168
|
+
}
|
|
1169
|
+
if (!satisfiesContract(edge, observed)) {
|
|
1170
|
+
demoteEdge(context, edge);
|
|
1171
|
+
return { current: observed, failedEdge: edge };
|
|
1172
|
+
}
|
|
1173
|
+
recordEdgeSuccess(context, current.variant, observed.variant, edge.command, args, false);
|
|
1174
|
+
current = observed;
|
|
1175
|
+
}
|
|
1176
|
+
return { current };
|
|
1177
|
+
}
|
|
1178
|
+
function safeTapCandidates(variant, options = {}) {
|
|
1179
|
+
const candidates = [];
|
|
1180
|
+
const seen = new Set();
|
|
1181
|
+
const currentNavigationTarget = options.skipCurrentNavigation ? preferredNavigationTarget(variant) : null;
|
|
1182
|
+
const elements = [...variant.elements].sort((left, right) => {
|
|
1183
|
+
const priority = tapElementPriority(left, variant, options) - tapElementPriority(right, variant, options);
|
|
1184
|
+
if (priority !== 0) {
|
|
1185
|
+
return priority;
|
|
1186
|
+
}
|
|
1187
|
+
return (left.y ?? 0) - (right.y ?? 0) || (left.x ?? 0) - (right.x ?? 0);
|
|
1188
|
+
});
|
|
1189
|
+
for (const element of elements) {
|
|
1190
|
+
if (!(element.safety === 'safe' && element.stable && isLikelyTapElement(element))) {
|
|
1191
|
+
continue;
|
|
1192
|
+
}
|
|
1193
|
+
const target = plainTapTargets(element)[0] ?? element.selector;
|
|
1194
|
+
if (!target) {
|
|
1195
|
+
continue;
|
|
1196
|
+
}
|
|
1197
|
+
const normalizedTarget = normalizeControlTarget(target);
|
|
1198
|
+
if (options.excludeNavigation && isNavigationTarget(variant, normalizedTarget)) {
|
|
1199
|
+
continue;
|
|
1200
|
+
}
|
|
1201
|
+
if (currentNavigationTarget && normalizedTarget === currentNavigationTarget) {
|
|
1202
|
+
continue;
|
|
1203
|
+
}
|
|
1204
|
+
const coordinateArgs = coordinateArgsForElement(element);
|
|
1205
|
+
const args = coordinateArgs ?? { target };
|
|
1206
|
+
const key = coordinateArgs ? coordinateTarget(coordinateArgs) : target;
|
|
1207
|
+
if (seen.has(key)) {
|
|
1208
|
+
continue;
|
|
1209
|
+
}
|
|
1210
|
+
seen.add(key);
|
|
1211
|
+
candidates.push({ target, args, key });
|
|
1212
|
+
}
|
|
1213
|
+
return candidates;
|
|
1214
|
+
}
|
|
1215
|
+
function tapElementPriority(element, variant, options = {}) {
|
|
1216
|
+
const target = plainTapTargets(element)[0] ?? element.selector;
|
|
1217
|
+
const normalized = normalizeControlTarget(target);
|
|
1218
|
+
if (options.preferContent && isHighValueContentCard(element)) {
|
|
1219
|
+
return 5;
|
|
1220
|
+
}
|
|
1221
|
+
const navPriority = navigationControlPriority(variant, normalized);
|
|
1222
|
+
if (navPriority !== undefined) {
|
|
1223
|
+
if (options.preferContent) {
|
|
1224
|
+
return 80 + navPriority;
|
|
1225
|
+
}
|
|
1226
|
+
return navPriority;
|
|
1227
|
+
}
|
|
1228
|
+
if (normalized === 'see all') {
|
|
1229
|
+
return 10;
|
|
1230
|
+
}
|
|
1231
|
+
if (isBackControl(element)) {
|
|
1232
|
+
return 90;
|
|
1233
|
+
}
|
|
1234
|
+
if (element.tag.toLowerCase().includes('button')) {
|
|
1235
|
+
return 40;
|
|
1236
|
+
}
|
|
1237
|
+
if (element.tag.toLowerCase().includes('other')) {
|
|
1238
|
+
return 60;
|
|
1239
|
+
}
|
|
1240
|
+
return 50;
|
|
1241
|
+
}
|
|
1242
|
+
function isHighValueContentCard(element) {
|
|
1243
|
+
if (!isLikelyTapElement(element)) {
|
|
1244
|
+
return false;
|
|
1245
|
+
}
|
|
1246
|
+
const target = plainTapTargets(element)[0] ?? element.selector;
|
|
1247
|
+
const normalizedTarget = normalizeControlTarget(target);
|
|
1248
|
+
if (normalizedTarget === 'see all' || /^(back|close|dismiss|done)$/i.test(target)) {
|
|
1249
|
+
return false;
|
|
1250
|
+
}
|
|
1251
|
+
const lines = unique(element.labels.flatMap((label) => label.split(/\n+/).map((line) => line.trim())));
|
|
1252
|
+
const area = (element.width ?? 0) * (element.height ?? 0);
|
|
1253
|
+
return (lines.length >= 2 ||
|
|
1254
|
+
(element.tag.toLowerCase().includes('other') && area >= 3000) ||
|
|
1255
|
+
((element.width ?? 0) >= 120 && (element.height ?? 0) >= 64 && element.labels.length > 0));
|
|
1256
|
+
}
|
|
1257
|
+
function actionAffordancesForElements(elements) {
|
|
1258
|
+
const navigationElements = new Set(navigationControlsForElements(elements).map((control) => control.element));
|
|
1259
|
+
const navigationTarget = preferredNavigationTargetForElements(elements) ?? undefined;
|
|
1260
|
+
const actions = [];
|
|
1261
|
+
const seen = new Set();
|
|
1262
|
+
const candidates = [...elements]
|
|
1263
|
+
.filter((element) => element.safety === 'safe' && element.stable && isLikelyTapElement(element))
|
|
1264
|
+
.filter((element) => !navigationElements.has(element) && !isBackControl(element))
|
|
1265
|
+
.sort((left, right) => (left.y ?? 0) - (right.y ?? 0) || (left.x ?? 0) - (right.x ?? 0));
|
|
1266
|
+
for (const element of candidates) {
|
|
1267
|
+
const label = actionLabel(element);
|
|
1268
|
+
if (!label) {
|
|
1269
|
+
continue;
|
|
1270
|
+
}
|
|
1271
|
+
const coordinateArgs = coordinateArgsForElement(element);
|
|
1272
|
+
const stableTarget = plainTapTargets(element)[0] ?? element.selector;
|
|
1273
|
+
const args = coordinateArgs ?? { target: stableTarget };
|
|
1274
|
+
const target = coordinateArgs ? coordinateTarget(coordinateArgs) : stableTarget;
|
|
1275
|
+
const scope = actionScopeForElement(elements, element);
|
|
1276
|
+
const dedupeKey = [
|
|
1277
|
+
actionIntent(label),
|
|
1278
|
+
target,
|
|
1279
|
+
scope?.kind ?? '',
|
|
1280
|
+
scope?.label ?? ''
|
|
1281
|
+
].join('|');
|
|
1282
|
+
if (seen.has(dedupeKey)) {
|
|
1283
|
+
continue;
|
|
1284
|
+
}
|
|
1285
|
+
seen.add(dedupeKey);
|
|
1286
|
+
actions.push({
|
|
1287
|
+
command: 'tap',
|
|
1288
|
+
intent: actionIntent(label),
|
|
1289
|
+
label,
|
|
1290
|
+
target,
|
|
1291
|
+
args,
|
|
1292
|
+
safety: element.safety,
|
|
1293
|
+
...(scope ? { scope } : {}),
|
|
1294
|
+
...(navigationTarget ? { navigation_target: navigationTarget } : {}),
|
|
1295
|
+
source: {
|
|
1296
|
+
tag: element.tag,
|
|
1297
|
+
x: element.x,
|
|
1298
|
+
y: element.y,
|
|
1299
|
+
width: element.width,
|
|
1300
|
+
height: element.height
|
|
1301
|
+
}
|
|
1302
|
+
});
|
|
1303
|
+
}
|
|
1304
|
+
return actions;
|
|
1305
|
+
}
|
|
1306
|
+
function actionLabel(element) {
|
|
1307
|
+
const target = plainTapTargets(element)[0] ?? element.labels[0] ?? element.selector;
|
|
1308
|
+
return target.split(/\n+/).map((line) => line.trim()).filter(Boolean)[0] ?? target.trim();
|
|
1309
|
+
}
|
|
1310
|
+
function actionIntent(label) {
|
|
1311
|
+
const normalized = normalizeControlTarget(label);
|
|
1312
|
+
const intentPatterns = [
|
|
1313
|
+
['like', /\blike\b/],
|
|
1314
|
+
['comment', /\bcomment\b/],
|
|
1315
|
+
['share', /\bshare\b/],
|
|
1316
|
+
['save', /\b(save|bookmark)\b/],
|
|
1317
|
+
['follow', /\bfollow\b/],
|
|
1318
|
+
['watchlist', /\bwatchlist\b/],
|
|
1319
|
+
['search', /\bsearch\b/],
|
|
1320
|
+
['create', /\b(create|new)\b/],
|
|
1321
|
+
['filter', /\bfilter\b/],
|
|
1322
|
+
['sort', /\bsort\b/],
|
|
1323
|
+
['dismiss', /\b(close|dismiss|skip)\b/]
|
|
1324
|
+
];
|
|
1325
|
+
const matched = intentPatterns.find(([, pattern]) => pattern.test(normalized));
|
|
1326
|
+
if (matched) {
|
|
1327
|
+
return matched[0];
|
|
1328
|
+
}
|
|
1329
|
+
return normalized.split(/[^a-z0-9]+/).filter(Boolean)[0] ?? 'tap';
|
|
1330
|
+
}
|
|
1331
|
+
function actionScopeForElement(elements, element) {
|
|
1332
|
+
const content = nearestContentScopeElement(elements, element);
|
|
1333
|
+
if (content) {
|
|
1334
|
+
return {
|
|
1335
|
+
kind: 'content',
|
|
1336
|
+
label: content.labels[0]
|
|
1337
|
+
};
|
|
1338
|
+
}
|
|
1339
|
+
const section = nearestActionSectionElement(elements, element);
|
|
1340
|
+
if (section) {
|
|
1341
|
+
return {
|
|
1342
|
+
kind: 'section',
|
|
1343
|
+
label: section.labels[0]
|
|
1344
|
+
};
|
|
1345
|
+
}
|
|
1346
|
+
return { kind: 'screen' };
|
|
1347
|
+
}
|
|
1348
|
+
function nearestContentScopeElement(elements, element) {
|
|
1349
|
+
if (element.y === null) {
|
|
1350
|
+
return null;
|
|
1351
|
+
}
|
|
1352
|
+
return elements
|
|
1353
|
+
.filter((candidate) => candidate !== element && candidate.y !== null && candidate.height !== null)
|
|
1354
|
+
.filter((candidate) => isHighValueContentCard(candidate))
|
|
1355
|
+
.filter((candidate) => {
|
|
1356
|
+
const candidateTop = candidate.y;
|
|
1357
|
+
const candidateBottom = candidateTop + Math.max(0, candidate.height ?? 0);
|
|
1358
|
+
return candidateTop <= element.y && candidateBottom + 96 >= element.y;
|
|
1359
|
+
})
|
|
1360
|
+
.sort((left, right) => (right.y ?? 0) - (left.y ?? 0))[0] ?? null;
|
|
1361
|
+
}
|
|
1362
|
+
function nearestActionSectionElement(elements, element) {
|
|
1363
|
+
if (element.y === null) {
|
|
1364
|
+
return null;
|
|
1365
|
+
}
|
|
1366
|
+
return elements
|
|
1367
|
+
.filter((candidate) => candidate.y !== null && candidate.y < element.y && isActionSectionElement(candidate))
|
|
1368
|
+
.sort((left, right) => (right.y ?? 0) - (left.y ?? 0))[0] ?? null;
|
|
1369
|
+
}
|
|
1370
|
+
function isActionSectionElement(element) {
|
|
1371
|
+
if (isLikelyTapElement(element) || element.labels.length === 0) {
|
|
1372
|
+
return false;
|
|
1373
|
+
}
|
|
1374
|
+
return element.labels.some((label) => {
|
|
1375
|
+
const normalized = normalizeControlTarget(label);
|
|
1376
|
+
if (!/[a-z]/i.test(normalized)) {
|
|
1377
|
+
return false;
|
|
1378
|
+
}
|
|
1379
|
+
const words = normalized.split(/\s+/).filter(Boolean);
|
|
1380
|
+
return words.length <= 8;
|
|
1381
|
+
});
|
|
1382
|
+
}
|
|
1383
|
+
function normalizeControlTarget(target) {
|
|
1384
|
+
return target.replace(/\s+/g, ' ').trim().toLowerCase();
|
|
1385
|
+
}
|
|
1386
|
+
function sectionNavigationHint(value, map) {
|
|
1387
|
+
const normalized = normalizeControlTarget(value);
|
|
1388
|
+
const targets = unique(map.variants.flatMap((variant) => navigationControls(variant).map((control) => control.target)))
|
|
1389
|
+
.sort((left, right) => right.length - left.length);
|
|
1390
|
+
for (const target of targets) {
|
|
1391
|
+
if (normalized.includes(target)) {
|
|
1392
|
+
return target;
|
|
1393
|
+
}
|
|
1394
|
+
}
|
|
1395
|
+
return null;
|
|
1396
|
+
}
|
|
1397
|
+
function crawlCandidateOptions(variant) {
|
|
1398
|
+
const preferredTarget = preferredNavigationTarget(variant);
|
|
1399
|
+
const preferContent = preferredTarget !== null && !isPrimaryNavigationTarget(variant, preferredTarget);
|
|
1400
|
+
return {
|
|
1401
|
+
preferContent,
|
|
1402
|
+
excludeNavigation: preferContent,
|
|
1403
|
+
skipCurrentNavigation: true
|
|
1404
|
+
};
|
|
1405
|
+
}
|
|
1406
|
+
function crawlTemplateKey(variant) {
|
|
1407
|
+
const navigationTarget = preferredNavigationTarget(variant) ?? 'none';
|
|
1408
|
+
const elementKeys = unique(variant.elements.flatMap((element) => crawlTemplateElementKeys(variant, element))).sort();
|
|
1409
|
+
return signatureFor([`nav=${navigationTarget}`, ...elementKeys]);
|
|
1410
|
+
}
|
|
1411
|
+
function crawlTemplateElementKeys(variant, element) {
|
|
1412
|
+
if (element.visible === false || element.stable === false) {
|
|
1413
|
+
return [];
|
|
1414
|
+
}
|
|
1415
|
+
const tag = element.tag.toLowerCase();
|
|
1416
|
+
const layout = `${bucketNumber(element.width, 24)}x${bucketNumber(element.height, 24)}`;
|
|
1417
|
+
if (isScrollableTag(tag)) {
|
|
1418
|
+
return [`scroll:${layout}`];
|
|
1419
|
+
}
|
|
1420
|
+
if (isNavigationElement(variant, element)) {
|
|
1421
|
+
return [`nav-control:${normalizeControlTarget(plainTapTargets(element)[0] ?? element.selector)}:${layout}`];
|
|
1422
|
+
}
|
|
1423
|
+
if (isHighValueContentCard(element)) {
|
|
1424
|
+
return [`content-card:${contentLineCount(element)}:${layout}`];
|
|
1425
|
+
}
|
|
1426
|
+
if (isLikelyTapElement(element)) {
|
|
1427
|
+
return [`tap:${tag}:${templateControlLabel(element)}:${layout}`];
|
|
1428
|
+
}
|
|
1429
|
+
return [];
|
|
1430
|
+
}
|
|
1431
|
+
function templateControlLabel(element) {
|
|
1432
|
+
if (isBackControl(element)) {
|
|
1433
|
+
return 'back';
|
|
1434
|
+
}
|
|
1435
|
+
const target = normalizeControlTarget(plainTapTargets(element)[0] ?? element.selector);
|
|
1436
|
+
const words = target.split(/\s+/).filter(Boolean);
|
|
1437
|
+
if (target && words.length <= 4 && target.length <= 48) {
|
|
1438
|
+
return target;
|
|
1439
|
+
}
|
|
1440
|
+
return 'control';
|
|
1441
|
+
}
|
|
1442
|
+
function repeatedContentCandidateKey(variant, candidate) {
|
|
1443
|
+
const element = elementForTapCandidate(variant, candidate);
|
|
1444
|
+
if (!element || !isHighValueContentCard(element)) {
|
|
1445
|
+
return null;
|
|
1446
|
+
}
|
|
1447
|
+
const section = nearestSectionHeadingLabel(variant, element) ?? 'section=none';
|
|
1448
|
+
const tag = element.tag.toLowerCase();
|
|
1449
|
+
const shape = [
|
|
1450
|
+
crawlTemplateKey(variant),
|
|
1451
|
+
section,
|
|
1452
|
+
tag,
|
|
1453
|
+
`lines=${contentLineCount(element)}`,
|
|
1454
|
+
`size=${bucketNumber(element.width, 24)}x${bucketNumber(element.height, 24)}`
|
|
1455
|
+
];
|
|
1456
|
+
return signatureFor(shape);
|
|
1457
|
+
}
|
|
1458
|
+
function elementForTapCandidate(variant, candidate) {
|
|
1459
|
+
const candidateCoordinate = typeof candidate.args.x === 'number' && typeof candidate.args.y === 'number'
|
|
1460
|
+
? coordinateTarget(candidate.args)
|
|
1461
|
+
: null;
|
|
1462
|
+
for (const element of variant.elements) {
|
|
1463
|
+
if (candidateCoordinate) {
|
|
1464
|
+
const elementCoordinate = coordinateArgsForElement(element);
|
|
1465
|
+
if (elementCoordinate && coordinateTarget(elementCoordinate) === candidateCoordinate) {
|
|
1466
|
+
return element;
|
|
1467
|
+
}
|
|
1468
|
+
}
|
|
1469
|
+
const target = plainTapTargets(element)[0] ?? element.selector;
|
|
1470
|
+
if (target === candidate.target || element.targets.includes(candidate.target)) {
|
|
1471
|
+
return element;
|
|
1472
|
+
}
|
|
1473
|
+
}
|
|
1474
|
+
return null;
|
|
1475
|
+
}
|
|
1476
|
+
function nearestSectionHeadingLabel(variant, element) {
|
|
1477
|
+
if (element.y === null) {
|
|
1478
|
+
return null;
|
|
1479
|
+
}
|
|
1480
|
+
const heading = variant.elements
|
|
1481
|
+
.filter((candidate) => candidate.y !== null && candidate.y < element.y && isLikelySectionHeading(variant, candidate))
|
|
1482
|
+
.sort((left, right) => (right.y ?? 0) - (left.y ?? 0))[0];
|
|
1483
|
+
const label = heading?.labels[0];
|
|
1484
|
+
return label ? `section=${normalizeControlTarget(label)}` : null;
|
|
1485
|
+
}
|
|
1486
|
+
function contentLineCount(element) {
|
|
1487
|
+
const lines = unique(element.labels.flatMap((label) => label.split(/\n+/).map((line) => line.trim()).filter(Boolean)));
|
|
1488
|
+
return lines.length;
|
|
1489
|
+
}
|
|
1490
|
+
function bucketNumber(value, size) {
|
|
1491
|
+
if (value === null || !Number.isFinite(value)) {
|
|
1492
|
+
return 0;
|
|
1493
|
+
}
|
|
1494
|
+
return Math.round(value / size) * size;
|
|
1495
|
+
}
|
|
1496
|
+
function crawlScrollCandidates(context, variant) {
|
|
1497
|
+
if (!context.adapter.capability().commands.includes('scroll') || !shouldExploreScroll(variant)) {
|
|
1498
|
+
return [];
|
|
1499
|
+
}
|
|
1500
|
+
const args = { direction: 'down', percent: 70 };
|
|
1501
|
+
return [
|
|
1502
|
+
{
|
|
1503
|
+
target: scrollTarget(args),
|
|
1504
|
+
args,
|
|
1505
|
+
key: 'scroll:down:70'
|
|
1506
|
+
}
|
|
1507
|
+
];
|
|
1508
|
+
}
|
|
1509
|
+
function shouldExploreScroll(variant) {
|
|
1510
|
+
return hasScrollableSurface(variant) || hasBottomReachableContent(variant);
|
|
1511
|
+
}
|
|
1512
|
+
function hasScrollableSurface(variant) {
|
|
1513
|
+
return elementsHaveScrollableSurface(variant.elements);
|
|
1514
|
+
}
|
|
1515
|
+
function elementsHaveScrollableSurface(elements) {
|
|
1516
|
+
return elements.some((element) => isScrollableTag(element.tag));
|
|
1517
|
+
}
|
|
1518
|
+
function hasBottomReachableContent(variant) {
|
|
1519
|
+
const content = scrollProbeElements(variant);
|
|
1520
|
+
if (content.length < 3) {
|
|
1521
|
+
return false;
|
|
1522
|
+
}
|
|
1523
|
+
const navControls = navigationControls(variant);
|
|
1524
|
+
const navTop = navControls
|
|
1525
|
+
.map((control) => control.element.y)
|
|
1526
|
+
.filter((value) => value !== null)
|
|
1527
|
+
.sort((left, right) => left - right)[0] ?? null;
|
|
1528
|
+
if (navTop === null) {
|
|
1529
|
+
return false;
|
|
1530
|
+
}
|
|
1531
|
+
const viewportBottom = navTop ?? Math.max(...content.map((element) => (element.y ?? 0) + (element.height ?? 0)));
|
|
1532
|
+
return content.some((element) => ((element.y ?? 0) + (element.height ?? 0)) >= viewportBottom - 48);
|
|
1533
|
+
}
|
|
1534
|
+
function scrollProbeElements(variant) {
|
|
1535
|
+
const navigationElements = new Set(navigationControls(variant).map((control) => control.element));
|
|
1536
|
+
return variant.elements.filter((element) => {
|
|
1537
|
+
if (element.visible === false || element.y === null || element.height === null || navigationElements.has(element)) {
|
|
1538
|
+
return false;
|
|
1539
|
+
}
|
|
1540
|
+
if (isBackControl(element)) {
|
|
1541
|
+
return false;
|
|
1542
|
+
}
|
|
1543
|
+
const tag = element.tag.toLowerCase();
|
|
1544
|
+
return !isScrollableTag(tag);
|
|
1545
|
+
});
|
|
1546
|
+
}
|
|
1547
|
+
function preferredNavigationTarget(variant) {
|
|
1548
|
+
return preferredNavigationTargetForElements(variant.elements);
|
|
1549
|
+
}
|
|
1550
|
+
function preferredNavigationTargetForElements(elements) {
|
|
1551
|
+
const controls = navigationControlsForElements(elements);
|
|
1552
|
+
if (controls.length === 0) {
|
|
1553
|
+
return null;
|
|
1554
|
+
}
|
|
1555
|
+
const labels = contentLabelsForElements(elements);
|
|
1556
|
+
const matched = controls.find((control) => labels.some((label) => labelMatchesNavigationTarget(label, control.target)));
|
|
1557
|
+
if (matched) {
|
|
1558
|
+
return matched.target;
|
|
1559
|
+
}
|
|
1560
|
+
return controls[0]?.target ?? null;
|
|
1561
|
+
}
|
|
1562
|
+
function hasBottomNavigation(variant) {
|
|
1563
|
+
return navigationControls(variant).length >= 2;
|
|
1564
|
+
}
|
|
1565
|
+
function contentLabels(variant) {
|
|
1566
|
+
return contentLabelsForElements(variant.elements);
|
|
1567
|
+
}
|
|
1568
|
+
function contentLabelsForElements(elements) {
|
|
1569
|
+
const navigationElements = new Set(navigationControlsForElements(elements).map((control) => control.element));
|
|
1570
|
+
return elements.flatMap((element) => element.labels
|
|
1571
|
+
.filter(() => !navigationElements.has(element))
|
|
1572
|
+
.filter((label) => !isGlobalIdentityLabel(label, element))
|
|
1573
|
+
.map(normalizeControlTarget));
|
|
1574
|
+
}
|
|
1575
|
+
function navigationControls(variant) {
|
|
1576
|
+
return navigationControlsForElements(variant.elements);
|
|
1577
|
+
}
|
|
1578
|
+
function navigationControlsForElements(elements) {
|
|
1579
|
+
const candidates = elements
|
|
1580
|
+
.filter((element) => element.safety === 'safe' && element.stable && isLikelyTapElement(element))
|
|
1581
|
+
.filter((element) => !isBackControl(element))
|
|
1582
|
+
.filter((element) => coordinateArgsForElement(element) !== null)
|
|
1583
|
+
.map((element) => ({ element, target: normalizeControlTarget(plainTapTargets(element)[0] ?? element.selector) }))
|
|
1584
|
+
.filter((candidate) => isCompactNavigationLabel(candidate.element, candidate.target));
|
|
1585
|
+
if (candidates.length < 2) {
|
|
1586
|
+
return [];
|
|
1587
|
+
}
|
|
1588
|
+
const maxY = Math.max(...candidates.map((candidate) => candidate.element.y ?? 0));
|
|
1589
|
+
const bottomRow = candidates
|
|
1590
|
+
.filter((candidate) => (candidate.element.y ?? 0) >= maxY - 12)
|
|
1591
|
+
.sort((left, right) => (left.element.x ?? 0) - (right.element.x ?? 0));
|
|
1592
|
+
if (bottomRow.length < 2) {
|
|
1593
|
+
return [];
|
|
1594
|
+
}
|
|
1595
|
+
return bottomRow.map((candidate, priority) => ({
|
|
1596
|
+
...candidate,
|
|
1597
|
+
priority
|
|
1598
|
+
}));
|
|
1599
|
+
}
|
|
1600
|
+
function isCompactNavigationLabel(element, target) {
|
|
1601
|
+
if (!target || target.includes('\n') || target.length > 32 || target === 'see all') {
|
|
1602
|
+
return false;
|
|
1603
|
+
}
|
|
1604
|
+
if ((element.height ?? 0) > 72) {
|
|
1605
|
+
return false;
|
|
1606
|
+
}
|
|
1607
|
+
const lines = unique(element.labels.flatMap((label) => label.split(/\n+/).map((line) => normalizeControlTarget(line))));
|
|
1608
|
+
return lines.length <= 2 && lines.some((line) => line === target);
|
|
1609
|
+
}
|
|
1610
|
+
function isNavigationElement(variant, element) {
|
|
1611
|
+
return navigationControls(variant).some((control) => control.element === element);
|
|
1612
|
+
}
|
|
1613
|
+
function isNavigationTarget(variant, normalizedTarget) {
|
|
1614
|
+
return navigationControls(variant).some((control) => control.target === normalizedTarget);
|
|
1615
|
+
}
|
|
1616
|
+
function navigationControlPriority(variant, normalizedTarget) {
|
|
1617
|
+
return navigationControls(variant).find((control) => control.target === normalizedTarget)?.priority;
|
|
1618
|
+
}
|
|
1619
|
+
function isPrimaryNavigationTarget(variant, normalizedTarget) {
|
|
1620
|
+
return navigationControls(variant)[0]?.target === normalizedTarget;
|
|
1621
|
+
}
|
|
1622
|
+
function labelMatchesNavigationTarget(label, target) {
|
|
1623
|
+
if (label === target) {
|
|
1624
|
+
return true;
|
|
1625
|
+
}
|
|
1626
|
+
const labelWords = new Set(label.split(/[^a-z0-9]+/).filter(Boolean));
|
|
1627
|
+
const targetWords = target.split(/[^a-z0-9]+/).filter(Boolean);
|
|
1628
|
+
if (targetWords.length === 1) {
|
|
1629
|
+
const words = Array.from(labelWords);
|
|
1630
|
+
return words.length <= 2 && words[0] === targetWords[0];
|
|
1631
|
+
}
|
|
1632
|
+
return targetWords.length > 0 && targetWords.every((word) => labelWords.has(word));
|
|
1633
|
+
}
|
|
1634
|
+
function currentScreenTapArgs(variant, descriptor) {
|
|
1635
|
+
if (descriptor.kind === 'section-first') {
|
|
1636
|
+
return sectionFirstTapArgs(variant, descriptor);
|
|
1637
|
+
}
|
|
1638
|
+
const action = matchingAction(variant, descriptor);
|
|
1639
|
+
if (action) {
|
|
1640
|
+
return structuredClone(action.args);
|
|
1641
|
+
}
|
|
1642
|
+
for (const element of variant.elements) {
|
|
1643
|
+
if (!elementContainsTarget(element, descriptor)) {
|
|
1644
|
+
continue;
|
|
1645
|
+
}
|
|
1646
|
+
if (!(element.safety === 'safe' && element.stable && isLikelyTapElement(element))) {
|
|
1647
|
+
continue;
|
|
1648
|
+
}
|
|
1649
|
+
const coordinateArgs = coordinateArgsForElement(element);
|
|
1650
|
+
if (coordinateArgs) {
|
|
1651
|
+
return coordinateArgs;
|
|
1652
|
+
}
|
|
1653
|
+
const stableTarget = plainTapTargets(element)[0] ?? element.selector;
|
|
1654
|
+
if (stableTarget && shouldUseStableTargetFallback(element, descriptor)) {
|
|
1655
|
+
return { target: stableTarget };
|
|
1656
|
+
}
|
|
1657
|
+
}
|
|
1658
|
+
return null;
|
|
1659
|
+
}
|
|
1660
|
+
function currentScreenLiveTapArgs(variant, descriptor) {
|
|
1661
|
+
const action = matchingAction(variant, descriptor);
|
|
1662
|
+
return action && shouldUseActionForLiveSemanticTap(action)
|
|
1663
|
+
? structuredClone(action.args)
|
|
1664
|
+
: null;
|
|
1665
|
+
}
|
|
1666
|
+
function matchingAction(variant, descriptor) {
|
|
1667
|
+
return variant.actions.find((candidate) => actionMatchesDescriptor(candidate, descriptor));
|
|
1668
|
+
}
|
|
1669
|
+
function actionMatchesDescriptor(action, descriptor) {
|
|
1670
|
+
if (action.command !== 'tap' || action.safety !== 'safe') {
|
|
1671
|
+
return false;
|
|
1672
|
+
}
|
|
1673
|
+
if (descriptor.kind === 'text-contains') {
|
|
1674
|
+
return false;
|
|
1675
|
+
}
|
|
1676
|
+
const target = descriptor.target ?? '';
|
|
1677
|
+
const value = descriptor.value ?? target;
|
|
1678
|
+
const normalizedValue = normalizeControlTarget(value);
|
|
1679
|
+
const normalizedTarget = normalizeControlTarget(target);
|
|
1680
|
+
const normalizedActionLabel = normalizeControlTarget(action.label);
|
|
1681
|
+
const normalizedActionTarget = normalizeControlTarget(action.target ?? '');
|
|
1682
|
+
if (!normalizedValue && !normalizedTarget) {
|
|
1683
|
+
return false;
|
|
1684
|
+
}
|
|
1685
|
+
return [normalizedActionLabel, normalizedActionTarget].some((candidate) => candidate !== '' && (candidate === normalizedValue || candidate === normalizedTarget));
|
|
1686
|
+
}
|
|
1687
|
+
function shouldUseActionForLiveSemanticTap(action) {
|
|
1688
|
+
return (typeof action.args.x === 'number' &&
|
|
1689
|
+
typeof action.args.y === 'number' &&
|
|
1690
|
+
action.source.tag.toLowerCase().includes('xcuielementtype'));
|
|
1691
|
+
}
|
|
1692
|
+
function shouldUseStableTargetFallback(element, descriptor) {
|
|
1693
|
+
if (descriptor.kind !== 'text-contains') {
|
|
1694
|
+
return false;
|
|
1695
|
+
}
|
|
1696
|
+
const value = descriptor.value ?? '';
|
|
1697
|
+
return value !== '' && !element.labels.some((label) => label.includes(value));
|
|
1698
|
+
}
|
|
1699
|
+
function sectionFirstTapArgs(variant, descriptor) {
|
|
1700
|
+
const element = firstItemInSection(variant, descriptor);
|
|
1701
|
+
return element ? coordinateArgsForElement(element) : null;
|
|
1702
|
+
}
|
|
1703
|
+
function firstItemInSection(variant, descriptor) {
|
|
1704
|
+
const value = descriptor.value ?? descriptor.target ?? '';
|
|
1705
|
+
if (!/[a-z]/i.test(value)) {
|
|
1706
|
+
return null;
|
|
1707
|
+
}
|
|
1708
|
+
const section = sectionHeadingElement(variant, value);
|
|
1709
|
+
if (!section || section.y === null || section.height === null) {
|
|
1710
|
+
return null;
|
|
1711
|
+
}
|
|
1712
|
+
const sectionY = section.y;
|
|
1713
|
+
const sectionBottom = sectionY + Math.max(0, section.height);
|
|
1714
|
+
const nextSectionY = variant.elements
|
|
1715
|
+
.filter((element) => element !== section &&
|
|
1716
|
+
element.y !== null &&
|
|
1717
|
+
element.y > sectionY &&
|
|
1718
|
+
isLikelySectionHeading(variant, element))
|
|
1719
|
+
.map((element) => element.y)
|
|
1720
|
+
.sort((left, right) => left - right)[0] ?? null;
|
|
1721
|
+
const candidates = variant.elements
|
|
1722
|
+
.filter((element) => isSectionItemCandidate(variant, element, value) &&
|
|
1723
|
+
element.y !== null &&
|
|
1724
|
+
element.y > sectionBottom &&
|
|
1725
|
+
(nextSectionY === null || element.y < nextSectionY))
|
|
1726
|
+
.sort((left, right) => (left.y ?? 0) - (right.y ?? 0) || (left.x ?? 0) - (right.x ?? 0));
|
|
1727
|
+
return candidates[0] ?? null;
|
|
1728
|
+
}
|
|
1729
|
+
function sectionHeadingElement(variant, value) {
|
|
1730
|
+
const matches = variant.elements
|
|
1731
|
+
.filter((element) => element.visible !== false &&
|
|
1732
|
+
element.y !== null &&
|
|
1733
|
+
element.height !== null &&
|
|
1734
|
+
elementLabelMatchesValue(element, value))
|
|
1735
|
+
.sort((left, right) => (left.y ?? 0) - (right.y ?? 0) || (left.x ?? 0) - (right.x ?? 0));
|
|
1736
|
+
return matches[0] ?? null;
|
|
1737
|
+
}
|
|
1738
|
+
function isSectionItemCandidate(variant, element, sectionValue) {
|
|
1739
|
+
if (!(element.safety === 'safe' && element.stable && isLikelyTapElement(element))) {
|
|
1740
|
+
return false;
|
|
1741
|
+
}
|
|
1742
|
+
if (!coordinateArgsForElement(element)) {
|
|
1743
|
+
return false;
|
|
1744
|
+
}
|
|
1745
|
+
if (isBackControl(element) || elementLabelMatchesValue(element, sectionValue)) {
|
|
1746
|
+
return false;
|
|
1747
|
+
}
|
|
1748
|
+
const target = normalizeControlTarget(plainTapTargets(element)[0] ?? element.selector);
|
|
1749
|
+
if (isNavigationTarget(variant, target) || target === 'see all') {
|
|
1750
|
+
return false;
|
|
1751
|
+
}
|
|
1752
|
+
return true;
|
|
1753
|
+
}
|
|
1754
|
+
function isLikelySectionHeading(variant, element) {
|
|
1755
|
+
if (isHighValueContentCard(element)) {
|
|
1756
|
+
return false;
|
|
1757
|
+
}
|
|
1758
|
+
const labels = element.labels.map(normalizeControlTarget);
|
|
1759
|
+
if (labels.some((label) => label === 'see all' || isNavigationTarget(variant, label))) {
|
|
1760
|
+
return false;
|
|
1761
|
+
}
|
|
1762
|
+
return labels.some((label) => {
|
|
1763
|
+
if (!/[a-z]/i.test(label)) {
|
|
1764
|
+
return false;
|
|
1765
|
+
}
|
|
1766
|
+
const words = label.split(/\s+/).filter(Boolean);
|
|
1767
|
+
const lineCount = label.split(/\n+/).filter(Boolean).length;
|
|
1768
|
+
return words.length <= 8 && lineCount <= 2;
|
|
1769
|
+
});
|
|
1770
|
+
}
|
|
1771
|
+
function elementLabelMatchesValue(element, value) {
|
|
1772
|
+
const normalizedValue = normalizeControlTarget(value);
|
|
1773
|
+
if (!normalizedValue) {
|
|
1774
|
+
return false;
|
|
1775
|
+
}
|
|
1776
|
+
return element.labels.some((label) => {
|
|
1777
|
+
const normalizedLabel = normalizeControlTarget(label);
|
|
1778
|
+
return (normalizedLabel === normalizedValue ||
|
|
1779
|
+
normalizedLabel.includes(normalizedValue) ||
|
|
1780
|
+
label.split(/\n+/).some((line) => normalizeControlTarget(line) === normalizedValue));
|
|
1781
|
+
});
|
|
1782
|
+
}
|
|
1783
|
+
function coordinateArgsForElement(element) {
|
|
1784
|
+
if (element.x === null ||
|
|
1785
|
+
element.y === null ||
|
|
1786
|
+
element.width === null ||
|
|
1787
|
+
element.height === null ||
|
|
1788
|
+
element.width <= 0 ||
|
|
1789
|
+
element.height <= 0) {
|
|
1790
|
+
return null;
|
|
1791
|
+
}
|
|
1792
|
+
if (isBackControl(element)) {
|
|
1793
|
+
return {
|
|
1794
|
+
x: Math.round(element.x + Math.min(40, element.width / 2)),
|
|
1795
|
+
y: Math.round(element.y + element.height / 2)
|
|
1796
|
+
};
|
|
1797
|
+
}
|
|
1798
|
+
return {
|
|
1799
|
+
x: Math.round(element.x + element.width / 2),
|
|
1800
|
+
y: Math.round(element.y + element.height / 2)
|
|
1801
|
+
};
|
|
1802
|
+
}
|
|
1803
|
+
function isBackControl(element) {
|
|
1804
|
+
if (!element.tag.toLowerCase().includes('button')) {
|
|
1805
|
+
return false;
|
|
1806
|
+
}
|
|
1807
|
+
return element.labels.some((label) => label.split(/\n+/).some((line) => line.trim().toLowerCase() === 'back'));
|
|
1808
|
+
}
|
|
1809
|
+
function plainTapTargets(element) {
|
|
1810
|
+
return element.targets
|
|
1811
|
+
.filter((target) => !target.includes('='))
|
|
1812
|
+
.sort((left, right) => tapTargetScore(left) - tapTargetScore(right));
|
|
1813
|
+
}
|
|
1814
|
+
function tapTargetScore(target) {
|
|
1815
|
+
let score = target.length;
|
|
1816
|
+
if (target.includes('\n')) {
|
|
1817
|
+
score += 1000;
|
|
1818
|
+
}
|
|
1819
|
+
return score;
|
|
1820
|
+
}
|
|
1821
|
+
function isLikelyTapElement(element) {
|
|
1822
|
+
if (element.enabled === false || element.visible === false) {
|
|
1823
|
+
return false;
|
|
1824
|
+
}
|
|
1825
|
+
if (element.width !== null &&
|
|
1826
|
+
element.height !== null &&
|
|
1827
|
+
(element.width <= 0 || element.height <= 0)) {
|
|
1828
|
+
return false;
|
|
1829
|
+
}
|
|
1830
|
+
const tag = element.tag.toLowerCase();
|
|
1831
|
+
if (tag.includes('statictext') ||
|
|
1832
|
+
tag.includes('image') ||
|
|
1833
|
+
tag.includes('textfield') ||
|
|
1834
|
+
isScrollableTag(tag)) {
|
|
1835
|
+
return false;
|
|
1836
|
+
}
|
|
1837
|
+
if (tag.includes('button') || element.clickable) {
|
|
1838
|
+
return true;
|
|
1839
|
+
}
|
|
1840
|
+
if (tag.includes('other')) {
|
|
1841
|
+
const width = element.width ?? 0;
|
|
1842
|
+
const height = element.height ?? 0;
|
|
1843
|
+
return width >= 20 && height >= 20 && element.labels.length > 0;
|
|
1844
|
+
}
|
|
1845
|
+
return false;
|
|
1846
|
+
}
|
|
1847
|
+
async function repairToTarget(context, current, descriptor, depth, deadline, blockedTargets) {
|
|
1848
|
+
if (Date.now() > deadline || depth <= 0) {
|
|
1849
|
+
return null;
|
|
1850
|
+
}
|
|
1851
|
+
const attemptedTargets = new Map();
|
|
1852
|
+
const attemptFrom = async (start, remainingDepth, route) => {
|
|
1853
|
+
let cursor = start;
|
|
1854
|
+
let depthLeft = remainingDepth;
|
|
1855
|
+
let routeSoFar = route;
|
|
1856
|
+
while (Date.now() <= deadline) {
|
|
1857
|
+
if (variantContainsTarget(cursor.variant, descriptor)) {
|
|
1858
|
+
return { found: true, current: cursor, route: routeSoFar, remainingDepth: depthLeft };
|
|
1859
|
+
}
|
|
1860
|
+
if (depthLeft <= 0) {
|
|
1861
|
+
return { found: false, current: cursor, route: routeSoFar, remainingDepth: depthLeft };
|
|
1862
|
+
}
|
|
1863
|
+
const attemptsForVariant = attemptedTargets.get(cursor.variant.id) ?? new Set();
|
|
1864
|
+
attemptedTargets.set(cursor.variant.id, attemptsForVariant);
|
|
1865
|
+
const candidate = safeTapCandidates(cursor.variant).find((tapCandidate) => !blockedTargets.has(tapCandidate.target) &&
|
|
1866
|
+
!blockedTargets.has(tapCandidate.key) &&
|
|
1867
|
+
!attemptsForVariant.has(tapCandidate.key));
|
|
1868
|
+
if (!candidate) {
|
|
1869
|
+
return { found: false, current: cursor, route: routeSoFar, remainingDepth: depthLeft };
|
|
1870
|
+
}
|
|
1871
|
+
attemptsForVariant.add(candidate.key);
|
|
1872
|
+
const args = candidate.args;
|
|
1873
|
+
let observed;
|
|
1874
|
+
try {
|
|
1875
|
+
await context.adapter.tap(args);
|
|
1876
|
+
observed = await observeCurrentScreen(context);
|
|
1877
|
+
}
|
|
1878
|
+
catch {
|
|
1879
|
+
try {
|
|
1880
|
+
cursor = await observeCurrentScreen(context);
|
|
1881
|
+
}
|
|
1882
|
+
catch {
|
|
1883
|
+
// If both the exploratory tap and source read fail, keep the last observed screen.
|
|
1884
|
+
}
|
|
1885
|
+
continue;
|
|
1886
|
+
}
|
|
1887
|
+
recordEdgeSuccess(context, cursor.variant, observed.variant, 'tap', args, true);
|
|
1888
|
+
const nextRoute = [...routeSoFar, summarizeControl('tap', candidate.target, 0.45)];
|
|
1889
|
+
const nested = await attemptFrom(observed, depthLeft - 1, nextRoute);
|
|
1890
|
+
if (nested.found) {
|
|
1891
|
+
return nested;
|
|
1892
|
+
}
|
|
1893
|
+
if (nested.current.variant.id === cursor.variant.id) {
|
|
1894
|
+
cursor = nested.current;
|
|
1895
|
+
routeSoFar = nested.route;
|
|
1896
|
+
continue;
|
|
1897
|
+
}
|
|
1898
|
+
cursor = nested.current;
|
|
1899
|
+
depthLeft = nested.remainingDepth;
|
|
1900
|
+
routeSoFar = nested.route;
|
|
1901
|
+
}
|
|
1902
|
+
return { found: false, current: cursor, route: routeSoFar, remainingDepth: depthLeft };
|
|
1903
|
+
};
|
|
1904
|
+
const attempt = await attemptFrom(current, depth, []);
|
|
1905
|
+
return attempt.found ? { current: attempt.current, route: attempt.route } : null;
|
|
1906
|
+
}
|
|
1907
|
+
export async function runMappedCommand(context, command, args) {
|
|
1908
|
+
const descriptor = targetDescriptor(command, args);
|
|
1909
|
+
const metadata = {
|
|
1910
|
+
enabled: true,
|
|
1911
|
+
routed: false,
|
|
1912
|
+
route: [],
|
|
1913
|
+
repaired: false,
|
|
1914
|
+
repairs: 0,
|
|
1915
|
+
path: context.filePath
|
|
1916
|
+
};
|
|
1917
|
+
if (descriptor.kind === 'unknown') {
|
|
1918
|
+
const details = await context.adapter[command](args);
|
|
1919
|
+
return { ...details, map: metadata };
|
|
1920
|
+
}
|
|
1921
|
+
let before = await observeCurrentScreen(context);
|
|
1922
|
+
if (descriptor.kind === 'coordinate' || command === 'scroll') {
|
|
1923
|
+
const details = await context.adapter[command](args);
|
|
1924
|
+
const after = await observeCurrentScreen(context);
|
|
1925
|
+
assertObservedEffect(command, args, before, after, descriptor);
|
|
1926
|
+
recordEdgeSuccess(context, before.variant, after.variant, command, args, false);
|
|
1927
|
+
return { ...details, map: metadata };
|
|
1928
|
+
}
|
|
1929
|
+
let liveTargetVisible = await liveContainsTarget(context, descriptor);
|
|
1930
|
+
if (!variantContainsTarget(before.variant, descriptor) && !liveTargetVisible) {
|
|
1931
|
+
if (before.variant.auth_required) {
|
|
1932
|
+
throw new Error(`Target '${descriptor.target ?? descriptor.value}' requires authentication from the current screen`);
|
|
1933
|
+
}
|
|
1934
|
+
if (routeableDescriptor(descriptor)) {
|
|
1935
|
+
const route = planRoute(context.map, before.variant.id, descriptor);
|
|
1936
|
+
if (route.ambiguous) {
|
|
1937
|
+
throw new Error(`Target '${descriptor.target ?? descriptor.value}' is ambiguous in the app map`);
|
|
1938
|
+
}
|
|
1939
|
+
if (route.authRequired) {
|
|
1940
|
+
throw new Error(`Target '${descriptor.target ?? descriptor.value}' is behind an auth-required screen`);
|
|
1941
|
+
}
|
|
1942
|
+
if (route.edges.length > 0) {
|
|
1943
|
+
context.summary.used = true;
|
|
1944
|
+
metadata.routed = true;
|
|
1945
|
+
metadata.route = route.edges.map(summarizeEdge);
|
|
1946
|
+
if (route.diagnostics) {
|
|
1947
|
+
metadata.diagnostics = route.diagnostics;
|
|
1948
|
+
}
|
|
1949
|
+
const driven = await driveRoute(context, before, route.edges);
|
|
1950
|
+
before = driven.current;
|
|
1951
|
+
liveTargetVisible = await liveContainsTarget(context, descriptor);
|
|
1952
|
+
const targetReachedDespiteStaleContract = variantContainsTarget(before.variant, descriptor) || liveTargetVisible;
|
|
1953
|
+
if (driven.failedEdge && !targetReachedDespiteStaleContract) {
|
|
1954
|
+
if (!context.options.repair) {
|
|
1955
|
+
throw new Error(`Cached app-map route for '${driven.failedEdge.target ?? driven.failedEdge.command}' failed; rerun with repair enabled to explore an alternate route`);
|
|
1956
|
+
}
|
|
1957
|
+
const repair = await repairToTarget(context, before, descriptor, context.options.repairDepth, Date.now() + context.options.repairTimeoutMs, new Set(driven.failedEdge.target ? [driven.failedEdge.target] : []));
|
|
1958
|
+
if (!repair) {
|
|
1959
|
+
throw new Error(`Cached app-map route for '${driven.failedEdge.target ?? driven.failedEdge.command}' could not be repaired within the bounded repair budget`);
|
|
1960
|
+
}
|
|
1961
|
+
context.summary.repaired = true;
|
|
1962
|
+
context.summary.repairs += 1;
|
|
1963
|
+
metadata.repaired = true;
|
|
1964
|
+
metadata.repairs = context.summary.repairs;
|
|
1965
|
+
metadata.route = [...metadata.route, ...repair.route];
|
|
1966
|
+
before = repair.current;
|
|
1967
|
+
liveTargetVisible = await liveContainsTarget(context, descriptor);
|
|
1968
|
+
}
|
|
1969
|
+
}
|
|
1970
|
+
else if (context.options.repair) {
|
|
1971
|
+
const repair = await repairToTarget(context, before, descriptor, context.options.repairDepth, Date.now() + context.options.repairTimeoutMs, new Set());
|
|
1972
|
+
if (repair) {
|
|
1973
|
+
context.summary.used = true;
|
|
1974
|
+
context.summary.repaired = true;
|
|
1975
|
+
context.summary.repairs += 1;
|
|
1976
|
+
metadata.routed = repair.route.length > 0;
|
|
1977
|
+
metadata.repaired = true;
|
|
1978
|
+
metadata.repairs = context.summary.repairs;
|
|
1979
|
+
metadata.route = repair.route;
|
|
1980
|
+
before = repair.current;
|
|
1981
|
+
liveTargetVisible = await liveContainsTarget(context, descriptor);
|
|
1982
|
+
}
|
|
1983
|
+
}
|
|
1984
|
+
}
|
|
1985
|
+
}
|
|
1986
|
+
if (!variantContainsTarget(before.variant, descriptor) && !liveTargetVisible) {
|
|
1987
|
+
throw new Error(`Target '${descriptor.target ?? descriptor.value}' is not reachable from the current app-map state`);
|
|
1988
|
+
}
|
|
1989
|
+
const executableArgs = command === 'tap'
|
|
1990
|
+
? liveTargetVisible
|
|
1991
|
+
? currentScreenLiveTapArgs(before.variant, descriptor) ?? args
|
|
1992
|
+
: currentScreenTapArgs(before.variant, descriptor) ?? args
|
|
1993
|
+
: args;
|
|
1994
|
+
const details = await context.adapter[command](executableArgs);
|
|
1995
|
+
const after = await observeCurrentScreen(context);
|
|
1996
|
+
assertObservedEffect(command, executableArgs, before, after, descriptor);
|
|
1997
|
+
recordEdgeSuccess(context, before.variant, after.variant, command, executableArgs, false);
|
|
1998
|
+
return { ...details, map: metadata };
|
|
1999
|
+
}
|
|
2000
|
+
function restoreAttemptDiagnostic(strategy, result, observed, match, target, command, error) {
|
|
2001
|
+
return {
|
|
2002
|
+
strategy,
|
|
2003
|
+
result,
|
|
2004
|
+
...(command ? { command } : {}),
|
|
2005
|
+
...(target ? { target } : {}),
|
|
2006
|
+
...(observed
|
|
2007
|
+
? {
|
|
2008
|
+
observed_variant_id: observed.variant.id,
|
|
2009
|
+
observed_screen_id: observed.variant.screen_id
|
|
2010
|
+
}
|
|
2011
|
+
: {}),
|
|
2012
|
+
...(match ? { accepted_by: match.acceptedBy } : {}),
|
|
2013
|
+
...(error instanceof Error ? { error: error.message } : {})
|
|
2014
|
+
};
|
|
2015
|
+
}
|
|
2016
|
+
function makeRestoreDiagnostic(from, target) {
|
|
2017
|
+
return {
|
|
2018
|
+
from_variant_id: from.variant.id,
|
|
2019
|
+
target_variant_id: target.variant.id,
|
|
2020
|
+
result: 'failed',
|
|
2021
|
+
attempts: []
|
|
2022
|
+
};
|
|
2023
|
+
}
|
|
2024
|
+
async function tryTapRestoreCandidate(context, current, target, candidate, strategy, diagnostic) {
|
|
2025
|
+
const args = candidate.args;
|
|
2026
|
+
try {
|
|
2027
|
+
await context.adapter.tap(args);
|
|
2028
|
+
const observed = await observeAfterAction(context, target);
|
|
2029
|
+
recordEdgeSuccess(context, current.variant, observed.variant, 'tap', args, true);
|
|
2030
|
+
const match = restoreMatch(observed, target);
|
|
2031
|
+
diagnostic.attempts.push(restoreAttemptDiagnostic(strategy, match ? 'matched' : 'mismatched', observed, match, targetDescriptor('tap', args).target ?? candidate.target, 'tap'));
|
|
2032
|
+
return { current: observed, match };
|
|
2033
|
+
}
|
|
2034
|
+
catch (error) {
|
|
2035
|
+
diagnostic.attempts.push(restoreAttemptDiagnostic(strategy, 'error', undefined, null, candidate.target, 'tap', error));
|
|
2036
|
+
try {
|
|
2037
|
+
return { current: await observeCurrentScreen(context), match: null };
|
|
2038
|
+
}
|
|
2039
|
+
catch {
|
|
2040
|
+
return { current, match: null };
|
|
2041
|
+
}
|
|
2042
|
+
}
|
|
2043
|
+
}
|
|
2044
|
+
async function tryScrollRestoreCandidate(context, current, target, candidate, diagnostic) {
|
|
2045
|
+
const args = candidate.args;
|
|
2046
|
+
try {
|
|
2047
|
+
await context.adapter.scroll(args);
|
|
2048
|
+
const observed = await observeAfterAction(context, target);
|
|
2049
|
+
recordEdgeSuccess(context, current.variant, observed.variant, 'scroll', args, true);
|
|
2050
|
+
const match = restoreMatch(observed, target);
|
|
2051
|
+
diagnostic.attempts.push(restoreAttemptDiagnostic('reverse-scroll', match ? 'matched' : 'mismatched', observed, match, candidate.target, 'scroll'));
|
|
2052
|
+
return { current: observed, match };
|
|
2053
|
+
}
|
|
2054
|
+
catch (error) {
|
|
2055
|
+
diagnostic.attempts.push(restoreAttemptDiagnostic('reverse-scroll', 'error', undefined, null, candidate.target, 'scroll', error));
|
|
2056
|
+
try {
|
|
2057
|
+
return { current: await observeCurrentScreen(context), match: null };
|
|
2058
|
+
}
|
|
2059
|
+
catch {
|
|
2060
|
+
return { current, match: null };
|
|
2061
|
+
}
|
|
2062
|
+
}
|
|
2063
|
+
}
|
|
2064
|
+
function targetRestoreCandidates(current, target) {
|
|
2065
|
+
const preferredTarget = preferredNavigationTarget(target.variant);
|
|
2066
|
+
return safeTapCandidates(current.variant).filter((candidate) => {
|
|
2067
|
+
if (/^(back|close|dismiss|done)$/i.test(candidate.target)) {
|
|
2068
|
+
return false;
|
|
2069
|
+
}
|
|
2070
|
+
const normalizedTarget = normalizeControlTarget(candidate.target);
|
|
2071
|
+
if (preferredTarget) {
|
|
2072
|
+
return normalizedTarget === preferredTarget;
|
|
2073
|
+
}
|
|
2074
|
+
if (isGlobalIdentityLabel(candidate.target)) {
|
|
2075
|
+
return false;
|
|
2076
|
+
}
|
|
2077
|
+
const descriptor = targetDescriptor('tap', { target: candidate.target });
|
|
2078
|
+
return descriptor.kind !== 'unknown' && variantContainsTarget(target.variant, descriptor);
|
|
2079
|
+
});
|
|
2080
|
+
}
|
|
2081
|
+
function reverseScrollRestoreCandidates(current, target) {
|
|
2082
|
+
if (!shouldTryReverseScrollRestore(current.variant, target.variant)) {
|
|
2083
|
+
return [];
|
|
2084
|
+
}
|
|
2085
|
+
const args = { direction: 'up', percent: 70 };
|
|
2086
|
+
return [
|
|
2087
|
+
{
|
|
2088
|
+
target: scrollTarget(args),
|
|
2089
|
+
args,
|
|
2090
|
+
key: 'scroll:up:70'
|
|
2091
|
+
}
|
|
2092
|
+
];
|
|
2093
|
+
}
|
|
2094
|
+
function shouldTryReverseScrollRestore(current, target) {
|
|
2095
|
+
if (hasScrollableSurface(current) || hasScrollableSurface(target)) {
|
|
2096
|
+
return true;
|
|
2097
|
+
}
|
|
2098
|
+
const currentNavigationTarget = preferredNavigationTarget(current);
|
|
2099
|
+
const targetNavigationTarget = preferredNavigationTarget(target);
|
|
2100
|
+
if (currentNavigationTarget && targetNavigationTarget && currentNavigationTarget === targetNavigationTarget) {
|
|
2101
|
+
return true;
|
|
2102
|
+
}
|
|
2103
|
+
return similarity(current.element_keys, target.element_keys) >= 0.2;
|
|
2104
|
+
}
|
|
2105
|
+
function navigationResetCandidates(current, root) {
|
|
2106
|
+
const rootTarget = preferredNavigationTarget(root.variant);
|
|
2107
|
+
if (!rootTarget) {
|
|
2108
|
+
return [];
|
|
2109
|
+
}
|
|
2110
|
+
return safeTapCandidates(current.variant).filter((candidate) => normalizeControlTarget(candidate.target) === rootTarget);
|
|
2111
|
+
}
|
|
2112
|
+
function shortestRouteToVariant(map, fromVariantId, toVariantId) {
|
|
2113
|
+
if (fromVariantId === toVariantId) {
|
|
2114
|
+
return [];
|
|
2115
|
+
}
|
|
2116
|
+
const visited = new Set([fromVariantId]);
|
|
2117
|
+
const queue = [{ variantId: fromVariantId, edges: [] }];
|
|
2118
|
+
while (queue.length > 0) {
|
|
2119
|
+
const current = queue.shift();
|
|
2120
|
+
if (!current) {
|
|
2121
|
+
break;
|
|
2122
|
+
}
|
|
2123
|
+
const variant = map.variants.find((candidate) => candidate.id === current.variantId);
|
|
2124
|
+
const outgoing = [
|
|
2125
|
+
...map.edges,
|
|
2126
|
+
...navigationVirtualEdges(map, variant)
|
|
2127
|
+
]
|
|
2128
|
+
.filter((edge) => edge.from_variant_id === current.variantId && !edge.stale && edge.confidence >= 0.25 && replayableEdge(edge))
|
|
2129
|
+
.sort((left, right) => right.confidence - left.confidence);
|
|
2130
|
+
for (const edge of outgoing) {
|
|
2131
|
+
if (visited.has(edge.to_variant_id)) {
|
|
2132
|
+
continue;
|
|
2133
|
+
}
|
|
2134
|
+
const edges = [...current.edges, edge];
|
|
2135
|
+
if (edge.to_variant_id === toVariantId) {
|
|
2136
|
+
return edges;
|
|
2137
|
+
}
|
|
2138
|
+
visited.add(edge.to_variant_id);
|
|
2139
|
+
queue.push({ variantId: edge.to_variant_id, edges });
|
|
2140
|
+
}
|
|
2141
|
+
}
|
|
2142
|
+
return null;
|
|
2143
|
+
}
|
|
2144
|
+
async function restoreViaRootRoute(context, current, target, root, diagnostic) {
|
|
2145
|
+
let resetCurrent = current;
|
|
2146
|
+
const rootMatch = restoreMatch(resetCurrent, root);
|
|
2147
|
+
if (!rootMatch) {
|
|
2148
|
+
for (const candidate of navigationResetCandidates(resetCurrent, root)) {
|
|
2149
|
+
const restored = await tryTapRestoreCandidate(context, resetCurrent, root, candidate, 'root-navigation', diagnostic);
|
|
2150
|
+
resetCurrent = restored.current;
|
|
2151
|
+
if (restored.match) {
|
|
2152
|
+
break;
|
|
2153
|
+
}
|
|
2154
|
+
}
|
|
2155
|
+
}
|
|
2156
|
+
const matchedRoot = restoreMatch(resetCurrent, root);
|
|
2157
|
+
if (!matchedRoot) {
|
|
2158
|
+
return null;
|
|
2159
|
+
}
|
|
2160
|
+
if (restoreMatch(resetCurrent, target)) {
|
|
2161
|
+
diagnostic.result = 'restored';
|
|
2162
|
+
diagnostic.accepted_by = 'root';
|
|
2163
|
+
return { current: target, restored: true, acceptedBy: 'root', diagnostic };
|
|
2164
|
+
}
|
|
2165
|
+
const route = shortestRouteToVariant(context.map, root.variant.id, target.variant.id);
|
|
2166
|
+
if (!route || route.length === 0) {
|
|
2167
|
+
return null;
|
|
2168
|
+
}
|
|
2169
|
+
const driven = await driveRoute(context, root, route);
|
|
2170
|
+
const match = restoreMatch(driven.current, target);
|
|
2171
|
+
diagnostic.attempts.push(restoreAttemptDiagnostic('root-route', match ? 'matched' : 'mismatched', driven.current, match ? { ...match, acceptedBy: 'root-route' } : null, route.map((edge) => edge.target ?? edge.command).join(' -> '), 'tap'));
|
|
2172
|
+
if (!match) {
|
|
2173
|
+
return { current: driven.current, restored: false, diagnostic };
|
|
2174
|
+
}
|
|
2175
|
+
diagnostic.result = 'restored';
|
|
2176
|
+
diagnostic.accepted_by = 'root-route';
|
|
2177
|
+
return { current: target, restored: true, acceptedBy: 'root-route', diagnostic };
|
|
2178
|
+
}
|
|
2179
|
+
async function restoreToVariant(context, current, target, root) {
|
|
2180
|
+
const diagnostic = makeRestoreDiagnostic(current, target);
|
|
2181
|
+
const initialMatch = restoreMatch(current, target);
|
|
2182
|
+
if (initialMatch) {
|
|
2183
|
+
diagnostic.result = 'restored';
|
|
2184
|
+
diagnostic.accepted_by = initialMatch.acceptedBy;
|
|
2185
|
+
return { current: target, restored: true, acceptedBy: initialMatch.acceptedBy, diagnostic };
|
|
2186
|
+
}
|
|
2187
|
+
const restoreTargets = safeTapCandidates(current.variant).filter((candidate) => /^(back|close|dismiss|done)$/i.test(candidate.target));
|
|
2188
|
+
for (const restoreCandidate of restoreTargets) {
|
|
2189
|
+
const restored = await tryTapRestoreCandidate(context, current, target, restoreCandidate, 'dismiss-control', diagnostic);
|
|
2190
|
+
current = restored.current;
|
|
2191
|
+
if (restored.match) {
|
|
2192
|
+
diagnostic.result = 'restored';
|
|
2193
|
+
diagnostic.accepted_by = restored.match.acceptedBy;
|
|
2194
|
+
return { current: target, restored: true, acceptedBy: restored.match.acceptedBy, diagnostic };
|
|
2195
|
+
}
|
|
2196
|
+
}
|
|
2197
|
+
for (const restoreCandidate of targetRestoreCandidates(current, target)) {
|
|
2198
|
+
const restored = await tryTapRestoreCandidate(context, current, target, restoreCandidate, 'target-navigation', diagnostic);
|
|
2199
|
+
current = restored.current;
|
|
2200
|
+
if (restored.match) {
|
|
2201
|
+
diagnostic.result = 'restored';
|
|
2202
|
+
diagnostic.accepted_by = restored.match.acceptedBy;
|
|
2203
|
+
return { current: target, restored: true, acceptedBy: restored.match.acceptedBy, diagnostic };
|
|
2204
|
+
}
|
|
2205
|
+
}
|
|
2206
|
+
for (const restoreCandidate of reverseScrollRestoreCandidates(current, target)) {
|
|
2207
|
+
const restored = await tryScrollRestoreCandidate(context, current, target, restoreCandidate, diagnostic);
|
|
2208
|
+
current = restored.current;
|
|
2209
|
+
if (restored.match) {
|
|
2210
|
+
diagnostic.result = 'restored';
|
|
2211
|
+
diagnostic.accepted_by = restored.match.acceptedBy;
|
|
2212
|
+
return { current: target, restored: true, acceptedBy: restored.match.acceptedBy, diagnostic };
|
|
2213
|
+
}
|
|
2214
|
+
}
|
|
2215
|
+
try {
|
|
2216
|
+
await context.adapter.act({ name: 'back' });
|
|
2217
|
+
const observed = await observeAfterAction(context, target);
|
|
2218
|
+
const match = restoreMatch(observed, target);
|
|
2219
|
+
diagnostic.attempts.push(restoreAttemptDiagnostic('platform-back', match ? 'matched' : 'mismatched', observed, match, 'back', 'act'));
|
|
2220
|
+
if (match) {
|
|
2221
|
+
diagnostic.result = 'restored';
|
|
2222
|
+
diagnostic.accepted_by = match.acceptedBy;
|
|
2223
|
+
return { current: target, restored: true, acceptedBy: match.acceptedBy, diagnostic };
|
|
2224
|
+
}
|
|
2225
|
+
current = observed;
|
|
2226
|
+
}
|
|
2227
|
+
catch (error) {
|
|
2228
|
+
diagnostic.attempts.push(restoreAttemptDiagnostic('platform-back', 'error', undefined, null, 'back', 'act', error));
|
|
2229
|
+
}
|
|
2230
|
+
if (root) {
|
|
2231
|
+
const rooted = await restoreViaRootRoute(context, current, target, root, diagnostic);
|
|
2232
|
+
if (rooted) {
|
|
2233
|
+
return rooted;
|
|
2234
|
+
}
|
|
2235
|
+
}
|
|
2236
|
+
return { current, restored: false, diagnostic };
|
|
2237
|
+
}
|
|
2238
|
+
async function crawlAppMap(context, start) {
|
|
2239
|
+
const visited = new Set();
|
|
2240
|
+
const visitedTemplates = new Set();
|
|
2241
|
+
const exploredRepeatedContentCandidates = new Set();
|
|
2242
|
+
const attemptedTargets = new Map();
|
|
2243
|
+
const restoreDiagnostics = [];
|
|
2244
|
+
let restoreFailures = 0;
|
|
2245
|
+
let actions = 0;
|
|
2246
|
+
let stoppedReason = 'complete';
|
|
2247
|
+
const restore = async (current, target) => {
|
|
2248
|
+
const result = await restoreToVariant(context, current, target, start);
|
|
2249
|
+
if (result.diagnostic.attempts.length > 0 || result.diagnostic.accepted_by) {
|
|
2250
|
+
restoreDiagnostics.push(result.diagnostic);
|
|
2251
|
+
}
|
|
2252
|
+
if (!result.restored) {
|
|
2253
|
+
restoreFailures += 1;
|
|
2254
|
+
}
|
|
2255
|
+
return result;
|
|
2256
|
+
};
|
|
2257
|
+
const markPartial = () => {
|
|
2258
|
+
if (stoppedReason === 'complete') {
|
|
2259
|
+
stoppedReason = 'partial';
|
|
2260
|
+
}
|
|
2261
|
+
};
|
|
2262
|
+
const visit = async (origin, depth, allowScrollFirst = true) => {
|
|
2263
|
+
if (depth <= 0) {
|
|
2264
|
+
return origin;
|
|
2265
|
+
}
|
|
2266
|
+
if (actions >= context.options.crawlLimit) {
|
|
2267
|
+
stoppedReason = 'limit';
|
|
2268
|
+
return origin;
|
|
2269
|
+
}
|
|
2270
|
+
const originTemplateKey = crawlTemplateKey(origin.variant);
|
|
2271
|
+
if (visitedTemplates.has(originTemplateKey) && !visited.has(origin.variant.id)) {
|
|
2272
|
+
return origin;
|
|
2273
|
+
}
|
|
2274
|
+
visitedTemplates.add(originTemplateKey);
|
|
2275
|
+
visited.add(origin.variant.id);
|
|
2276
|
+
const attemptsForVariant = attemptedTargets.get(origin.variant.id) ?? new Set();
|
|
2277
|
+
attemptedTargets.set(origin.variant.id, attemptsForVariant);
|
|
2278
|
+
const candidates = safeTapCandidates(origin.variant, crawlCandidateOptions(origin.variant))
|
|
2279
|
+
.filter((candidate) => !attemptsForVariant.has(candidate.key));
|
|
2280
|
+
let current = origin;
|
|
2281
|
+
const exploreScrollCandidates = async (scrollCandidatesToExplore) => {
|
|
2282
|
+
for (const candidate of scrollCandidatesToExplore) {
|
|
2283
|
+
if (actions >= context.options.crawlLimit) {
|
|
2284
|
+
stoppedReason = 'limit';
|
|
2285
|
+
return true;
|
|
2286
|
+
}
|
|
2287
|
+
if (!restoreMatch(current, origin)) {
|
|
2288
|
+
if (restoreMatch(current, start)) {
|
|
2289
|
+
markPartial();
|
|
2290
|
+
current = start;
|
|
2291
|
+
return true;
|
|
2292
|
+
}
|
|
2293
|
+
stoppedReason = 'restore_failed';
|
|
2294
|
+
return true;
|
|
2295
|
+
}
|
|
2296
|
+
current = origin;
|
|
2297
|
+
attemptsForVariant.add(candidate.key);
|
|
2298
|
+
const args = candidate.args;
|
|
2299
|
+
let observed;
|
|
2300
|
+
try {
|
|
2301
|
+
await context.adapter.scroll(args);
|
|
2302
|
+
actions += 1;
|
|
2303
|
+
observed = await observeAfterCrawlAction(context);
|
|
2304
|
+
}
|
|
2305
|
+
catch {
|
|
2306
|
+
try {
|
|
2307
|
+
current = await observeCurrentScreen(context);
|
|
2308
|
+
}
|
|
2309
|
+
catch {
|
|
2310
|
+
// Stay on the last known origin if a failed scroll also prevents source capture.
|
|
2311
|
+
}
|
|
2312
|
+
continue;
|
|
2313
|
+
}
|
|
2314
|
+
if (observed.variant.id === origin.variant.id || observed.observation.fingerprint === origin.observation.fingerprint) {
|
|
2315
|
+
current = origin;
|
|
2316
|
+
continue;
|
|
2317
|
+
}
|
|
2318
|
+
recordEdgeSuccess(context, origin.variant, observed.variant, 'scroll', args, true);
|
|
2319
|
+
visited.add(observed.variant.id);
|
|
2320
|
+
if (observed.variant.id !== origin.variant.id && depth > 1) {
|
|
2321
|
+
const nestedCurrent = await visit(observed, depth - 1, false);
|
|
2322
|
+
const restored = await restore(nestedCurrent, origin);
|
|
2323
|
+
current = restored.current;
|
|
2324
|
+
if (!restored.restored) {
|
|
2325
|
+
if (restoreMatch(current, start)) {
|
|
2326
|
+
markPartial();
|
|
2327
|
+
current = start;
|
|
2328
|
+
return true;
|
|
2329
|
+
}
|
|
2330
|
+
stoppedReason = 'restore_failed';
|
|
2331
|
+
return true;
|
|
2332
|
+
}
|
|
2333
|
+
}
|
|
2334
|
+
else {
|
|
2335
|
+
const restored = await restore(observed, origin);
|
|
2336
|
+
current = restored.current;
|
|
2337
|
+
if (!restored.restored) {
|
|
2338
|
+
if (restoreMatch(current, start)) {
|
|
2339
|
+
markPartial();
|
|
2340
|
+
if (restoreMatch(origin, start)) {
|
|
2341
|
+
current = origin;
|
|
2342
|
+
continue;
|
|
2343
|
+
}
|
|
2344
|
+
current = start;
|
|
2345
|
+
return true;
|
|
2346
|
+
}
|
|
2347
|
+
stoppedReason = 'restore_failed';
|
|
2348
|
+
return true;
|
|
2349
|
+
}
|
|
2350
|
+
}
|
|
2351
|
+
}
|
|
2352
|
+
return false;
|
|
2353
|
+
};
|
|
2354
|
+
const scrollCandidates = crawlScrollCandidates(context, origin.variant)
|
|
2355
|
+
.filter((candidate) => !attemptsForVariant.has(candidate.key));
|
|
2356
|
+
if (allowScrollFirst && hasScrollableSurface(origin.variant) && await exploreScrollCandidates(scrollCandidates)) {
|
|
2357
|
+
return current;
|
|
2358
|
+
}
|
|
2359
|
+
for (const candidate of candidates) {
|
|
2360
|
+
const repeatedCandidateKey = repeatedContentCandidateKey(origin.variant, candidate);
|
|
2361
|
+
if (repeatedCandidateKey && exploredRepeatedContentCandidates.has(repeatedCandidateKey)) {
|
|
2362
|
+
continue;
|
|
2363
|
+
}
|
|
2364
|
+
if (actions >= context.options.crawlLimit) {
|
|
2365
|
+
stoppedReason = 'limit';
|
|
2366
|
+
return current;
|
|
2367
|
+
}
|
|
2368
|
+
if (!restoreMatch(current, origin)) {
|
|
2369
|
+
if (restoreMatch(current, start)) {
|
|
2370
|
+
markPartial();
|
|
2371
|
+
return start;
|
|
2372
|
+
}
|
|
2373
|
+
stoppedReason = 'restore_failed';
|
|
2374
|
+
return current;
|
|
2375
|
+
}
|
|
2376
|
+
current = origin;
|
|
2377
|
+
attemptsForVariant.add(candidate.key);
|
|
2378
|
+
const args = candidate.args;
|
|
2379
|
+
let observed;
|
|
2380
|
+
try {
|
|
2381
|
+
await context.adapter.tap(args);
|
|
2382
|
+
actions += 1;
|
|
2383
|
+
observed = await observeAfterCrawlAction(context);
|
|
2384
|
+
}
|
|
2385
|
+
catch {
|
|
2386
|
+
try {
|
|
2387
|
+
current = await observeCurrentScreen(context);
|
|
2388
|
+
}
|
|
2389
|
+
catch {
|
|
2390
|
+
// Stay on the last known origin if a failed tap also prevents source capture.
|
|
2391
|
+
}
|
|
2392
|
+
continue;
|
|
2393
|
+
}
|
|
2394
|
+
recordEdgeSuccess(context, origin.variant, observed.variant, 'tap', args, true);
|
|
2395
|
+
visited.add(observed.variant.id);
|
|
2396
|
+
if (repeatedCandidateKey && observed.variant.id !== origin.variant.id) {
|
|
2397
|
+
exploredRepeatedContentCandidates.add(repeatedCandidateKey);
|
|
2398
|
+
}
|
|
2399
|
+
if (observed.variant.id !== origin.variant.id && depth > 1) {
|
|
2400
|
+
const nestedCurrent = await visit(observed, depth - 1);
|
|
2401
|
+
const restored = await restore(nestedCurrent, origin);
|
|
2402
|
+
current = restored.current;
|
|
2403
|
+
if (!restored.restored) {
|
|
2404
|
+
if (restoreMatch(current, start)) {
|
|
2405
|
+
markPartial();
|
|
2406
|
+
return start;
|
|
2407
|
+
}
|
|
2408
|
+
stoppedReason = 'restore_failed';
|
|
2409
|
+
return current;
|
|
2410
|
+
}
|
|
2411
|
+
}
|
|
2412
|
+
else {
|
|
2413
|
+
const restored = restoreMatch(observed, origin)
|
|
2414
|
+
? { current: origin, restored: true }
|
|
2415
|
+
: await restore(observed, origin);
|
|
2416
|
+
current = restored.current;
|
|
2417
|
+
if (!restored.restored) {
|
|
2418
|
+
if (restoreMatch(current, start)) {
|
|
2419
|
+
markPartial();
|
|
2420
|
+
if (restoreMatch(origin, start)) {
|
|
2421
|
+
current = origin;
|
|
2422
|
+
continue;
|
|
2423
|
+
}
|
|
2424
|
+
return start;
|
|
2425
|
+
}
|
|
2426
|
+
stoppedReason = 'restore_failed';
|
|
2427
|
+
return current;
|
|
2428
|
+
}
|
|
2429
|
+
}
|
|
2430
|
+
}
|
|
2431
|
+
const remainingScrollCandidates = crawlScrollCandidates(context, origin.variant)
|
|
2432
|
+
.filter((candidate) => !attemptsForVariant.has(candidate.key));
|
|
2433
|
+
if (await exploreScrollCandidates(remainingScrollCandidates)) {
|
|
2434
|
+
return current;
|
|
2435
|
+
}
|
|
2436
|
+
return current;
|
|
2437
|
+
};
|
|
2438
|
+
await visit(start, context.options.crawlDepth);
|
|
2439
|
+
return {
|
|
2440
|
+
enabled: true,
|
|
2441
|
+
actions,
|
|
2442
|
+
variants: visited.size,
|
|
2443
|
+
stopped_reason: stoppedReason,
|
|
2444
|
+
...(restoreFailures > 0 ? { restore_failures: restoreFailures } : {}),
|
|
2445
|
+
...(restoreDiagnostics.length > 0 ? { restore_diagnostics: restoreDiagnostics } : {})
|
|
2446
|
+
};
|
|
2447
|
+
}
|
|
2448
|
+
export async function discoverAppMap(adapter, options) {
|
|
2449
|
+
const context = createAppMapContext(adapter, options);
|
|
2450
|
+
if (!context) {
|
|
2451
|
+
return {
|
|
2452
|
+
action: 'discover',
|
|
2453
|
+
map: createMapSummary(adapter, options),
|
|
2454
|
+
screen: null
|
|
2455
|
+
};
|
|
2456
|
+
}
|
|
2457
|
+
try {
|
|
2458
|
+
const observed = await observeCurrentScreen(context);
|
|
2459
|
+
const crawl = context.options.crawl ? await crawlAppMap(context, observed) : undefined;
|
|
2460
|
+
const summary = persistAppMapContext(context);
|
|
2461
|
+
return {
|
|
2462
|
+
action: 'discover',
|
|
2463
|
+
map: summary,
|
|
2464
|
+
...(crawl ? { crawl } : {}),
|
|
2465
|
+
screen: {
|
|
2466
|
+
variant_id: observed.variant.id,
|
|
2467
|
+
screen_id: observed.variant.screen_id,
|
|
2468
|
+
element_count: observed.variant.elements.length,
|
|
2469
|
+
auth_required: observed.variant.auth_required,
|
|
2470
|
+
created: observed.created
|
|
2471
|
+
}
|
|
2472
|
+
};
|
|
2473
|
+
}
|
|
2474
|
+
finally {
|
|
2475
|
+
persistAppMapContext(context);
|
|
2476
|
+
}
|
|
2477
|
+
}
|
|
2478
|
+
export function mapDebugSnapshot(rootDir, platform, appId) {
|
|
2479
|
+
const identity = mapIdentity(platform, appId);
|
|
2480
|
+
const filePath = mapFilePath(rootDir, identity);
|
|
2481
|
+
if (!fs.existsSync(filePath)) {
|
|
2482
|
+
return null;
|
|
2483
|
+
}
|
|
2484
|
+
const parsed = JSON.parse(fs.readFileSync(filePath, 'utf8'));
|
|
2485
|
+
return JSON.parse(canonicalJson(parsed));
|
|
2486
|
+
}
|