simframe 0.7.2 → 0.9.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +21 -4
- package/flows/hpi-suite.json +68 -0
- package/native/simframed/Sources/PrivateAPI/CoreSimulatorPlatform.swift +49 -1
- package/native/simframed/Sources/PrivateAPI/PrivateAPI.swift +7 -0
- package/native/simframed/Sources/PrivateAPI/StubPlatform.swift +11 -0
- package/native/simframed/Sources/SimframeCore/CaptureRecovery.swift +48 -0
- package/native/simframed/Sources/simframed/main.swift +128 -73
- package/native/simframed/Tests/SimframeCoreTests/HashingTests.swift +40 -0
- package/package.json +2 -1
- package/scripts/bench-hpi.mjs +254 -0
- package/scripts/check-package.mjs +7 -0
- package/scripts/ci-memory.mjs +15 -2
- package/src/actions.js +184 -4
- package/src/baseline.js +333 -0
- package/src/cli.js +336 -2
- package/src/daemon.js +9 -0
- package/src/fingerprint.js +7 -1
- package/src/graph.js +162 -1
- package/src/index.js +118 -20
- package/src/input.js +111 -1
- package/src/intent.js +11 -2
- package/src/matching.js +81 -2
- package/src/mcp.js +14 -1
- package/src/metrics.js +499 -0
- package/src/navigate.js +44 -7
- package/src/platform/android.js +27 -0
- package/src/platform/index.js +3 -0
- package/src/platform/ios.js +63 -0
- package/src/screenmap.js +36 -14
- package/src/view.js +4 -3
package/src/index.js
CHANGED
|
@@ -19,6 +19,7 @@ import * as graph from './graph.js';
|
|
|
19
19
|
import * as matching from './matching.js';
|
|
20
20
|
import * as refs from './refs.js';
|
|
21
21
|
import * as screenmap from './screenmap.js';
|
|
22
|
+
import * as metrics from './metrics.js';
|
|
22
23
|
import { capabilitiesFor, resolveDevice, resize, screenshot } from './platform/index.js';
|
|
23
24
|
import * as store from './store.js';
|
|
24
25
|
|
|
@@ -314,10 +315,24 @@ function stallNote(health) {
|
|
|
314
315
|
const parts = [`capture: stalled — the display surface has been unreadable for ${Math.round(forMs / 1000)}s`];
|
|
315
316
|
if (health.reattaches) parts.push(`${health.reattaches} re-attach${health.reattaches === 1 ? '' : 'es'} did not help`);
|
|
316
317
|
if (health.reason) parts.push(String(health.reason).slice(0, 120));
|
|
317
|
-
//
|
|
318
|
-
//
|
|
319
|
-
//
|
|
320
|
-
|
|
318
|
+
// What to do about it, and this line has been wrong until now.
|
|
319
|
+
//
|
|
320
|
+
// It said "only restarting the device is known to cure it". Then the same
|
|
321
|
+
// daemon's log turned out to contain nine "capture recovered on its own"
|
|
322
|
+
// lines, and a wedge that had survived 223 port re-resolves and 6 device
|
|
323
|
+
// rebinds cleared by itself while nobody touched it. Meanwhile Apple's own
|
|
324
|
+
// `simctl io screenshot` on a wedged device returns a valid PNG whose every
|
|
325
|
+
// pixel is black, in 16 s — so the display pipeline has stopped rendering
|
|
326
|
+
// and simframe's read is an accurate report of that, not a bug in it.
|
|
327
|
+
//
|
|
328
|
+
// So: it is the simulator, it often comes back, and restarting the device
|
|
329
|
+
// also cures it. `simframe doctor` runs a screenshot probe when this is
|
|
330
|
+
// published and says which of the two faults it is.
|
|
331
|
+
parts.push(
|
|
332
|
+
"this is the simulator's display, not simframe's read of it: Apple's own screenshot path "
|
|
333
|
+
+ 'returns an all-black image on a wedged device. It frequently recovers on its own; '
|
|
334
|
+
+ 'restarting the device also cures it, and `simframe doctor` will confirm which fault this is',
|
|
335
|
+
);
|
|
321
336
|
return parts.join('; ');
|
|
322
337
|
}
|
|
323
338
|
|
|
@@ -473,7 +488,7 @@ function pngSize(png) {
|
|
|
473
488
|
return { width: png.readUInt32BE(16), height: png.readUInt32BE(20) };
|
|
474
489
|
}
|
|
475
490
|
|
|
476
|
-
export async function getState(deviceQuery, { since, options } = {}) {
|
|
491
|
+
export async function getState(deviceQuery, { since, options, inputHealth = false } = {}) {
|
|
477
492
|
const { device, state } = await ensureDaemon(deviceQuery, options);
|
|
478
493
|
return {
|
|
479
494
|
device,
|
|
@@ -482,9 +497,54 @@ export async function getState(deviceQuery, { since, options } = {}) {
|
|
|
482
497
|
map: regionMap(state.regions || [], REGION_COLS),
|
|
483
498
|
since: compareToBaseline(state, resolveBaseline(state, since)),
|
|
484
499
|
live: liveness(device.udid, state),
|
|
500
|
+
// Off by default and asked for by the state commands only. A flow step
|
|
501
|
+
// calls getState twice, and the fix for a stale session runs before every
|
|
502
|
+
// action anyway (input.ensureFreshSession) — this is the report, not the
|
|
503
|
+
// repair.
|
|
504
|
+
input: inputHealth ? await input.sessionHealth(device.udid) : undefined,
|
|
505
|
+
timing: inputHealth ? timingOfNow(device.udid, state) : undefined,
|
|
485
506
|
};
|
|
486
507
|
}
|
|
487
508
|
|
|
509
|
+
/**
|
|
510
|
+
* How long this screen has been moving, against how long it usually takes.
|
|
511
|
+
*
|
|
512
|
+
* Research §7 asks `sim_state` for this, and the point is a sentence an agent
|
|
513
|
+
* can act on: a screen 400 ms into a transition that normally takes 350 ms is
|
|
514
|
+
* fine, and the same screen four seconds in is not. Elapsed comes from the
|
|
515
|
+
* frame store's own clock; the distribution comes from the edge that was most
|
|
516
|
+
* recently traversed into this screen.
|
|
517
|
+
*
|
|
518
|
+
* Reads no frames and takes no perception pass: the layout hash and the
|
|
519
|
+
* structural hash screen memory filed under it are both already on disk.
|
|
520
|
+
*/
|
|
521
|
+
function timingOfNow(udid, state) {
|
|
522
|
+
try {
|
|
523
|
+
const near = screenmap.recallNearest(udid, state.layoutHash);
|
|
524
|
+
const edge = graph.timingInto(udid, near?.entry?.structuralHash);
|
|
525
|
+
const elapsed = Number.isFinite(state.lastChangeAt) ? Date.now() - state.lastChangeAt : null;
|
|
526
|
+
const settled = state.stableForMs >= STRUCTURAL_SETTLE_MS;
|
|
527
|
+
const slower = graph.slowerThanUsual({
|
|
528
|
+
elapsedMs: elapsed,
|
|
529
|
+
p95: edge?.p95,
|
|
530
|
+
settled,
|
|
531
|
+
kind: state.transition?.kind,
|
|
532
|
+
});
|
|
533
|
+
return {
|
|
534
|
+
edge_p50: edge?.p50 ?? null,
|
|
535
|
+
edge_p95: edge?.p95 ?? null,
|
|
536
|
+
samples: edge?.samples ?? 0,
|
|
537
|
+
elapsed_ms: elapsed,
|
|
538
|
+
stable_for_ms: state.stableForMs ?? null,
|
|
539
|
+
slower_than_usual: Boolean(slower.slower),
|
|
540
|
+
note: slower.slower ? slower.note : null,
|
|
541
|
+
};
|
|
542
|
+
} catch {
|
|
543
|
+
// Timing is commentary. A screen with no history still has a state.
|
|
544
|
+
return null;
|
|
545
|
+
}
|
|
546
|
+
}
|
|
547
|
+
|
|
488
548
|
/**
|
|
489
549
|
* Wait for the screen to settle (`mode: 'stable'`) or to move away from what it
|
|
490
550
|
* shows right now (`mode: 'change'`). Removes the screenshot-retry loop.
|
|
@@ -516,6 +576,21 @@ export async function waitFor(
|
|
|
516
576
|
const deadline = startedAt + timeoutMs;
|
|
517
577
|
const startSeq = first.seq;
|
|
518
578
|
let last = first;
|
|
579
|
+
/**
|
|
580
|
+
* The longest pause *inside* this transition, in ms.
|
|
581
|
+
*
|
|
582
|
+
* This is the statistic a stillness window exists to defeat: a transition
|
|
583
|
+
* that pauses for 180 ms mid-flight will be mistaken for a finished screen
|
|
584
|
+
* by any window shorter than that. Nobody was measuring it, so the window
|
|
585
|
+
* was a constant — 500 ms, chosen once, paid by every step forever.
|
|
586
|
+
*
|
|
587
|
+
* Tracked as the highest `stableForMs` observed before the screen moved
|
|
588
|
+
* again. Reported so a caller can learn it per edge; a settle that ends
|
|
589
|
+
* without a second change has no gap to report and says 0.
|
|
590
|
+
*/
|
|
591
|
+
let quietGapMs = 0;
|
|
592
|
+
let quietRun = 0;
|
|
593
|
+
let lastHash = first.hash;
|
|
519
594
|
let sawChange = mode === 'stable' || first.hash !== baselineHashValue;
|
|
520
595
|
const changedAtStart = sawChange && mode !== 'stable';
|
|
521
596
|
|
|
@@ -524,6 +599,7 @@ export async function waitFor(
|
|
|
524
599
|
state: last,
|
|
525
600
|
satisfied,
|
|
526
601
|
mode,
|
|
602
|
+
quietGapMs,
|
|
527
603
|
sawChange,
|
|
528
604
|
changedBeforeWait: changedAtStart,
|
|
529
605
|
baselineHash: baselineHashValue,
|
|
@@ -543,6 +619,17 @@ export async function waitFor(
|
|
|
543
619
|
|
|
544
620
|
if (!sawChange && state.hash !== baselineHashValue) sawChange = true;
|
|
545
621
|
|
|
622
|
+
// A pause that turned out not to be the end of the transition. Only
|
|
623
|
+
// pauses followed by more movement count: the quiet at the end of a
|
|
624
|
+
// settle is the answer, not a gap.
|
|
625
|
+
if (state.hash !== lastHash) {
|
|
626
|
+
if (quietRun > quietGapMs) quietGapMs = quietRun;
|
|
627
|
+
quietRun = 0;
|
|
628
|
+
lastHash = state.hash;
|
|
629
|
+
} else if (Number.isFinite(state.stableForMs)) {
|
|
630
|
+
quietRun = Math.max(quietRun, state.stableForMs);
|
|
631
|
+
}
|
|
632
|
+
|
|
546
633
|
// Some controls barely move the screen at all — a radio dot, a checkbox,
|
|
547
634
|
// a button changing state. Waiting the full timeout for a change that
|
|
548
635
|
// will never be visible turns a 100ms action into a 12s one, so give up
|
|
@@ -890,8 +977,15 @@ export async function locate(
|
|
|
890
977
|
const list = outcome.alternatives
|
|
891
978
|
.map((a, i) => `[${i}] "${a.label}" (${a.x},${a.y}) ${a.region ?? 'content'} ${a.score}`)
|
|
892
979
|
.join(', ');
|
|
893
|
-
|
|
894
|
-
|
|
980
|
+
// Tagged, not just thrown: the reason an escalation happened is known
|
|
981
|
+
// here and nowhere above here. See metrics.tag — it adds a property and
|
|
982
|
+
// changes nothing else about the error.
|
|
983
|
+
throw metrics.tag(
|
|
984
|
+
new Error(
|
|
985
|
+
`"${query}" matches ${outcome.alternatives.length} things on this screen — say which, or pass index: ${list}`,
|
|
986
|
+
),
|
|
987
|
+
'ambiguous_intent',
|
|
988
|
+
{ candidates: outcome.alternatives },
|
|
895
989
|
);
|
|
896
990
|
}
|
|
897
991
|
if (outcome.status === 'ok') {
|
|
@@ -905,24 +999,28 @@ export async function locate(
|
|
|
905
999
|
// here undoes every guard above — it has no off-screen filter and no
|
|
906
1000
|
// coverage weighting, and it is what returned a scrolled-away list row for
|
|
907
1001
|
// "back". "Not found" is the correct answer.
|
|
908
|
-
const
|
|
909
|
-
|
|
910
|
-
|
|
911
|
-
|
|
912
|
-
|
|
913
|
-
|
|
1002
|
+
const visible = entry.targets.filter((t) => t.label && t.y >= 0 && t.y <= points.height);
|
|
1003
|
+
const sample = visible.slice(0, 12).map((t) => t.label.slice(0, 24)).join(', ');
|
|
1004
|
+
// Which escalation this is depends on whether the screen was recognised.
|
|
1005
|
+
// Screen memory had nothing for it (`from` is one of the built values) and
|
|
1006
|
+
// the target is missing: that is not knowing the screen. On a screen
|
|
1007
|
+
// recalled from memory, the screen is known and the intent did not resolve.
|
|
1008
|
+
throw metrics.tag(
|
|
1009
|
+
new Error(`"${query}" is not on this screen. Visible: ${sample || '(nothing readable)'}`),
|
|
1010
|
+
from === 'memory' ? 'ambiguous_intent' : 'unknown_screen',
|
|
1011
|
+
{ candidates: visible.slice(0, 8) },
|
|
1012
|
+
);
|
|
914
1013
|
}
|
|
915
1014
|
|
|
916
1015
|
const candidates = screenmap.rank(entry, query);
|
|
917
1016
|
const target = index != null ? candidates[index] : candidates[0];
|
|
918
1017
|
if (!target) {
|
|
919
|
-
const
|
|
920
|
-
|
|
921
|
-
|
|
922
|
-
|
|
923
|
-
|
|
924
|
-
|
|
925
|
-
`"${query}" is not on this screen. Visible: ${sample || '(nothing readable)'}`,
|
|
1018
|
+
const visible = entry.targets.filter((t) => t.label);
|
|
1019
|
+
const sample = visible.slice(0, 12).map((t) => t.label).join(', ');
|
|
1020
|
+
throw metrics.tag(
|
|
1021
|
+
new Error(`"${query}" is not on this screen. Visible: ${sample || '(nothing readable)'}`),
|
|
1022
|
+
from === 'memory' ? 'ambiguous_intent' : 'unknown_screen',
|
|
1023
|
+
{ candidates: visible.slice(0, 8) },
|
|
926
1024
|
);
|
|
927
1025
|
}
|
|
928
1026
|
return { device, state: current, entry, target, from, distance, settled, screens: screenmap.stats(udid).screens };
|
package/src/input.js
CHANGED
|
@@ -2,8 +2,11 @@
|
|
|
2
2
|
// capability layered on top, so every entry point here has to answer "is this
|
|
3
3
|
// even available?" before it answers anything else.
|
|
4
4
|
import { execFile } from 'node:child_process';
|
|
5
|
+
import fs from 'node:fs';
|
|
6
|
+
import path from 'node:path';
|
|
5
7
|
import * as control from './control.js';
|
|
6
|
-
import
|
|
8
|
+
import * as store from './store.js';
|
|
9
|
+
import { bootedAtFor, capabilitiesFor, geometryFor, inputDriverFor, setPasteboard } from './platform/index.js';
|
|
7
10
|
import { promisify } from 'node:util';
|
|
8
11
|
|
|
9
12
|
const run = promisify(execFile);
|
|
@@ -323,6 +326,7 @@ export function centerOf(node) {
|
|
|
323
326
|
}
|
|
324
327
|
|
|
325
328
|
export async function tapPoint(udid, x, y, { durationMs } = {}) {
|
|
329
|
+
await ensureFreshSession(udid);
|
|
326
330
|
const point = { x: Math.round(x), y: Math.round(y) };
|
|
327
331
|
const own = inputDriverFor(udid);
|
|
328
332
|
if (own) {
|
|
@@ -347,6 +351,7 @@ export async function tapLabel(udid, query, { index, durationMs } = {}) {
|
|
|
347
351
|
}
|
|
348
352
|
|
|
349
353
|
export async function typeText(udid, value) {
|
|
354
|
+
await ensureFreshSession(udid);
|
|
350
355
|
const own = inputDriverFor(udid);
|
|
351
356
|
if (own) {
|
|
352
357
|
// No pasteboard on Android (docs/DEFERRED.md), so exact text goes through
|
|
@@ -379,6 +384,7 @@ export async function typeText(udid, value) {
|
|
|
379
384
|
* to say so; `type` still works on every device.
|
|
380
385
|
*/
|
|
381
386
|
export async function pasteText(udid, value) {
|
|
387
|
+
await ensureFreshSession(udid);
|
|
382
388
|
const own = inputDriverFor(udid);
|
|
383
389
|
if (own?.key) {
|
|
384
390
|
// The clipboard goes over gRPC; KEYCODE_PASTE is what puts it in the field.
|
|
@@ -413,6 +419,7 @@ export async function typeKeys(udid, value) {
|
|
|
413
419
|
}
|
|
414
420
|
|
|
415
421
|
export async function pressKey(udid, keycode) {
|
|
422
|
+
await ensureFreshSession(udid);
|
|
416
423
|
const own = inputDriverFor(udid);
|
|
417
424
|
if (own) {
|
|
418
425
|
await own.key(udid, keycode);
|
|
@@ -433,10 +440,111 @@ export async function pressKey(udid, keycode) {
|
|
|
433
440
|
*
|
|
434
441
|
* @returns {Promise<boolean>} whether a session was actually reset.
|
|
435
442
|
*/
|
|
443
|
+
/**
|
|
444
|
+
* Is the daemon's HID session older than the device it talks to?
|
|
445
|
+
*
|
|
446
|
+
* Pure, so the comparison is testable without a device. `graceMs` covers the
|
|
447
|
+
* ordinary case where a daemon is started immediately after a boot and the two
|
|
448
|
+
* timestamps land within milliseconds of each other in either order.
|
|
449
|
+
*/
|
|
450
|
+
export function sessionStaleness({ bootedAt, sessionSince, graceMs = 2000 }) {
|
|
451
|
+
if (!Number.isFinite(bootedAt)) {
|
|
452
|
+
return { stale: false, reason: 'cannot tell when the device booted' };
|
|
453
|
+
}
|
|
454
|
+
if (!Number.isFinite(sessionSince)) {
|
|
455
|
+
return { stale: false, reason: 'no capture daemon has recorded a start time, so there is no session to compare against' };
|
|
456
|
+
}
|
|
457
|
+
if (bootedAt <= sessionSince + graceMs) return { stale: false, reason: null };
|
|
458
|
+
return {
|
|
459
|
+
stale: true,
|
|
460
|
+
reason: `the device booted ${Math.round((bootedAt - sessionSince) / 1000)}s after the capture daemon started, `
|
|
461
|
+
+ 'so the daemon holds an HID session for a device session that no longer exists',
|
|
462
|
+
bootedAt,
|
|
463
|
+
sessionSince,
|
|
464
|
+
};
|
|
465
|
+
}
|
|
466
|
+
|
|
467
|
+
/**
|
|
468
|
+
* What state the input path is in, for doctor and sim_state.
|
|
469
|
+
*
|
|
470
|
+
* Reads two timestamps off disk — the device's boot marker and the daemon's
|
|
471
|
+
* own `startedAt` — and costs a stat each. No input is dispatched to find out,
|
|
472
|
+
* because the whole failure being detected is input that reports success and
|
|
473
|
+
* does nothing.
|
|
474
|
+
*/
|
|
475
|
+
const bootCache = new Map();
|
|
476
|
+
/**
|
|
477
|
+
* Boot time, cached for a moment.
|
|
478
|
+
*
|
|
479
|
+
* iOS answers with a stat; Android runs `adb shell cat /proc/uptime`, a
|
|
480
|
+
* subprocess of 20-40 ms, and `getState` runs twice per flow step. A few
|
|
481
|
+
* seconds of staleness in the staleness detector costs nothing — a device that
|
|
482
|
+
* rebooted three seconds ago is still rebooted at the next check.
|
|
483
|
+
*/
|
|
484
|
+
async function bootedAtCached(udid, ttlMs = 3000) {
|
|
485
|
+
const hit = bootCache.get(udid);
|
|
486
|
+
if (hit && Date.now() - hit.at < ttlMs) return hit.value;
|
|
487
|
+
const value = await bootedAtFor(udid);
|
|
488
|
+
bootCache.set(udid, { at: Date.now(), value });
|
|
489
|
+
return value;
|
|
490
|
+
}
|
|
491
|
+
|
|
492
|
+
export async function sessionHealth(udid) {
|
|
493
|
+
if (!udid || !control.available(udid)) return { stale: false, reason: null };
|
|
494
|
+
let bootedAt = null;
|
|
495
|
+
try {
|
|
496
|
+
bootedAt = await bootedAtCached(udid);
|
|
497
|
+
} catch (err) {
|
|
498
|
+
return { stale: false, reason: `cannot tell when the device booted: ${err.message}` };
|
|
499
|
+
}
|
|
500
|
+
const meta = store.readJson(store.paths(udid).meta);
|
|
501
|
+
const rebuilt = store.readJson(sessionFile(udid))?.rebuiltAt;
|
|
502
|
+
// The newer of the two: the daemon starting creates a session, and rebuilding
|
|
503
|
+
// it replaces one. Either makes the session current as of that moment.
|
|
504
|
+
const sessionSince = Math.max(meta?.startedAt ?? 0, rebuilt ?? 0) || undefined;
|
|
505
|
+
return sessionStaleness({ bootedAt, sessionSince });
|
|
506
|
+
}
|
|
507
|
+
|
|
508
|
+
/**
|
|
509
|
+
* Rebuild the session if the device outlived it. Once per process, per device.
|
|
510
|
+
*
|
|
511
|
+
* Rebuild, and retry nothing: this runs *before* the action, so the action is
|
|
512
|
+
* delivered on a session known to be current. Retrying afterwards is how an
|
|
513
|
+
* action fires twice, which is the hazard the verify barrier exists to
|
|
514
|
+
* prevent — and it is why the existing recovery covers hardware buttons only.
|
|
515
|
+
*/
|
|
516
|
+
const freshened = new Set();
|
|
517
|
+
export async function ensureFreshSession(udid) {
|
|
518
|
+
if (!udid || freshened.has(udid)) return null;
|
|
519
|
+
freshened.add(udid);
|
|
520
|
+
const health = await sessionHealth(udid);
|
|
521
|
+
if (!health.stale) return null;
|
|
522
|
+
const rebuilt = await resetSession(udid);
|
|
523
|
+
return { ...health, rebuilt };
|
|
524
|
+
}
|
|
525
|
+
|
|
526
|
+
/**
|
|
527
|
+
* When the HID session was last rebuilt, if it has been.
|
|
528
|
+
*
|
|
529
|
+
* The daemon's `startedAt` is the wrong clock on its own: rebuilding the
|
|
530
|
+
* session makes it current again without restarting the daemon, so comparing
|
|
531
|
+
* against the daemon's start left `doctor` reporting `stale` about a session
|
|
532
|
+
* that had just been rebuilt and was demonstrably working. It is a file rather
|
|
533
|
+
* than a variable because every CLI command is a new process and the daemon
|
|
534
|
+
* holding the session outlives all of them.
|
|
535
|
+
*/
|
|
536
|
+
const sessionFile = (udid) => path.join(store.deviceDir(udid), 'input-session.json');
|
|
537
|
+
|
|
436
538
|
export async function resetSession(udid) {
|
|
437
539
|
if (!control.available(udid)) return false;
|
|
438
540
|
try {
|
|
439
541
|
await control.resetInput(udid);
|
|
542
|
+
try {
|
|
543
|
+
fs.mkdirSync(store.deviceDir(udid), { recursive: true });
|
|
544
|
+
store.writeAtomic(sessionFile(udid), JSON.stringify({ rebuiltAt: Date.now() }));
|
|
545
|
+
} catch {
|
|
546
|
+
/* the rebuild happened; failing to write it down only costs a stale report */
|
|
547
|
+
}
|
|
440
548
|
return true;
|
|
441
549
|
} catch {
|
|
442
550
|
return false;
|
|
@@ -444,6 +552,7 @@ export async function resetSession(udid) {
|
|
|
444
552
|
}
|
|
445
553
|
|
|
446
554
|
export async function pressButton(udid, name) {
|
|
555
|
+
await ensureFreshSession(udid);
|
|
447
556
|
const own = inputDriverFor(udid);
|
|
448
557
|
if (own) {
|
|
449
558
|
// Android's whole key vocabulary is safe to offer: `input keyevent` takes
|
|
@@ -464,6 +573,7 @@ export async function pressButton(udid, name) {
|
|
|
464
573
|
}
|
|
465
574
|
|
|
466
575
|
export async function swipe(udid, from, to, { durationMs = 300 } = {}) {
|
|
576
|
+
await ensureFreshSession(udid);
|
|
467
577
|
const own = inputDriverFor(udid);
|
|
468
578
|
if (own) {
|
|
469
579
|
await own.swipe(udid, from, to, { durationMs });
|
package/src/intent.js
CHANGED
|
@@ -5,6 +5,7 @@
|
|
|
5
5
|
// instinct removes the expensive part of driving a UI with an agent: the model
|
|
6
6
|
// round trip spent reasoning about controls it has never seen.
|
|
7
7
|
import * as input from './input.js';
|
|
8
|
+
import * as metrics from './metrics.js';
|
|
8
9
|
|
|
9
10
|
/** Confirming words, most specific first. The first tier with a hit wins. */
|
|
10
11
|
const CONFIRM_TIERS = [
|
|
@@ -77,7 +78,11 @@ export function findOptions(nodes, geo) {
|
|
|
77
78
|
export async function chooseAny(udid, { prefer, geo } = {}) {
|
|
78
79
|
const nodes = await input.describeAll(udid);
|
|
79
80
|
const options = findOptions(nodes, geo);
|
|
80
|
-
if (!options.length)
|
|
81
|
+
if (!options.length) {
|
|
82
|
+
throw metrics.tag(new Error('no selectable options found on screen'), 'novel_dialog', {
|
|
83
|
+
candidates: nodes.filter((n) => n.label).slice(0, 8).map((n) => ({ label: n.label })),
|
|
84
|
+
});
|
|
85
|
+
}
|
|
81
86
|
const chosen =
|
|
82
87
|
(prefer && options.find((o) => o.label.toLowerCase().includes(String(prefer).toLowerCase()))) ||
|
|
83
88
|
options[0];
|
|
@@ -89,7 +94,11 @@ export async function chooseAny(udid, { prefer, geo } = {}) {
|
|
|
89
94
|
export async function confirm(udid, { geo } = {}) {
|
|
90
95
|
const nodes = await input.describeAll(udid);
|
|
91
96
|
const target = findConfirm(nodes, geo);
|
|
92
|
-
if (!target)
|
|
97
|
+
if (!target) {
|
|
98
|
+
throw metrics.tag(new Error('no confirming control on screen'), 'novel_dialog', {
|
|
99
|
+
candidates: nodes.filter((n) => n.label).slice(0, 8).map((n) => ({ label: n.label })),
|
|
100
|
+
});
|
|
101
|
+
}
|
|
93
102
|
const point = input.centerOf(target);
|
|
94
103
|
await input.tapPoint(udid, point.x, point.y);
|
|
95
104
|
return { label: target.label, point, enabled: target.enabled };
|
package/src/matching.js
CHANGED
|
@@ -182,6 +182,77 @@ export const SAME_CONTROL_POINTS = 12;
|
|
|
182
182
|
*/
|
|
183
183
|
const INTERACTIVE_ROLE = /button|field|cell|row|link|switch|slider|tab|menu|segment|checkbox/i;
|
|
184
184
|
|
|
185
|
+
/**
|
|
186
|
+
* Is this target the accessibility tree's reading of a control?
|
|
187
|
+
*
|
|
188
|
+
* A merged target carries `ax|ocr`, because both sensors saw it. Every
|
|
189
|
+
* comparison against the string `'ax'` had to become this: an element does not
|
|
190
|
+
* stop being the tree's element because OCR agreed with it, and treating
|
|
191
|
+
* `ax|ocr` as "not ax" would have quietly demoted exactly the elements the
|
|
192
|
+
* merge is most confident about.
|
|
193
|
+
*/
|
|
194
|
+
export const isAxTarget = (t) => /(^|\|)ax(\||$)/.test(t?.source ?? '');
|
|
195
|
+
|
|
196
|
+
/** How much of `inner` lies inside `outer`, as a fraction of inner's own area. */
|
|
197
|
+
export function containedFraction(inner, outer) {
|
|
198
|
+
if (!inner || !outer) return 0;
|
|
199
|
+
const iw = Math.max(0, inner.width ?? 0);
|
|
200
|
+
const ih = Math.max(0, inner.height ?? 0);
|
|
201
|
+
const innerArea = iw * ih;
|
|
202
|
+
if (innerArea <= 0) return 0;
|
|
203
|
+
const x = Math.max(inner.x, outer.x);
|
|
204
|
+
const y = Math.max(inner.y, outer.y);
|
|
205
|
+
const right = Math.min(inner.x + iw, outer.x + (outer.width ?? 0));
|
|
206
|
+
const bottom = Math.min(inner.y + ih, outer.y + (outer.height ?? 0));
|
|
207
|
+
const overlap = Math.max(0, right - x) * Math.max(0, bottom - y);
|
|
208
|
+
// Clamped: float arithmetic on sub-pixel OCR frames put a fully contained
|
|
209
|
+
// box at 1.0000000000000007, and a fraction of an area cannot exceed 1.
|
|
210
|
+
return Math.min(1, overlap / innerArea);
|
|
211
|
+
}
|
|
212
|
+
|
|
213
|
+
/**
|
|
214
|
+
* How much of the smaller box must sit inside the larger one to be the same
|
|
215
|
+
* element. OCR boxes sit a pixel or two outside the row they are printed on
|
|
216
|
+
* often enough that 1.0 would miss them.
|
|
217
|
+
*/
|
|
218
|
+
export const CONTAINMENT = 0.9;
|
|
219
|
+
|
|
220
|
+
/**
|
|
221
|
+
* Do these two strings name the same thing?
|
|
222
|
+
*
|
|
223
|
+
* The discriminator that makes containment safe. A tab bar contains all five
|
|
224
|
+
* of its tab labels, and merging a container with its contents is the failure
|
|
225
|
+
* the old size cap was defending against — but a tab bar's own label is not
|
|
226
|
+
* "Assets", so the text test refuses that merge while allowing a row labelled
|
|
227
|
+
* "Kate Bell" to absorb OCR's reading of "Kate Bell".
|
|
228
|
+
*
|
|
229
|
+
* Substring counts because iOS labels carry state the printed text does not:
|
|
230
|
+
* a row reads "Larger Text" on screen and publishes "Larger Text, Off".
|
|
231
|
+
*/
|
|
232
|
+
export function sameText(a, b) {
|
|
233
|
+
const x = norm(a);
|
|
234
|
+
const y = norm(b);
|
|
235
|
+
if (!x || !y) return false;
|
|
236
|
+
if (x === y) return true;
|
|
237
|
+
if (x.includes(y) || y.includes(x)) return Math.min(x.length, y.length) >= 3;
|
|
238
|
+
// Fuzzy, because OCR misreads a letter or two — measured: "Location (AII)"
|
|
239
|
+
// for "Location (All)", and a Cyrillic К for a K in a monogram.
|
|
240
|
+
return nameScore(x, y) >= 0.5;
|
|
241
|
+
}
|
|
242
|
+
|
|
243
|
+
/**
|
|
244
|
+
* The same element, seen by both sensors.
|
|
245
|
+
*
|
|
246
|
+
* `ax` is the tree's element, `ocr` a text box. True when the text sits
|
|
247
|
+
* (almost) wholly inside the element AND says the same thing as its label or
|
|
248
|
+
* value.
|
|
249
|
+
*/
|
|
250
|
+
export function sameElementSeenTwice(ax, ocr) {
|
|
251
|
+
if (!ax?.frame || !ocr?.frame) return false;
|
|
252
|
+
if (containedFraction(ocr.frame, ax.frame) < CONTAINMENT) return false;
|
|
253
|
+
return sameText(ax.label, ocr.label ?? ocr.text) || sameText(ax.value, ocr.label ?? ocr.text);
|
|
254
|
+
}
|
|
255
|
+
|
|
185
256
|
const contains = (frame, target) =>
|
|
186
257
|
Boolean(frame)
|
|
187
258
|
&& target.x >= frame.x && target.x <= frame.x + (frame.width ?? 0)
|
|
@@ -205,6 +276,14 @@ function sameControl(a, b) {
|
|
|
205
276
|
// tab bar from absorbing its own tabs.
|
|
206
277
|
if (INTERACTIVE_ROLE.test(a.type ?? '') && contains(a.frame, b)) return true;
|
|
207
278
|
if (INTERACTIVE_ROLE.test(b.type ?? '') && contains(b.frame, a)) return true;
|
|
279
|
+
// And a labelled accessibility element that is not an interactive role —
|
|
280
|
+
// a list row published as StaticText — with OCR's reading of its own label
|
|
281
|
+
// inside it. This is the pair that made `tap "Kate Bell"` refuse on every
|
|
282
|
+
// Contacts list: 16 escalations in the first instrumented run, all one
|
|
283
|
+
// screen. Defence in depth: the screen map now merges this pair at fusion,
|
|
284
|
+
// and a map built before that still resolves.
|
|
285
|
+
if (isAxTarget(a) && !isAxTarget(b) && sameElementSeenTwice(a, b)) return true;
|
|
286
|
+
if (isAxTarget(b) && !isAxTarget(a) && sameElementSeenTwice(b, a)) return true;
|
|
208
287
|
return false;
|
|
209
288
|
}
|
|
210
289
|
|
|
@@ -219,8 +298,8 @@ function collapseSamePlace(ranked) {
|
|
|
219
298
|
// Prefer the real hit target: an accessibility element over OCR's reading of
|
|
220
299
|
// it, and an interactive role over a caption sitting inside it.
|
|
221
300
|
const better = (candidate, incumbent) => {
|
|
222
|
-
if (candidate.target
|
|
223
|
-
if (candidate.target
|
|
301
|
+
if (isAxTarget(candidate.target) && !isAxTarget(incumbent.target)) return true;
|
|
302
|
+
if (!isAxTarget(candidate.target) && isAxTarget(incumbent.target)) return false;
|
|
224
303
|
return INTERACTIVE_ROLE.test(candidate.target.type ?? '')
|
|
225
304
|
&& !INTERACTIVE_ROLE.test(incumbent.target.type ?? '');
|
|
226
305
|
};
|
package/src/mcp.js
CHANGED
|
@@ -480,7 +480,7 @@ async function look(target, args, options) {
|
|
|
480
480
|
async function state(target, args, options) {
|
|
481
481
|
const { device } = await api.ensureDaemon(target, options);
|
|
482
482
|
const requested = baselineFor(device.udid, args.since);
|
|
483
|
-
const res = await api.getState(target, { since: requested, options });
|
|
483
|
+
const res = await api.getState(target, { since: requested, options, inputHealth: true });
|
|
484
484
|
const s = res.state;
|
|
485
485
|
const lines = [
|
|
486
486
|
header(res.device, s, res.ageMs),
|
|
@@ -500,6 +500,19 @@ async function state(target, args, options) {
|
|
|
500
500
|
}
|
|
501
501
|
const warn = livenessLine(res.live);
|
|
502
502
|
if (warn) lines.unshift(warn);
|
|
503
|
+
// An agent that cannot see this spends five turns wondering why a correct
|
|
504
|
+
// tap on a correct element did nothing. The repair happens before the next
|
|
505
|
+
// action either way; this is so the cause is visible when it does.
|
|
506
|
+
if (res.input?.stale) lines.unshift(`input: stale — ${res.input.reason}`);
|
|
507
|
+
// §7's timing line. Worth a line because "is it still coming or is it done"
|
|
508
|
+
// is a question an agent otherwise answers by waiting and guessing.
|
|
509
|
+
if (res.timing?.samples) {
|
|
510
|
+
lines.push(
|
|
511
|
+
`timing: usually ${res.timing.edge_p50}ms to arrive here (p95 ${res.timing.edge_p95}ms over `
|
|
512
|
+
+ `${res.timing.samples} samples), ${res.timing.elapsed_ms}ms since the last change`
|
|
513
|
+
+ (res.timing.slower_than_usual ? ` — ${res.timing.note}` : ''),
|
|
514
|
+
);
|
|
515
|
+
}
|
|
503
516
|
remember(device.udid, s);
|
|
504
517
|
return { content: [text(lines.filter(Boolean).join('\n'))] };
|
|
505
518
|
}
|