simframe 0.14.0 → 0.14.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -16,11 +16,20 @@ public struct AXNode: Sendable {
16
16
  public var selected: Bool?
17
17
  public var focused: Bool?
18
18
  public var frame: CGRect
19
+ /// Where the control actuates, when the app says so.
20
+ ///
21
+ /// Not the same as the centre of `frame`, and a switch is the proof: UIKit
22
+ /// publishes a row-wide frame for it, so the geometric centre is the label
23
+ /// and iOS does not actuate a switch from there. Measured — tapping the
24
+ /// centre flipped it 0 of 3 times and the control itself 3 of 3. UIKit has
25
+ /// `accessibilityActivationPoint` for exactly this question, and nil means
26
+ /// the app did not answer it, never a guess of ours.
27
+ public var activationPoint: CGPoint?
19
28
  public var depth: Int
20
29
 
21
30
  public init(role: String, subrole: String? = nil, label: String? = nil, value: String? = nil,
22
31
  identifier: String? = nil, enabled: Bool? = nil, selected: Bool? = nil,
23
- focused: Bool? = nil, frame: CGRect, depth: Int) {
32
+ focused: Bool? = nil, frame: CGRect, activationPoint: CGPoint? = nil, depth: Int) {
24
33
  self.role = role
25
34
  self.subrole = subrole
26
35
  self.label = label
@@ -30,6 +39,7 @@ public struct AXNode: Sendable {
30
39
  self.selected = selected
31
40
  self.focused = focused
32
41
  self.frame = frame
42
+ self.activationPoint = activationPoint
33
43
  self.depth = depth
34
44
  }
35
45
  }
@@ -251,7 +261,8 @@ public final class AccessibilityBridge {
251
261
 
252
262
  /// Everything scalar about a node, asked for in one go.
253
263
  private static let batched = ["AXRole", "AXSubrole", "AXDescription", "AXValue",
254
- "AXIdentifier", "AXEnabled", "AXSelected", "AXFocused"]
264
+ "AXIdentifier", "AXEnabled", "AXSelected", "AXFocused",
265
+ "AXActivationPoint"]
255
266
 
256
267
  private func node(from element: NSObject, depth: Int) -> AXNode {
257
268
  // One bridge round trip for eight attributes, not eight.
@@ -283,6 +294,7 @@ public final class AccessibilityBridge {
283
294
  selected: (value("AXSelected") as? NSNumber)?.boolValue,
284
295
  focused: (value("AXFocused") as? NSNumber)?.boolValue,
285
296
  frame: frame(of: element),
297
+ activationPoint: point(value("AXActivationPoint")) ?? activationPoint(of: element),
286
298
  depth: depth)
287
299
  }
288
300
 
@@ -310,6 +322,41 @@ public final class AccessibilityBridge {
310
322
  return unsafeBitCast(imp, to: RectFn.self)(element, sel)
311
323
  }
312
324
 
325
+ /// The activation point as the attribute bag reports it.
326
+ ///
327
+ /// Defensive about the shape because this is a private bridge and the
328
+ /// answer arrives as whatever the translator chose: an NSValue wrapping a
329
+ /// CGPoint on one path, a stringified `{x, y}` on another. Anything it does
330
+ /// not recognise is nil, which costs nothing — the caller falls back to the
331
+ /// frame centre, which is what it did before this attribute was asked for.
332
+ private func point(_ value: Any?) -> CGPoint? {
333
+ switch value {
334
+ case let v as NSValue:
335
+ // `pointValue`, not `cgPointValue`: this daemon is a macOS binary
336
+ // driving a guest, and the iOS-only accessor does not exist here.
337
+ // NSPoint and CGPoint are the same type on 64-bit.
338
+ let p = v.pointValue
339
+ return p.x.isFinite && p.y.isFinite ? p : nil
340
+ case let s as String:
341
+ let parts = s.trimmingCharacters(in: CharacterSet(charactersIn: "{} "))
342
+ .split(separator: ",")
343
+ .compactMap { Double($0.trimmingCharacters(in: .whitespaces)) }
344
+ guard parts.count == 2 else { return nil }
345
+ return CGPoint(x: parts[0], y: parts[1])
346
+ default:
347
+ return nil
348
+ }
349
+ }
350
+
351
+ /// The direct selector, for elements that answer it but do not batch it.
352
+ private func activationPoint(of element: NSObject) -> CGPoint? {
353
+ let sel = NSSelectorFromString("accessibilityActivationPoint")
354
+ guard element.responds(to: sel), let imp = element.method(for: sel) else { return nil }
355
+ typealias PointFn = @convention(c) (AnyObject, Selector) -> CGPoint
356
+ let p = unsafeBitCast(imp, to: PointFn.self)(element, sel)
357
+ return p.x.isFinite && p.y.isFinite ? p : nil
358
+ }
359
+
313
360
  /// A value may be a string, a number, or something with no useful text.
314
361
  private func string(_ value: Any?) -> String? {
315
362
  switch value {
@@ -60,10 +60,20 @@ public struct Element: Sendable {
60
60
  public var state: ElementState
61
61
  public var source: ElementSource
62
62
  public var confidence: Double
63
+ /// Where the control actuates, when the app publishes it.
64
+ ///
65
+ /// Deliberately NOT folded into `center`. `center` is the element's place,
66
+ /// and two readings of one control are matched by how close their places
67
+ /// are — 12pt apart, per `SAME_CONTROL_POINTS` above the boundary. A switch
68
+ /// actuates at the far end of a row-wide frame, so making `center` mean
69
+ /// "where to tap" would move the tree's reading ~145pt away from OCR's and
70
+ /// stop the two collapsing into one control. Aiming and identity are
71
+ /// different questions; this is the answer to the first one only.
72
+ public var activationPoint: CGPoint?
63
73
 
64
74
  public init(id: Int, frame: CGRect, role: String, label: String? = nil, value: String? = nil,
65
75
  identifier: String? = nil, state: ElementState = ElementState(),
66
- source: ElementSource, confidence: Double = 1) {
76
+ source: ElementSource, confidence: Double = 1, activationPoint: CGPoint? = nil) {
67
77
  self.id = id
68
78
  self.frame = frame
69
79
  self.role = role
@@ -73,6 +83,7 @@ public struct Element: Sendable {
73
83
  self.state = state
74
84
  self.source = source
75
85
  self.confidence = confidence
86
+ self.activationPoint = activationPoint
76
87
  }
77
88
 
78
89
  public var center: CGPoint { CGPoint(x: frame.midX, y: frame.midY) }
@@ -89,6 +100,11 @@ public struct Element: Sendable {
89
100
  if let label { out["label"] = label }
90
101
  if let value { out["value"] = value }
91
102
  if let identifier { out["identifier"] = identifier }
103
+ // Only when the app answered. Absent means "no opinion", and the layer
104
+ // above then aims at the centre exactly as it always has.
105
+ if let activationPoint {
106
+ out["activationPoint"] = ["x": Int(activationPoint.x.rounded()), "y": Int(activationPoint.y.rounded())]
107
+ }
92
108
  let state = state.json
93
109
  if !state.isEmpty { out["state"] = state }
94
110
  return out
@@ -112,7 +128,8 @@ public extension Element {
112
128
  state: ElementState(enabled: node.enabled, selected: node.selected,
113
129
  checked: nil, focused: node.focused),
114
130
  source: .accessibility,
115
- confidence: 1)
131
+ confidence: 1,
132
+ activationPoint: node.activationPoint)
116
133
  }
117
134
  }
118
135
 
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "simframe",
3
- "version": "0.14.0",
3
+ "version": "0.14.2",
4
4
  "mcpName": "io.github.lvlrSajjad/simframe",
5
5
  "description": "Always-warm iOS Simulator and Android emulator frames: agents read the screen in ~20ms instead of waiting on screenshots. MCP server + CLI.",
6
6
  "keywords": [
@@ -0,0 +1,89 @@
1
+ #!/usr/bin/env node
2
+ /**
3
+ * Of the `verification_failed` escalations, how many happened on a
4
+ * `(screen, action)` the graph had already seen behave consistently?
5
+ * EXPERIMENTS §17. The answer was 3% and 0%, which is why this is kept: it is
6
+ * the measurement that stopped a phase being spent on remembering edge
7
+ * outcomes.
8
+ *
9
+ * Two thirds of those escalations are `no-visible-change`, and 83-100% of them
10
+ * are on a screen or an action the graph has never seen — memory cannot answer
11
+ * a question about a place it has never been.
12
+ *
13
+ * The graph is read as it is NOW, so an edge matched here may have been learned
14
+ * after the escalation. That makes the "addressable" figure an upper bound, and
15
+ * it is already near zero.
16
+ *
17
+ * Aggregate by design, and stricter than it looks: `detail` is app content, so
18
+ * verdicts are classified against a CLOSED vocabulary and the raw string is
19
+ * never printed. The first version of this printed the prefix and put a
20
+ * client's screen labels and a customer email into a terminal.
21
+ *
22
+ * Usage: node scripts/analyse-escalations.mjs <udid-prefix> [<udid-prefix>...] */
23
+ import fs from 'node:fs';
24
+ import path from 'node:path';
25
+ import os from 'node:os';
26
+
27
+ const root = process.env.SIMFRAME_HOME || path.join(os.homedir(), '.simframe');
28
+
29
+ for (const dev of process.argv.slice(2)) {
30
+ const dir = fs.readdirSync(root).find((d) => d.startsWith(dev));
31
+ if (!dir) continue;
32
+ const base = path.join(root, dir);
33
+ const gdir = path.join(base, 'graph');
34
+ if (!fs.existsSync(gdir)) continue;
35
+
36
+ const nodes = fs.readdirSync(gdir).filter((f) => f.endsWith('.json'))
37
+ .map((f) => { try { return JSON.parse(fs.readFileSync(path.join(gdir, f), 'utf8')); } catch { return null; } })
38
+ .filter(Boolean);
39
+ const byHash = new Map();
40
+ for (const n of nodes) { byHash.set(n.hash, n); for (const v of n.variants ?? []) if (!byHash.has(v.hash)) byHash.set(v.hash, n); }
41
+
42
+ const esc = [];
43
+ const p = path.join(base, 'escalations.jsonl');
44
+ if (!fs.existsSync(p)) continue;
45
+ for (const line of fs.readFileSync(p, 'utf8').split('\n')) {
46
+ const t = line.trim(); if (!t) continue;
47
+ try { esc.push(JSON.parse(t)); } catch {}
48
+ }
49
+ const vf = esc.filter((e) => e.reason === 'verification_failed');
50
+
51
+ const bucket = { noScreen: 0, noEdge: 0, seenOnce: 0, consistent: 0, nondet: 0 };
52
+ const verdicts = new Map();
53
+ for (const e of vf) {
54
+ // Classify against a CLOSED vocabulary and never emit the raw text. The
55
+ // first version split on ':' and printed the prefix, which put a client's
56
+ // screen labels and an email address into a terminal. `detail` is app
57
+ // content; only its shape is ours to report.
58
+ const d = String(e.detail ?? '').toLowerCase();
59
+ const v = d.includes('no-visible-change') || d.includes('did not change') ? 'no-visible-change'
60
+ : d.includes('unexpected-screen') ? 'unexpected-screen'
61
+ : d.includes('did not settle') ? 'settle-timeout'
62
+ : d.includes('waited') ? 'wait-timeout'
63
+ : d.includes('matches') && d.includes('things') ? 'ambiguous'
64
+ : d.includes('numbered on a different') ? 'stale-ref'
65
+ : d.includes('disabled') ? 'disabled-control'
66
+ : d ? 'other' : '(none)';
67
+ verdicts.set(v, (verdicts.get(v) ?? 0) + 1);
68
+ const node = byHash.get(e.screen_fingerprint ?? '');
69
+ if (!node) { bucket.noScreen++; continue; }
70
+ const want = String(e.intent ?? '').toLowerCase();
71
+ const hit = (node.edges ?? []).find((x) => {
72
+ const sig = String(x.action ?? '').toLowerCase();
73
+ return want && (sig.endsWith(`:${want}`) || sig.includes(want));
74
+ });
75
+ if (!hit) { bucket.noEdge++; continue; }
76
+ if ((hit.changedOutcomes ?? 0) > 0) bucket.nondet++;
77
+ else if ((hit.count ?? 0) > 1) bucket.consistent++;
78
+ else bucket.seenOnce++;
79
+ }
80
+ const pct = (n) => (vf.length ? Math.round((100 * n) / vf.length) : 0);
81
+ console.log(`--- ${dev} ---`);
82
+ console.log(` verification_failed: ${vf.length}`);
83
+ console.log(` screen not in graph at all: ${bucket.noScreen} (${pct(bucket.noScreen)}%)`);
84
+ console.log(` screen known, this action never seen: ${bucket.noEdge} (${pct(bucket.noEdge)}%)`);
85
+ console.log(` edge seen exactly once: ${bucket.seenOnce} (${pct(bucket.seenOnce)}%)`);
86
+ console.log(` edge NONDETERMINISTIC (rightly asked): ${bucket.nondet} (${pct(bucket.nondet)}%)`);
87
+ console.log(` edge repeated and CONSISTENT: ${bucket.consistent} (${pct(bucket.consistent)}%) <- addressable`);
88
+ console.log(` verdict words: ${[...verdicts].sort((a,b)=>b[1]-a[1]).slice(0,6).map(([k,n])=>`${k}=${n}`).join(', ')}`);
89
+ }
@@ -0,0 +1,96 @@
1
+ #!/usr/bin/env node
2
+ /**
3
+ * Would routing by measured time differ from routing by hop count, and by how
4
+ * much? EXPERIMENTS §16, kept runnable because the answer was "barely" and that
5
+ * is the kind of result that gets re-litigated by someone's intuition.
6
+ *
7
+ * `route()` in graph.js is a plain BFS over hop count, while `record()` has
8
+ * been storing a settle sample on nearly every edge all along (97% and 85% of
9
+ * edges on the two real-app graphs measured). This compares the two.
10
+ *
11
+ * Measured: the fastest route differs from the fewest-hops route on 9%, 0% and
12
+ * 7% of reachable pairs, saving a median of 1853ms, — and 615ms. Real, unused,
13
+ * and pointed at the wrong prize — it saves seconds, not round trips, and round
14
+ * trips are what the human-parity series exists to reduce.
15
+ *
16
+ * Output is aggregate by design: this reads a real device's memory of real
17
+ * third-party apps, so it prints counts and never a label, a screen or an app
18
+ * name. Same standing rule as scripts/phase17-corpus.mjs.
19
+ *
20
+ * Usage: node scripts/analyse-routes.mjs <udid-prefix> [<udid-prefix>...] */
21
+ import fs from 'node:fs';
22
+ import path from 'node:path';
23
+ import os from 'node:os';
24
+
25
+ const root = process.env.SIMFRAME_HOME || path.join(os.homedir(), '.simframe');
26
+ const median = (a) => { const s=[...a].sort((x,y)=>x-y); return s.length? (s.length%2? s[(s.length-1)/2] : (s[s.length/2-1]+s[s.length/2])/2) : null; };
27
+
28
+ for (const dev of process.argv.slice(2)) {
29
+ const dir = fs.readdirSync(root).find((d) => d.startsWith(dev));
30
+ if (!dir) continue;
31
+ const gdir = path.join(root, dir, 'graph');
32
+ if (!fs.existsSync(gdir)) continue;
33
+ const nodes = fs.readdirSync(gdir).filter(f=>f.endsWith('.json'))
34
+ .map(f => { try { return JSON.parse(fs.readFileSync(path.join(gdir,f),'utf8')); } catch { return null; } })
35
+ .filter(Boolean);
36
+
37
+ const byHash = new Map();
38
+ for (const n of nodes) { byHash.set(n.hash, n); for (const v of n.variants ?? []) if(!byHash.has(v.hash)) byHash.set(v.hash, n); }
39
+ const canonical = (h) => byHash.get(h)?.hash ?? h;
40
+
41
+ // Edge costs: the median settle actually measured on that edge.
42
+ const all = [];
43
+ for (const n of nodes) for (const e of n.edges ?? []) { const m = median(e.settles ?? []); if (m!=null) all.push(m); }
44
+ const fallback = median(all) ?? 500;
45
+ const cost = (e) => median(e.settles ?? []) ?? fallback;
46
+
47
+ // BFS exactly as graph.js does it: first path found, insertion order.
48
+ const bfs = (start, goal) => {
49
+ const seen = new Set([start]); const q = [{h:start, p:[]}];
50
+ while (q.length) { const {h,p} = q.shift();
51
+ if (p.length >= 12) continue;
52
+ for (const e of byHash.get(h)?.edges ?? []) {
53
+ const to = canonical(e.to); const next=[...p,e];
54
+ if (to === goal) return next;
55
+ if (seen.has(to)) continue; seen.add(to); q.push({h:to,p:next});
56
+ } }
57
+ return null;
58
+ };
59
+ // Dijkstra on measured time.
60
+ const fastest = (start, goal) => {
61
+ const dist = new Map([[start,0]]); const prev = new Map(); const done = new Set();
62
+ while (true) {
63
+ let cur=null, best=Infinity;
64
+ for (const [h,d] of dist) if (!done.has(h) && d<best) { best=d; cur=h; }
65
+ if (cur==null) break;
66
+ if (cur===goal) break;
67
+ done.add(cur);
68
+ for (const e of byHash.get(cur)?.edges ?? []) {
69
+ const to = canonical(e.to); const nd = best + cost(e);
70
+ if (nd < (dist.get(to) ?? Infinity)) { dist.set(to,nd); prev.set(to,[cur,e]); }
71
+ } }
72
+ if (!dist.has(goal)) return null;
73
+ const out=[]; let at=goal;
74
+ while (prev.has(at)) { const [from,e]=prev.get(at); out.unshift(e); at=from; }
75
+ return out;
76
+ };
77
+
78
+ const canon = [...new Set(nodes.map(n=>n.hash))];
79
+ let pairs=0, differ=0, saved=[], hopsUp=0;
80
+ for (const s of canon) for (const g of canon) {
81
+ if (s===g) continue;
82
+ const b = bfs(s,g); if (!b) continue;
83
+ const f = fastest(s,g); if (!f) continue;
84
+ pairs++;
85
+ const cb = b.reduce((t,e)=>t+cost(e),0), cf = f.reduce((t,e)=>t+cost(e),0);
86
+ if (cf < cb - 1) { differ++; saved.push(cb-cf); if (f.length > b.length) hopsUp++; }
87
+ }
88
+ saved.sort((a,b)=>a-b);
89
+ console.log(`--- ${dev} ---`);
90
+ console.log(` reachable pairs: ${pairs}`);
91
+ console.log(` where the fastest route differs from the fewest-hops route: ${differ} (${pairs?Math.round(100*differ/pairs):0}%)`);
92
+ if (saved.length) {
93
+ console.log(` time saved per such route — median ${Math.round(median(saved))}ms, max ${Math.round(saved[saved.length-1])}ms`);
94
+ console.log(` ...of those, routes that take MORE hops to be faster: ${hopsUp}`);
95
+ }
96
+ }
@@ -225,10 +225,28 @@ function writeFlow(name, steps) {
225
225
  return file;
226
226
  }
227
227
 
228
- // A closed loop: openUrl puts Safari in front, home leaves it. Both ends are
229
- // screens the graph can learn, and every pass starts where the last one ended.
228
+ // A closed loop: launching an app puts it in front, home leaves it. Both ends
229
+ // are screens the graph can learn, and every pass starts where the last one
230
+ // ended.
231
+ //
232
+ // **It used to be `openUrl https://example.com`, and that was the bug.** Item
233
+ // 142 catalogued `simctl openurl` timing out on a loaded runner as a failure
234
+ // class, marked it "fixed — vehicle changed", and changed the vehicle in the
235
+ // *workflow's* step only. This loop was left on it, and so was the novel action
236
+ // below. That is the same class-versus-symptom error the `waitFor`/`assert`
237
+ // twin recorded: the fix went where the report pointed instead of everywhere
238
+ // the cause reached.
239
+ //
240
+ // It came back on 2026-09-14: passes 2 and 3 halted at step 0, the graph never
241
+ // got the chance to predict, and the failure read as "the outcome is predicted
242
+ // — pass 0", which names the graph for something Safari did.
243
+ //
244
+ // The replacement is the vehicle item 142 measured and proved for exactly this:
245
+ // **from inside an app, pressing home always changes the screen.** Settings is
246
+ // already installed everywhere this runs, the launch needs no network, and
247
+ // neither end depends on a browser cold-starting on a shared machine.
230
248
  const LOOP = writeFlow('simframe-ci-loop.json', [
231
- { openUrl: 'https://example.com' },
249
+ { launch: { value: 'com.apple.Preferences', relaunch: true } },
232
250
  { button: 'home' },
233
251
  ]);
234
252
  // Leaving whatever screen the map was read on.
@@ -537,10 +555,23 @@ if (novelRan && novelMoved) {
537
555
  // so a run in which every pass failed to dispatch says nothing about
538
556
  // prediction. It failed the build as `pass 0` while the real cause was a
539
557
  // simctl launch timing out, three checks upstream.
540
- const anyPassRan = passes.some((p) => Array.isArray(p.run?.results) && p.run.results.some((r) => r.ok !== false));
541
- if (!anyPassRan) {
558
+ //
559
+ // **The rule was right and the test of it was too coarse.** `anyPassRan` asks
560
+ // whether *any* pass ran, but prediction can only be observed on a pass AFTER
561
+ // the one that taught the edge — so pass 1 running is not enough. On
562
+ // 2026-09-14 pass 1 ran, passes 2 and 3 halted at step 0, and this reported
563
+ // `pass 0` as though the graph had declined to predict. Nothing had asked it
564
+ // to. Same sentence as the novel action three checks above: untested is not
565
+ // broken, and the guard has to test the pass the claim actually depends on.
566
+ const dispatched = (p) => Array.isArray(p.run?.results) && p.run.results.some((r) => r.ok !== false);
567
+ const laterPassRan = passes.slice(1).some(dispatched);
568
+ if (!passes.some(dispatched)) {
542
569
  skip('and once the graph has seen it, the outcome is predicted',
543
570
  'no pass dispatched a step, so the graph was never given anything to learn');
571
+ } else if (!laterPassRan) {
572
+ skip('and once the graph has seen it, the outcome is predicted',
573
+ `only the first pass dispatched a step (${passes.slice(1).map((p, i) => `pass ${i + 2}: [${p.verdicts.join(', ')}]`).join('; ')})`
574
+ + ' — prediction is only observable on a pass after the one that taught the edge');
544
575
  } else {
545
576
  check(passes.some((p) => p.verdicts.includes('ok')),
546
577
  'and once the graph has seen it, the outcome is predicted',
@@ -88,6 +88,8 @@ const readings = [];
88
88
  const arrivalFailures = [];
89
89
  /** Readings too bare to be a screen — see the guard where this is used. */
90
90
  const sparseReadings = [];
91
+ /** `name|round` of every reading that stayed too sparse, so the report below can name the cause. */
92
+ const sparseAt = new Set();
91
93
  /**
92
94
  * Below this, a reading cannot distinguish its screen from any other bare one.
93
95
  *
@@ -144,6 +146,14 @@ const save = (extra = {}) => {
144
146
  * rather than hanging the job.
145
147
  */
146
148
  const STALE_READ_RETRIES = 3;
149
+ /**
150
+ * How hard to try for a reading rich enough to tell its screen apart.
151
+ *
152
+ * Backed off rather than fixed, because the thing being waited for is a screen
153
+ * finishing its draw, and the runner that needs this is the slow one.
154
+ */
155
+ const SPARSE_READ_RETRIES = 3;
156
+ const SPARSE_READ_WAIT_MS = 1500;
147
157
  const STALE_READ_WAIT_MS = 1000;
148
158
 
149
159
  /** When the steps that were supposed to change the screen finished. */
@@ -151,6 +161,52 @@ let navigatedAt = 0;
151
161
  /** Readings that never got a frame newer than their own navigation. */
152
162
  const staleReadings = [];
153
163
 
164
+ /**
165
+ * Visit every screen once before measuring, taking no readings.
166
+ *
167
+ * **The rounds are supposed to be repeat measurements of one thing.** They were
168
+ * not: round 1 systematically differed from rounds 2 and 3 whenever an app was
169
+ * cold, because an app that has just launched has not finished publishing its
170
+ * accessibility tree. Measured on CI — round 1 of `browser` read `e00725fce7`
171
+ * with **0 named** elements where rounds 2 and 3 read `9589eb2471` with **5**,
172
+ * while every other screen matched across all three. That is the eval measuring
173
+ * app cold-start, which it never set out to measure and does not report.
174
+ *
175
+ * It was invisible for a long time because something else was paying for it:
176
+ * the memory-layer step runs first on CI and drives the same apps for 504s,
177
+ * Safari included, so the eval always met a warm device. Sharding the job
178
+ * removed that neighbour and the dependency surfaced immediately. The defect
179
+ * was always here; the neighbour was hiding it.
180
+ *
181
+ * This is **not** the same as making the readings warm. Every reading is still
182
+ * taken with `fresh: true` against cold screen memory, which is what "cold"
183
+ * means in this harness — the graph must not have seen the screen before. What
184
+ * the warm-up removes is a variable about the *operating system* that the
185
+ * fingerprint has nothing to do with.
186
+ *
187
+ * Skippable with `--no-warmup`, because the comparison is the evidence: run it
188
+ * both ways to see whether round 1 still disagrees with its own repeats.
189
+ */
190
+ async function warmUp() {
191
+ const started = Date.now();
192
+ console.log('warming: visiting each screen once, taking no readings');
193
+ for (const screen of tour) {
194
+ if (!screen.steps?.length) continue;
195
+ try {
196
+ await actions.runScript(device, { steps: screen.steps, verify: false });
197
+ } catch (err) {
198
+ // A warm-up failure is not a result. The measured rounds below will meet
199
+ // the same screen and fail there with the harness's own reporting, which
200
+ // says which screen and writes the readings out. Failing here would cost
201
+ // that and report a screen that was never measured.
202
+ console.log(` (warm-up could not reach "${screen.name}": ${err.message})`);
203
+ }
204
+ }
205
+ console.log(`warming: done in ${Math.round((Date.now() - started) / 1000)}s\n`);
206
+ }
207
+
208
+ if (!process.argv.includes('--no-warmup')) await warmUp();
209
+
154
210
  for (let round = 1; round <= rounds; round += 1) {
155
211
  for (const screen of tour) {
156
212
  if (screen.steps?.length) {
@@ -176,7 +232,30 @@ for (let round = 1; round <= rounds; round += 1) {
176
232
  process.exit(1);
177
233
  }
178
234
  }
179
- let id = await api.screenIdentity(device, { fresh: true, confirmNovel: false });
235
+ // A read can throw, and when it does the readings so far are the evidence.
236
+ //
237
+ // This file already says so — *"Everything read so far is written out
238
+ // first, because a failing run is the one whose evidence matters"* — and
239
+ // then left this call unguarded, so a runner whose OCR overran its budget
240
+ // produced a raw stack trace, no `--out` file, and nothing for
241
+ // `analyse-fingerprint.mjs` to read. Exactly the failure the `save()` above
242
+ // was written to prevent, one line away from it.
243
+ //
244
+ // Not retried here. The read budget is already the client's own give-up
245
+ // point, so a read that overran it is a statement about the machine, and
246
+ // the honest thing is to say which screen and how far the tour got.
247
+ let id;
248
+ try {
249
+ id = await api.screenIdentity(device, { fresh: true, confirmNovel: false });
250
+ } catch (err) {
251
+ save({ abandonedAt: { screen: screen.name, round, error: err.message, phase: 'reading' } });
252
+ console.error(`\nFAIL round ${round}, "${screen.name}" could not be read: ${err.message}`);
253
+ console.error(`${readings.length} reading(s) taken before this were written out.`);
254
+ console.error('A read that overran its budget is a fact about the host, not about the');
255
+ console.error('fingerprint — the budget is already the client\'s own give-up point, so');
256
+ console.error('raising it would only move where the wait ends.');
257
+ process.exit(1);
258
+ }
180
259
  // A reading off a frame older than the navigation is not a reading either.
181
260
  //
182
261
  // Same rule as the sparseness guard below, on the other axis, and it took a
@@ -227,13 +306,36 @@ for (let round = 1; round <= rounds; round += 1) {
227
306
  // "untested is not passed" rule the memory harness learned. A screen that
228
307
  // is genuinely this bare after a second look is a real finding.
229
308
  if ((id.tokens ?? []).length < MIN_TOKENS_FOR_A_READING) {
309
+ // Read again, and keep the BEST reading rather than the last one.
310
+ //
311
+ // One retry was not enough and the failure it produced pointed at the
312
+ // wrong thing. On CI, `settings-general` read 4 tokens twice and then
313
+ // scored 1.00 against the *Settings root* — because the General screen's
314
+ // back button is labelled "Settings", so a General seen only down to its
315
+ // nav bar is structurally the same screen as the root. The harness
316
+ // reported it as a fingerprint that does not resemble its own screen. The
317
+ // fingerprint was fine; the look was too short. `settings-general` reads
318
+ // 5 tokens when it is read properly.
319
+ //
320
+ // Best-of, not last, because these reads are samples of a screen that is
321
+ // still drawing: a later read is usually richer but not reliably so, and
322
+ // throwing away a 5-token reading because the retry saw 4 would be the
323
+ // same bug with more steps.
230
324
  const before = (id.tokens ?? []).length;
231
- await new Promise((r) => setTimeout(r, 1500));
232
- id = await api.screenIdentity(device, { fresh: true, confirmNovel: false });
325
+ const seen = [before];
326
+ for (let attempt = 1; attempt <= SPARSE_READ_RETRIES; attempt += 1) {
327
+ await new Promise((r) => setTimeout(r, SPARSE_READ_WAIT_MS * attempt));
328
+ const again = await api.screenIdentity(device, { fresh: true, confirmNovel: false });
329
+ seen.push((again.tokens ?? []).length);
330
+ if ((again.tokens ?? []).length > (id.tokens ?? []).length) id = again;
331
+ if ((id.tokens ?? []).length >= MIN_TOKENS_FOR_A_READING) break;
332
+ }
233
333
  const after = (id.tokens ?? []).length;
234
- console.log(` ("${screen.name}" read ${before} token(s) — too sparse to compare; read again: ${after})`);
334
+ console.log(` ("${screen.name}" read ${before} token(s) — too sparse to compare;`
335
+ + ` read again: ${seen.slice(1).join(', ')} — kept ${after})`);
235
336
  if (after < MIN_TOKENS_FOR_A_READING) {
236
- sparseReadings.push(`${screen.name} round ${round}: ${after} token(s) after two reads`);
337
+ sparseReadings.push(`${screen.name} round ${round}: ${after} token(s) after ${seen.length} reads`);
338
+ sparseAt.add(`${screen.name}|${round}`);
237
339
  }
238
340
  }
239
341
  // Did we actually arrive? Two differently-named screens reading the same
@@ -419,6 +521,38 @@ if (strays.length) {
419
521
  const matchNamed = match ? namedTokens(match.tokens) : [];
420
522
  const collided = bestOther >= 0.99 && named.length === 0 && matchNamed.length === 0;
421
523
  if (collided) collisions += 1;
524
+ // The third cause, and the one that produced this report on 2026-09-15.
525
+ //
526
+ // A reading that stayed under the token floor did not fail to resemble its
527
+ // screen — it never saw enough of the screen to resemble anything. On CI
528
+ // `settings-general` read 4 tokens and scored 1.00 against the Settings
529
+ // root, because the General screen's back button is labelled "Settings", so
530
+ // a General seen only down to its nav bar IS the root structurally. The
531
+ // report called that a wrong turn and sent the reader to fix the tour.
532
+ //
533
+ // Not folded into `collided` above: that one means the fingerprint had
534
+ // nothing to work with, which is this harness's subject. This means we did
535
+ // not look long enough, which is the harness's own fault and a different
536
+ // remedy.
537
+ //
538
+ // **And sparseness alone is not enough to claim it**, which this
539
+ // misdiagnosed once. A wrong turn that lands on a screen which legitimately
540
+ // reads sparse gets flagged sparse too, and was then reported as our
541
+ // instrument's fault rather than the tour's. Measured: `settings-general`
542
+ // r3 came back with tokens byte-identical to the Settings root, including
543
+ // `heading:nav-bar:"settings"` — a *heading*, which is the root's title,
544
+ // where General publishes a *button* with the same word. It was read on the
545
+ // wrong screen, and this called it an under-read.
546
+ //
547
+ // The discriminator is the screen's own other rounds: if `settings-general`
548
+ // read differently and richly in rounds 1 and 2, then the screen IS
549
+ // distinguishable and a round matching another screen exactly went
550
+ // somewhere else. Only when no round of this screen can tell itself apart
551
+ // is "we did not look long enough" the honest reading.
552
+ const distinguishable = (byName.get(reading.name) ?? [])
553
+ .some((o) => o !== reading && fingerprint.similarity(o.tokens, match?.tokens ?? []) < 0.99);
554
+ const underRead = sparseAt.has(`${reading.name}|${reading.round}`) && !distinguishable;
555
+ const wrongScreen = bestOther >= 0.99 && distinguishable;
422
556
  console.error(` ${reading.name} r${reading.round}: own screen ${bestSelf.toFixed(2)}, `
423
557
  + `${match ? `${match.name} r${match.round}` : 'another screen'} ${bestOther.toFixed(2)} `
424
558
  + `(${reading.count} tokens, ${named.length} named, sources ${reading.sources.join('+') || 'none'})`);
@@ -426,6 +560,15 @@ if (strays.length) {
426
560
  console.error(' ^ a COLLISION, not a wrong turn: neither reading carries a chrome');
427
561
  console.error(' label, so both are structure with no name and the fingerprint has');
428
562
  console.error(' nothing left to tell two list screens apart.');
563
+ } else if (wrongScreen) {
564
+ console.error(` ^ WRONG SCREEN: these tokens are ${match ? `identical to ${match.name}` : 'another screen'},`);
565
+ console.error(' and this screen reads differently in its other rounds — so it is');
566
+ console.error(' distinguishable and the tour was simply somewhere else. A tap that');
567
+ console.error(' missed, or a screen that went back before it was read.');
568
+ } else if (underRead) {
569
+ console.error(' ^ UNDER-READ, not a wrong turn: this reading stayed below the token');
570
+ console.error(' floor after every retry, so it never saw enough of its screen to');
571
+ console.error(' resemble one. Two screens read this thinly are the same screen.');
429
572
  }
430
573
  if (named.length) console.error(` names: ${named.map((t) => t.slice(t.indexOf('"'), t.lastIndexOf('"') + 1)).join(' ')}`);
431
574
  }
@@ -433,6 +576,10 @@ if (strays.length) {
433
576
  console.error(`\n${collisions} of ${strays.length} are fingerprint collisions. That is this harness's own subject,`);
434
577
  console.error('not a tour fault: a reading whose chrome label went missing cannot establish');
435
578
  console.error('identity, and comparing it as though it could is what produced the verdict above.');
579
+ } else if (strays.every((x) => sparseAt.has(`${x.reading.name}|${x.reading.round}`))) {
580
+ console.error('\nEvery stray above was UNDER-READ, so this says nothing about the tour or the');
581
+ console.error('fingerprint — the harness scored a look that was too short. The retries are in');
582
+ console.error('SPARSE_READ_RETRIES; a runner that needs more than they allow is the finding.');
436
583
  } else {
437
584
  console.error('\nThat is the tour going somewhere unintended, not the fingerprint drifting, and');
438
585
  console.error('measuring it as either distribution poisons both ends. Fix the tour — a tap that');
package/src/actions.js CHANGED
@@ -90,16 +90,49 @@ const ACTION_STEPS = new Set([
90
90
  'launch', 'terminate', 'openUrl', 'confirm', 'chooseAny', 'permission',
91
91
  ]);
92
92
 
93
+ /**
94
+ * Step keys that are really the MCP tool names, accepted as aliases.
95
+ *
96
+ * Two field reports guessed these independently and each wrong guess cost a
97
+ * round trip and aborted the rest of the batch. There is a `sim_wait` tool and
98
+ * a `sim_type_into` tool, so `wait` and `type_into` are what a caller reaches
99
+ * for inside `sim_do` — and the vocabulary answered "unknown step" without
100
+ * saying what the words are. A tool surface that names an action one way and
101
+ * accepts it another is charging the caller for our inconsistency.
102
+ */
103
+ const STEP_ALIASES = Object.freeze({
104
+ wait: 'settle',
105
+ type_into: 'type',
106
+ typeInto: 'type',
107
+ scroll_to: 'scrollTo',
108
+ wait_for: 'waitFor',
109
+ waitfor: 'waitFor',
110
+ tap_at: 'tapAt',
111
+ open_url: 'openUrl',
112
+ assert_gone: 'assertGone',
113
+ assert_text: 'assertText',
114
+ });
115
+
116
+ /** Every step key `runStep` understands, for an error that can be acted on. */
117
+ export const STEP_KEYS = Object.freeze([
118
+ 'tap', 'tapAt', 'type', 'paste', 'clear', 'swipe', 'scroll', 'scrollTo',
119
+ 'button', 'key', 'launch', 'terminate', 'openUrl', 'confirm', 'chooseAny',
120
+ 'permission', 'settle', 'waitFor', 'waitText', 'assert', 'assertGone',
121
+ 'assertText', 'visible', 'gone', 'enabled', 'disabled', 'value', 'look',
122
+ 'seek', 'sweep', 'pause',
123
+ ]);
124
+
93
125
  /** Accept both `{tap: "Save"}` shorthand and `{action: "tap", target: "Save"}`. */
94
126
  export function normalizeStep(raw) {
95
- if (typeof raw === 'string') return { action: raw };
96
- if (raw.action) return { ...raw };
127
+ const canonical = (a) => STEP_ALIASES[a] ?? a;
128
+ if (typeof raw === 'string') return { action: canonical(raw) };
129
+ if (raw.action) return { ...raw, action: canonical(raw.action) };
97
130
  const [key] = Object.keys(raw);
98
131
  if (!key) throw new Error('empty step');
99
132
  // Siblings like timeoutMs sit alongside the shorthand key and must survive.
100
133
  const { [key]: value, ...rest } = raw;
101
134
  const inline = value && typeof value === 'object' && !Array.isArray(value) ? value : { value };
102
- const step = { ...rest, ...inline, action: key };
135
+ const step = { ...rest, ...inline, action: canonical(key) };
103
136
  // Drop keys that are present but undefined. `simframe tap X` used to pass
104
137
  // `index: undefined`, which survived here and then crashed the signature
105
138
  // builder before the tap was ever sent.
@@ -2486,11 +2519,47 @@ async function runStep(deviceQuery, udid, step, ctx) {
2486
2519
  return null;
2487
2520
  }
2488
2521
  };
2522
+ /**
2523
+ * Is this target actually on the screen?
2524
+ *
2525
+ * Resolving is not the same as being in view, and this returned `"X" is
2526
+ * in view at 201,-35 after 1 scroll down` — 35pt ABOVE the viewport, so
2527
+ * the tap that followed missed. Reported from the field. The tree carries
2528
+ * scrolled-away rows with out-of-bounds coordinates, which is exactly the
2529
+ * case `scrollTo` exists to resolve, so claiming success on one is the
2530
+ * one answer it must never give.
2531
+ *
2532
+ * Generous rather than strict: any overlap with the viewport counts, so a
2533
+ * row half over an edge is still reachable and still reported. Only a
2534
+ * target entirely outside keeps the search going.
2535
+ */
2536
+ const inViewport = (t) => {
2537
+ const w = points?.width;
2538
+ const h = points?.height;
2539
+ if (!Number.isFinite(w) || !Number.isFinite(h)) return true;
2540
+ const f = t?.frame;
2541
+ if (f && Number.isFinite(f.y) && Number.isFinite(f.height)) {
2542
+ return f.y + f.height > 0 && f.y < h && f.x + (f.width ?? 0) > 0 && f.x < w;
2543
+ }
2544
+ return Number.isFinite(t?.y) && t.y >= 0 && t.y <= h
2545
+ && Number.isFinite(t?.x) && t.x >= 0 && t.x <= w;
2546
+ };
2547
+ // The direction actually scrolled, which is not always the one asked for:
2548
+ // `offsetSays()` may reverse it, and the message used to name the request
2549
+ // rather than the act — reported as "after 1 scroll down" on a request
2550
+ // for "up".
2551
+ const scrolled = [];
2489
2552
  for (let i = 0; i <= max; i += 1) {
2490
2553
  try {
2491
2554
  const found = await api.locate(deviceQuery, query, { index: step.index, refresh: i > 0 });
2492
- return `"${found.target.label}" is in view at ${found.target.x},${found.target.y}` +
2493
- (i ? ` after ${i} scroll${i === 1 ? '' : 's'} ${dir}` : ' already');
2555
+ if (!inViewport(found.target)) throw new Error(
2556
+ `"${query}" is in the tree but not in view (at ${found.target.x},${found.target.y}`
2557
+ + ` on a ${Math.round(points?.width ?? 0)}x${Math.round(points?.height ?? 0)}pt screen)`);
2558
+ const how = scrolled.length
2559
+ ? ` after ${scrolled.length} scroll${scrolled.length === 1 ? '' : 's'} `
2560
+ + (new Set(scrolled).size === 1 ? scrolled[0] : scrolled.join(' then '))
2561
+ : ' already';
2562
+ return `"${found.target.label ?? query}" is in view at ${found.target.x},${found.target.y}${how}`;
2494
2563
  } catch (err) {
2495
2564
  if (i === max) {
2496
2565
  throw new Error(`scrolled ${dir} ${max}x without finding ${query}: ${err.message}`);
@@ -2499,6 +2568,7 @@ async function runStep(deviceQuery, udid, step, ctx) {
2499
2568
  const evidence = await offsetSays();
2500
2569
  if (evidence) dir = evidence;
2501
2570
  const wasAt = await hashNow(deviceQuery, ctx.options);
2571
+ scrolled.push(dir);
2502
2572
  await runStep(deviceQuery, udid, { action: 'scroll', value: dir }, ctx);
2503
2573
  // A scroll either moves immediately or not at all, so it does not need a
2504
2574
  // transition's budget. Six iterations at 2,500ms was most of why this
@@ -2763,8 +2833,20 @@ async function runStep(deviceQuery, udid, step, ctx) {
2763
2833
  await sleep(ms);
2764
2834
  return `paused ${ms}ms`;
2765
2835
  }
2766
- default:
2767
- throw new Error(`unknown step "${step.action}"`);
2836
+ default: {
2837
+ // Say what the words are. "unknown step" on its own sent two testers
2838
+ // guessing, and a guess costs a round trip and the rest of the batch.
2839
+ const near = STEP_KEYS.filter((k) => {
2840
+ const a = String(step.action ?? '').toLowerCase().replace(/[_-]/g, '');
2841
+ const b = k.toLowerCase();
2842
+ return a && (b.startsWith(a) || a.startsWith(b) || b.includes(a));
2843
+ }).slice(0, 3);
2844
+ throw new Error(
2845
+ `unknown step "${step.action}"`
2846
+ + (near.length ? ` — did you mean ${near.map((k) => `"${k}"`).join(' or ')}?` : '')
2847
+ + ` Valid steps: ${STEP_KEYS.join(', ')}.`,
2848
+ );
2849
+ }
2768
2850
  }
2769
2851
  }
2770
2852
 
package/src/cli.js CHANGED
@@ -34,7 +34,7 @@ const USAGE = `simframe — always-warm iOS Simulator frames
34
34
  simframe tap <selector> tap #3, "Save", or @120,400
35
35
  simframe do <script.json> run a scripted flow (see below)
36
36
  simframe screens [device] list screens this device has learned
37
- simframe storage [bundle-id] what the app saved (works on a shut-down device)
37
+ simframe storage [bundle-id] [--device=<name|udid>] what the app saved (no boot needed)
38
38
  simframe goto <screen> walk to a known screen through known steps
39
39
  simframe flow save <name> <script.json> run a flow and save it if every step verifies
40
40
  simframe flow run <name> replay a saved flow
@@ -867,7 +867,44 @@ async function main() {
867
867
  // whole value of this command is that it answers on a device that is not
868
868
  // running — measured: simctl itself cannot, on this Xcode. Routing it
869
869
  // through the daemon would throw that away for no gain.
870
- const device = await resolveDevice(flags.device);
870
+ // Resolving a device here must NOT require one to be running, and it did.
871
+ //
872
+ // The whole claim of this command is that it answers before a boot, and
873
+ // `resolveDevice(undefined)` resolves against *booted* devices — so the
874
+ // documented form, `simframe storage <bundle-id>`, failed with "no booted
875
+ // simulator" at exactly the feature's headline use case. Reported by an
876
+ // external tester, who found it worked only when a device was named,
877
+ // which the help line did not mention either.
878
+ //
879
+ // Named: resolve as usual, booted or not. Unnamed: prefer a booted device
880
+ // when there is one, otherwise the only device that has app containers —
881
+ // and when that is ambiguous, say so and name the candidates rather than
882
+ // complaining about a boot nobody needs.
883
+ const named = flags.device ?? flags.udid ?? process.env.SIMFRAME_DEVICE;
884
+ let device;
885
+ if (named) {
886
+ device = await resolveDevice(String(named));
887
+ } else {
888
+ device = await resolveDevice(undefined).catch(() => null);
889
+ if (!device) {
890
+ const all = await listDevices({ all: true });
891
+ const withApps = [];
892
+ for (const d of all) {
893
+ const apps = await storage.apps(d.udid).catch(() => []);
894
+ if (apps.length) withApps.push({ device: d, apps: apps.length });
895
+ }
896
+ if (withApps.length === 1) {
897
+ device = withApps[0].device;
898
+ } else {
899
+ throw new Error(
900
+ withApps.length
901
+ ? 'storage reads a device that does not have to be running, so name which one:\n'
902
+ + withApps.map((w) => ` --device=${w.device.udid} ${w.device.name} (${w.apps} app(s))`).join('\n')
903
+ : 'no device on this host has any app data to read',
904
+ );
905
+ }
906
+ }
907
+ }
871
908
  const [bundleId] = positional;
872
909
  if (!bundleId) {
873
910
  const list = await storage.apps(device.udid);
package/src/index.js CHANGED
@@ -1489,6 +1489,17 @@ export function sensorMode(options) {
1489
1489
  return raw === 'ax-first' || raw === 'axfirst' ? 'ax-first' : 'full';
1490
1490
  }
1491
1491
 
1492
+ /**
1493
+ * What to call an element in a "Visible:" list.
1494
+ *
1495
+ * The label, or the accessibility identifier when there is no label. Both are
1496
+ * now matchable, and the list had been filtered to `t.label` alone — so an
1497
+ * element `sim_ui` had just printed by identifier was missing from the list of
1498
+ * what is on screen, in the same reply that refused to resolve it. Reported
1499
+ * from the field, three times in one session.
1500
+ */
1501
+ const nameFor = (t) => t.label || t.identifier || null;
1502
+
1492
1503
  export async function locate(deviceQuery, query, opts = {}) {
1493
1504
  if (sensorMode(opts.options) !== 'ax-first' || opts.useOcr === false || opts.escalated) {
1494
1505
  return locateWith(deviceQuery, query, opts);
@@ -1701,8 +1712,8 @@ async function locateWith(
1701
1712
  // here undoes every guard above — it has no off-screen filter and no
1702
1713
  // coverage weighting, and it is what returned a scrolled-away list row for
1703
1714
  // "back". "Not found" is the correct answer.
1704
- const visible = entry.targets.filter((t) => t.label && !regions.offViewport(t, points));
1705
- const sample = visible.slice(0, 12).map((t) => t.label.slice(0, 24)).join(', ');
1715
+ const visible = entry.targets.filter((t) => nameFor(t) && !regions.offViewport(t, points));
1716
+ const sample = visible.slice(0, 12).map((t) => nameFor(t).slice(0, 24)).join(', ');
1706
1717
  // "Not on this screen" and "not in view" are different answers, and giving
1707
1718
  // the first for the second cost a reported 15 seconds: a `waitFor REVIEW`
1708
1719
  // burned its whole timeout while REVIEW sat one scroll below the fold, and
@@ -1734,8 +1745,8 @@ async function locateWith(
1734
1745
  const candidates = screenmap.rank(entry, query);
1735
1746
  const target = index != null ? candidates[index] : candidates[0];
1736
1747
  if (!target) {
1737
- const visible = entry.targets.filter((t) => t.label);
1738
- const shown = visible.slice(0, 12).map((t) => t.label);
1748
+ const visible = entry.targets.filter((t) => nameFor(t));
1749
+ const shown = visible.slice(0, 12).map((t) => nameFor(t));
1739
1750
  // Say when the list is cut. A field report found `"Work Orders" is not on
1740
1751
  // this screen. Visible: …` on a screen whose own element map, three lines
1741
1752
  // below in the same reply, listed `#25 text 200,836 Work Orders` — it was
package/src/input.js CHANGED
@@ -254,6 +254,14 @@ export function elementToNode(e) {
254
254
  selected: e.state?.selected ?? null,
255
255
  focused: e.state?.focused ?? null,
256
256
  frame: e.frame ?? null,
257
+ // Where the control actuates, when the app publishes it. See `centerOf`.
258
+ //
259
+ // Added here as well as in `normalizeNode`, and the comment four lines up
260
+ // is the reason: this is the converter that actually runs, and a field set
261
+ // only in the idb fallback is a field that is never set. That is exactly
262
+ // how AXSelected and AXFocused came to be batched by the daemon for four
263
+ // versions and dropped on the way in.
264
+ activationPoint: e.activationPoint ?? null,
257
265
  raw: e,
258
266
  };
259
267
  }
@@ -292,6 +300,8 @@ function normalizeNode(node) {
292
300
  height: frame.height ?? frame.Height ?? 0,
293
301
  }
294
302
  : null,
303
+ // Where the control actuates, when the app says so. See `centerOf`.
304
+ activationPoint: node.activationPoint ?? node.AXActivationPoint ?? null,
295
305
  raw: node,
296
306
  };
297
307
  }
@@ -329,11 +339,33 @@ export function matchElement(nodes, query, { index } = {}) {
329
339
  throw new Error(`no element matching "${query}" is on screen`);
330
340
  }
331
341
 
342
+ /**
343
+ * Where to aim at this element.
344
+ *
345
+ * The geometric centre, unless the app has published somewhere better. UIKit
346
+ * exposes `accessibilityActivationPoint` for controls whose hit target is not
347
+ * the middle of what they publish as their frame, and a switch is the case that
348
+ * forced this: it reports a row-wide frame, so the centre is the *label*, and
349
+ * iOS does not actuate a switch from there. Measured on Settings →
350
+ * Accessibility → Hover Text — the frame centre flipped it **0 of 3** times and
351
+ * the control itself **3 of 3**, while the step reported `[no visible change]`
352
+ * and was telling the truth.
353
+ *
354
+ * The app's answer is preferred whenever it lands inside the frame. Outside it
355
+ * is not trusted: a point that is not on the element is not a better guess than
356
+ * the middle of one, and this runs on the tap path, where a wrong guess is the
357
+ * one thing that does damage.
358
+ */
332
359
  export function centerOf(node) {
333
- return {
360
+ const middle = {
334
361
  x: Math.round(node.frame.x + node.frame.width / 2),
335
362
  y: Math.round(node.frame.y + node.frame.height / 2),
336
363
  };
364
+ const p = node.activationPoint;
365
+ if (!p || !Number.isFinite(p.x) || !Number.isFinite(p.y)) return middle;
366
+ const f = node.frame;
367
+ const inside = p.x >= f.x && p.x <= f.x + f.width && p.y >= f.y && p.y <= f.y + f.height;
368
+ return inside ? { x: Math.round(p.x), y: Math.round(p.y) } : middle;
337
369
  }
338
370
 
339
371
  export async function tapPoint(udid, x, y, { durationMs } = {}) {
package/src/matching.js CHANGED
@@ -157,7 +157,17 @@ export function rank(targets, intent, { screen } = {}) {
157
157
 
158
158
  const scored = [];
159
159
  for (const t of visible) {
160
- const names = [t.label, ...(t.aliases ?? [])].filter(Boolean);
160
+ // The accessibility identifier is a name a caller can legitimately write,
161
+ // and this list did not contain it.
162
+ //
163
+ // `sim_ui` prints elements BY identifier — in React Native a `testID`
164
+ // becomes one, so it is most interactive controls in an RN app — and the
165
+ // resolver then rejected that exact string, in the same response that had
166
+ // just printed it, with the identifier also absent from the "Visible:"
167
+ // list. An external tester reproduced it three times and called it the
168
+ // single biggest friction of their session. `screenmap.rank` had learned
169
+ // this; this ranker, which is the one `resolve()` uses, had not.
170
+ const names = [t.label, t.identifier, ...(t.aliases ?? [])].filter(Boolean);
161
171
  let base = 0;
162
172
  let matched = null;
163
173
  for (const name of names) {
package/src/mcp.js CHANGED
@@ -405,7 +405,8 @@ const TOOLS = [
405
405
  'What the app saved, as text: its UserDefaults and (for React Native) its AsyncStorage, read straight out of'
406
406
  + ' the data container. sim_ui says what is drawn; sim_storage says what the app believes — use it when the'
407
407
  + ' screen and the behaviour disagree, or to check a value without driving the UI to it.'
408
- + ' Works on a device that is NOT running, so it can answer before anything is booted.'
408
+ + ' Works on a device that is NOT running, so it can answer before anything is booted —'
409
+ + ' pass device when nothing is booted and the host has more than one simulator with data.'
409
410
  + ' Call with no bundleId to list the apps that have a container (match filters that list).',
410
411
  inputSchema: {
411
412
  type: 'object',
@@ -772,6 +773,23 @@ async function look(target, args, options) {
772
773
  lines.push(`could not crop that region (${cropped.note}) — this is the whole screen`);
773
774
  }
774
775
  }
776
+ // Say what space the image is in, and the factor.
777
+ //
778
+ // A field report lost time to this and had to derive the number from
779
+ // landmarks: `sim_ui` and `sim_tap` speak 402x874 POINTS, the returned image
780
+ // is 322x700 PIXELS, and `simctl io screenshot` is a third space again at 3x.
781
+ // Reading a coordinate off this image and feeding it to a tap is silently
782
+ // wrong by 1.248x — silently, because nothing in the reply relates the two.
783
+ // The header already prints the frame size; it just never said what it was
784
+ // relative to.
785
+ const geometry = await api.screenIdentity(target, { options, confirmNovel: false }).catch(() => null);
786
+ const pts = geometry?.points;
787
+ if (pts?.width && res.state?.width) {
788
+ const factor = res.state.width / pts.width;
789
+ lines.push(`this image is ${res.state.width}x${res.state.height} px = `
790
+ + `${Math.round(pts.width)}x${Math.round(pts.height)}pt — divide image coordinates by `
791
+ + `${factor.toFixed(3)} before tapping (sim_tap and sim_ui speak points)`);
792
+ }
775
793
  return { content: [text(lines.filter(Boolean).join('\n')), image(png)] };
776
794
  }
777
795
 
@@ -1200,7 +1218,30 @@ function listStateDirs() {
1200
1218
  * the host filesystem. That is what the backend does.
1201
1219
  */
1202
1220
  async function appStorage({ bundleId, match: query, device } = {}) {
1203
- const resolved = await resolveDevice(device);
1221
+ // Same correction as the CLI: this must not require a booted device, because
1222
+ // answering before a boot is the point of the tool. A bare call used to fail
1223
+ // with "no booted simulator" — reported from the field — so an unnamed device
1224
+ // falls back to the only one that has app data before it gives up.
1225
+ let resolved = device
1226
+ ? await resolveDevice(String(device))
1227
+ : await resolveDevice(undefined).catch(() => null);
1228
+ if (!resolved) {
1229
+ const all = await listDevices({ all: true });
1230
+ const withApps = [];
1231
+ for (const d of all) {
1232
+ const apps = await storage.apps(d.udid).catch(() => []);
1233
+ if (apps.length) withApps.push({ device: d, apps: apps.length });
1234
+ }
1235
+ if (withApps.length === 1) resolved = withApps[0].device;
1236
+ else {
1237
+ throw new Error(
1238
+ withApps.length
1239
+ ? 'storage reads a device that does not have to be running, so say which one: '
1240
+ + withApps.map((w) => `${w.device.name} (${w.device.udid}, ${w.apps} app(s))`).join('; ')
1241
+ : 'no device on this host has any app data to read',
1242
+ );
1243
+ }
1244
+ }
1204
1245
  if (!bundleId) {
1205
1246
  const list = await storage.apps(resolved.udid);
1206
1247
  const needle = query ? String(query).toLowerCase() : null;
@@ -13,6 +13,46 @@ import * as plist from './plist.js';
13
13
 
14
14
  const run = promisify(execFile);
15
15
 
16
+ /**
17
+ * How long a `simctl` verb may take before we stop waiting.
18
+ *
19
+ * **It was 20s, and that was below what a loaded runner actually needs.** Item
20
+ * 142 recorded `simctl launch` "taking 47-55s" on a hosted runner and filed it
21
+ * under a boot that had not finished; the launches were real and the budget was
22
+ * simply shorter than they were. The same 20s sat on `openurl`, which is the
23
+ * failure class that item listed four runs of, and on `terminate`. One number,
24
+ * three symptoms, and every one of them read as the command refusing rather
25
+ * than as us leaving.
26
+ *
27
+ * 90s is chosen against that measurement — comfortably past the observed 55s
28
+ * worst case — and not for feel. A `simctl` verb that has not returned in
29
+ * ninety seconds is genuinely wrong, and says so below instead of being
30
+ * indistinguishable from a rejection.
31
+ */
32
+ const SIMCTL_TIMEOUT_MS = 90_000;
33
+
34
+ /** A local file decode, which owes nothing to device latency. */
35
+ const PLUTIL_TIMEOUT_MS = 20_000;
36
+
37
+ /**
38
+ * What actually went wrong, including the case that has been invisible.
39
+ *
40
+ * On a timeout `execFile` kills the child, so `stderr` is EMPTY and `message`
41
+ * is the bare "Command failed: xcrun simctl ..." — which reads exactly like
42
+ * simctl rejecting the request. Three separate investigations have started from
43
+ * that sentence and gone looking for a broken device. The timeout has to name
44
+ * itself, or the next one starts in the same wrong place.
45
+ */
46
+ function simctlFailure(err, what) {
47
+ const detail = (err.stderr || '').trim().split('\n').filter(Boolean).pop();
48
+ if (detail) return `${what}: ${detail}`;
49
+ if (err.killed || err.signal === 'SIGTERM') {
50
+ return `${what}: simctl did not return within ${Math.round(SIMCTL_TIMEOUT_MS / 1000)}s`
51
+ + ' (killed by simframe, not refused by simctl — the host is loaded or the device is not answering)';
52
+ }
53
+ return `${what}: ${err.message}`;
54
+ }
55
+
16
56
  // `simctl list` costs ~130ms, which would otherwise dominate every warm read,
17
57
  // so the parsed list is cached for a few seconds.
18
58
  const DEVICE_CACHE_MS = 4000;
@@ -240,24 +280,33 @@ async function launchApp(udid, bundleId, { args = [], env = {}, terminateFirst =
240
280
  for (const [k, v] of Object.entries(env)) childEnv[`SIMCTL_CHILD_${k}`] = String(v);
241
281
  try {
242
282
  await run('xcrun', ['simctl', 'launch', udid, bundleId, ...args.map(String)], {
243
- timeout: 20_000,
283
+ timeout: SIMCTL_TIMEOUT_MS,
244
284
  env: childEnv,
245
285
  });
246
286
  } catch (err) {
247
287
  // execFile's message is just "Command failed: ..." with simctl's actual
248
288
  // complaint left in stderr. A CI run failed here and said nothing about
249
289
  // why, which is the same sin as a silent fallback.
250
- const detail = (err.stderr || '').trim().split('\n').filter(Boolean).pop();
251
- throw new Error(detail ? `could not launch ${bundleId}: ${detail}` : `could not launch ${bundleId}: ${err.message}`);
290
+ throw new Error(simctlFailure(err, `could not launch ${bundleId}`));
252
291
  }
253
292
  }
254
293
 
255
294
  async function terminateApp(udid, bundleId) {
256
- await run('xcrun', ['simctl', 'terminate', udid, bundleId], { timeout: 20_000 });
295
+ try {
296
+ await run('xcrun', ['simctl', 'terminate', udid, bundleId], { timeout: SIMCTL_TIMEOUT_MS });
297
+ } catch (err) {
298
+ throw new Error(simctlFailure(err, `could not terminate ${bundleId}`));
299
+ }
257
300
  }
258
301
 
259
302
  async function openUrl(udid, url) {
260
- await run('xcrun', ['simctl', 'openurl', udid, url], { timeout: 20_000 });
303
+ // The same budget and the same reporting as `launch`, because it was the same
304
+ // 20s and it is the failure class item 142 counted four runs of.
305
+ try {
306
+ await run('xcrun', ['simctl', 'openurl', udid, url], { timeout: SIMCTL_TIMEOUT_MS });
307
+ } catch (err) {
308
+ throw new Error(simctlFailure(err, 'could not open the url'));
309
+ }
261
310
  }
262
311
 
263
312
  /**
@@ -305,10 +354,9 @@ async function setPermission(udid, action, service, bundleId) {
305
354
  const args = ['simctl', 'privacy', udid, verb, service];
306
355
  if (bundleId) args.push(bundleId);
307
356
  try {
308
- await run('xcrun', args, { timeout: 20_000 });
357
+ await run('xcrun', args, { timeout: SIMCTL_TIMEOUT_MS });
309
358
  } catch (err) {
310
- const detail = (err.stderr || '').trim().split('\n').filter(Boolean).pop();
311
- throw new Error(`could not ${verb} ${service}: ${detail || err.message}`);
359
+ throw new Error(simctlFailure(err, `could not ${verb} ${service}`));
312
360
  }
313
361
  return `${verb === 'reset' ? 'reset' : verb + 'ed'} ${service}${bundleId ? ` for ${bundleId}` : ''}`;
314
362
  }
@@ -422,8 +470,12 @@ async function appContainer(udid, bundleId) {
422
470
  * the note at the top of plist.js.
423
471
  */
424
472
  async function readPropertyList(file) {
473
+ // Its own budget, deliberately not the simctl one. This reads a local file
474
+ // and never speaks to a device, so it has none of the latency the simctl
475
+ // budget exists to absorb — and a plist that takes twenty seconds to decode
476
+ // is a problem worth hearing about promptly.
425
477
  const { stdout } = await run('plutil', ['-convert', 'xml1', '-o', '-', file], {
426
- timeout: 20_000,
478
+ timeout: PLUTIL_TIMEOUT_MS,
427
479
  maxBuffer: 64 * 1024 * 1024,
428
480
  });
429
481
  return plist.parse(stdout);
package/src/storage.js CHANGED
@@ -39,6 +39,33 @@ export const VALUE_PREVIEW_BYTES = 4096;
39
39
  const ASYNC_STORAGE_DIR = 'RCTAsyncLocalStorage_V1';
40
40
  const ASYNC_STORAGE_MANIFEST = 'manifest.json';
41
41
 
42
+ /**
43
+ * Where AsyncStorage actually lives, newest layout first.
44
+ *
45
+ * **`Documents/` alone was wrong, and wrong in the worst available way.** That
46
+ * is the *legacy* React Native location. The community package every current RN
47
+ * app uses — `@react-native-async-storage/async-storage` — writes to
48
+ * `Library/Application Support/<bundle-id>/`, and on a real app measured by an
49
+ * external tester the `Documents/` path **did not exist at all**. So
50
+ * `readAsyncStorage` returned null, `format()` omitted the section, and the
51
+ * output read as "this app has no AsyncStorage" while 25 keys sat on disk,
52
+ * including a 1.5 MB MobX-State-Tree root store.
53
+ *
54
+ * Their diagnosis was exact: *"The decoder is correct; only the path is wrong."*
55
+ * Which makes this the precise class of confident wrong answer the whole feature
56
+ * exists to prevent, shipped inside it.
57
+ *
58
+ * Both are tried because both are real — an older app still writes to
59
+ * `Documents/` — and every path looked in is reported, because "found nothing"
60
+ * and "did not look there" are different facts and only one of them is about
61
+ * the app.
62
+ */
63
+ const asyncStorageDirs = (container, bundleId) => [
64
+ path.join(container, 'Library', 'Application Support', String(bundleId ?? ''), ASYNC_STORAGE_DIR),
65
+ path.join(container, 'Library', 'Application Support', ASYNC_STORAGE_DIR),
66
+ path.join(container, 'Documents', ASYNC_STORAGE_DIR),
67
+ ];
68
+
42
69
  /** Files that are plainly a store but that nothing here can decode yet. */
43
70
  const OPAQUE_STORES = /\.(sqlite3?|db|realm|leveldb|mmkv)$/i;
44
71
 
@@ -59,10 +86,13 @@ export async function apps(udid) {
59
86
  * reporting it as one would be the exact class of wrong answer this feature
60
87
  * exists to stop.
61
88
  */
62
- export function readAsyncStorage(container) {
63
- const dir = path.join(container, 'Documents', ASYNC_STORAGE_DIR);
89
+ export function readAsyncStorage(container, bundleId) {
90
+ const looked = asyncStorageDirs(container, bundleId);
91
+ const dir = looked.find((d) => fs.existsSync(path.join(d, ASYNC_STORAGE_MANIFEST)));
92
+ // The paths travel with the miss. A reader told only "no AsyncStorage" cannot
93
+ // tell a bare app from a store we failed to find, and the second is ours.
94
+ if (!dir) return { missing: true, looked: looked.map((d) => path.relative(container, d)) };
64
95
  const manifestPath = path.join(dir, ASYNC_STORAGE_MANIFEST);
65
- if (!fs.existsSync(manifestPath)) return null;
66
96
  const manifest = readJson(manifestPath);
67
97
  const entries = [];
68
98
  for (const [key, inline] of Object.entries(manifest)) {
@@ -151,9 +181,15 @@ export function typeOf(value) {
151
181
  export async function read(udid, bundleId) {
152
182
  const container = await platform.appContainer(udid, bundleId);
153
183
  const stores = await readPreferences(udid, container, bundleId);
154
- const async_ = readAsyncStorage(container);
155
- if (async_) stores.push(async_);
156
- return { bundleId, container, stores, opaque: opaqueStores(container) };
184
+ const async_ = readAsyncStorage(container, bundleId);
185
+ if (async_ && !async_.missing) stores.push(async_);
186
+ return {
187
+ bundleId,
188
+ container,
189
+ stores,
190
+ opaque: opaqueStores(container),
191
+ asyncStorageMissing: async_?.missing ? async_.looked : null,
192
+ };
157
193
  }
158
194
 
159
195
  /** One value, rendered for reading, saying so whenever it is not the whole thing. */
@@ -186,11 +222,24 @@ export function format(result) {
186
222
  lines.push(` ${renderValue(e.value).split('\n').join('\n ')}`);
187
223
  }
188
224
  }
225
+ // Say where we looked and did not find it. See `asyncStorageDirs`.
226
+ if (result.asyncStorageMissing) {
227
+ lines.push('');
228
+ lines.push(' no AsyncStorage found. Looked in:');
229
+ for (const d of result.asyncStorageMissing) lines.push(` ${d}`);
230
+ lines.push(' (an app that does not use AsyncStorage will have none of these)');
231
+ }
189
232
  if (result.opaque?.length) {
190
233
  lines.push('');
191
234
  lines.push(` ${result.opaque.length} store(s) present that this cannot decode yet:`);
192
235
  for (const o of result.opaque) lines.push(` ${o.file} ${o.bytes} bytes`);
193
236
  }
237
+ // Asked for by the external tester, and they were right to: mid-session the
238
+ // app restored a session from the Keychain and walked past its own login
239
+ // screen, so "logged out" as read from storage was not the whole truth.
240
+ // Saying what is NOT readable sets expectations that silence does not.
241
+ lines.push('');
242
+ lines.push(' Keychain is not readable from here — auth state may differ from what is above.');
194
243
  return lines.join('\n');
195
244
  }
196
245
 
package/src/view.js CHANGED
@@ -579,7 +579,7 @@ export async function screenMap(deviceQuery, {
579
579
  const anonymous = rows.filter(unaddressable).length;
580
580
  const unnamed = anonymous
581
581
  ? `${anonymous} on-screen control(s) have no accessibility label — they are listed with their`
582
- + ' coordinates and can be tapped by point or by #ref, but not by name. If what you are'
582
+ + ' coordinates and can be tapped by #ref, or by point with tapAt. If what you are'
583
583
  + ' looking for is not in the list either, the app has views that were never declared'
584
584
  + ' accessible and only a screenshot will find those.'
585
585
  : null;