simframe 0.14.1 → 0.14.2

package/native/simframed/Sources/PrivateAPI/AccessibilityBridge.swift

@@ -16,11 +16,20 @@ public struct AXNode: Sendable {
     public var selected: Bool?
     public var focused: Bool?
     public var frame: CGRect
+    /// Where the control actuates, when the app says so.
+    ///
+    /// Not the same as the centre of `frame`, and a switch is the proof: UIKit
+    /// publishes a row-wide frame for it, so the geometric centre is the label
+    /// and iOS does not actuate a switch from there. Measured — tapping the
+    /// centre flipped it 0 of 3 times and the control itself 3 of 3. UIKit has
+    /// `accessibilityActivationPoint` for exactly this question, and nil means
+    /// the app did not answer it, never a guess of ours.
+    public var activationPoint: CGPoint?
     public var depth: Int
 
     public init(role: String, subrole: String? = nil, label: String? = nil, value: String? = nil,
                 identifier: String? = nil, enabled: Bool? = nil, selected: Bool? = nil,
-                focused: Bool? = nil, frame: CGRect, depth: Int) {
+                focused: Bool? = nil, frame: CGRect, activationPoint: CGPoint? = nil, depth: Int) {
         self.role = role
         self.subrole = subrole
         self.label = label
@@ -30,6 +39,7 @@ public struct AXNode: Sendable {
         self.selected = selected
         self.focused = focused
         self.frame = frame
+        self.activationPoint = activationPoint
         self.depth = depth
     }
 }
@@ -251,7 +261,8 @@ public final class AccessibilityBridge {
 
     /// Everything scalar about a node, asked for in one go.
     private static let batched = ["AXRole", "AXSubrole", "AXDescription", "AXValue",
-                                  "AXIdentifier", "AXEnabled", "AXSelected", "AXFocused"]
+                                  "AXIdentifier", "AXEnabled", "AXSelected", "AXFocused",
+                                  "AXActivationPoint"]
 
     private func node(from element: NSObject, depth: Int) -> AXNode {
         // One bridge round trip for eight attributes, not eight.
@@ -283,6 +294,7 @@ public final class AccessibilityBridge {
             selected: (value("AXSelected") as? NSNumber)?.boolValue,
             focused: (value("AXFocused") as? NSNumber)?.boolValue,
             frame: frame(of: element),
+            activationPoint: point(value("AXActivationPoint")) ?? activationPoint(of: element),
             depth: depth)
     }
 
@@ -310,6 +322,41 @@ public final class AccessibilityBridge {
         return unsafeBitCast(imp, to: RectFn.self)(element, sel)
     }
 
+    /// The activation point as the attribute bag reports it.
+    ///
+    /// Defensive about the shape because this is a private bridge and the
+    /// answer arrives as whatever the translator chose: an NSValue wrapping a
+    /// CGPoint on one path, a stringified `{x, y}` on another. Anything it does
+    /// not recognise is nil, which costs nothing — the caller falls back to the
+    /// frame centre, which is what it did before this attribute was asked for.
+    private func point(_ value: Any?) -> CGPoint? {
+        switch value {
+        case let v as NSValue:
+            // `pointValue`, not `cgPointValue`: this daemon is a macOS binary
+            // driving a guest, and the iOS-only accessor does not exist here.
+            // NSPoint and CGPoint are the same type on 64-bit.
+            let p = v.pointValue
+            return p.x.isFinite && p.y.isFinite ? p : nil
+        case let s as String:
+            let parts = s.trimmingCharacters(in: CharacterSet(charactersIn: "{} "))
+                .split(separator: ",")
+                .compactMap { Double($0.trimmingCharacters(in: .whitespaces)) }
+            guard parts.count == 2 else { return nil }
+            return CGPoint(x: parts[0], y: parts[1])
+        default:
+            return nil
+        }
+    }
+
+    /// The direct selector, for elements that answer it but do not batch it.
+    private func activationPoint(of element: NSObject) -> CGPoint? {
+        let sel = NSSelectorFromString("accessibilityActivationPoint")
+        guard element.responds(to: sel), let imp = element.method(for: sel) else { return nil }
+        typealias PointFn = @convention(c) (AnyObject, Selector) -> CGPoint
+        let p = unsafeBitCast(imp, to: PointFn.self)(element, sel)
+        return p.x.isFinite && p.y.isFinite ? p : nil
+    }
+
     /// A value may be a string, a number, or something with no useful text.
     private func string(_ value: Any?) -> String? {
         switch value {

package/native/simframed/Sources/SimframeCore/Element.swift

@@ -60,10 +60,20 @@ public struct Element: Sendable {
     public var state: ElementState
     public var source: ElementSource
     public var confidence: Double
+    /// Where the control actuates, when the app publishes it.
+    ///
+    /// Deliberately NOT folded into `center`. `center` is the element's place,
+    /// and two readings of one control are matched by how close their places
+    /// are — 12pt apart, per `SAME_CONTROL_POINTS` above the boundary. A switch
+    /// actuates at the far end of a row-wide frame, so making `center` mean
+    /// "where to tap" would move the tree's reading ~145pt away from OCR's and
+    /// stop the two collapsing into one control. Aiming and identity are
+    /// different questions; this is the answer to the first one only.
+    public var activationPoint: CGPoint?
 
     public init(id: Int, frame: CGRect, role: String, label: String? = nil, value: String? = nil,
                 identifier: String? = nil, state: ElementState = ElementState(),
-                source: ElementSource, confidence: Double = 1) {
+                source: ElementSource, confidence: Double = 1, activationPoint: CGPoint? = nil) {
         self.id = id
         self.frame = frame
         self.role = role
@@ -73,6 +83,7 @@ public struct Element: Sendable {
         self.state = state
         self.source = source
         self.confidence = confidence
+        self.activationPoint = activationPoint
     }
 
     public var center: CGPoint { CGPoint(x: frame.midX, y: frame.midY) }
@@ -89,6 +100,11 @@ public struct Element: Sendable {
         if let label { out["label"] = label }
         if let value { out["value"] = value }
         if let identifier { out["identifier"] = identifier }
+        // Only when the app answered. Absent means "no opinion", and the layer
+        // above then aims at the centre exactly as it always has.
+        if let activationPoint {
+            out["activationPoint"] = ["x": Int(activationPoint.x.rounded()), "y": Int(activationPoint.y.rounded())]
+        }
         let state = state.json
         if !state.isEmpty { out["state"] = state }
         return out
@@ -112,7 +128,8 @@ public extension Element {
             state: ElementState(enabled: node.enabled, selected: node.selected,
                                 checked: nil, focused: node.focused),
             source: .accessibility,
-            confidence: 1)
+            confidence: 1,
+            activationPoint: node.activationPoint)
     }
 }
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "simframe",
-  "version": "0.14.1",
+  "version": "0.14.2",
   "mcpName": "io.github.lvlrSajjad/simframe",
   "description": "Always-warm iOS Simulator and Android emulator frames: agents read the screen in ~20ms instead of waiting on screenshots. MCP server + CLI.",
   "keywords": [

package/scripts/analyse-escalations.mjs

@@ -0,0 +1,89 @@
+#!/usr/bin/env node
+/**
+ * Of the `verification_failed` escalations, how many happened on a
+ * `(screen, action)` the graph had already seen behave consistently?
+ * EXPERIMENTS §17. The answer was 3% and 0%, which is why this is kept: it is
+ * the measurement that stopped a phase being spent on remembering edge
+ * outcomes.
+ *
+ * Two thirds of those escalations are `no-visible-change`, and 83-100% of them
+ * are on a screen or an action the graph has never seen — memory cannot answer
+ * a question about a place it has never been.
+ *
+ * The graph is read as it is NOW, so an edge matched here may have been learned
+ * after the escalation. That makes the "addressable" figure an upper bound, and
+ * it is already near zero.
+ *
+ * Aggregate by design, and stricter than it looks: `detail` is app content, so
+ * verdicts are classified against a CLOSED vocabulary and the raw string is
+ * never printed. The first version of this printed the prefix and put a
+ * client's screen labels and a customer email into a terminal.
+ *
+ * Usage: node scripts/analyse-escalations.mjs <udid-prefix> [<udid-prefix>...] */
+import fs from 'node:fs';
+import path from 'node:path';
+import os from 'node:os';
+
+const root = process.env.SIMFRAME_HOME || path.join(os.homedir(), '.simframe');
+
+for (const dev of process.argv.slice(2)) {
+  const dir = fs.readdirSync(root).find((d) => d.startsWith(dev));
+  if (!dir) continue;
+  const base = path.join(root, dir);
+  const gdir = path.join(base, 'graph');
+  if (!fs.existsSync(gdir)) continue;
+
+  const nodes = fs.readdirSync(gdir).filter((f) => f.endsWith('.json'))
+    .map((f) => { try { return JSON.parse(fs.readFileSync(path.join(gdir, f), 'utf8')); } catch { return null; } })
+    .filter(Boolean);
+  const byHash = new Map();
+  for (const n of nodes) { byHash.set(n.hash, n); for (const v of n.variants ?? []) if (!byHash.has(v.hash)) byHash.set(v.hash, n); }
+
+  const esc = [];
+  const p = path.join(base, 'escalations.jsonl');
+  if (!fs.existsSync(p)) continue;
+  for (const line of fs.readFileSync(p, 'utf8').split('\n')) {
+    const t = line.trim(); if (!t) continue;
+    try { esc.push(JSON.parse(t)); } catch {}
+  }
+  const vf = esc.filter((e) => e.reason === 'verification_failed');
+
+  const bucket = { noScreen: 0, noEdge: 0, seenOnce: 0, consistent: 0, nondet: 0 };
+  const verdicts = new Map();
+  for (const e of vf) {
+    // Classify against a CLOSED vocabulary and never emit the raw text. The
+    // first version split on ':' and printed the prefix, which put a client's
+    // screen labels and an email address into a terminal. `detail` is app
+    // content; only its shape is ours to report.
+    const d = String(e.detail ?? '').toLowerCase();
+    const v = d.includes('no-visible-change') || d.includes('did not change') ? 'no-visible-change'
+      : d.includes('unexpected-screen') ? 'unexpected-screen'
+      : d.includes('did not settle') ? 'settle-timeout'
+      : d.includes('waited') ? 'wait-timeout'
+      : d.includes('matches') && d.includes('things') ? 'ambiguous'
+      : d.includes('numbered on a different') ? 'stale-ref'
+      : d.includes('disabled') ? 'disabled-control'
+      : d ? 'other' : '(none)';
+    verdicts.set(v, (verdicts.get(v) ?? 0) + 1);
+    const node = byHash.get(e.screen_fingerprint ?? '');
+    if (!node) { bucket.noScreen++; continue; }
+    const want = String(e.intent ?? '').toLowerCase();
+    const hit = (node.edges ?? []).find((x) => {
+      const sig = String(x.action ?? '').toLowerCase();
+      return want && (sig.endsWith(`:${want}`) || sig.includes(want));
+    });
+    if (!hit) { bucket.noEdge++; continue; }
+    if ((hit.changedOutcomes ?? 0) > 0) bucket.nondet++;
+    else if ((hit.count ?? 0) > 1) bucket.consistent++;
+    else bucket.seenOnce++;
+  }
+  const pct = (n) => (vf.length ? Math.round((100 * n) / vf.length) : 0);
+  console.log(`--- ${dev} ---`);
+  console.log(`  verification_failed: ${vf.length}`);
+  console.log(`  screen not in graph at all:            ${bucket.noScreen} (${pct(bucket.noScreen)}%)`);
+  console.log(`  screen known, this action never seen:  ${bucket.noEdge} (${pct(bucket.noEdge)}%)`);
+  console.log(`  edge seen exactly once:                ${bucket.seenOnce} (${pct(bucket.seenOnce)}%)`);
+  console.log(`  edge NONDETERMINISTIC (rightly asked): ${bucket.nondet} (${pct(bucket.nondet)}%)`);
+  console.log(`  edge repeated and CONSISTENT:          ${bucket.consistent} (${pct(bucket.consistent)}%)  <- addressable`);
+  console.log(`  verdict words: ${[...verdicts].sort((a,b)=>b[1]-a[1]).slice(0,6).map(([k,n])=>`${k}=${n}`).join(', ')}`);
+}

package/scripts/analyse-routes.mjs

@@ -0,0 +1,96 @@
+#!/usr/bin/env node
+/**
+ * Would routing by measured time differ from routing by hop count, and by how
+ * much? EXPERIMENTS §16, kept runnable because the answer was "barely" and that
+ * is the kind of result that gets re-litigated by someone's intuition.
+ *
+ * `route()` in graph.js is a plain BFS over hop count, while `record()` has
+ * been storing a settle sample on nearly every edge all along (97% and 85% of
+ * edges on the two real-app graphs measured). This compares the two.
+ *
+ * Measured: the fastest route differs from the fewest-hops route on 9%, 0% and
+ * 7% of reachable pairs, saving a median of 1853ms, — and 615ms. Real, unused,
+ * and pointed at the wrong prize — it saves seconds, not round trips, and round
+ * trips are what the human-parity series exists to reduce.
+ *
+ * Output is aggregate by design: this reads a real device's memory of real
+ * third-party apps, so it prints counts and never a label, a screen or an app
+ * name. Same standing rule as scripts/phase17-corpus.mjs.
+ *
+ * Usage: node scripts/analyse-routes.mjs <udid-prefix> [<udid-prefix>...] */
+import fs from 'node:fs';
+import path from 'node:path';
+import os from 'node:os';
+
+const root = process.env.SIMFRAME_HOME || path.join(os.homedir(), '.simframe');
+const median = (a) => { const s=[...a].sort((x,y)=>x-y); return s.length? (s.length%2? s[(s.length-1)/2] : (s[s.length/2-1]+s[s.length/2])/2) : null; };
+
+for (const dev of process.argv.slice(2)) {
+  const dir = fs.readdirSync(root).find((d) => d.startsWith(dev));
+  if (!dir) continue;
+  const gdir = path.join(root, dir, 'graph');
+  if (!fs.existsSync(gdir)) continue;
+  const nodes = fs.readdirSync(gdir).filter(f=>f.endsWith('.json'))
+    .map(f => { try { return JSON.parse(fs.readFileSync(path.join(gdir,f),'utf8')); } catch { return null; } })
+    .filter(Boolean);
+
+  const byHash = new Map();
+  for (const n of nodes) { byHash.set(n.hash, n); for (const v of n.variants ?? []) if(!byHash.has(v.hash)) byHash.set(v.hash, n); }
+  const canonical = (h) => byHash.get(h)?.hash ?? h;
+
+  // Edge costs: the median settle actually measured on that edge.
+  const all = [];
+  for (const n of nodes) for (const e of n.edges ?? []) { const m = median(e.settles ?? []); if (m!=null) all.push(m); }
+  const fallback = median(all) ?? 500;
+  const cost = (e) => median(e.settles ?? []) ?? fallback;
+
+  // BFS exactly as graph.js does it: first path found, insertion order.
+  const bfs = (start, goal) => {
+    const seen = new Set([start]); const q = [{h:start, p:[]}];
+    while (q.length) { const {h,p} = q.shift();
+      if (p.length >= 12) continue;
+      for (const e of byHash.get(h)?.edges ?? []) {
+        const to = canonical(e.to); const next=[...p,e];
+        if (to === goal) return next;
+        if (seen.has(to)) continue; seen.add(to); q.push({h:to,p:next});
+      } }
+    return null;
+  };
+  // Dijkstra on measured time.
+  const fastest = (start, goal) => {
+    const dist = new Map([[start,0]]); const prev = new Map(); const done = new Set();
+    while (true) {
+      let cur=null, best=Infinity;
+      for (const [h,d] of dist) if (!done.has(h) && d<best) { best=d; cur=h; }
+      if (cur==null) break;
+      if (cur===goal) break;
+      done.add(cur);
+      for (const e of byHash.get(cur)?.edges ?? []) {
+        const to = canonical(e.to); const nd = best + cost(e);
+        if (nd < (dist.get(to) ?? Infinity)) { dist.set(to,nd); prev.set(to,[cur,e]); }
+      } }
+    if (!dist.has(goal)) return null;
+    const out=[]; let at=goal;
+    while (prev.has(at)) { const [from,e]=prev.get(at); out.unshift(e); at=from; }
+    return out;
+  };
+
+  const canon = [...new Set(nodes.map(n=>n.hash))];
+  let pairs=0, differ=0, saved=[], hopsUp=0;
+  for (const s of canon) for (const g of canon) {
+    if (s===g) continue;
+    const b = bfs(s,g); if (!b) continue;
+    const f = fastest(s,g); if (!f) continue;
+    pairs++;
+    const cb = b.reduce((t,e)=>t+cost(e),0), cf = f.reduce((t,e)=>t+cost(e),0);
+    if (cf < cb - 1) { differ++; saved.push(cb-cf); if (f.length > b.length) hopsUp++; }
+  }
+  saved.sort((a,b)=>a-b);
+  console.log(`--- ${dev} ---`);
+  console.log(`  reachable pairs: ${pairs}`);
+  console.log(`  where the fastest route differs from the fewest-hops route: ${differ} (${pairs?Math.round(100*differ/pairs):0}%)`);
+  if (saved.length) {
+    console.log(`  time saved per such route — median ${Math.round(median(saved))}ms, max ${Math.round(saved[saved.length-1])}ms`);
+    console.log(`  ...of those, routes that take MORE hops to be faster: ${hopsUp}`);
+  }
+}

package/scripts/eval-fingerprint.mjs

@@ -534,7 +534,25 @@ if (strays.length) {
     // nothing to work with, which is this harness's subject. This means we did
     // not look long enough, which is the harness's own fault and a different
     // remedy.
-    const underRead = sparseAt.has(`${reading.name}|${reading.round}`);
+    //
+    // **And sparseness alone is not enough to claim it**, which this
+    // misdiagnosed once. A wrong turn that lands on a screen which legitimately
+    // reads sparse gets flagged sparse too, and was then reported as our
+    // instrument's fault rather than the tour's. Measured: `settings-general`
+    // r3 came back with tokens byte-identical to the Settings root, including
+    // `heading:nav-bar:"settings"` — a *heading*, which is the root's title,
+    // where General publishes a *button* with the same word. It was read on the
+    // wrong screen, and this called it an under-read.
+    //
+    // The discriminator is the screen's own other rounds: if `settings-general`
+    // read differently and richly in rounds 1 and 2, then the screen IS
+    // distinguishable and a round matching another screen exactly went
+    // somewhere else. Only when no round of this screen can tell itself apart
+    // is "we did not look long enough" the honest reading.
+    const distinguishable = (byName.get(reading.name) ?? [])
+      .some((o) => o !== reading && fingerprint.similarity(o.tokens, match?.tokens ?? []) < 0.99);
+    const underRead = sparseAt.has(`${reading.name}|${reading.round}`) && !distinguishable;
+    const wrongScreen = bestOther >= 0.99 && distinguishable;
     console.error(`       ${reading.name} r${reading.round}: own screen ${bestSelf.toFixed(2)}, `
       + `${match ? `${match.name} r${match.round}` : 'another screen'} ${bestOther.toFixed(2)} `
       + `(${reading.count} tokens, ${named.length} named, sources ${reading.sources.join('+') || 'none'})`);
@@ -542,6 +560,11 @@ if (strays.length) {
       console.error('              ^ a COLLISION, not a wrong turn: neither reading carries a chrome');
       console.error('                label, so both are structure with no name and the fingerprint has');
       console.error('                nothing left to tell two list screens apart.');
+    } else if (wrongScreen) {
+      console.error(`              ^ WRONG SCREEN: these tokens are ${match ? `identical to ${match.name}` : 'another screen'},`);
+      console.error('                and this screen reads differently in its other rounds — so it is');
+      console.error('                distinguishable and the tour was simply somewhere else. A tap that');
+      console.error('                missed, or a screen that went back before it was read.');
     } else if (underRead) {
       console.error('              ^ UNDER-READ, not a wrong turn: this reading stayed below the token');
       console.error('                floor after every retry, so it never saw enough of its screen to');

package/src/actions.js

@@ -90,16 +90,49 @@ const ACTION_STEPS = new Set([
   'launch', 'terminate', 'openUrl', 'confirm', 'chooseAny', 'permission',
 ]);
 
+/**
+ * Step keys that are really the MCP tool names, accepted as aliases.
+ *
+ * Two field reports guessed these independently and each wrong guess cost a
+ * round trip and aborted the rest of the batch. There is a `sim_wait` tool and
+ * a `sim_type_into` tool, so `wait` and `type_into` are what a caller reaches
+ * for inside `sim_do` — and the vocabulary answered "unknown step" without
+ * saying what the words are. A tool surface that names an action one way and
+ * accepts it another is charging the caller for our inconsistency.
+ */
+const STEP_ALIASES = Object.freeze({
+  wait: 'settle',
+  type_into: 'type',
+  typeInto: 'type',
+  scroll_to: 'scrollTo',
+  wait_for: 'waitFor',
+  waitfor: 'waitFor',
+  tap_at: 'tapAt',
+  open_url: 'openUrl',
+  assert_gone: 'assertGone',
+  assert_text: 'assertText',
+});
+
+/** Every step key `runStep` understands, for an error that can be acted on. */
+export const STEP_KEYS = Object.freeze([
+  'tap', 'tapAt', 'type', 'paste', 'clear', 'swipe', 'scroll', 'scrollTo',
+  'button', 'key', 'launch', 'terminate', 'openUrl', 'confirm', 'chooseAny',
+  'permission', 'settle', 'waitFor', 'waitText', 'assert', 'assertGone',
+  'assertText', 'visible', 'gone', 'enabled', 'disabled', 'value', 'look',
+  'seek', 'sweep', 'pause',
+]);
+
 /** Accept both `{tap: "Save"}` shorthand and `{action: "tap", target: "Save"}`. */
 export function normalizeStep(raw) {
-  if (typeof raw === 'string') return { action: raw };
-  if (raw.action) return { ...raw };
+  const canonical = (a) => STEP_ALIASES[a] ?? a;
+  if (typeof raw === 'string') return { action: canonical(raw) };
+  if (raw.action) return { ...raw, action: canonical(raw.action) };
   const [key] = Object.keys(raw);
   if (!key) throw new Error('empty step');
   // Siblings like timeoutMs sit alongside the shorthand key and must survive.
   const { [key]: value, ...rest } = raw;
   const inline = value && typeof value === 'object' && !Array.isArray(value) ? value : { value };
-  const step = { ...rest, ...inline, action: key };
+  const step = { ...rest, ...inline, action: canonical(key) };
   // Drop keys that are present but undefined. `simframe tap X` used to pass
   // `index: undefined`, which survived here and then crashed the signature
   // builder before the tap was ever sent.
@@ -2486,11 +2519,47 @@ async function runStep(deviceQuery, udid, step, ctx) {
           return null;
         }
       };
+      /**
+       * Is this target actually on the screen?
+       *
+       * Resolving is not the same as being in view, and this returned `"X" is
+       * in view at 201,-35 after 1 scroll down` — 35pt ABOVE the viewport, so
+       * the tap that followed missed. Reported from the field. The tree carries
+       * scrolled-away rows with out-of-bounds coordinates, which is exactly the
+       * case `scrollTo` exists to resolve, so claiming success on one is the
+       * one answer it must never give.
+       *
+       * Generous rather than strict: any overlap with the viewport counts, so a
+       * row half over an edge is still reachable and still reported. Only a
+       * target entirely outside keeps the search going.
+       */
+      const inViewport = (t) => {
+        const w = points?.width;
+        const h = points?.height;
+        if (!Number.isFinite(w) || !Number.isFinite(h)) return true;
+        const f = t?.frame;
+        if (f && Number.isFinite(f.y) && Number.isFinite(f.height)) {
+          return f.y + f.height > 0 && f.y < h && f.x + (f.width ?? 0) > 0 && f.x < w;
+        }
+        return Number.isFinite(t?.y) && t.y >= 0 && t.y <= h
+          && Number.isFinite(t?.x) && t.x >= 0 && t.x <= w;
+      };
+      // The direction actually scrolled, which is not always the one asked for:
+      // `offsetSays()` may reverse it, and the message used to name the request
+      // rather than the act — reported as "after 1 scroll down" on a request
+      // for "up".
+      const scrolled = [];
       for (let i = 0; i <= max; i += 1) {
         try {
           const found = await api.locate(deviceQuery, query, { index: step.index, refresh: i > 0 });
-          return `"${found.target.label}" is in view at ${found.target.x},${found.target.y}` +
-            (i ? ` after ${i} scroll${i === 1 ? '' : 's'} ${dir}` : ' already');
+          if (!inViewport(found.target)) throw new Error(
+            `"${query}" is in the tree but not in view (at ${found.target.x},${found.target.y}`
+            + ` on a ${Math.round(points?.width ?? 0)}x${Math.round(points?.height ?? 0)}pt screen)`);
+          const how = scrolled.length
+            ? ` after ${scrolled.length} scroll${scrolled.length === 1 ? '' : 's'} `
+              + (new Set(scrolled).size === 1 ? scrolled[0] : scrolled.join(' then '))
+            : ' already';
+          return `"${found.target.label ?? query}" is in view at ${found.target.x},${found.target.y}${how}`;
         } catch (err) {
           if (i === max) {
             throw new Error(`scrolled ${dir} ${max}x without finding ${query}: ${err.message}`);
@@ -2499,6 +2568,7 @@ async function runStep(deviceQuery, udid, step, ctx) {
         const evidence = await offsetSays();
         if (evidence) dir = evidence;
         const wasAt = await hashNow(deviceQuery, ctx.options);
+        scrolled.push(dir);
         await runStep(deviceQuery, udid, { action: 'scroll', value: dir }, ctx);
         // A scroll either moves immediately or not at all, so it does not need a
         // transition's budget. Six iterations at 2,500ms was most of why this
@@ -2763,8 +2833,20 @@ async function runStep(deviceQuery, udid, step, ctx) {
       await sleep(ms);
       return `paused ${ms}ms`;
     }
-    default:
-      throw new Error(`unknown step "${step.action}"`);
+    default: {
+      // Say what the words are. "unknown step" on its own sent two testers
+      // guessing, and a guess costs a round trip and the rest of the batch.
+      const near = STEP_KEYS.filter((k) => {
+        const a = String(step.action ?? '').toLowerCase().replace(/[_-]/g, '');
+        const b = k.toLowerCase();
+        return a && (b.startsWith(a) || a.startsWith(b) || b.includes(a));
+      }).slice(0, 3);
+      throw new Error(
+        `unknown step "${step.action}"`
+        + (near.length ? ` — did you mean ${near.map((k) => `"${k}"`).join(' or ')}?` : '')
+        + ` Valid steps: ${STEP_KEYS.join(', ')}.`,
+      );
+    }
   }
 }
 

package/src/cli.js

@@ -34,7 +34,7 @@ const USAGE = `simframe — always-warm iOS Simulator frames
   simframe tap     <selector>        tap #3, "Save", or @120,400
   simframe do      <script.json>     run a scripted flow (see below)
   simframe screens [device]          list screens this device has learned
-  simframe storage [bundle-id]       what the app saved (works on a shut-down device)
+  simframe storage [bundle-id] [--device=<name|udid>]   what the app saved (no boot needed)
   simframe goto    <screen>          walk to a known screen through known steps
   simframe flow    save <name> <script.json>   run a flow and save it if every step verifies
   simframe flow    run  <name>       replay a saved flow
@@ -867,7 +867,44 @@ async function main() {
       // whole value of this command is that it answers on a device that is not
       // running — measured: simctl itself cannot, on this Xcode. Routing it
       // through the daemon would throw that away for no gain.
-      const device = await resolveDevice(flags.device);
+      // Resolving a device here must NOT require one to be running, and it did.
+      //
+      // The whole claim of this command is that it answers before a boot, and
+      // `resolveDevice(undefined)` resolves against *booted* devices — so the
+      // documented form, `simframe storage <bundle-id>`, failed with "no booted
+      // simulator" at exactly the feature's headline use case. Reported by an
+      // external tester, who found it worked only when a device was named,
+      // which the help line did not mention either.
+      //
+      // Named: resolve as usual, booted or not. Unnamed: prefer a booted device
+      // when there is one, otherwise the only device that has app containers —
+      // and when that is ambiguous, say so and name the candidates rather than
+      // complaining about a boot nobody needs.
+      const named = flags.device ?? flags.udid ?? process.env.SIMFRAME_DEVICE;
+      let device;
+      if (named) {
+        device = await resolveDevice(String(named));
+      } else {
+        device = await resolveDevice(undefined).catch(() => null);
+        if (!device) {
+          const all = await listDevices({ all: true });
+          const withApps = [];
+          for (const d of all) {
+            const apps = await storage.apps(d.udid).catch(() => []);
+            if (apps.length) withApps.push({ device: d, apps: apps.length });
+          }
+          if (withApps.length === 1) {
+            device = withApps[0].device;
+          } else {
+            throw new Error(
+              withApps.length
+                ? 'storage reads a device that does not have to be running, so name which one:\n'
+                  + withApps.map((w) => `  --device=${w.device.udid}   ${w.device.name} (${w.apps} app(s))`).join('\n')
+                : 'no device on this host has any app data to read',
+            );
+          }
+        }
+      }
       const [bundleId] = positional;
       if (!bundleId) {
         const list = await storage.apps(device.udid);

package/src/index.js

@@ -1489,6 +1489,17 @@ export function sensorMode(options) {
   return raw === 'ax-first' || raw === 'axfirst' ? 'ax-first' : 'full';
 }
 
+/**
+ * What to call an element in a "Visible:" list.
+ *
+ * The label, or the accessibility identifier when there is no label. Both are
+ * now matchable, and the list had been filtered to `t.label` alone — so an
+ * element `sim_ui` had just printed by identifier was missing from the list of
+ * what is on screen, in the same reply that refused to resolve it. Reported
+ * from the field, three times in one session.
+ */
+const nameFor = (t) => t.label || t.identifier || null;
+
 export async function locate(deviceQuery, query, opts = {}) {
   if (sensorMode(opts.options) !== 'ax-first' || opts.useOcr === false || opts.escalated) {
     return locateWith(deviceQuery, query, opts);
@@ -1701,8 +1712,8 @@ async function locateWith(
     // here undoes every guard above — it has no off-screen filter and no
     // coverage weighting, and it is what returned a scrolled-away list row for
     // "back". "Not found" is the correct answer.
-    const visible = entry.targets.filter((t) => t.label && !regions.offViewport(t, points));
-    const sample = visible.slice(0, 12).map((t) => t.label.slice(0, 24)).join(', ');
+    const visible = entry.targets.filter((t) => nameFor(t) && !regions.offViewport(t, points));
+    const sample = visible.slice(0, 12).map((t) => nameFor(t).slice(0, 24)).join(', ');
     // "Not on this screen" and "not in view" are different answers, and giving
     // the first for the second cost a reported 15 seconds: a `waitFor REVIEW`
     // burned its whole timeout while REVIEW sat one scroll below the fold, and
@@ -1734,8 +1745,8 @@ async function locateWith(
   const candidates = screenmap.rank(entry, query);
   const target = index != null ? candidates[index] : candidates[0];
   if (!target) {
-    const visible = entry.targets.filter((t) => t.label);
-    const shown = visible.slice(0, 12).map((t) => t.label);
+    const visible = entry.targets.filter((t) => nameFor(t));
+    const shown = visible.slice(0, 12).map((t) => nameFor(t));
     // Say when the list is cut. A field report found `"Work Orders" is not on
     // this screen. Visible: …` on a screen whose own element map, three lines
     // below in the same reply, listed `#25 text 200,836 Work Orders` — it was

package/src/input.js

@@ -254,6 +254,14 @@ export function elementToNode(e) {
     selected: e.state?.selected ?? null,
     focused: e.state?.focused ?? null,
     frame: e.frame ?? null,
+    // Where the control actuates, when the app publishes it. See `centerOf`.
+    //
+    // Added here as well as in `normalizeNode`, and the comment four lines up
+    // is the reason: this is the converter that actually runs, and a field set
+    // only in the idb fallback is a field that is never set. That is exactly
+    // how AXSelected and AXFocused came to be batched by the daemon for four
+    // versions and dropped on the way in.
+    activationPoint: e.activationPoint ?? null,
     raw: e,
   };
 }
@@ -292,6 +300,8 @@ function normalizeNode(node) {
           height: frame.height ?? frame.Height ?? 0,
         }
       : null,
+    // Where the control actuates, when the app says so. See `centerOf`.
+    activationPoint: node.activationPoint ?? node.AXActivationPoint ?? null,
     raw: node,
   };
 }
@@ -329,11 +339,33 @@ export function matchElement(nodes, query, { index } = {}) {
   throw new Error(`no element matching "${query}" is on screen`);
 }
 
+/**
+ * Where to aim at this element.
+ *
+ * The geometric centre, unless the app has published somewhere better. UIKit
+ * exposes `accessibilityActivationPoint` for controls whose hit target is not
+ * the middle of what they publish as their frame, and a switch is the case that
+ * forced this: it reports a row-wide frame, so the centre is the *label*, and
+ * iOS does not actuate a switch from there. Measured on Settings →
+ * Accessibility → Hover Text — the frame centre flipped it **0 of 3** times and
+ * the control itself **3 of 3**, while the step reported `[no visible change]`
+ * and was telling the truth.
+ *
+ * The app's answer is preferred whenever it lands inside the frame. Outside it
+ * is not trusted: a point that is not on the element is not a better guess than
+ * the middle of one, and this runs on the tap path, where a wrong guess is the
+ * one thing that does damage.
+ */
 export function centerOf(node) {
-  return {
+  const middle = {
     x: Math.round(node.frame.x + node.frame.width / 2),
     y: Math.round(node.frame.y + node.frame.height / 2),
   };
+  const p = node.activationPoint;
+  if (!p || !Number.isFinite(p.x) || !Number.isFinite(p.y)) return middle;
+  const f = node.frame;
+  const inside = p.x >= f.x && p.x <= f.x + f.width && p.y >= f.y && p.y <= f.y + f.height;
+  return inside ? { x: Math.round(p.x), y: Math.round(p.y) } : middle;
 }
 
 export async function tapPoint(udid, x, y, { durationMs } = {}) {

package/src/matching.js

@@ -157,7 +157,17 @@ export function rank(targets, intent, { screen } = {}) {
 
   const scored = [];
   for (const t of visible) {
-    const names = [t.label, ...(t.aliases ?? [])].filter(Boolean);
+    // The accessibility identifier is a name a caller can legitimately write,
+    // and this list did not contain it.
+    //
+    // `sim_ui` prints elements BY identifier — in React Native a `testID`
+    // becomes one, so it is most interactive controls in an RN app — and the
+    // resolver then rejected that exact string, in the same response that had
+    // just printed it, with the identifier also absent from the "Visible:"
+    // list. An external tester reproduced it three times and called it the
+    // single biggest friction of their session. `screenmap.rank` had learned
+    // this; this ranker, which is the one `resolve()` uses, had not.
+    const names = [t.label, t.identifier, ...(t.aliases ?? [])].filter(Boolean);
     let base = 0;
     let matched = null;
     for (const name of names) {

package/src/mcp.js

@@ -405,7 +405,8 @@ const TOOLS = [
       'What the app saved, as text: its UserDefaults and (for React Native) its AsyncStorage, read straight out of'
       + ' the data container. sim_ui says what is drawn; sim_storage says what the app believes — use it when the'
       + ' screen and the behaviour disagree, or to check a value without driving the UI to it.'
-      + ' Works on a device that is NOT running, so it can answer before anything is booted.'
+      + ' Works on a device that is NOT running, so it can answer before anything is booted —'
+      + ' pass device when nothing is booted and the host has more than one simulator with data.'
       + ' Call with no bundleId to list the apps that have a container (match filters that list).',
     inputSchema: {
       type: 'object',
@@ -772,6 +773,23 @@ async function look(target, args, options) {
       lines.push(`could not crop that region (${cropped.note}) — this is the whole screen`);
     }
   }
+  // Say what space the image is in, and the factor.
+  //
+  // A field report lost time to this and had to derive the number from
+  // landmarks: `sim_ui` and `sim_tap` speak 402x874 POINTS, the returned image
+  // is 322x700 PIXELS, and `simctl io screenshot` is a third space again at 3x.
+  // Reading a coordinate off this image and feeding it to a tap is silently
+  // wrong by 1.248x — silently, because nothing in the reply relates the two.
+  // The header already prints the frame size; it just never said what it was
+  // relative to.
+  const geometry = await api.screenIdentity(target, { options, confirmNovel: false }).catch(() => null);
+  const pts = geometry?.points;
+  if (pts?.width && res.state?.width) {
+    const factor = res.state.width / pts.width;
+    lines.push(`this image is ${res.state.width}x${res.state.height} px = `
+      + `${Math.round(pts.width)}x${Math.round(pts.height)}pt — divide image coordinates by `
+      + `${factor.toFixed(3)} before tapping (sim_tap and sim_ui speak points)`);
+  }
   return { content: [text(lines.filter(Boolean).join('\n')), image(png)] };
 }
 
@@ -1200,7 +1218,30 @@ function listStateDirs() {
  * the host filesystem. That is what the backend does.
  */
 async function appStorage({ bundleId, match: query, device } = {}) {
-  const resolved = await resolveDevice(device);
+  // Same correction as the CLI: this must not require a booted device, because
+  // answering before a boot is the point of the tool. A bare call used to fail
+  // with "no booted simulator" — reported from the field — so an unnamed device
+  // falls back to the only one that has app data before it gives up.
+  let resolved = device
+    ? await resolveDevice(String(device))
+    : await resolveDevice(undefined).catch(() => null);
+  if (!resolved) {
+    const all = await listDevices({ all: true });
+    const withApps = [];
+    for (const d of all) {
+      const apps = await storage.apps(d.udid).catch(() => []);
+      if (apps.length) withApps.push({ device: d, apps: apps.length });
+    }
+    if (withApps.length === 1) resolved = withApps[0].device;
+    else {
+      throw new Error(
+        withApps.length
+          ? 'storage reads a device that does not have to be running, so say which one: '
+            + withApps.map((w) => `${w.device.name} (${w.device.udid}, ${w.apps} app(s))`).join('; ')
+          : 'no device on this host has any app data to read',
+      );
+    }
+  }
   if (!bundleId) {
     const list = await storage.apps(resolved.udid);
     const needle = query ? String(query).toLowerCase() : null;

package/src/storage.js

@@ -39,6 +39,33 @@ export const VALUE_PREVIEW_BYTES = 4096;
 const ASYNC_STORAGE_DIR = 'RCTAsyncLocalStorage_V1';
 const ASYNC_STORAGE_MANIFEST = 'manifest.json';
 
+/**
+ * Where AsyncStorage actually lives, newest layout first.
+ *
+ * **`Documents/` alone was wrong, and wrong in the worst available way.** That
+ * is the *legacy* React Native location. The community package every current RN
+ * app uses — `@react-native-async-storage/async-storage` — writes to
+ * `Library/Application Support/<bundle-id>/`, and on a real app measured by an
+ * external tester the `Documents/` path **did not exist at all**. So
+ * `readAsyncStorage` returned null, `format()` omitted the section, and the
+ * output read as "this app has no AsyncStorage" while 25 keys sat on disk,
+ * including a 1.5 MB MobX-State-Tree root store.
+ *
+ * Their diagnosis was exact: *"The decoder is correct; only the path is wrong."*
+ * Which makes this the precise class of confident wrong answer the whole feature
+ * exists to prevent, shipped inside it.
+ *
+ * Both are tried because both are real — an older app still writes to
+ * `Documents/` — and every path looked in is reported, because "found nothing"
+ * and "did not look there" are different facts and only one of them is about
+ * the app.
+ */
+const asyncStorageDirs = (container, bundleId) => [
+  path.join(container, 'Library', 'Application Support', String(bundleId ?? ''), ASYNC_STORAGE_DIR),
+  path.join(container, 'Library', 'Application Support', ASYNC_STORAGE_DIR),
+  path.join(container, 'Documents', ASYNC_STORAGE_DIR),
+];
+
 /** Files that are plainly a store but that nothing here can decode yet. */
 const OPAQUE_STORES = /\.(sqlite3?|db|realm|leveldb|mmkv)$/i;
 
@@ -59,10 +86,13 @@ export async function apps(udid) {
  * reporting it as one would be the exact class of wrong answer this feature
  * exists to stop.
  */
-export function readAsyncStorage(container) {
-  const dir = path.join(container, 'Documents', ASYNC_STORAGE_DIR);
+export function readAsyncStorage(container, bundleId) {
+  const looked = asyncStorageDirs(container, bundleId);
+  const dir = looked.find((d) => fs.existsSync(path.join(d, ASYNC_STORAGE_MANIFEST)));
+  // The paths travel with the miss. A reader told only "no AsyncStorage" cannot
+  // tell a bare app from a store we failed to find, and the second is ours.
+  if (!dir) return { missing: true, looked: looked.map((d) => path.relative(container, d)) };
   const manifestPath = path.join(dir, ASYNC_STORAGE_MANIFEST);
-  if (!fs.existsSync(manifestPath)) return null;
   const manifest = readJson(manifestPath);
   const entries = [];
   for (const [key, inline] of Object.entries(manifest)) {
@@ -151,9 +181,15 @@ export function typeOf(value) {
 export async function read(udid, bundleId) {
   const container = await platform.appContainer(udid, bundleId);
   const stores = await readPreferences(udid, container, bundleId);
-  const async_ = readAsyncStorage(container);
-  if (async_) stores.push(async_);
-  return { bundleId, container, stores, opaque: opaqueStores(container) };
+  const async_ = readAsyncStorage(container, bundleId);
+  if (async_ && !async_.missing) stores.push(async_);
+  return {
+    bundleId,
+    container,
+    stores,
+    opaque: opaqueStores(container),
+    asyncStorageMissing: async_?.missing ? async_.looked : null,
+  };
 }
 
 /** One value, rendered for reading, saying so whenever it is not the whole thing. */
@@ -186,11 +222,24 @@ export function format(result) {
       lines.push(`      ${renderValue(e.value).split('\n').join('\n      ')}`);
     }
   }
+  // Say where we looked and did not find it. See `asyncStorageDirs`.
+  if (result.asyncStorageMissing) {
+    lines.push('');
+    lines.push('  no AsyncStorage found. Looked in:');
+    for (const d of result.asyncStorageMissing) lines.push(`    ${d}`);
+    lines.push('  (an app that does not use AsyncStorage will have none of these)');
+  }
   if (result.opaque?.length) {
     lines.push('');
     lines.push(`  ${result.opaque.length} store(s) present that this cannot decode yet:`);
     for (const o of result.opaque) lines.push(`    ${o.file}  ${o.bytes} bytes`);
   }
+  // Asked for by the external tester, and they were right to: mid-session the
+  // app restored a session from the Keychain and walked past its own login
+  // screen, so "logged out" as read from storage was not the whole truth.
+  // Saying what is NOT readable sets expectations that silence does not.
+  lines.push('');
+  lines.push('  Keychain is not readable from here — auth state may differ from what is above.');
   return lines.join('\n');
 }
 

package/src/view.js

@@ -579,7 +579,7 @@ export async function screenMap(deviceQuery, {
   const anonymous = rows.filter(unaddressable).length;
   const unnamed = anonymous
     ? `${anonymous} on-screen control(s) have no accessibility label — they are listed with their`
-      + ' coordinates and can be tapped by point or by #ref, but not by name. If what you are'
+      + ' coordinates and can be tapped by #ref, or by point with tapAt. If what you are'
       + ' looking for is not in the list either, the app has views that were never declared'
       + ' accessible and only a screenshot will find those.'
     : null;