simframe 0.14.1 → 0.14.3

package/src/index.js

@@ -1489,6 +1489,17 @@ export function sensorMode(options) {
   return raw === 'ax-first' || raw === 'axfirst' ? 'ax-first' : 'full';
 }
 
+/**
+ * What to call an element in a "Visible:" list.
+ *
+ * The label, or the accessibility identifier when there is no label. Both are
+ * now matchable, and the list had been filtered to `t.label` alone — so an
+ * element `sim_ui` had just printed by identifier was missing from the list of
+ * what is on screen, in the same reply that refused to resolve it. Reported
+ * from the field, three times in one session.
+ */
+const nameFor = (t) => t.label || t.identifier || null;
+
 export async function locate(deviceQuery, query, opts = {}) {
   if (sensorMode(opts.options) !== 'ax-first' || opts.useOcr === false || opts.escalated) {
     return locateWith(deviceQuery, query, opts);
@@ -1701,8 +1712,8 @@ async function locateWith(
     // here undoes every guard above — it has no off-screen filter and no
     // coverage weighting, and it is what returned a scrolled-away list row for
     // "back". "Not found" is the correct answer.
-    const visible = entry.targets.filter((t) => t.label && !regions.offViewport(t, points));
-    const sample = visible.slice(0, 12).map((t) => t.label.slice(0, 24)).join(', ');
+    const visible = entry.targets.filter((t) => nameFor(t) && !regions.offViewport(t, points));
+    const sample = visible.slice(0, 12).map((t) => nameFor(t).slice(0, 24)).join(', ');
     // "Not on this screen" and "not in view" are different answers, and giving
     // the first for the second cost a reported 15 seconds: a `waitFor REVIEW`
     // burned its whole timeout while REVIEW sat one scroll below the fold, and
@@ -1734,8 +1745,8 @@ async function locateWith(
   const candidates = screenmap.rank(entry, query);
   const target = index != null ? candidates[index] : candidates[0];
   if (!target) {
-    const visible = entry.targets.filter((t) => t.label);
-    const shown = visible.slice(0, 12).map((t) => t.label);
+    const visible = entry.targets.filter((t) => nameFor(t));
+    const shown = visible.slice(0, 12).map((t) => nameFor(t));
     // Say when the list is cut. A field report found `"Work Orders" is not on
     // this screen. Visible: …` on a screen whose own element map, three lines
     // below in the same reply, listed `#25 text 200,836 Work Orders` — it was

package/src/input.js

@@ -254,6 +254,14 @@ export function elementToNode(e) {
     selected: e.state?.selected ?? null,
     focused: e.state?.focused ?? null,
     frame: e.frame ?? null,
+    // Where the control actuates, when the app publishes it. See `centerOf`.
+    //
+    // Added here as well as in `normalizeNode`, and the comment four lines up
+    // is the reason: this is the converter that actually runs, and a field set
+    // only in the idb fallback is a field that is never set. That is exactly
+    // how AXSelected and AXFocused came to be batched by the daemon for four
+    // versions and dropped on the way in.
+    activationPoint: e.activationPoint ?? null,
     raw: e,
   };
 }
@@ -292,6 +300,8 @@ function normalizeNode(node) {
           height: frame.height ?? frame.Height ?? 0,
         }
       : null,
+    // Where the control actuates, when the app says so. See `centerOf`.
+    activationPoint: node.activationPoint ?? node.AXActivationPoint ?? null,
     raw: node,
   };
 }
@@ -329,11 +339,33 @@ export function matchElement(nodes, query, { index } = {}) {
   throw new Error(`no element matching "${query}" is on screen`);
 }
 
+/**
+ * Where to aim at this element.
+ *
+ * The geometric centre, unless the app has published somewhere better. UIKit
+ * exposes `accessibilityActivationPoint` for controls whose hit target is not
+ * the middle of what they publish as their frame, and a switch is the case that
+ * forced this: it reports a row-wide frame, so the centre is the *label*, and
+ * iOS does not actuate a switch from there. Measured on Settings →
+ * Accessibility → Hover Text — the frame centre flipped it **0 of 3** times and
+ * the control itself **3 of 3**, while the step reported `[no visible change]`
+ * and was telling the truth.
+ *
+ * The app's answer is preferred whenever it lands inside the frame. Outside it
+ * is not trusted: a point that is not on the element is not a better guess than
+ * the middle of one, and this runs on the tap path, where a wrong guess is the
+ * one thing that does damage.
+ */
 export function centerOf(node) {
-  return {
+  const middle = {
     x: Math.round(node.frame.x + node.frame.width / 2),
     y: Math.round(node.frame.y + node.frame.height / 2),
   };
+  const p = node.activationPoint;
+  if (!p || !Number.isFinite(p.x) || !Number.isFinite(p.y)) return middle;
+  const f = node.frame;
+  const inside = p.x >= f.x && p.x <= f.x + f.width && p.y >= f.y && p.y <= f.y + f.height;
+  return inside ? { x: Math.round(p.x), y: Math.round(p.y) } : middle;
 }
 
 export async function tapPoint(udid, x, y, { durationMs } = {}) {

package/src/matching.js

@@ -157,7 +157,17 @@ export function rank(targets, intent, { screen } = {}) {
 
   const scored = [];
   for (const t of visible) {
-    const names = [t.label, ...(t.aliases ?? [])].filter(Boolean);
+    // The accessibility identifier is a name a caller can legitimately write,
+    // and this list did not contain it.
+    //
+    // `sim_ui` prints elements BY identifier — in React Native a `testID`
+    // becomes one, so it is most interactive controls in an RN app — and the
+    // resolver then rejected that exact string, in the same response that had
+    // just printed it, with the identifier also absent from the "Visible:"
+    // list. An external tester reproduced it three times and called it the
+    // single biggest friction of their session. `screenmap.rank` had learned
+    // this; this ranker, which is the one `resolve()` uses, had not.
+    const names = [t.label, t.identifier, ...(t.aliases ?? [])].filter(Boolean);
     let base = 0;
     let matched = null;
     for (const name of names) {

package/src/mcp.js

@@ -405,7 +405,8 @@ const TOOLS = [
       'What the app saved, as text: its UserDefaults and (for React Native) its AsyncStorage, read straight out of'
       + ' the data container. sim_ui says what is drawn; sim_storage says what the app believes — use it when the'
       + ' screen and the behaviour disagree, or to check a value without driving the UI to it.'
-      + ' Works on a device that is NOT running, so it can answer before anything is booted.'
+      + ' Works on a device that is NOT running, so it can answer before anything is booted —'
+      + ' pass device when nothing is booted and the host has more than one simulator with data.'
       + ' Call with no bundleId to list the apps that have a container (match filters that list).',
     inputSchema: {
       type: 'object',
@@ -772,6 +773,23 @@ async function look(target, args, options) {
       lines.push(`could not crop that region (${cropped.note}) — this is the whole screen`);
     }
   }
+  // Say what space the image is in, and the factor.
+  //
+  // A field report lost time to this and had to derive the number from
+  // landmarks: `sim_ui` and `sim_tap` speak 402x874 POINTS, the returned image
+  // is 322x700 PIXELS, and `simctl io screenshot` is a third space again at 3x.
+  // Reading a coordinate off this image and feeding it to a tap is silently
+  // wrong by 1.248x — silently, because nothing in the reply relates the two.
+  // The header already prints the frame size; it just never said what it was
+  // relative to.
+  const geometry = await api.screenIdentity(target, { options, confirmNovel: false }).catch(() => null);
+  const pts = geometry?.points;
+  if (pts?.width && res.state?.width) {
+    const factor = res.state.width / pts.width;
+    lines.push(`this image is ${res.state.width}x${res.state.height} px = `
+      + `${Math.round(pts.width)}x${Math.round(pts.height)}pt — divide image coordinates by `
+      + `${factor.toFixed(3)} before tapping (sim_tap and sim_ui speak points)`);
+  }
   return { content: [text(lines.filter(Boolean).join('\n')), image(png)] };
 }
 
@@ -1200,7 +1218,30 @@ function listStateDirs() {
  * the host filesystem. That is what the backend does.
  */
 async function appStorage({ bundleId, match: query, device } = {}) {
-  const resolved = await resolveDevice(device);
+  // Same correction as the CLI: this must not require a booted device, because
+  // answering before a boot is the point of the tool. A bare call used to fail
+  // with "no booted simulator" — reported from the field — so an unnamed device
+  // falls back to the only one that has app data before it gives up.
+  let resolved = device
+    ? await resolveDevice(String(device))
+    : await resolveDevice(undefined).catch(() => null);
+  if (!resolved) {
+    const all = await listDevices({ all: true });
+    const withApps = [];
+    for (const d of all) {
+      const apps = await storage.apps(d.udid).catch(() => []);
+      if (apps.length) withApps.push({ device: d, apps: apps.length });
+    }
+    if (withApps.length === 1) resolved = withApps[0].device;
+    else {
+      throw new Error(
+        withApps.length
+          ? 'storage reads a device that does not have to be running, so say which one: '
+            + withApps.map((w) => `${w.device.name} (${w.device.udid}, ${w.apps} app(s))`).join('; ')
+          : 'no device on this host has any app data to read',
+      );
+    }
+  }
   if (!bundleId) {
     const list = await storage.apps(resolved.udid);
     const needle = query ? String(query).toLowerCase() : null;

package/src/storage.js

@@ -39,6 +39,33 @@ export const VALUE_PREVIEW_BYTES = 4096;
 const ASYNC_STORAGE_DIR = 'RCTAsyncLocalStorage_V1';
 const ASYNC_STORAGE_MANIFEST = 'manifest.json';
 
+/**
+ * Where AsyncStorage actually lives, newest layout first.
+ *
+ * **`Documents/` alone was wrong, and wrong in the worst available way.** That
+ * is the *legacy* React Native location. The community package every current RN
+ * app uses — `@react-native-async-storage/async-storage` — writes to
+ * `Library/Application Support/<bundle-id>/`, and on a real app measured by an
+ * external tester the `Documents/` path **did not exist at all**. So
+ * `readAsyncStorage` returned null, `format()` omitted the section, and the
+ * output read as "this app has no AsyncStorage" while 25 keys sat on disk,
+ * including a 1.5 MB MobX-State-Tree root store.
+ *
+ * Their diagnosis was exact: *"The decoder is correct; only the path is wrong."*
+ * Which makes this the precise class of confident wrong answer the whole feature
+ * exists to prevent, shipped inside it.
+ *
+ * Both are tried because both are real — an older app still writes to
+ * `Documents/` — and every path looked in is reported, because "found nothing"
+ * and "did not look there" are different facts and only one of them is about
+ * the app.
+ */
+const asyncStorageDirs = (container, bundleId) => [
+  path.join(container, 'Library', 'Application Support', String(bundleId ?? ''), ASYNC_STORAGE_DIR),
+  path.join(container, 'Library', 'Application Support', ASYNC_STORAGE_DIR),
+  path.join(container, 'Documents', ASYNC_STORAGE_DIR),
+];
+
 /** Files that are plainly a store but that nothing here can decode yet. */
 const OPAQUE_STORES = /\.(sqlite3?|db|realm|leveldb|mmkv)$/i;
 
@@ -59,10 +86,13 @@ export async function apps(udid) {
  * reporting it as one would be the exact class of wrong answer this feature
  * exists to stop.
  */
-export function readAsyncStorage(container) {
-  const dir = path.join(container, 'Documents', ASYNC_STORAGE_DIR);
+export function readAsyncStorage(container, bundleId) {
+  const looked = asyncStorageDirs(container, bundleId);
+  const dir = looked.find((d) => fs.existsSync(path.join(d, ASYNC_STORAGE_MANIFEST)));
+  // The paths travel with the miss. A reader told only "no AsyncStorage" cannot
+  // tell a bare app from a store we failed to find, and the second is ours.
+  if (!dir) return { missing: true, looked: looked.map((d) => path.relative(container, d)) };
   const manifestPath = path.join(dir, ASYNC_STORAGE_MANIFEST);
-  if (!fs.existsSync(manifestPath)) return null;
   const manifest = readJson(manifestPath);
   const entries = [];
   for (const [key, inline] of Object.entries(manifest)) {
@@ -151,9 +181,15 @@ export function typeOf(value) {
 export async function read(udid, bundleId) {
   const container = await platform.appContainer(udid, bundleId);
   const stores = await readPreferences(udid, container, bundleId);
-  const async_ = readAsyncStorage(container);
-  if (async_) stores.push(async_);
-  return { bundleId, container, stores, opaque: opaqueStores(container) };
+  const async_ = readAsyncStorage(container, bundleId);
+  if (async_ && !async_.missing) stores.push(async_);
+  return {
+    bundleId,
+    container,
+    stores,
+    opaque: opaqueStores(container),
+    asyncStorageMissing: async_?.missing ? async_.looked : null,
+  };
 }
 
 /** One value, rendered for reading, saying so whenever it is not the whole thing. */
@@ -186,11 +222,24 @@ export function format(result) {
       lines.push(`      ${renderValue(e.value).split('\n').join('\n      ')}`);
     }
   }
+  // Say where we looked and did not find it. See `asyncStorageDirs`.
+  if (result.asyncStorageMissing) {
+    lines.push('');
+    lines.push('  no AsyncStorage found. Looked in:');
+    for (const d of result.asyncStorageMissing) lines.push(`    ${d}`);
+    lines.push('  (an app that does not use AsyncStorage will have none of these)');
+  }
   if (result.opaque?.length) {
     lines.push('');
     lines.push(`  ${result.opaque.length} store(s) present that this cannot decode yet:`);
     for (const o of result.opaque) lines.push(`    ${o.file}  ${o.bytes} bytes`);
   }
+  // Asked for by the external tester, and they were right to: mid-session the
+  // app restored a session from the Keychain and walked past its own login
+  // screen, so "logged out" as read from storage was not the whole truth.
+  // Saying what is NOT readable sets expectations that silence does not.
+  lines.push('');
+  lines.push('  Keychain is not readable from here — auth state may differ from what is above.');
   return lines.join('\n');
 }
 

package/src/view.js

@@ -579,7 +579,7 @@ export async function screenMap(deviceQuery, {
   const anonymous = rows.filter(unaddressable).length;
   const unnamed = anonymous
     ? `${anonymous} on-screen control(s) have no accessibility label — they are listed with their`
-      + ' coordinates and can be tapped by point or by #ref, but not by name. If what you are'
+      + ' coordinates and can be tapped by #ref, or by point with tapAt. If what you are'
       + ' looking for is not in the list either, the app has views that were never declared'
       + ' accessible and only a screenshot will find those.'
     : null;