simframe 0.6.1 → 0.6.2

package/native/simframed/Sources/PrivateAPI/AccessibilityBridge.swift

@@ -237,8 +237,13 @@ public final class AccessibilityBridge {
             cut = cut ?? "the read ran out of time"
             return
         }
-        out.append(node(from: element, depth: depth))
-        guard let children = attribute(element, "AXChildren") as? [NSObject] else { return }
+        // Each node's reads hand back autoreleased objects, and a 4000-node
+        // cap means 4000 nodes' worth of them living until the whole walk
+        // returns. Draining per node keeps the peak flat.
+        autoreleasepool {
+            out.append(node(from: element, depth: depth))
+        }
+        guard let children = autoreleasepool(invoking: { attribute(element, "AXChildren") as? [NSObject] }) else { return }
         for child in children {
             walk(child, depth: depth + 1, into: &out, deadline: deadline, cut: &cut)
         }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "simframe",
-  "version": "0.6.1",
+  "version": "0.6.2",
   "mcpName": "io.github.lvlrSajjad/simframe",
   "description": "Always-warm iOS Simulator frames: agents read the screen in ~20ms instead of waiting on screenshots. MCP server + CLI.",
   "keywords": [

package/scripts/ci-memory.mjs

@@ -41,6 +41,15 @@ const CONVERGE_PASSES = 3;
 const BETWEEN_PASSES_MS = 1200;
 
 let failures = 0;
+/**
+ * The device is gone, as distinct from having blinked. See `jsonRetry`, which
+ * is the only thing that sets it: a dropped frame is retryable and this file
+ * already treats it that way, so calling the first one fatal would fight the
+ * retry rather than help it. Exhausted attempts are the difference between a
+ * blink and a death.
+ */
+let deviceDied = null;
+
 function check(ok, label, detail = '') {
   if (!ok) failures += 1;
   console.log(`${ok ? 'ok  ' : 'FAIL'} ${label}${detail ? ` — ${detail}` : ''}`);
@@ -65,6 +74,10 @@ async function cli(args, { expectFail = false, allowFail = false } = {}) {
     if (err.unexpectedSuccess) throw err;
     if (expectFail || allowFail) return `${err.stdout ?? ''}${err.stderr ?? ''}`;
     const why = (err.stdout || err.stderr || err.message || '').trim();
+    // Noted here, not in `check`: by the time a failure reaches a check its
+    // detail has been truncated for legibility, and the first version of this
+    // guard looked for "did not produce a frame" in a string that had been cut
+    // to "simframe daemon di". The full text only exists at this boundary.
     throw new Error(`simframe ${full.join(' ')} failed: ${why.slice(0, 400)}`);
   }
 }
@@ -97,6 +110,23 @@ async function jsonRetry(args, opts, attempts = 3) {
       await new Promise((r) => setTimeout(r, 1500));
     }
   }
+  // Out of attempts on a capture error: the device is not blinking, it is gone.
+  //
+  // Diagnosed and exited here rather than flagged for a later `check` to
+  // notice, because most call sites do not wrap this — the throw escapes, the
+  // run dies on an unhandled rejection, and the operator gets a stack trace
+  // pointing at this file instead of a sentence about their simulator. Which is
+  // exactly what the first version of this did.
+  if (TRANSIENT.test(last?.message ?? '')) {
+    deviceDied = last.message;
+    console.error(`\nFAIL the device stopped producing frames, and did not come back after ${attempts} attempts:`);
+    console.error(`  ${String(last.message).split('\n')[0]}`);
+    console.error('\nEverything after this point would be testing a dead simulator, so the run');
+    console.error('stops here. This is not a memory-layer failure — it is the device-state');
+    console.error('problem in docs/DEFERRED.md. A device restart is the only known cure;');
+    console.error('on a hosted runner it means a retry.');
+    process.exit(1);
+  }
   throw last;
 }
 
@@ -394,9 +424,14 @@ if (check(forced.saved?.ok === true, 'and --force saves it anyway', `${forced.sa
 console.log('\n--- every command speaks JSON ---');
 // The --json plumbing is per-command and hand-written, so one command quietly
 // printing prose is exactly the kind of thing nothing else would catch.
+// Through `jsonRetry` like everything else. This loop used to call `cli`
+// directly, and it is the last section of a run that takes minutes — so a
+// capture dropout here failed five checks about `--json` plumbing that was
+// working perfectly, while every earlier section shrugged the same dropout off.
+// The one place that did not retry was the one place most likely to need it.
 for (const args of [['status'], ['state'], ['mark'], ['ui'], ['screens'], ['devices'], ['doctor'], ['flow', 'list'], ['recall']]) {
   try {
-    const parsed = JSON.parse(await cli([...args, '--json']));
+    const parsed = await jsonRetry([...args]);
     check(parsed !== null && parsed !== undefined, `simframe ${args.join(' ')} --json`);
   } catch (err) {
     check(false, `simframe ${args.join(' ')} --json`, err.message.slice(0, 120));

package/src/fingerprint.js

@@ -11,6 +11,18 @@
 import crypto from 'node:crypto';
 import * as regions from './regions.js';
 
+/**
+ * Bumped whenever the token rules change, and read by `graph.FINGERPRINT_VERSION`
+ * and `screenmap.MAP_VERSION` so stored hashes are discarded rather than
+ * compared against hashes computed by different rules. An old hash is a
+ * perfectly well-formed hash that never matches anything, which is the quietest
+ * kind of wrong.
+ *
+ * 2 — elements with no visible footprint, and containers holding two or more
+ *     others, no longer enter identity: only one sensor can see either.
+ */
+export const TOKEN_RULES_VERSION = 2;
+
 /** Frames are quantised to this, so sub-pixel drift and a nudged row do not matter. */
 export const GRID = 24;
 
@@ -108,13 +120,31 @@ export function tokens(targets, screen) {
   const keyboardTop = regions.detectKeyboardTop(targets, screen);
   const groups = new Map();
 
+  // Identity is what the screen *is*, not which sensor happened to see it, so
+  // two things that only one sensor can produce must not enter it: an element
+  // with no visible footprint, and a container that exists to hold others.
+  // Pixels cannot see either, and the accessibility tree reports both.
+  const encloses = (frame) => targets.filter((o) => {
+    const f = o.frame;
+    if (!f || f === frame) return false;
+    const cx = f.x + (f.width ?? 0) / 2;
+    const cy = f.y + (f.height ?? 0) / 2;
+    return cx > frame.x && cx < frame.x + (frame.width ?? 0)
+      && cy > frame.y && cy < frame.y + (frame.height ?? 0);
+  }).length;
+
   for (const t of targets) {
     const frame = t.frame ?? { x: t.x, y: t.y, width: 0, height: 0 };
     // Off-screen elements are not part of what this screen looks like.
     if (frame.y + (frame.height ?? 0) <= 0 || frame.y >= screen.height) continue;
+    // Nor is anything with no footprint to be seen.
+    if (!(frame.width > 0) || !(frame.height > 0)) continue;
     const region = t.region ?? regions.regionFor(frame, screen, { keyboardTop });
     if (region === 'status-bar') continue;
     if (keyboardTop != null && frame.y >= keyboardTop) continue;
+    // A thing that holds two or more other things is scenery, and only the
+    // tree can see it. Its children are already in the fingerprint.
+    if (/group|other|generic/i.test(String(t.type ?? '')) && encloses(frame) >= 2) continue;
 
     const role = roleOf(t);
     // Group by what a thing IS and how big it is, not where it is. Repeated

package/src/graph.js

@@ -7,11 +7,28 @@
 import fs from 'node:fs';
 import path from 'node:path';
 import { hashDistance } from './analyze.js';
+import { informative } from './refs.js';
 import * as fingerprint from './fingerprint.js';
 import * as matching from './matching.js';
 import * as store from './store.js';
 
-const GRAPH_VERSION = 2;
+const GRAPH_VERSION = 3;
+
+/**
+ * Which fingerprint produced the hashes in these files.
+ *
+ * Separate from `GRAPH_VERSION` because it answers a different question: not
+ * "is this file shaped the way I expect" but "were these hashes computed by the
+ * same rules I am about to compare them with". A stored graph whose hashes came
+ * from an older fingerprint is not stale, it is *incomparable* — and the failure
+ * is silent, because an old hash is a perfectly well-formed hash that simply
+ * never matches anything.
+ *
+ * On a mismatch the graph is discarded and rebuilt, never translated. A rebuild
+ * costs a few hundred milliseconds per screen and happens once. A mis-merged
+ * graph costs a wrong tap, and costs it for as long as the file survives.
+ */
+export const FINGERPRINT_VERSION = fingerprint.TOKEN_RULES_VERSION;
 /**
  * Screens are matched by structural hash, exactly, and then by how alike their
  * token sets are — which tolerates one optional element appearing (a badge, a
@@ -38,6 +55,14 @@ const GRAPH_VERSION = 2;
  * Most revisits match on a hash outright and never reach this at all.
  */
 export const SIMILARITY_THRESHOLD = 0.36;
+
+/**
+ * How many of the 288 layout bits may differ and still be the same arrangement
+ * of light. The same number `screenmap` recalls maps by, and for the same
+ * reason: measured, a revisit is usually identical and different screens sit at
+ * 74 and above, so 20 is well inside the gap.
+ */
+export const SAME_SCREEN_LAYOUT_BITS = 20;
 /**
  * A screen with three async sections has a few settled structures, not endless
  * ones. Capping this keeps a genuinely wrong merge bounded: if a node starts
@@ -105,11 +130,28 @@ function fingerprintsOf(node) {
 function load(udid, screen) {
   const key = typeof screen === 'string' ? { hash: screen, tokens: [] } : screen;
   const entry = store.readJson(path.join(graphDir(udid), `${key.hash}.json`));
-  if (entry?.version === GRAPH_VERSION) return entry;
+  if (entry?.version === GRAPH_VERSION && entry?.fingerprintVersion === FINGERPRINT_VERSION) return entry;
   // The hash may be a variant of a node filed under a different name.
   const byVariant = allNodes(udid).find((n) => (n.variants ?? []).some((v) => v.hash === key.hash));
   if (byVariant) return byVariant;
-  return { version: GRAPH_VERSION, hash: key.hash, tokens: key.tokens ?? [], variants: [], edges: [] };
+  return {
+    version: GRAPH_VERSION,
+    fingerprintVersion: FINGERPRINT_VERSION,
+    hash: key.hash,
+    tokens: key.tokens ?? [],
+    // What the pixels looked like here. Kept because it is the evidence that
+    // two structurally different readings are the same screen — see `record`.
+    layoutHash: key.layoutHash ?? null,
+    variants: [],
+    edges: [],
+  };
+}
+
+/** Keep the pixel baseline current for a screen we are standing on. */
+function noteLayout(node, reading) {
+  const now = typeof reading === 'string' ? null : reading?.layoutHash;
+  if (now && informative(now)) node.layoutHash = now;
+  return node;
 }
 
 function save(udid, node) {
@@ -124,7 +166,7 @@ export function allNodes(udid) {
       .readdirSync(graphDir(udid))
       .filter((f) => f.endsWith('.json'))
       .map((f) => store.readJson(path.join(graphDir(udid), f)))
-      .filter((n) => n?.version === GRAPH_VERSION);
+      .filter((n) => n?.version === GRAPH_VERSION && n?.fingerprintVersion === FINGERPRINT_VERSION);
   } catch {
     return [];
   }
@@ -243,6 +285,10 @@ export function record(udid, { from, action, to, kind }) {
   // arrive as — that is what variants are for, and rewriting it here would let
   // a node drift screen by screen into something it never was.
   if (!node.tokens?.length && fromKey.tokens?.length) node.tokens = fromKey.tokens;
+  // The pixel baseline, on the other hand, *should* track: it is the evidence
+  // for "same arrangement of light as last time I stood here", and a stale one
+  // answers a question about a screen as it was weeks ago.
+  noteLayout(node, fromKey);
   const to_ = toHash;
   const signature = actionSignature(action);
   const existing = node.edges.find((e) => e.action === signature);
@@ -272,9 +318,28 @@ export function record(udid, { from, action, to, kind }) {
       // So the reading has to positively look like the target before it is
       // called a face of it. Unclaimed is a necessary condition, not a
       // sufficient one.
-      const looksLikeTarget = resembles(target, reading);
+      // Two kinds of positive evidence that this reading is a face of the
+      // target, and either will do. What will not do is "nothing else claims
+      // it", which is an absence of evidence and used to be the whole test.
+      //
+      //  * It looks like the target — the tokens overlap enough to be the same
+      //    screen by the same measure used everywhere else.
+      //  * It *looks like* the target on screen. The pixels are within the
+      //    same-screen band of what was seen here before, and we arrived
+      //    through an edge that has led here. Identity belongs to the graph as
+      //    much as to the hash: the transition is evidence the fingerprint
+      //    cannot supply, and it is exactly the evidence needed when one
+      //    perception layer answered this time and not last time.
+      //
+      // That second route is what carries a screen whose structure genuinely
+      // differs between reads. Measured on four device-native screens, the
+      // accessibility tree and OCR agree on only 0.33–0.47 of a screen's
+      // tokens and never on its hash, and coarsening the vocabulary barely
+      // moved it — so this is not a residual case, it is the common one.
+      const looksLikeTarget = resembles(target, reading) || pixelsAgree(target, reading);
       if (unclaimed && looksLikeTarget && target.hash !== to_ && reading.tokens?.length) {
         addVariant(target, reading);
+        noteLayout(target, reading);
         save(udid, target);
         existing.count += 1;
         existing.lastSeen = Date.now();
@@ -306,6 +371,25 @@ export function record(udid, { from, action, to, kind }) {
 }
 
 /** What this action did last time, if we have ever seen it here. */
+/**
+ * Do the pixels say this is the same screen we have stood on here before?
+ *
+ * The layout hash is a poor answer to "which screen is this" on its own — that
+ * is why identity is structural — but it is a good answer to "is this the same
+ * arrangement of light", and combined with having arrived through a known edge
+ * it is the evidence that two structurally different readings are one screen.
+ *
+ * Guarded by `informative`, because a dark or uniform screen hashes to almost
+ * nothing and two of those are within any tolerance of each other while being
+ * evidence of nothing at all.
+ */
+function pixelsAgree(node, reading) {
+  const before = node?.layoutHash;
+  const now = reading?.layoutHash;
+  if (!before || !now || !informative(before) || !informative(now)) return false;
+  return hashDistance(before, now) <= SAME_SCREEN_LAYOUT_BITS;
+}
+
 /**
  * Does this reading look like a face of this screen, rather than a different
  * screen we happen not to have stored yet?

package/src/screenmap.js

@@ -16,7 +16,7 @@ import * as regions from './regions.js';
 import { informative } from './refs.js';
 import * as store from './store.js';
 
-const MAP_VERSION = 6; // dates, prices and phone numbers no longer contribute labels
+const MAP_VERSION = 7; // footprintless elements and containers no longer enter identity
 
 function mapDir(udid) {
   return path.join(store.deviceDir(udid), 'screens');