simframe 0.6.0 → 0.6.2

package/native/simframed/Sources/PrivateAPI/AccessibilityBridge.swift

@@ -130,6 +130,17 @@ public final class AccessibilityBridge {
             ? device.value(forKey: "accessibilityPlatformTranslationToken")
             : nil
 
+        // Guarded like its neighbours below, and for a sharper reason than
+        // tidiness: `setValue(_:forKey:)` on a key the class does not have
+        // raises `NSUnknownKeyException`, which is an Objective-C exception and
+        // therefore uncatchable from Swift. An Xcode that renames this property
+        // would not degrade the accessibility layer, it would crash the daemon
+        // — capture, input and OCR with it — which is the degrade-rather-than-
+        // fail rule exactly inverted, in the one file most likely to be
+        // invalidated by an upgrade.
+        guard shared.responds(to: NSSelectorFromString("setBridgeTokenDelegate:")) else {
+            throw AccessibilityError.unavailable("AXPTranslator has no bridgeTokenDelegate to install onto")
+        }
         shared.setValue(delegate, forKey: "bridgeTokenDelegate")
         if shared.responds(to: NSSelectorFromString("setSupportsDelegateTokens:")) {
             shared.setValue(true, forKey: "supportsDelegateTokens")
@@ -156,7 +167,16 @@ public final class AccessibilityBridge {
             .takeUnretainedValue() as? NSObject else {
             throw AccessibilityError.unavailable("the frontmost application did not translate to an element")
         }
-        delegate.resetTimeouts()
+        // A ticket per read rather than one counter reset per read.
+        //
+        // The caller bounds how long it will *wait*, not how long this runs, so
+        // an abandoned read can still be walking when the next one starts. With
+        // a single shared counter, one call's reset cleared timeouts the other
+        // had already accumulated — and a tree that had lost subtrees came back
+        // claiming to be whole, which is precisely the failure this reporting
+        // exists to prevent.
+        let ticket = delegate.beginRead()
+        defer { delegate.endRead(ticket) }
         var out: [AXNode] = []
         var cut: String?
         let deadline = Date().addingTimeInterval(budget)
@@ -164,7 +184,7 @@ public final class AccessibilityBridge {
         // A guest that missed the deadline answers `emptyResponse`, which makes
         // the subtree below it look genuinely childless. Nothing in the nodes
         // can show that, so the count has to.
-        let missed = delegate.timeouts
+        let missed = delegate.timeouts(for: ticket)
         if cut == nil, missed > 0 {
             cut = "\(missed) request(s) to the device timed out, so part of the tree is missing"
         }
@@ -192,6 +212,7 @@ public final class AccessibilityBridge {
     /// The pid of the app currently frontmost, or nil when the bridge cannot say.
     public func frontmostPid() -> Int32? {
         guard let app = frontmostApplication() else { return nil }
+        guard app.responds(to: NSSelectorFromString("pid")) else { return nil }
         return (app.value(forKey: "pid") as? NSNumber)?.int32Value
     }
 
@@ -216,8 +237,13 @@ public final class AccessibilityBridge {
             cut = cut ?? "the read ran out of time"
             return
         }
-        out.append(node(from: element, depth: depth))
-        guard let children = attribute(element, "AXChildren") as? [NSObject] else { return }
+        // Each node's reads hand back autoreleased objects, and a 4000-node
+        // cap means 4000 nodes' worth of them living until the whole walk
+        // returns. Draining per node keeps the peak flat.
+        autoreleasepool {
+            out.append(node(from: element, depth: depth))
+        }
+        guard let children = autoreleasepool(invoking: { attribute(element, "AXChildren") as? [NSObject] }) else { return }
         for child in children {
             walk(child, depth: depth + 1, into: &out, deadline: deadline, cut: &cut)
         }
@@ -318,27 +344,40 @@ private final class BridgeDelegate: NSObject {
     private let timeout: TimeInterval
     private let queue = DispatchQueue(label: "simframe.accessibility.bridge")
     private let counter = NSLock()
-    private var timedOut = 0
+    /// Timeouts per in-flight read, so two overlapping reads cannot clear or
+    /// inherit each other's count. A timeout with no read in flight belongs to
+    /// an abandoned one and is dropped rather than charged to a stranger.
+    private var timedOut: [Int: Int] = [:]
+    private var nextTicket = 0
 
     init(device: NSObject, timeout: TimeInterval) {
         self.device = device
         self.timeout = timeout
     }
 
-    /// How many requests the device failed to answer since the last reset.
-    var timeouts: Int {
+    func beginRead() -> Int {
+        counter.lock(); defer { counter.unlock() }
+        nextTicket += 1
+        timedOut[nextTicket] = 0
+        return nextTicket
+    }
+
+    /// How many requests the device failed to answer during this read.
+    func timeouts(for ticket: Int) -> Int {
         counter.lock(); defer { counter.unlock() }
-        return timedOut
+        return timedOut[ticket] ?? 0
     }
 
-    func resetTimeouts() {
+    func endRead(_ ticket: Int) {
         counter.lock(); defer { counter.unlock() }
-        timedOut = 0
+        timedOut.removeValue(forKey: ticket)
     }
 
     fileprivate func recordTimeout() {
         counter.lock(); defer { counter.unlock() }
-        timedOut += 1
+        // Charged to every read currently in flight: a request that timed out
+        // while two reads were walking cost both of them a subtree.
+        for key in timedOut.keys { timedOut[key, default: 0] += 1 }
     }
 
     @objc(accessibilityTranslationDelegateBridgeCallbackWithToken:)

package/native/simframed/Sources/PrivateAPI/CoreSimulatorPlatform.swift

@@ -28,6 +28,7 @@ public final class CoreSimulatorPlatform: SimulatorPlatform {
     // machine where the translation framework is missing must still capture.
     private var accessibility: AccessibilityBridge?
     private var accessibilityFailure: String?
+    private let bridgeLock = NSLock()
     private var simulatorKitHandle: UnsafeMutableRawPointer?
     // Read once at attach: spawning simctl per status call cost 300ms.
     private var cachedKeyboardWarning: [String]?
@@ -193,8 +194,10 @@ public final class CoreSimulatorPlatform: SimulatorPlatform {
             // process-wide translator, so it belongs to the device it was built
             // for. Binding to a different one must not inherit it.
             if self.device !== device {
+                bridgeLock.lock()
                 accessibility = nil
                 accessibilityFailure = nil
+                bridgeLock.unlock()
             }
             self.device = device
             let resolved = info(for: device)
@@ -229,7 +232,18 @@ public final class CoreSimulatorPlatform: SimulatorPlatform {
         try bridge().tree()
     }
 
+    /// Serialised, because building a bridge installs a delegate on a
+    /// **process-global** translator that holds it weakly.
+    ///
+    /// Two constructions racing both install; the loser's delegate is released
+    /// and the survivor is left holding a translator whose delegate has
+    /// deallocated, which answers nil for everything and sets no failure — the
+    /// exact silent-nil this file's header warns about. The control socket is
+    /// serial, but the capture loop's rebind path clears these same two fields,
+    /// so there is a genuine second writer.
     private func bridge() throws -> AccessibilityBridge {
+        bridgeLock.lock()
+        defer { bridgeLock.unlock() }
         if let accessibility { return accessibility }
         // A framework that is missing stays missing; re-probing it on every
         // screen read would cost a dlopen per call to learn the same thing.

package/native/simframed/Sources/PrivateAPI/StubPlatform.swift

@@ -39,8 +39,14 @@ public final class StubPlatform: SimulatorPlatform {
     public private(set) var resetInputCount = 0
     public func resetInput() throws { resetInputCount += 1 }
 
+    /// Test hook: make re-resolving the port fail, which is the case a real
+    /// device only reaches when something is badly wrong and therefore the one
+    /// hardest to observe.
+    public var failReattach = false
+
     public func reattachDisplay() throws -> DeviceInfo {
         reattachCount += 1
+        if failReattach { throw PrivateAPIError.noDisplayPort }
         return DeviceInfo(udid: attachedUdid ?? "STUB-0000", name: "Stub Device", runtime: "iOS 26.0")
     }
 

package/native/simframed/Sources/SimframeCore/CaptureRecovery.swift

@@ -0,0 +1,60 @@
+import Foundation
+import PrivateAPI
+
+/// When a run of failed captures means the display port itself is gone.
+///
+/// The port can be torn down and rebuilt under a running daemon, and every read
+/// on the old descriptor returns nil from then on. Observed on a device awake
+/// and visible the whole time: six minutes of "the display surface could not be
+/// read", cured instantly by restarting the daemon.
+///
+/// This lives in its own type for one reason: the recovery path only ever runs
+/// in the situation nobody is watching, and inline in the capture loop it could
+/// not be tested at all. A teardown cannot be induced on demand, but the
+/// decision to re-resolve and the act of re-resolving both can be.
+public struct CaptureRecovery {
+    /// Roughly three seconds of failed reads at the loop's half-second back-off.
+    /// Long enough not to thrash on a momentary hiccup, short enough that nobody
+    /// watches a dead capture loop and wonders.
+    public static let reattachAfterFailures = 6
+
+    public private(set) var consecutiveFailures = 0
+    private let threshold: Int
+
+    public init(threshold: Int = CaptureRecovery.reattachAfterFailures) {
+        self.threshold = threshold
+    }
+
+    public mutating func captureSucceeded() {
+        consecutiveFailures = 0
+    }
+
+    /// Records a failure and says whether the port is now due a re-resolve.
+    public mutating func captureFailed() -> Bool {
+        consecutiveFailures += 1
+        return consecutiveFailures >= threshold
+    }
+
+    /// Re-resolve the port and re-arm the damage callback.
+    ///
+    /// Both halves matter and only one is obvious: a fresh descriptor with no
+    /// callback registered on it produces a daemon that has recovered and will
+    /// never notice another change, which looks exactly like the failure it just
+    /// recovered from.
+    public mutating func reattach(
+        platform: SimulatorPlatform,
+        onDamage: @escaping () -> Void
+    ) -> Result<Int, Error> {
+        let failures = consecutiveFailures
+        do {
+            _ = try platform.reattachDisplay()
+            try platform.observeChanges(onDamage)
+            consecutiveFailures = 0
+            return .success(failures)
+        } catch {
+            // Deliberately not reset: if the port cannot be re-resolved, the
+            // next failure should try again rather than wait for another six.
+            return .failure(error)
+        }
+    }
+}

package/native/simframed/Sources/simframed/main.swift

@@ -9,6 +9,14 @@ func flag(_ name: String) -> String? {
     return String(a.dropFirst(name.count + 3))
 }
 
+/// How long a `ui` request may spend on its two reads before what has not
+/// arrived is reported as missing.
+///
+/// Just inside the client's own 30 s give-up, because a read the caller has
+/// already abandoned is worth nothing, and stopping any earlier than that
+/// converts a slow-but-correct read into a failure. See the `ui` handler.
+let readBudgetMs = 25_000
+
 let command = args.first(where: { !$0.hasPrefix("--") }) ?? "run"
 let platform: SimulatorPlatform = args.contains("--stub") ? StubPlatform() : CoreSimulatorPlatform()
 let longEdge = Int(flag("max-dim") ?? "") ?? 700
@@ -158,11 +166,7 @@ case "run":
 
         let lock = NSLock()
         var dirty = true
-        var consecutiveFailures = 0
-        /// Roughly three seconds of failed reads at the loop's half-second
-        /// back-off. Long enough not to thrash on a momentary hiccup, short
-        /// enough that nobody watches a dead capture loop and wonders.
-        let reattachAfterFailures = 6
+        var recovery = CaptureRecovery()
         var lastCapture = 0.0
         var frames = 0
         var lastReport = Date().timeIntervalSince1970
@@ -260,19 +264,29 @@ case "run":
                     }
                     // Bounded, because this blocks the control socket and the
                     // socket is serial: a read that runs long does not just
-                    // return late, it holds up every command behind it. A CI
-                    // runner spent 28 s inside one of these. Whatever has not
-                    // arrived by the deadline is reported as missing rather
-                    // than waited for — the layer that did answer is still
-                    // worth having.
-                    let timedOut = group.wait(timeout: .now() + .seconds(12)) == .timedOut
+                    // return late, it holds up every command behind it.
+                    //
+                    // The bound is "as long as the caller is willing to wait",
+                    // not a number picked for feel. The client gives up at 30 s
+                    // (`control.js`), so anything past that is lost either way,
+                    // and stopping earlier only converts a slow-but-correct
+                    // read into a failure. It was 12 s, chosen against a runner
+                    // that once spent 28 s inside a *tree* read — a cost the
+                    // attribute batching then removed — and a hosted runner
+                    // where Vision has no GPU promptly failed a screen read
+                    // with "text recognition did not finish within 12s". OCR is
+                    // 100–400 ms on a developer's machine and evidently much
+                    // slower on a shared one; guessing its ceiling was the
+                    // mistake.
+                    let timedOut = group.wait(timeout: .now() + .milliseconds(readBudgetMs)) == .timedOut
                     var (axTree, axError, axMs, ocrElements, ocrError, ocrMs) = reads.snapshot()
                     if timedOut {
+                        let secs = readBudgetMs / 1000
                         if wantAx, axTree.nodes.isEmpty, axError == nil {
-                            axError = "the accessibility read did not finish within 12s"
+                            axError = "the accessibility read did not finish within \(secs)s"
                         }
                         if wantOcr, ocrElements.isEmpty, ocrError == nil {
-                            ocrError = "text recognition did not finish within 12s"
+                            ocrError = "text recognition did not finish within \(secs)s"
                         }
                     }
                     // OCR failing is fatal to a screen read in a way a missing
@@ -419,11 +433,11 @@ case "run":
                     latencies.append(Double(DispatchTime.now().uptimeNanoseconds - t0) / 1e6)
                     frames += 1
                     lastCapture = now
-                    consecutiveFailures = 0
+                    recovery.captureSucceeded()
                 } catch {
-                    consecutiveFailures += 1
+                    let due = recovery.captureFailed()
                     FileHandle.standardError.write(
-                        "simframed: capture failed: \(error) (\(consecutiveFailures) in a row)\n".data(using: .utf8)!)
+                        "simframed: capture failed: \(error) (\(recovery.consecutiveFailures) in a row)\n".data(using: .utf8)!)
                     // The display port can be torn down and rebuilt under a
                     // running daemon, and every read on the old descriptor
                     // returns nil from then on. Observed on a device that was
@@ -432,18 +446,16 @@ case "run":
                     // by restarting the daemon. Reporting a failure loudly is
                     // right; never recovering from it is not, so re-resolve the
                     // port and re-arm the damage callback.
-                    if consecutiveFailures >= reattachAfterFailures {
-                        do {
-                            _ = try platform.reattachDisplay()
-                            try platform.observeChanges {
-                                lock.lock(); dirty = true; lock.unlock()
-                            }
+                    if due {
+                        switch recovery.reattach(platform: platform, onDamage: {
+                            lock.lock(); dirty = true; lock.unlock()
+                        }) {
+                        case .success(let after):
                             lock.lock(); dirty = true; lock.unlock()
                             FileHandle.standardError.write(
-                                "simframed: re-resolved the display port after \(consecutiveFailures) failed reads\n"
+                                "simframed: re-resolved the display port after \(after) failed reads\n"
                                     .data(using: .utf8)!)
-                            consecutiveFailures = 0
-                        } catch {
+                        case .failure(let error):
                             FileHandle.standardError.write(
                                 "simframed: could not re-resolve the display port: \(error)\n".data(using: .utf8)!)
                         }

package/native/simframed/Tests/SimframeCoreTests/HashingTests.swift

@@ -268,3 +268,54 @@ final class AccessibilityElementTests: XCTestCase {
         XCTAssertNil(element.json["identifier"], "an absent identifier is absent, not null")
     }
 }
+
+/// The recovery path that only ever runs when nobody is watching.
+///
+/// A display teardown cannot be induced on demand, which is why this went
+/// untested through two releases. The decision and the act can both be driven
+/// with a stub, and that covers everything except the teardown itself.
+final class CaptureRecoveryTests: XCTestCase {
+    func testAMomentaryHiccupDoesNotReattach() {
+        var recovery = CaptureRecovery(threshold: 6)
+        for _ in 0..<5 { XCTAssertFalse(recovery.captureFailed(), "five failures is a hiccup") }
+        XCTAssertEqual(recovery.consecutiveFailures, 5)
+        recovery.captureSucceeded()
+        XCTAssertEqual(recovery.consecutiveFailures, 0, "one good frame clears the run")
+        for _ in 0..<5 { XCTAssertFalse(recovery.captureFailed()) }
+    }
+
+    func testASustainedRunReattachesAndRearmsTheCallback() {
+        let platform = StubPlatform()
+        _ = try? platform.attach(udid: "STUB-1")
+        var recovery = CaptureRecovery(threshold: 3)
+        XCTAssertFalse(recovery.captureFailed())
+        XCTAssertFalse(recovery.captureFailed())
+        XCTAssertTrue(recovery.captureFailed(), "the third failure is due a re-resolve")
+
+        var damaged = false
+        let outcome = recovery.reattach(platform: platform) { damaged = true }
+        guard case .success(let after) = outcome else { return XCTFail("reattach should succeed on a live stub") }
+        XCTAssertEqual(after, 3, "it reports how many reads it lost")
+        XCTAssertEqual(platform.reattachCount, 1)
+        XCTAssertEqual(recovery.consecutiveFailures, 0)
+
+        // The callback half is the one that is easy to forget: a fresh
+        // descriptor with nothing registered on it gives a daemon that has
+        // recovered and will never notice another change.
+        platform.simulateChange()
+        XCTAssertTrue(damaged, "the damage callback was re-armed on the new descriptor")
+    }
+
+    func testAFailedReattachStaysDueRatherThanWaitingForAnotherSix() {
+        let platform = StubPlatform()
+        platform.failReattach = true
+        var recovery = CaptureRecovery(threshold: 2)
+        _ = recovery.captureFailed()
+        XCTAssertTrue(recovery.captureFailed())
+        guard case .failure = recovery.reattach(platform: platform, onDamage: {}) else {
+            return XCTFail("a stub told to fail should fail")
+        }
+        XCTAssertEqual(recovery.consecutiveFailures, 2, "the run is not cleared by an attempt that did not work")
+        XCTAssertTrue(recovery.captureFailed(), "so the next failure tries again immediately")
+    }
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "simframe",
-  "version": "0.6.0",
+  "version": "0.6.2",
   "mcpName": "io.github.lvlrSajjad/simframe",
   "description": "Always-warm iOS Simulator frames: agents read the screen in ~20ms instead of waiting on screenshots. MCP server + CLI.",
   "keywords": [

package/scripts/check-package.mjs

@@ -15,6 +15,7 @@
 // Swift source or a new module here and it becomes required automatically.
 import { execFileSync } from 'node:child_process';
 import fs from 'node:fs';
+import os from 'node:os';
 import path from 'node:path';
 import { fileURLToPath } from 'node:url';
 
@@ -83,8 +84,34 @@ function requiredFiles() {
 }
 
 const required = requiredFiles();
-const packed = JSON.parse(execFileSync('npm', ['pack', '--dry-run', '--json'], { cwd: ROOT, encoding: 'utf8' }));
-const shipped = new Set(packed[0].files.map((f) => f.path));
+// Read the tarball, not npm's description of it.
+//
+// This used to parse `npm pack --dry-run --json` as `packed[0].files`, which is
+// npm 10's shape. npm 12 reports something else, and the check died with
+// "Cannot read properties of undefined (reading 'files')" — a check that exists
+// to catch a silently broken package, itself silently broken by the tool it
+// asks. It had only ever run under the npm bundled with Node 18–22; the release
+// job runs a newer one.
+//
+// `tar -tzf` on the artifact npm actually produces has no shape to change, and
+// it answers the stronger question: what is *in* the file people download.
+const out = fs.mkdtempSync(path.join(os.tmpdir(), 'simframe-pack-'));
+const name = execFileSync('npm', ['pack', '--silent', '--pack-destination', out], { cwd: ROOT, encoding: 'utf8' })
+  .trim()
+  .split('\n')
+  .pop();
+const listing = execFileSync('tar', ['-tzf', path.join(out, name)], { encoding: 'utf8' });
+fs.rmSync(out, { recursive: true, force: true });
+const shipped = new Set(
+  listing
+    .split('\n')
+    .map((line) => line.trim())
+    .filter(Boolean)
+    // Every path in an npm tarball is prefixed `package/`, and directories
+    // arrive with a trailing slash.
+    .filter((line) => line.startsWith('package/') && !line.endsWith('/'))
+    .map((line) => line.slice('package/'.length)),
+);
 const missing = required.filter((f) => !shipped.has(f));
 
 console.log(`${required.length} build inputs required, ${shipped.size} files in the tarball`);

package/scripts/ci-memory.mjs

@@ -41,6 +41,15 @@ const CONVERGE_PASSES = 3;
 const BETWEEN_PASSES_MS = 1200;
 
 let failures = 0;
+/**
+ * The device is gone, as distinct from having blinked. See `jsonRetry`, which
+ * is the only thing that sets it: a dropped frame is retryable and this file
+ * already treats it that way, so calling the first one fatal would fight the
+ * retry rather than help it. Exhausted attempts are the difference between a
+ * blink and a death.
+ */
+let deviceDied = null;
+
 function check(ok, label, detail = '') {
   if (!ok) failures += 1;
   console.log(`${ok ? 'ok  ' : 'FAIL'} ${label}${detail ? ` — ${detail}` : ''}`);
@@ -65,6 +74,10 @@ async function cli(args, { expectFail = false, allowFail = false } = {}) {
     if (err.unexpectedSuccess) throw err;
     if (expectFail || allowFail) return `${err.stdout ?? ''}${err.stderr ?? ''}`;
     const why = (err.stdout || err.stderr || err.message || '').trim();
+    // Noted here, not in `check`: by the time a failure reaches a check its
+    // detail has been truncated for legibility, and the first version of this
+    // guard looked for "did not produce a frame" in a string that had been cut
+    // to "simframe daemon di". The full text only exists at this boundary.
     throw new Error(`simframe ${full.join(' ')} failed: ${why.slice(0, 400)}`);
   }
 }
@@ -97,6 +110,23 @@ async function jsonRetry(args, opts, attempts = 3) {
       await new Promise((r) => setTimeout(r, 1500));
     }
   }
+  // Out of attempts on a capture error: the device is not blinking, it is gone.
+  //
+  // Diagnosed and exited here rather than flagged for a later `check` to
+  // notice, because most call sites do not wrap this — the throw escapes, the
+  // run dies on an unhandled rejection, and the operator gets a stack trace
+  // pointing at this file instead of a sentence about their simulator. Which is
+  // exactly what the first version of this did.
+  if (TRANSIENT.test(last?.message ?? '')) {
+    deviceDied = last.message;
+    console.error(`\nFAIL the device stopped producing frames, and did not come back after ${attempts} attempts:`);
+    console.error(`  ${String(last.message).split('\n')[0]}`);
+    console.error('\nEverything after this point would be testing a dead simulator, so the run');
+    console.error('stops here. This is not a memory-layer failure — it is the device-state');
+    console.error('problem in docs/DEFERRED.md. A device restart is the only known cure;');
+    console.error('on a hosted runner it means a retry.');
+    process.exit(1);
+  }
   throw last;
 }
 
@@ -394,9 +424,14 @@ if (check(forced.saved?.ok === true, 'and --force saves it anyway', `${forced.sa
 console.log('\n--- every command speaks JSON ---');
 // The --json plumbing is per-command and hand-written, so one command quietly
 // printing prose is exactly the kind of thing nothing else would catch.
+// Through `jsonRetry` like everything else. This loop used to call `cli`
+// directly, and it is the last section of a run that takes minutes — so a
+// capture dropout here failed five checks about `--json` plumbing that was
+// working perfectly, while every earlier section shrugged the same dropout off.
+// The one place that did not retry was the one place most likely to need it.
 for (const args of [['status'], ['state'], ['mark'], ['ui'], ['screens'], ['devices'], ['doctor'], ['flow', 'list'], ['recall']]) {
   try {
-    const parsed = JSON.parse(await cli([...args, '--json']));
+    const parsed = await jsonRetry([...args]);
     check(parsed !== null && parsed !== undefined, `simframe ${args.join(' ')} --json`);
   } catch (err) {
     check(false, `simframe ${args.join(' ')} --json`, err.message.slice(0, 120));

package/src/fingerprint.js

@@ -11,6 +11,18 @@
 import crypto from 'node:crypto';
 import * as regions from './regions.js';
 
+/**
+ * Bumped whenever the token rules change, and read by `graph.FINGERPRINT_VERSION`
+ * and `screenmap.MAP_VERSION` so stored hashes are discarded rather than
+ * compared against hashes computed by different rules. An old hash is a
+ * perfectly well-formed hash that never matches anything, which is the quietest
+ * kind of wrong.
+ *
+ * 2 — elements with no visible footprint, and containers holding two or more
+ *     others, no longer enter identity: only one sensor can see either.
+ */
+export const TOKEN_RULES_VERSION = 2;
+
 /** Frames are quantised to this, so sub-pixel drift and a nudged row do not matter. */
 export const GRID = 24;
 
@@ -108,13 +120,31 @@ export function tokens(targets, screen) {
   const keyboardTop = regions.detectKeyboardTop(targets, screen);
   const groups = new Map();
 
+  // Identity is what the screen *is*, not which sensor happened to see it, so
+  // two things that only one sensor can produce must not enter it: an element
+  // with no visible footprint, and a container that exists to hold others.
+  // Pixels cannot see either, and the accessibility tree reports both.
+  const encloses = (frame) => targets.filter((o) => {
+    const f = o.frame;
+    if (!f || f === frame) return false;
+    const cx = f.x + (f.width ?? 0) / 2;
+    const cy = f.y + (f.height ?? 0) / 2;
+    return cx > frame.x && cx < frame.x + (frame.width ?? 0)
+      && cy > frame.y && cy < frame.y + (frame.height ?? 0);
+  }).length;
+
   for (const t of targets) {
     const frame = t.frame ?? { x: t.x, y: t.y, width: 0, height: 0 };
     // Off-screen elements are not part of what this screen looks like.
     if (frame.y + (frame.height ?? 0) <= 0 || frame.y >= screen.height) continue;
+    // Nor is anything with no footprint to be seen.
+    if (!(frame.width > 0) || !(frame.height > 0)) continue;
     const region = t.region ?? regions.regionFor(frame, screen, { keyboardTop });
     if (region === 'status-bar') continue;
     if (keyboardTop != null && frame.y >= keyboardTop) continue;
+    // A thing that holds two or more other things is scenery, and only the
+    // tree can see it. Its children are already in the fingerprint.
+    if (/group|other|generic/i.test(String(t.type ?? '')) && encloses(frame) >= 2) continue;
 
     const role = roleOf(t);
     // Group by what a thing IS and how big it is, not where it is. Repeated
@@ -157,6 +187,19 @@ export function tokens(targets, screen) {
 }
 
 export function hashTokens(list) {
+  // No tokens is not an identity. It is the absence of one.
+  //
+  // sha256 of the empty string is a constant, so every unreadable screen used
+  // to hash to `e3b0c442…` — one identity shared by a dark screen, a screen
+  // whose OCR failed, and a screen read mid-transition. Different screens
+  // collapsing onto a single hash is a wrong *merge*, which is worse than a
+  // missed match: a ref numbered on one screen resolved happily on another,
+  // and an edge learned on one predicted the other, with a verdict of `ok`.
+  //
+  // The pixel layout hash had exactly this degeneracy and got an
+  // `informative()` guard for it. The structural hash did not, and the guard
+  // could not have helped: a constant is perfectly informative-looking.
+  if (!list.length) return null;
   return crypto.createHash('sha256').update(list.join('\n')).digest('hex').slice(0, 32);
 }
 
@@ -174,7 +217,11 @@ export function fingerprint(targets, screen) {
  * everything.
  */
 export function similarity(a = [], b = []) {
-  if (!a.length && !b.length) return 1;
+  // Two empty token sets are not a match, they are two absences of evidence.
+  // Returning 1 here made unreadability *self-confirming*: two consecutive
+  // unreadable reads "agreed", which promoted the non-identity to a confirmed
+  // screen and let it be written into memory and learned as an edge.
+  if (!a.length || !b.length) return 0;
   const setA = new Set(a);
   const setB = new Set(b);
   let shared = 0;

package/src/graph.js

@@ -7,11 +7,28 @@
 import fs from 'node:fs';
 import path from 'node:path';
 import { hashDistance } from './analyze.js';
+import { informative } from './refs.js';
 import * as fingerprint from './fingerprint.js';
 import * as matching from './matching.js';
 import * as store from './store.js';
 
-const GRAPH_VERSION = 2;
+const GRAPH_VERSION = 3;
+
+/**
+ * Which fingerprint produced the hashes in these files.
+ *
+ * Separate from `GRAPH_VERSION` because it answers a different question: not
+ * "is this file shaped the way I expect" but "were these hashes computed by the
+ * same rules I am about to compare them with". A stored graph whose hashes came
+ * from an older fingerprint is not stale, it is *incomparable* — and the failure
+ * is silent, because an old hash is a perfectly well-formed hash that simply
+ * never matches anything.
+ *
+ * On a mismatch the graph is discarded and rebuilt, never translated. A rebuild
+ * costs a few hundred milliseconds per screen and happens once. A mis-merged
+ * graph costs a wrong tap, and costs it for as long as the file survives.
+ */
+export const FINGERPRINT_VERSION = fingerprint.TOKEN_RULES_VERSION;
 /**
  * Screens are matched by structural hash, exactly, and then by how alike their
  * token sets are — which tolerates one optional element appearing (a badge, a
@@ -38,6 +55,14 @@ const GRAPH_VERSION = 2;
  * Most revisits match on a hash outright and never reach this at all.
  */
 export const SIMILARITY_THRESHOLD = 0.36;
+
+/**
+ * How many of the 288 layout bits may differ and still be the same arrangement
+ * of light. The same number `screenmap` recalls maps by, and for the same
+ * reason: measured, a revisit is usually identical and different screens sit at
+ * 74 and above, so 20 is well inside the gap.
+ */
+export const SAME_SCREEN_LAYOUT_BITS = 20;
 /**
  * A screen with three async sections has a few settled structures, not endless
  * ones. Capping this keeps a genuinely wrong merge bounded: if a node starts
@@ -105,11 +130,28 @@ function fingerprintsOf(node) {
 function load(udid, screen) {
   const key = typeof screen === 'string' ? { hash: screen, tokens: [] } : screen;
   const entry = store.readJson(path.join(graphDir(udid), `${key.hash}.json`));
-  if (entry?.version === GRAPH_VERSION) return entry;
+  if (entry?.version === GRAPH_VERSION && entry?.fingerprintVersion === FINGERPRINT_VERSION) return entry;
   // The hash may be a variant of a node filed under a different name.
   const byVariant = allNodes(udid).find((n) => (n.variants ?? []).some((v) => v.hash === key.hash));
   if (byVariant) return byVariant;
-  return { version: GRAPH_VERSION, hash: key.hash, tokens: key.tokens ?? [], variants: [], edges: [] };
+  return {
+    version: GRAPH_VERSION,
+    fingerprintVersion: FINGERPRINT_VERSION,
+    hash: key.hash,
+    tokens: key.tokens ?? [],
+    // What the pixels looked like here. Kept because it is the evidence that
+    // two structurally different readings are the same screen — see `record`.
+    layoutHash: key.layoutHash ?? null,
+    variants: [],
+    edges: [],
+  };
+}
+
+/** Keep the pixel baseline current for a screen we are standing on. */
+function noteLayout(node, reading) {
+  const now = typeof reading === 'string' ? null : reading?.layoutHash;
+  if (now && informative(now)) node.layoutHash = now;
+  return node;
 }
 
 function save(udid, node) {
@@ -124,7 +166,7 @@ export function allNodes(udid) {
       .readdirSync(graphDir(udid))
       .filter((f) => f.endsWith('.json'))
       .map((f) => store.readJson(path.join(graphDir(udid), f)))
-      .filter((n) => n?.version === GRAPH_VERSION);
+      .filter((n) => n?.version === GRAPH_VERSION && n?.fingerprintVersion === FINGERPRINT_VERSION);
   } catch {
     return [];
   }
@@ -243,6 +285,10 @@ export function record(udid, { from, action, to, kind }) {
   // arrive as — that is what variants are for, and rewriting it here would let
   // a node drift screen by screen into something it never was.
   if (!node.tokens?.length && fromKey.tokens?.length) node.tokens = fromKey.tokens;
+  // The pixel baseline, on the other hand, *should* track: it is the evidence
+  // for "same arrangement of light as last time I stood here", and a stale one
+  // answers a question about a screen as it was weeks ago.
+  noteLayout(node, fromKey);
   const to_ = toHash;
   const signature = actionSignature(action);
   const existing = node.edges.find((e) => e.action === signature);
@@ -260,8 +306,40 @@ export function record(udid, { from, action, to, kind }) {
       const claimant = nearestScreen(udid, reading)?.node;
       const target = load(udid, existing.to);
       const unclaimed = !claimant || claimant.hash === target.hash;
-      if (unclaimed && target.hash !== to_ && reading.tokens?.length) {
+      // "Nothing we have stored claims this reading" is not evidence that the
+      // target grew a second face. It is equally consistent with this action
+      // being state-dependent and having gone somewhere genuinely new — and
+      // treating the two the same welds an unrelated screen onto a learned
+      // edge. Reproduced: a screen sharing *zero* tokens with the target got
+      // merged into it, after which landing there returned `ok` ("matches the
+      // outcome seen 3x before") and a flow kept walking, tapping real controls
+      // on a screen its plan never contained.
+      //
+      // So the reading has to positively look like the target before it is
+      // called a face of it. Unclaimed is a necessary condition, not a
+      // sufficient one.
+      // Two kinds of positive evidence that this reading is a face of the
+      // target, and either will do. What will not do is "nothing else claims
+      // it", which is an absence of evidence and used to be the whole test.
+      //
+      //  * It looks like the target — the tokens overlap enough to be the same
+      //    screen by the same measure used everywhere else.
+      //  * It *looks like* the target on screen. The pixels are within the
+      //    same-screen band of what was seen here before, and we arrived
+      //    through an edge that has led here. Identity belongs to the graph as
+      //    much as to the hash: the transition is evidence the fingerprint
+      //    cannot supply, and it is exactly the evidence needed when one
+      //    perception layer answered this time and not last time.
+      //
+      // That second route is what carries a screen whose structure genuinely
+      // differs between reads. Measured on four device-native screens, the
+      // accessibility tree and OCR agree on only 0.33–0.47 of a screen's
+      // tokens and never on its hash, and coarsening the vocabulary barely
+      // moved it — so this is not a residual case, it is the common one.
+      const looksLikeTarget = resembles(target, reading) || pixelsAgree(target, reading);
+      if (unclaimed && looksLikeTarget && target.hash !== to_ && reading.tokens?.length) {
         addVariant(target, reading);
+        noteLayout(target, reading);
         save(udid, target);
         existing.count += 1;
         existing.lastSeen = Date.now();
@@ -293,6 +371,39 @@ export function record(udid, { from, action, to, kind }) {
 }
 
 /** What this action did last time, if we have ever seen it here. */
+/**
+ * Do the pixels say this is the same screen we have stood on here before?
+ *
+ * The layout hash is a poor answer to "which screen is this" on its own — that
+ * is why identity is structural — but it is a good answer to "is this the same
+ * arrangement of light", and combined with having arrived through a known edge
+ * it is the evidence that two structurally different readings are one screen.
+ *
+ * Guarded by `informative`, because a dark or uniform screen hashes to almost
+ * nothing and two of those are within any tolerance of each other while being
+ * evidence of nothing at all.
+ */
+function pixelsAgree(node, reading) {
+  const before = node?.layoutHash;
+  const now = reading?.layoutHash;
+  if (!before || !now || !informative(before) || !informative(now)) return false;
+  return hashDistance(before, now) <= SAME_SCREEN_LAYOUT_BITS;
+}
+
+/**
+ * Does this reading look like a face of this screen, rather than a different
+ * screen we happen not to have stored yet?
+ *
+ * Compared against every face the screen already wears, because a screen with
+ * two structures is exactly the case variants exist for and a new reading may
+ * resemble the second one rather than the first.
+ */
+function resembles(node, reading) {
+  if (!node || !reading?.tokens?.length) return false;
+  const faces = [node.tokens ?? [], ...(node.variants ?? []).map((v) => v.tokens ?? [])];
+  return faces.some((face) => face.length && fingerprint.similarity(face, reading.tokens) >= SIMILARITY_THRESHOLD);
+}
+
 export function predict(udid, from, action) {
   const found = nearestScreen(udid, from);
   if (!found) return null;
@@ -323,10 +434,27 @@ export function forget(udid) {
 export function route(udid, fromHash, toHash, { maxDepth = 8 } = {}) {
   const start = nearestScreen(udid, fromHash);
   if (!start) return null;
-  const goal = (h) => h === toHash;
-  if (goal(start.node.hash)) return [];
+  if (start.node.hash === toHash) return [];
 
-  const byHash = new Map(allNodes(udid).map((n) => [n.hash, n]));
+  // Variants have to resolve here the same way they resolve everywhere else.
+  //
+  // A screen may wear more than one structure, and an edge records whichever
+  // one it arrived on — so an edge whose `to` is a variant hash was a dead end
+  // in this search while `nearestScreen` was perfectly happy to say that hash
+  // *is* the node. The graph then had a route it could not find, `goto`
+  // answered `no-route` for somewhere it had been, and a flow that should have
+  // replayed from memory was re-explored instead. That is at least one
+  // mechanical cause of the convergence flakiness in DEFERRED.
+  const byHash = new Map();
+  for (const node of allNodes(udid)) {
+    byHash.set(node.hash, node);
+    for (const variant of node.variants ?? []) if (!byHash.has(variant.hash)) byHash.set(variant.hash, node);
+  }
+  // Canonical identity, so a variant and its screen are one place in the search
+  // rather than two, and reaching either counts as reaching the goal.
+  const canonical = (h) => byHash.get(h)?.hash ?? h;
+  const goalHash = canonical(toHash);
+  const reached = (h) => canonical(h) === goalHash;
   const seen = new Set([start.node.hash]);
   const queue = [{ hash: start.node.hash, path: [] }];
   while (queue.length) {
@@ -334,11 +462,12 @@ export function route(udid, fromHash, toHash, { maxDepth = 8 } = {}) {
     if (taken.length >= maxDepth) continue;
     const node = byHash.get(hash);
     for (const edge of node?.edges ?? []) {
-      if (seen.has(edge.to)) continue;
+      const to = canonical(edge.to);
+      if (seen.has(to)) continue;
       const next = [...taken, edge];
-      if (goal(edge.to)) return next;
-      seen.add(edge.to);
-      queue.push({ hash: edge.to, path: next });
+      if (reached(edge.to)) return next;
+      seen.add(to);
+      queue.push({ hash: to, path: next });
     }
   }
   return null;
@@ -375,6 +504,18 @@ function sameScreen(udid, a, b) {
   return Boolean(nodeA && nodeB && nodeA.hash === nodeB.hash);
 }
 
+/**
+ * How many times an edge must have been observed before a mismatch counts as a
+ * wrong turn rather than as "we do not know yet".
+ *
+ * Two, because the difference between one and two observations is the
+ * difference between a coincidence and a pattern, and the cost of being wrong
+ * is asymmetric: halting a correct run is visible and annoying, while
+ * continuing one extra step past a genuinely wrong turn is caught by the next
+ * step's own verdict.
+ */
+export const CONFIDENT_OBSERVATIONS = 2;
+
 export function verdict({ udid, prediction, before, after, kind }) {
   if (!before || !after) return { verdict: 'unverified', detail: 'no state to compare' };
   const moved = before !== after;
@@ -387,6 +528,46 @@ export function verdict({ udid, prediction, before, after, kind }) {
     return { verdict: 'no-visible-change', detail: `expected to reach a different screen (seen ${prediction.count}x)` };
   }
   if (!sameScreen(udid, prediction.to, after)) {
+    // One observation is not a prediction, and halting on it is what made a new
+    // user's *second* run worse than their first.
+    //
+    // The first run learns every edge at count 1 and cannot contradict itself,
+    // so it reports `unverified` throughout and completes. The second run then
+    // has an expectation for every step, and any screen whose identity wobbles
+    // — a read taken while the tree was still arriving, a screen with more than
+    // one legitimate structure — contradicts it and stops the run. Measured
+    // from the published package: run 1 halted at 3/10, runs 2 and 3 went
+    // 10/10. The halt was not protecting anyone from anything.
+    //
+    // So a single-observation miss is reported as what it actually is: we do
+    // not know yet. It must not be `unexpected-screen`, because that verdict is
+    // what halts a run and what "a run that reported a wrong turn never also
+    // reports success" is asserted over — and both of those should stay true.
+    // Once the same edge has been seen twice, a miss is a real wrong turn.
+    if (prediction.count < CONFIDENT_OBSERVATIONS) {
+      return {
+        verdict: 'unverified',
+        detail: `seen here once before and went somewhere else that time`
+          + ` — one observation is not enough to call this a wrong turn`,
+        weakPrediction: { count: prediction.count, to: prediction.to },
+      };
+    }
+    // This action has already led somewhere different at least once, so its
+    // destination is not a fact about the screen — it is a distribution. An app
+    // relaunch that lands on restored state, a list whose first row depends on
+    // what happened last time: these genuinely have more than one outcome, and
+    // calling the second one a wrong turn is calling the world wrong.
+    //
+    // The graph has always counted this as `changedOutcomes` and nothing ever
+    // read it.
+    if (prediction.changedOutcomes > 0) {
+      return {
+        verdict: 'unverified',
+        detail: `this action has reached ${prediction.changedOutcomes + 1} different screens from here`
+          + ` — its outcome is not predictable, so this is not a wrong turn`,
+        nondeterministic: { outcomes: prediction.changedOutcomes + 1, count: prediction.count },
+      };
+    }
     return {
       verdict: 'unexpected-screen',
       detail: `expected the screen this action reached ${prediction.count}x before, and landed somewhere else`,

package/src/screenmap.js

@@ -13,9 +13,10 @@ import * as fingerprint from './fingerprint.js';
 import * as input from './input.js';
 import * as ocr from './ocr.js';
 import * as regions from './regions.js';
+import { informative } from './refs.js';
 import * as store from './store.js';
 
-const MAP_VERSION = 6; // dates, prices and phone numbers no longer contribute labels
+const MAP_VERSION = 7; // footprintless elements and containers no longer enter identity
 
 function mapDir(udid) {
   return path.join(store.deviceDir(udid), 'screens');
@@ -60,6 +61,12 @@ function loadAll(udid) {
  */
 export function recallNearest(udid, layoutHash, { tolerance = DEFAULT_TOLERANCE } = {}) {
   if (!layoutHash) return null;
+  // A hash of almost no set bits is a dark or uniform screen, and two of them
+  // are within any tolerance of each other while being evidence of nothing.
+  // `refs.js` documents this and guards for it; this function fed that one's
+  // `screenKnown` and `structuralHash` inputs without a guard of its own, so a
+  // near-uniform screen could hand back a different screen's element map.
+  if (!informative(layoutHash)) return null;
   let best = null;
   let bestDistance = Infinity;
   for (const entry of loadAll(udid)) {