simframe 0.5.0 → 0.6.0

package/native/simframed/Sources/PrivateAPI/CoreSimulatorPlatform.swift

@@ -24,6 +24,10 @@ public final class CoreSimulatorPlatform: SimulatorPlatform {
     private var changeUUID: NSUUID?
     private var device: NSObject?
     private var hid: IndigoHID?
+    // Built on first use, not at attach: reading the tree is optional, and a
+    // machine where the translation framework is missing must still capture.
+    private var accessibility: AccessibilityBridge?
+    private var accessibilityFailure: String?
     private var simulatorKitHandle: UnsafeMutableRawPointer?
     // Read once at attach: spawning simctl per status call cost 300ms.
     private var cachedKeyboardWarning: [String]?
@@ -149,6 +153,20 @@ public final class CoreSimulatorPlatform: SimulatorPlatform {
             device = booted[0]
         }
 
+        return try resolveDisplay(on: device, warmInput: true)
+    }
+
+    /// Re-resolve the display port on the device we are already bound to.
+    ///
+    /// Input is deliberately left alone: the HID session is independent of the
+    /// display port, it survives the port being replaced, and tearing it down
+    /// to fix capture would break the half that still worked.
+    public func reattachDisplay() throws -> DeviceInfo {
+        guard let device else { throw PrivateAPIError.noDisplayPort }
+        return try resolveDisplay(on: device, warmInput: false)
+    }
+
+    private func resolveDisplay(on device: NSObject, warmInput: Bool) throws -> DeviceInfo {
         guard let io = device.value(forKey: "io") as? NSObject,
               let ports = io.value(forKey: "ioPorts") as? [NSObject] else {
             throw PrivateAPIError.noDisplayPort
@@ -171,18 +189,62 @@ public final class CoreSimulatorPlatform: SimulatorPlatform {
             let size = unsafeBitCast(sizeImp, to: SizeFn.self)(descriptor, sizeSel)
             guard size.width > 0, size.height > 0 else { continue }
             display = descriptor
+            // The bridge captures one device's token and installs itself on a
+            // process-wide translator, so it belongs to the device it was built
+            // for. Binding to a different one must not inherit it.
+            if self.device !== device {
+                accessibility = nil
+                accessibilityFailure = nil
+            }
             self.device = device
             let resolved = info(for: device)
             attached = resolved
-            // Warm the HID session once, so the first gesture is not slower
-            // than the rest. Input being unavailable must not stop capture.
-            hid = try? IndigoHID(device: device, simulatorKit: Self.simulatorKitHandles)
-            cachedKeyboardWarning = nonEnglishKeyboards()
+            if warmInput {
+                // Warm the HID session once, so the first gesture is not slower
+                // than the rest. Input being unavailable must not stop capture.
+                hid = try? IndigoHID(device: device, simulatorKit: Self.simulatorKitHandles)
+                cachedKeyboardWarning = nonEnglishKeyboards()
+            }
             return resolved
         }
         throw PrivateAPIError.noDisplayPort
     }
 
+    // MARK: - Accessibility
+
+    public func accessibilityStatus() -> (available: Bool, detail: String) {
+        guard device != nil else { return (false, "no device attached") }
+        do {
+            // Constructing the bridge only proves the symbols are there. The
+            // failure this path took two attempts to get past was a bridge that
+            // constructed perfectly and read nothing, so this reads.
+            try bridge().probe()
+            return (true, "AXPTranslator, host-side")
+        } catch {
+            return (false, "\(error)")
+        }
+    }
+
+    public func accessibilityTree() throws -> AXTree {
+        try bridge().tree()
+    }
+
+    private func bridge() throws -> AccessibilityBridge {
+        if let accessibility { return accessibility }
+        // A framework that is missing stays missing; re-probing it on every
+        // screen read would cost a dlopen per call to learn the same thing.
+        if let accessibilityFailure { throw AccessibilityError.unavailable(accessibilityFailure) }
+        guard let device else { throw PrivateAPIError.noBootedDevice }
+        do {
+            let made = try AccessibilityBridge(device: device)
+            accessibility = made
+            return made
+        } catch {
+            accessibilityFailure = "\(error)"
+            throw error
+        }
+    }
+
     public func withFrame<T>(_ body: (RawFrame) throws -> T) throws -> T {
         guard let display else { throw PrivateAPIError.noDisplayPort }
         guard let raw = display.perform(NSSelectorFromString("framebufferSurface"))?.takeUnretainedValue(),
@@ -437,6 +499,18 @@ extension CoreSimulatorPlatform {
         try simctl(args)
     }
 
+    /// Reset the HID session, and rebuild the client if the reset does not take.
+    ///
+    /// `resetHIDSession` is the framework's own selector; recreating the client
+    /// is the bigger hammer for the case where the client itself is the stale
+    /// thing. Cheap either way — constructing one has no effect on the device.
+    public func resetInput() throws {
+        guard let device else { throw PrivateAPIError.hidUnavailable("not attached to a device") }
+        hid?.resetSession()
+        hid = try? IndigoHID(device: device, simulatorKit: Self.simulatorKitHandles)
+        guard hid != nil else { throw PrivateAPIError.hidUnavailable("could not rebuild the HID client") }
+    }
+
     public func press(_ button: HardwareButton) throws {
         let (hid, _) = try requireHID()
         guard let code = HIDKeyboard.buttonCode(button) else {

package/native/simframed/Sources/PrivateAPI/PrivateAPI.swift

@@ -74,6 +74,16 @@ public protocol SimulatorPlatform: AnyObject {
     func bootedDevices() throws -> [DeviceInfo]
     /// Bind to a device's display. Must be called before `withFrame`.
     func attach(udid: String?) throws -> DeviceInfo
+    /// Re-resolve the display port on the device already attached.
+    ///
+    /// The descriptor handed out by `attach` can die under a live daemon: the
+    /// simulator tears its display port down and builds a new one, and every
+    /// `framebufferSurface` read on the old object returns nil from then on.
+    /// Measured on one long session — capture reported "the display surface
+    /// could not be read" for six minutes on a device that was awake and
+    /// perfectly visible, and a daemon restart fixed it instantly. Without a
+    /// way to re-resolve, a restart is the only cure.
+    func reattachDisplay() throws -> DeviceInfo
     /// Borrow the current framebuffer. The pointer is only valid inside `body`.
     func withFrame<T>(_ body: (RawFrame) throws -> T) throws -> T
     /// Called whenever the display reports damage — the per-redraw signal, so a
@@ -105,6 +115,28 @@ public protocol SimulatorPlatform: AnyObject {
     /// it is the only reliable route for content that must be exact.
     func paste(_ text: String) throws
     func press(_ button: HardwareButton) throws
+    /// Rebuild the HID session.
+    ///
+    /// Input has no feedback channel: a dispatched Indigo message reports
+    /// success when the send succeeds, and there is no way to ask the device
+    /// whether anything happened. Measured on a long-running daemon, a HOME
+    /// press returned in 66ms and the screen never moved, while the same press
+    /// on a freshly started daemon worked — so the session can stop delivering
+    /// while still accepting. This is the recovery for that, and the caller
+    /// that notices is the one with the frames.
+    func resetInput() throws
+
+    // MARK: Accessibility. Optional in the same way input is: a platform may
+    // see the screen without being able to read the app's own tree.
+
+    /// Whether the accessibility tree can be read, and why not when it cannot.
+    func accessibilityStatus() -> (available: Bool, detail: String)
+    /// The frontmost application's accessibility tree, flattened depth-first,
+    /// carrying whether it is all of one.
+    ///
+    /// An app still launching has no tree yet, and this reports that as it is —
+    /// one node, no children — rather than retrying until it looks populated.
+    func accessibilityTree() throws -> AXTree
 
     // MARK: App lifecycle. These are simctl, not private API — no HID needed.
 

package/native/simframed/Sources/PrivateAPI/StubPlatform.swift

@@ -11,6 +11,9 @@ public final class StubPlatform: SimulatorPlatform {
     private var buffer: [UInt8] = []
     /// What was asked of it, so tests can assert on gestures without a device.
     public var recorded: [String] = []
+    /// What `accessibilityTree()` should answer. Empty by default, which is
+    /// what a device with no app in the foreground genuinely looks like.
+    public var stubTree: AXTree = AXTree(nodes: [])
 
     public init(width: Int = 1206, height: Int = 2622, tint: UInt8 = 0) {
         self.width = width
@@ -23,7 +26,22 @@ public final class StubPlatform: SimulatorPlatform {
     }
 
     public func attach(udid: String?) throws -> DeviceInfo {
-        DeviceInfo(udid: udid ?? "STUB-0000", name: "Stub Device", runtime: "iOS 26.0")
+        attachedUdid = udid
+        return DeviceInfo(udid: udid ?? "STUB-0000", name: "Stub Device", runtime: "iOS 26.0")
+    }
+
+    /// Counted so a test can assert the capture loop actually tries to recover
+    /// rather than logging the same failure forever.
+    public private(set) var reattachCount = 0
+    private var attachedUdid: String?
+
+    /// Counted, so a test can assert that a dead input path is actually retried.
+    public private(set) var resetInputCount = 0
+    public func resetInput() throws { resetInputCount += 1 }
+
+    public func reattachDisplay() throws -> DeviceInfo {
+        reattachCount += 1
+        return DeviceInfo(udid: attachedUdid ?? "STUB-0000", name: "Stub Device", runtime: "iOS 26.0")
     }
 
     public func withFrame<T>(_ body: (RawFrame) throws -> T) throws -> T {
@@ -86,4 +104,9 @@ extension StubPlatform {
     public func permission(action: String, service: String, bundleId: String?) throws {
         recorded.append("permission(\(action),\(service),\(bundleId ?? "-"))")
     }
+    public func accessibilityStatus() -> (available: Bool, detail: String) { (true, "stub") }
+    public func accessibilityTree() throws -> AXTree {
+        recorded.append("accessibilityTree()")
+        return stubTree
+    }
 }

package/native/simframed/Sources/SimframeCore/Element.swift

@@ -1,4 +1,5 @@
 import CoreGraphics
+import PrivateAPI
 import Foundation
 
 /// Where an element's information came from. A single element can be seen by
@@ -53,17 +54,22 @@ public struct Element: Sendable {
     public var role: String
     public var label: String?
     public var value: String?
+    /// The app's own name for the control, when it publishes one. Never shown
+    /// to a reader, but it is what a test written against the app matches on.
+    public var identifier: String?
     public var state: ElementState
     public var source: ElementSource
     public var confidence: Double
 
     public init(id: Int, frame: CGRect, role: String, label: String? = nil, value: String? = nil,
-                state: ElementState = ElementState(), source: ElementSource, confidence: Double = 1) {
+                identifier: String? = nil, state: ElementState = ElementState(),
+                source: ElementSource, confidence: Double = 1) {
         self.id = id
         self.frame = frame
         self.role = role
         self.label = label
         self.value = value
+        self.identifier = identifier
         self.state = state
         self.source = source
         self.confidence = confidence
@@ -82,12 +88,34 @@ public struct Element: Sendable {
         ]
         if let label { out["label"] = label }
         if let value { out["value"] = value }
+        if let identifier { out["identifier"] = identifier }
         let state = state.json
         if !state.isEmpty { out["state"] = state }
         return out
     }
 }
 
+public extension Element {
+    /// An accessibility node as an element.
+    ///
+    /// The role carries the subrole when there is one — a search field is a
+    /// `TextField` to the tree and a `SearchField` to anyone reading the map,
+    /// and the distinction is worth keeping.
+    init(id: Int, node: AXNode) {
+        self.init(
+            id: id,
+            frame: node.frame,
+            role: node.subrole ?? node.role,
+            label: node.label,
+            value: node.value,
+            identifier: node.identifier,
+            state: ElementState(enabled: node.enabled, selected: node.selected,
+                                checked: nil, focused: node.focused),
+            source: .accessibility,
+            confidence: 1)
+    }
+}
+
 /// Everything known about one screen.
 public struct ScreenMap: Sendable {
     public var fingerprint: String        // the layout hash

package/native/simframed/Sources/simframed/main.swift

@@ -158,6 +158,11 @@ case "run":
 
         let lock = NSLock()
         var dirty = true
+        var consecutiveFailures = 0
+        /// Roughly three seconds of failed reads at the loop's half-second
+        /// back-off. Long enough not to thrash on a momentary hiccup, short
+        /// enough that nobody watches a dead capture loop and wonders.
+        let reattachAfterFailures = 6
         var lastCapture = 0.0
         var frames = 0
         var lastReport = Date().timeIntervalSince1970
@@ -190,32 +195,124 @@ case "run":
                     return done(["device": device.name, "udid": device.udid])
                 case "status":
                     let input = platform.inputStatus()
+                    let ax = platform.accessibilityStatus()
                     return done([
                         "input": ["available": input.available, "detail": input.detail],
+                        "accessibility": ["available": ax.available, "detail": ax.detail],
                         "device": ["name": device.name, "udid": device.udid,
                                    "pointWidth": device.pointWidth, "pointHeight": device.pointHeight,
                                    "scale": device.scale],
                         "engine": "simframed",
                     ])
                 case "ui":
-                    // OCR straight off the framebuffer: no PNG encode, no file,
-                    // no process spawn. Runs on the socket queue, so a slow
-                    // recognition pass cannot stall frame capture.
-                    let started = DispatchTime.now().uptimeNanoseconds
-                    let elements = try platform.withFrame { frame in
-                        try VisionOCR.recognise(frame: frame, scale: device.scale)
+                    // The accessibility tree and OCR read the same instant of
+                    // the screen and neither needs the other, so they run
+                    // together: in series the tree read is simply added to the
+                    // OCR pass. Both run off the capture loop.
+                    let wantAx = request["ax"] as? Bool ?? true
+                    let wantOcr = request["ocr"] as? Bool ?? true
+                    // Shared with two background blocks, and a bounded wait
+                    // means they can still be running when this reads them —
+                    // so every touch goes through the lock. Timing out and
+                    // reading racing variables would trade a slow answer for a
+                    // wrong one.
+                    final class Reads {
+                        let lock = NSLock()
+                        var axTree = AXTree(nodes: [])
+                        var axError: String?
+                        var axMs = 0.0
+                        var ocrElements: [Element] = []
+                        var ocrError: String?
+                        var ocrMs = 0.0
+                        func set(_ body: (Reads) -> Void) { lock.lock(); body(self); lock.unlock() }
+                        func snapshot() -> (AXTree, String?, Double, [Element], String?, Double) {
+                            lock.lock(); defer { lock.unlock() }
+                            return (axTree, axError, axMs, ocrElements, ocrError, ocrMs)
+                        }
                     }
+                    let reads = Reads()
+
+                    let group = DispatchGroup()
+                    if wantAx {
+                        DispatchQueue.global(qos: .userInitiated).async(group: group) {
+                            let t = DispatchTime.now().uptimeNanoseconds
+                            var tree = AXTree(nodes: [])
+                            var failure: String?
+                            do { tree = try platform.accessibilityTree() }
+                            catch { failure = "\(error)" }
+                            let ms = Double(DispatchTime.now().uptimeNanoseconds - t) / 1e6
+                            reads.set { $0.axTree = tree; $0.axError = failure; $0.axMs = ms }
+                        }
+                    }
+                    if wantOcr {
+                        DispatchQueue.global(qos: .userInitiated).async(group: group) {
+                            let t = DispatchTime.now().uptimeNanoseconds
+                            var found: [Element] = []
+                            var failure: String?
+                            do {
+                                found = try platform.withFrame { frame in
+                                    try VisionOCR.recognise(frame: frame, scale: device.scale)
+                                }
+                            } catch { failure = "\(error)" }
+                            let ms = Double(DispatchTime.now().uptimeNanoseconds - t) / 1e6
+                            reads.set { $0.ocrElements = found; $0.ocrError = failure; $0.ocrMs = ms }
+                        }
+                    }
+                    // Bounded, because this blocks the control socket and the
+                    // socket is serial: a read that runs long does not just
+                    // return late, it holds up every command behind it. A CI
+                    // runner spent 28 s inside one of these. Whatever has not
+                    // arrived by the deadline is reported as missing rather
+                    // than waited for — the layer that did answer is still
+                    // worth having.
+                    let timedOut = group.wait(timeout: .now() + .seconds(12)) == .timedOut
+                    var (axTree, axError, axMs, ocrElements, ocrError, ocrMs) = reads.snapshot()
+                    if timedOut {
+                        if wantAx, axTree.nodes.isEmpty, axError == nil {
+                            axError = "the accessibility read did not finish within 12s"
+                        }
+                        if wantOcr, ocrElements.isEmpty, ocrError == nil {
+                            ocrError = "text recognition did not finish within 12s"
+                        }
+                    }
+                    // OCR failing is fatal to a screen read in a way a missing
+                    // tree is not: without pixels there is nothing to report.
+                    if wantOcr, let ocrError, axTree.nodes.isEmpty { return ["ok": false, "error": ocrError] }
+
+                    var elements = axTree.nodes.enumerated().map { Element(id: $0.offset, node: $0.element) }
+                    for (i, var e) in ocrElements.enumerated() {
+                        e.id = elements.count + i
+                        elements.append(e)
+                    }
+                    var sources: [String] = []
+                    // A tree that stopped early is reported as nodes, not as a
+                    // source. The layer above treats accessibility elements as
+                    // the real hit targets and writes them into screen memory,
+                    // and half a screen remembered as a whole one is worse than
+                    // a screen read again from pixels.
+                    if wantAx, axError == nil, axTree.truncated == nil { sources.append("ax") }
+                    if wantOcr, ocrError == nil { sources.append("ocr") }
                     let latest = store.latestState()
                     let map = ScreenMap(
                         fingerprint: latest?["layoutHash"] as? String ?? "",
                         hash: latest?["hash"] as? String ?? "",
                         size: CGSize(width: device.pointWidth, height: device.pointHeight),
                         elements: elements,
-                        sources: ["ocr"],
+                        sources: sources,
                         capturedAt: FrameStore.nowMs()
                     )
                     var payload = map.json
-                    payload["ocrMs"] = (Double(DispatchTime.now().uptimeNanoseconds - started) / 1e6 * 100).rounded() / 100
+                    let round = { (v: Double) in (v * 100).rounded() / 100 }
+                    if wantOcr { payload["ocrMs"] = round(ocrMs) }
+                    if wantAx {
+                        payload["axMs"] = round(axMs)
+                        payload["axCount"] = axTree.nodes.count
+                        if let truncated = axTree.truncated { payload["axTruncated"] = truncated }
+                        // Why the tree is missing matters: a launching app and a
+                        // framework that will not load look identical otherwise.
+                        if let axError { payload["axError"] = axError }
+                    }
+                    if let ocrError { payload["ocrError"] = ocrError }
                     return done(["screen": payload])
                 case "tap":
                     guard let p = point("x", "y") else { return ["ok": false, "error": "tap needs x and y"] }
@@ -273,6 +370,10 @@ case "run":
                     }
                     try platform.press(button)
                     return done()
+                case "resetInput":
+                    try platform.resetInput()
+                    FileHandle.standardError.write("simframed: HID session reset on request\n".data(using: .utf8)!)
+                    return done()
                 default:
                     return ["ok": false, "error": "unknown action '\(action)'"]
                 }
@@ -318,8 +419,35 @@ case "run":
                     latencies.append(Double(DispatchTime.now().uptimeNanoseconds - t0) / 1e6)
                     frames += 1
                     lastCapture = now
+                    consecutiveFailures = 0
                 } catch {
-                    FileHandle.standardError.write("simframed: capture failed: \(error)\n".data(using: .utf8)!)
+                    consecutiveFailures += 1
+                    FileHandle.standardError.write(
+                        "simframed: capture failed: \(error) (\(consecutiveFailures) in a row)\n".data(using: .utf8)!)
+                    // The display port can be torn down and rebuilt under a
+                    // running daemon, and every read on the old descriptor
+                    // returns nil from then on. Observed on a device that was
+                    // awake and visible the whole time: six minutes of
+                    // "the display surface could not be read", cured instantly
+                    // by restarting the daemon. Reporting a failure loudly is
+                    // right; never recovering from it is not, so re-resolve the
+                    // port and re-arm the damage callback.
+                    if consecutiveFailures >= reattachAfterFailures {
+                        do {
+                            _ = try platform.reattachDisplay()
+                            try platform.observeChanges {
+                                lock.lock(); dirty = true; lock.unlock()
+                            }
+                            lock.lock(); dirty = true; lock.unlock()
+                            FileHandle.standardError.write(
+                                "simframed: re-resolved the display port after \(consecutiveFailures) failed reads\n"
+                                    .data(using: .utf8)!)
+                            consecutiveFailures = 0
+                        } catch {
+                            FileHandle.standardError.write(
+                                "simframed: could not re-resolve the display port: \(error)\n".data(using: .utf8)!)
+                        }
+                    }
                     Thread.sleep(forTimeInterval: 0.5)
                 }
             }

package/native/simframed/Tests/SimframeCoreTests/HashingTests.swift

@@ -238,3 +238,33 @@ final class MotionTests: XCTestCase {
         XCTAssertTrue([.sheetPresent, .alertPresent].contains(t.kind), "got \(t.kind)")
     }
 }
+
+/// An accessibility node becoming an element. The subrole matters: a search
+/// field is a TextField to the tree and a SearchField to anyone reading the
+/// map, and losing that costs the reader the only word that identifies it.
+final class AccessibilityElementTests: XCTestCase {
+    func testSubroleWinsWhenThereIsOne() {
+        let node = AXNode(role: "TextField", subrole: "SearchField", label: nil, value: "Search",
+                          identifier: "search", enabled: true, selected: false, focused: nil,
+                          frame: CGRect(x: 32, y: 803, width: 336, height: 38), depth: 2)
+        let element = Element(id: 0, node: node)
+        XCTAssertEqual(element.role, "SearchField")
+        XCTAssertEqual(element.value, "Search")
+        XCTAssertEqual(element.identifier, "search")
+        XCTAssertEqual(element.state.enabled, true)
+        XCTAssertEqual(element.state.selected, false)
+        XCTAssertNil(element.state.focused, "unknown stays unknown rather than becoming false")
+        XCTAssertEqual(element.source.names, ["ax"])
+        XCTAssertEqual(element.center, CGPoint(x: 200, y: 822))
+    }
+
+    func testPlainRoleSurvivesWithNoSubrole() {
+        let node = AXNode(role: "Button", label: "Continue",
+                          frame: CGRect(x: 38, y: 784, width: 326, height: 52), depth: 1)
+        let element = Element(id: 3, node: node)
+        XCTAssertEqual(element.role, "Button")
+        XCTAssertEqual(element.label, "Continue")
+        XCTAssertNil(element.identifier)
+        XCTAssertNil(element.json["identifier"], "an absent identifier is absent, not null")
+    }
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "simframe",
-  "version": "0.5.0",
+  "version": "0.6.0",
   "mcpName": "io.github.lvlrSajjad/simframe",
   "description": "Always-warm iOS Simulator frames: agents read the screen in ~20ms instead of waiting on screenshots. MCP server + CLI.",
   "keywords": [
@@ -44,6 +44,7 @@
     "native/simframed/Package.swift",
     "native/simframed/Sources",
     "native/simframed/Tests",
+    "skills",
     "scripts",
     "README.md",
     "LICENSE"
@@ -53,7 +54,8 @@
     "mcp": "node src/cli.js mcp",
     "smoke": "node scripts/smoke.mjs",
     "bench": "./scripts/bench.sh",
-    "check:package": "node scripts/check-package.mjs"
+    "check:package": "node scripts/check-package.mjs",
+    "version": "node scripts/sync-server-version.mjs"
   },
   "dependencies": {
     "@modelcontextprotocol/sdk": "^1.0.0"

package/scripts/check-package.mjs

@@ -66,6 +66,14 @@ function requiredFiles() {
   // one catches anybody narrowing that later.
   for (const f of walk(path.join(ROOT, 'src'), (p) => p.endsWith('.js'))) required.add(rel(f));
 
+  // The Claude Code skill. It is the low-token path this tool is designed
+  // around, and an installed copy without it is an installation of half the
+  // idea — silently, which is the failure mode this whole check exists for.
+  for (const f of walk(path.join(ROOT, 'skills'), (p) => p.endsWith('.md'))) required.add(rel(f));
+  if (!walk(path.join(ROOT, 'skills'), (p) => p.endsWith('.md')).length) {
+    throw new Error('no skill found under skills/ — has the layout moved? This check would silently pass.');
+  }
+
   // Anything package.json points at to run.
   const pkg = JSON.parse(fs.readFileSync(path.join(ROOT, 'package.json'), 'utf8'));
   for (const target of Object.values(pkg.bin ?? {})) required.add(target.replace(/^\.\//, ''));