simframe 0.4.1 → 0.5.0

package/native/simframed/Sources/PrivateAPI/PrivateAPI.swift

@@ -0,0 +1,117 @@
+import Foundation
+
+/// A frame as the simulator hands it to us: tightly-packed BGRA is not
+/// guaranteed, so `bytesPerRow` must always be honoured rather than assumed
+/// to equal `width * 4`.
+public struct RawFrame {
+    public let width: Int
+    public let height: Int
+    public let bytesPerRow: Int
+    public let pixels: UnsafePointer<UInt8>
+
+    public init(width: Int, height: Int, bytesPerRow: Int, pixels: UnsafePointer<UInt8>) {
+        self.width = width
+        self.height = height
+        self.bytesPerRow = bytesPerRow
+        self.pixels = pixels
+    }
+}
+
+public struct DeviceInfo: Sendable {
+    public let udid: String
+    public let name: String
+    public let runtime: String
+    /// Native pixels, as the framebuffer reports them.
+    public let pixelWidth: Int
+    public let pixelHeight: Int
+    /// Points per pixel. Input coordinates are in points, frames are in pixels.
+    public let scale: Double
+
+    public var pointWidth: Int { scale > 0 ? Int((Double(pixelWidth) / scale).rounded()) : pixelWidth }
+    public var pointHeight: Int { scale > 0 ? Int((Double(pixelHeight) / scale).rounded()) : pixelHeight }
+
+    public init(udid: String, name: String, runtime: String,
+                pixelWidth: Int = 0, pixelHeight: Int = 0, scale: Double = 1) {
+        self.udid = udid
+        self.name = name
+        self.runtime = runtime
+        self.pixelWidth = pixelWidth
+        self.pixelHeight = pixelHeight
+        self.scale = scale
+    }
+}
+
+/// Hardware buttons, named rather than numbered at call sites.
+public enum HardwareButton: String, Sendable, CaseIterable {
+    case home, lock, siri, volumeUp, volumeDown
+}
+
+public enum PrivateAPIError: Error, CustomStringConvertible {
+    case frameworksUnavailable(String)
+    case noBootedDevice
+    case deviceNotFound(String)
+    case noDisplayPort
+    case surfaceUnavailable
+    case hidUnavailable(String)
+    case simctlFailed(String)
+
+    public var description: String {
+        switch self {
+        case .frameworksUnavailable(let d): return "simulator frameworks unavailable: \(d)"
+        case .noBootedDevice: return "no booted simulator"
+        case .deviceNotFound(let u): return "no simulator matching \(u)"
+        case .noDisplayPort: return "the device exposes no active display port"
+        case .surfaceUnavailable: return "the display surface could not be read"
+        case .hidUnavailable(let d): return "input is unavailable: \(d)"
+        case .simctlFailed(let d): return "simctl \(d)"
+        }
+    }
+}
+
+/// The whole private surface simframe depends on. Anything that talks to
+/// CoreSimulator or SimulatorKit goes through this.
+public protocol SimulatorPlatform: AnyObject {
+    func bootedDevices() throws -> [DeviceInfo]
+    /// Bind to a device's display. Must be called before `withFrame`.
+    func attach(udid: String?) throws -> DeviceInfo
+    /// Borrow the current framebuffer. The pointer is only valid inside `body`.
+    func withFrame<T>(_ body: (RawFrame) throws -> T) throws -> T
+    /// Called whenever the display reports damage — the per-redraw signal, so a
+    /// capture loop can be driven by the screen rather than by a timer.
+    func observeChanges(_ handler: @escaping () -> Void) throws
+    func detach()
+
+    // MARK: Input. Optional: a platform may observe without being able to touch.
+
+    /// Whether input is available, and why not when it is not.
+    func inputStatus() -> (available: Bool, detail: String)
+    /// A press and release at one point.
+    func tap(at point: CGPoint, durationMs: Double) throws
+    /// A deliberate hold. Named separately from `tap` because callers mean
+    /// something different by it, even though the mechanism is a longer hold.
+    func longPress(at point: CGPoint, durationMs: Double) throws
+    /// Press, hold in place, then move and release. The hold is what separates
+    /// a drag from a swipe: reorderable lists and drag-and-drop need the UI to
+    /// register a pick-up before movement starts.
+    func drag(from: CGPoint, to: CGPoint, holdMs: Double, durationMs: Double) throws
+    /// A real down, interpolated moves, then up — never a teleporting jump.
+    func swipe(from: CGPoint, to: CGPoint, durationMs: Double) throws
+    func type(_ text: String) throws
+    /// Put text on the device pasteboard and paste it.
+    ///
+    /// Key events are mapped by whatever keyboard layout iOS has active, so
+    /// typing "Fryer 3" against a Persian layout yields Persian text with no
+    /// error. The pasteboard carries characters rather than key positions, so
+    /// it is the only reliable route for content that must be exact.
+    func paste(_ text: String) throws
+    func press(_ button: HardwareButton) throws
+
+    // MARK: App lifecycle. These are simctl, not private API — no HID needed.
+
+    @discardableResult
+    func launch(bundleId: String, arguments: [String], environment: [String: String]) throws -> Int32
+    func terminate(bundleId: String) throws
+    func openURL(_ url: String) throws
+    /// `action` is grant/revoke/reset, `service` one of simctl's privacy services.
+    func permission(action: String, service: String, bundleId: String?) throws
+}

package/native/simframed/Sources/PrivateAPI/StubPlatform.swift

@@ -0,0 +1,89 @@
+import CoreGraphics
+import Foundation
+
+/// A deterministic platform for tests and for machines without Xcode.
+/// Renders a synthetic screen so hashing and frame-store logic can be tested
+/// with no simulator present.
+public final class StubPlatform: SimulatorPlatform {
+    public var width: Int
+    public var height: Int
+    public var tint: UInt8
+    private var buffer: [UInt8] = []
+    /// What was asked of it, so tests can assert on gestures without a device.
+    public var recorded: [String] = []
+
+    public init(width: Int = 1206, height: Int = 2622, tint: UInt8 = 0) {
+        self.width = width
+        self.height = height
+        self.tint = tint
+    }
+
+    public func bootedDevices() throws -> [DeviceInfo] {
+        [DeviceInfo(udid: "STUB-0000", name: "Stub Device", runtime: "iOS 26.0")]
+    }
+
+    public func attach(udid: String?) throws -> DeviceInfo {
+        DeviceInfo(udid: udid ?? "STUB-0000", name: "Stub Device", runtime: "iOS 26.0")
+    }
+
+    public func withFrame<T>(_ body: (RawFrame) throws -> T) throws -> T {
+        let bytesPerRow = width * 4 + 40   // deliberate padding: mirrors real surfaces
+        if buffer.count != bytesPerRow * height {
+            buffer = [UInt8](repeating: 0, count: bytesPerRow * height)
+        }
+        for y in 0..<height {
+            for x in 0..<width {
+                let i = y * bytesPerRow + x * 4
+                let band: UInt8 = x < width / 2 ? 40 : 200
+                buffer[i] = band &+ tint          // B
+                buffer[i + 1] = band              // G
+                buffer[i + 2] = band              // R
+                buffer[i + 3] = 255               // A
+            }
+        }
+        return try buffer.withUnsafeBufferPointer { ptr in
+            try body(RawFrame(width: width, height: height, bytesPerRow: bytesPerRow, pixels: ptr.baseAddress!))
+        }
+    }
+
+    public func observeChanges(_ handler: @escaping () -> Void) throws {
+        // The stub never changes on its own; tests drive `tint` and call this.
+        stubHandler = handler
+    }
+
+    /// Test hook: mutate the synthetic screen and announce it.
+    public func simulateChange() {
+        tint = tint &+ 40
+        stubHandler?()
+    }
+
+    private var stubHandler: (() -> Void)?
+
+    public func detach() { stubHandler = nil }
+}
+
+extension StubPlatform {
+    public func inputStatus() -> (available: Bool, detail: String) { (true, "stub") }
+    public func tap(at point: CGPoint, durationMs: Double) throws { recorded.append("tap(\(Int(point.x)),\(Int(point.y)))") }
+    public func swipe(from: CGPoint, to: CGPoint, durationMs: Double) throws {
+        recorded.append("swipe(\(Int(from.x)),\(Int(from.y))->\(Int(to.x)),\(Int(to.y)))")
+    }
+    public func type(_ text: String) throws { recorded.append("type(\(text))") }
+    public func paste(_ text: String) throws { recorded.append("paste(\(text))") }
+    public func press(_ button: HardwareButton) throws { recorded.append("press(\(button.rawValue))") }
+    public func longPress(at point: CGPoint, durationMs: Double) throws {
+        recorded.append("longPress(\(Int(point.x)),\(Int(point.y)),\(Int(durationMs)))")
+    }
+    public func drag(from: CGPoint, to: CGPoint, holdMs: Double, durationMs: Double) throws {
+        recorded.append("drag(\(Int(from.x)),\(Int(from.y))->\(Int(to.x)),\(Int(to.y)),hold=\(Int(holdMs)))")
+    }
+    @discardableResult
+    public func launch(bundleId: String, arguments: [String], environment: [String: String]) throws -> Int32 {
+        recorded.append("launch(\(bundleId))"); return 1234
+    }
+    public func terminate(bundleId: String) throws { recorded.append("terminate(\(bundleId))") }
+    public func openURL(_ url: String) throws { recorded.append("openURL(\(url))") }
+    public func permission(action: String, service: String, bundleId: String?) throws {
+        recorded.append("permission(\(action),\(service),\(bundleId ?? "-"))")
+    }
+}

package/native/simframed/Sources/SimframeCore/Bitmap.swift

@@ -0,0 +1,61 @@
+import Foundation
+import PrivateAPI
+
+/// A tightly-packed RGBA bitmap, which is what every algorithm below expects.
+/// The simulator hands us padded BGRA, so conversion happens exactly once.
+public struct Bitmap {
+    public let width: Int
+    public let height: Int
+    public var data: [UInt8]   // RGBA, 4 bytes per pixel, no row padding
+
+    public init(width: Int, height: Int, data: [UInt8]) {
+        self.width = width
+        self.height = height
+        self.data = data
+    }
+
+    /// Convert a padded BGRA surface into packed RGBA, downscaling by box
+    /// average in the same pass. Box averaging is what `grayGrid` does anyway,
+    /// so doing it here keeps the scaled image consistent with the hashes.
+    public static func from(_ frame: RawFrame, targetLongEdge: Int) -> Bitmap {
+        let (tw, th) = Self.fit(width: frame.width, height: frame.height, longEdge: targetLongEdge)
+        var out = [UInt8](repeating: 0, count: tw * th * 4)
+        let src = frame.pixels
+
+        for ty in 0..<th {
+            let y0 = (ty * frame.height) / th
+            let y1 = max(y0 + 1, ((ty + 1) * frame.height) / th)
+            for tx in 0..<tw {
+                let x0 = (tx * frame.width) / tw
+                let x1 = max(x0 + 1, ((tx + 1) * frame.width) / tw)
+                var r = 0, g = 0, b = 0, n = 0
+                for y in y0..<y1 {
+                    let row = y * frame.bytesPerRow
+                    for x in x0..<x1 {
+                        let i = row + x * 4
+                        b += Int(src[i])          // surfaces are BGRA
+                        g += Int(src[i + 1])
+                        r += Int(src[i + 2])
+                        n += 1
+                    }
+                }
+                let o = (ty * tw + tx) * 4
+                out[o] = UInt8(r / n)
+                out[o + 1] = UInt8(g / n)
+                out[o + 2] = UInt8(b / n)
+                out[o + 3] = 255
+            }
+        }
+        return Bitmap(width: tw, height: th, data: out)
+    }
+
+    /// Match `sips -Z`: the longer edge becomes `longEdge`, aspect preserved.
+    public static func fit(width: Int, height: Int, longEdge: Int) -> (Int, Int) {
+        guard width > 0, height > 0 else { return (1, 1) }
+        if max(width, height) <= longEdge { return (width, height) }
+        if height >= width {
+            return (max(1, Int((Double(width) * Double(longEdge) / Double(height)).rounded())), longEdge)
+        }
+        return (longEdge, max(1, Int((Double(height) * Double(longEdge) / Double(width)).rounded())))
+    }
+}

package/native/simframed/Sources/SimframeCore/ControlSocket.swift

@@ -0,0 +1,122 @@
+import Foundation
+import PrivateAPI
+
+/// A per-device Unix domain socket carrying one JSON object per line.
+///
+/// Unix sockets rather than TCP because the permission model is the file
+/// system's: mode 0600 means only this user can drive the simulator, with no
+/// port to collide and nothing reachable from the network.
+public final class ControlSocket {
+    public typealias Handler = ([String: Any]) -> [String: Any]
+
+    private let path: String
+    private let handler: Handler
+    private var listenFD: Int32 = -1
+    private var acceptSource: DispatchSourceRead?
+    // Requests run off the capture loop: a swipe sleeps for its whole duration
+    // and must not stall frame capture.
+    private let workQueue = DispatchQueue(label: "simframe.control", qos: .userInitiated)
+    /// Identity of the socket file this instance created, so shutdown can tell
+    /// its own socket from one a successor has since bound at the same path.
+    private var boundIno: ino_t = 0
+    private var boundDev: dev_t = 0
+
+    public init(path: String, handler: @escaping Handler) {
+        self.path = path
+        self.handler = handler
+    }
+
+    public func start() throws {
+        unlink(path)
+        listenFD = socket(AF_UNIX, SOCK_STREAM, 0)
+        guard listenFD >= 0 else { throw SocketError.failed("socket(): \(errno)") }
+
+        var addr = sockaddr_un()
+        addr.sun_family = sa_family_t(AF_UNIX)
+        let maxLen = MemoryLayout.size(ofValue: addr.sun_path)
+        guard path.utf8.count < maxLen else { throw SocketError.failed("socket path too long") }
+        withUnsafeMutablePointer(to: &addr.sun_path) { p in
+            p.withMemoryRebound(to: CChar.self, capacity: maxLen) { dst in
+                _ = strcpy(dst, path)
+            }
+        }
+        let size = socklen_t(MemoryLayout<sockaddr_un>.size)
+        let bound = withUnsafePointer(to: &addr) {
+            $0.withMemoryRebound(to: sockaddr.self, capacity: 1) { bind(listenFD, $0, size) }
+        }
+        guard bound == 0 else { close(listenFD); throw SocketError.failed("bind(): \(errno)") }
+        // Only this user may drive the device.
+        chmod(path, 0o600)
+        var bornStat = stat()
+        if stat(path, &bornStat) == 0 {
+            boundIno = bornStat.st_ino
+            boundDev = bornStat.st_dev
+        }
+        guard listen(listenFD, 8) == 0 else { close(listenFD); throw SocketError.failed("listen(): \(errno)") }
+
+        let source = DispatchSource.makeReadSource(fileDescriptor: listenFD, queue: workQueue)
+        source.setEventHandler { [weak self] in self?.acceptOne() }
+        source.resume()
+        acceptSource = source
+    }
+
+    private func acceptOne() {
+        let fd = accept(listenFD, nil, nil)
+        guard fd >= 0 else { return }
+        defer { close(fd) }
+        guard let request = readLine(fd) else { return }
+
+        var response: [String: Any]
+        if let data = request.data(using: .utf8),
+           let object = try? JSONSerialization.jsonObject(with: data) as? [String: Any] {
+            response = handler(object)
+            if let id = object["id"] { response["id"] = id }
+        } else {
+            response = ["ok": false, "error": "malformed request"]
+        }
+        if let out = try? JSONSerialization.data(withJSONObject: response) {
+            var line = out
+            line.append(0x0A)
+            _ = line.withUnsafeBytes { write(fd, $0.baseAddress, $0.count) }
+        }
+    }
+
+    private func readLine(_ fd: Int32) -> String? {
+        var buffer = [UInt8]()
+        var byte: UInt8 = 0
+        while read(fd, &byte, 1) == 1 {
+            if byte == 0x0A { break }
+            buffer.append(byte)
+            if buffer.count > 1 << 20 { return nil }
+        }
+        return buffer.isEmpty ? nil : String(decoding: buffer, as: UTF8.self)
+    }
+
+    public func stop() {
+        acceptSource?.cancel()
+        acceptSource = nil
+        if listenFD >= 0 { close(listenFD) }
+        listenFD = -1
+        // Only remove the socket if it is still the one we created.
+        //
+        // A restart binds a fresh socket at the same path, and a dying daemon
+        // that unlinks blindly deletes its *successor's* socket. Capture keeps
+        // working — that is file-based — so the only symptom is input quietly
+        // dropping to idb, which is slower and has different semantics. That is
+        // exactly the kind of invisible downgrade this tool is supposed to
+        // refuse to have.
+        var current = stat()
+        if stat(path, &current) == 0, current.st_ino == boundIno, current.st_dev == boundDev {
+            unlink(path)
+        }
+        boundIno = 0
+        boundDev = 0
+    }
+
+    public enum SocketError: Error, CustomStringConvertible {
+        case failed(String)
+        public var description: String {
+            switch self { case .failed(let d): return "control socket: \(d)" }
+        }
+    }
+}

package/native/simframed/Sources/SimframeCore/CoreGraphicsScaler.swift

@@ -0,0 +1,70 @@
+import CoreGraphics
+import ImageIO
+import Foundation
+import PrivateAPI
+
+/// Downscale through CoreGraphics rather than by box average.
+///
+/// `sips` — which the JavaScript capture loop uses — is a thin wrapper over
+/// ImageIO/CoreGraphics, so resampling the same way is the closest we can get
+/// to producing byte-identical hashes without reimplementing its filter.
+public enum CoreGraphicsScaler {
+    public static func bitmap(from frame: RawFrame, targetLongEdge: Int) -> Bitmap? {
+        let (tw, th) = Bitmap.fit(width: frame.width, height: frame.height, longEdge: targetLongEdge)
+        let srcBytes = frame.bytesPerRow * frame.height
+        guard let provider = CGDataProvider(dataInfo: nil, data: frame.pixels, size: srcBytes, releaseData: { _, _, _ in }) else {
+            return nil
+        }
+        // Surfaces are BGRA; this bitmapInfo is how CoreGraphics is told so.
+        let srcInfo = CGBitmapInfo(rawValue: CGImageAlphaInfo.noneSkipFirst.rawValue).union(.byteOrder32Little)
+        guard let cgImage = CGImage(
+            width: frame.width, height: frame.height,
+            bitsPerComponent: 8, bitsPerPixel: 32,
+            bytesPerRow: frame.bytesPerRow,
+            space: CGColorSpaceCreateDeviceRGB(),
+            bitmapInfo: srcInfo,
+            provider: provider, decode: nil,
+            shouldInterpolate: false, intent: .defaultIntent
+        ) else { return nil }
+
+        var out = [UInt8](repeating: 0, count: tw * th * 4)
+        let ok = out.withUnsafeMutableBytes { buf -> Bool in
+            guard let ctx = CGContext(
+                data: buf.baseAddress, width: tw, height: th,
+                bitsPerComponent: 8, bytesPerRow: tw * 4,
+                space: CGColorSpaceCreateDeviceRGB(),
+                bitmapInfo: CGImageAlphaInfo.premultipliedLast.rawValue
+            ) else { return false }
+            ctx.interpolationQuality = .high
+            ctx.draw(cgImage, in: CGRect(x: 0, y: 0, width: tw, height: th))
+            return true
+        }
+        return ok ? Bitmap(width: tw, height: th, data: out) : nil
+    }
+}
+
+extension CoreGraphicsScaler {
+    /// Re-encode an existing PNG at a fraction of its size. Frames that have
+    /// aged out of the fine window stay legible enough to tell which screen was
+    /// showing, at roughly a quarter of the bytes.
+    public static func downscalePNG(at url: URL, factor: Double) -> Bitmap? {
+        guard factor > 0, factor < 1,
+              let src = CGImageSourceCreateWithURL(url as CFURL, nil),
+              let image = CGImageSourceCreateImageAtIndex(src, 0, nil) else { return nil }
+        let tw = max(1, Int(Double(image.width) * factor))
+        let th = max(1, Int(Double(image.height) * factor))
+        var out = [UInt8](repeating: 0, count: tw * th * 4)
+        let ok = out.withUnsafeMutableBytes { buf -> Bool in
+            guard let ctx = CGContext(
+                data: buf.baseAddress, width: tw, height: th,
+                bitsPerComponent: 8, bytesPerRow: tw * 4,
+                space: CGColorSpaceCreateDeviceRGB(),
+                bitmapInfo: CGImageAlphaInfo.premultipliedLast.rawValue
+            ) else { return false }
+            ctx.interpolationQuality = .high
+            ctx.draw(image, in: CGRect(x: 0, y: 0, width: tw, height: th))
+            return true
+        }
+        return ok ? Bitmap(width: tw, height: th, data: out) : nil
+    }
+}

package/native/simframed/Sources/SimframeCore/Element.swift

@@ -0,0 +1,120 @@
+import CoreGraphics
+import Foundation
+
+/// Where an element's information came from. A single element can be seen by
+/// more than one source, which is why this is a set rather than a tag.
+public struct ElementSource: OptionSet, Sendable, Hashable {
+    public let rawValue: Int
+    public init(rawValue: Int) { self.rawValue = rawValue }
+
+    public static let accessibility = ElementSource(rawValue: 1 << 0)
+    public static let ocr = ElementSource(rawValue: 1 << 1)
+    public static let contour = ElementSource(rawValue: 1 << 2)
+    public static let template = ElementSource(rawValue: 1 << 3)
+
+    public var names: [String] {
+        var out: [String] = []
+        if contains(.accessibility) { out.append("ax") }
+        if contains(.ocr) { out.append("ocr") }
+        if contains(.contour) { out.append("cv") }
+        if contains(.template) { out.append("template") }
+        return out
+    }
+}
+
+public struct ElementState: Sendable, Equatable {
+    public var enabled: Bool?
+    public var selected: Bool?
+    public var checked: Bool?
+    public var focused: Bool?
+
+    public init(enabled: Bool? = nil, selected: Bool? = nil, checked: Bool? = nil, focused: Bool? = nil) {
+        self.enabled = enabled
+        self.selected = selected
+        self.checked = checked
+        self.focused = focused
+    }
+
+    public var json: [String: Any] {
+        var out: [String: Any] = [:]
+        if let enabled { out["enabled"] = enabled }
+        if let selected { out["selected"] = selected }
+        if let checked { out["checked"] = checked }
+        if let focused { out["focused"] = focused }
+        return out
+    }
+}
+
+/// One thing on screen. Frames are in **points**, because that is what input
+/// speaks; the capture side works in pixels and converts once, here.
+public struct Element: Sendable {
+    public var id: Int
+    public var frame: CGRect
+    public var role: String
+    public var label: String?
+    public var value: String?
+    public var state: ElementState
+    public var source: ElementSource
+    public var confidence: Double
+
+    public init(id: Int, frame: CGRect, role: String, label: String? = nil, value: String? = nil,
+                state: ElementState = ElementState(), source: ElementSource, confidence: Double = 1) {
+        self.id = id
+        self.frame = frame
+        self.role = role
+        self.label = label
+        self.value = value
+        self.state = state
+        self.source = source
+        self.confidence = confidence
+    }
+
+    public var center: CGPoint { CGPoint(x: frame.midX, y: frame.midY) }
+
+    public var json: [String: Any] {
+        var out: [String: Any] = [
+            "id": id,
+            "role": role,
+            "frame": ["x": frame.origin.x, "y": frame.origin.y, "width": frame.width, "height": frame.height],
+            "center": ["x": Int(center.x.rounded()), "y": Int(center.y.rounded())],
+            "source": source.names,
+            "confidence": (confidence * 1000).rounded() / 1000,
+        ]
+        if let label { out["label"] = label }
+        if let value { out["value"] = value }
+        let state = state.json
+        if !state.isEmpty { out["state"] = state }
+        return out
+    }
+}
+
+/// Everything known about one screen.
+public struct ScreenMap: Sendable {
+    public var fingerprint: String        // the layout hash
+    public var hash: String               // the content hash
+    public var size: CGSize               // points
+    public var elements: [Element]
+    public var sources: [String]
+    public var capturedAt: Double
+
+    public init(fingerprint: String, hash: String, size: CGSize, elements: [Element],
+                sources: [String], capturedAt: Double) {
+        self.fingerprint = fingerprint
+        self.hash = hash
+        self.size = size
+        self.elements = elements
+        self.sources = sources
+        self.capturedAt = capturedAt
+    }
+
+    public var json: [String: Any] {
+        [
+            "fingerprint": fingerprint,
+            "hash": hash,
+            "size": ["width": Int(size.width), "height": Int(size.height)],
+            "sources": sources,
+            "capturedAt": capturedAt,
+            "elements": elements.map(\.json),
+        ]
+    }
+}