zeno-mobile-runner 0.1.2

package/shims/android/protocol.md

@@ -0,0 +1,18 @@
+# Android Shim Protocol
+
+The Android shim protocol is internal and may change before `v1.0.0`.
+
+The first implementation should mirror the public ZMR action model:
+
+- `snapshot`
+- `tap`
+- `type`
+- `eraseText`
+- `hideKeyboard`
+- `swipe`
+- `pressBack`
+- `settle`
+- `appState`
+
+The Zig adapter must keep ADB/UI Automator fallback behavior so the runner can
+still operate when the shim is not installed.

package/shims/ios/README.md

@@ -0,0 +1,50 @@
+# ZMR iOS Shim
+
+This directory contains the XCTest/XCUIAutomation shim scaffold used for
+selector-grade iOS automation.
+
+The public ZMR API remains the scenario file and JSON-RPC protocol. The shim
+protocol is an internal local transport between the Zig runner and an app-local
+UI test runner.
+
+Current status:
+
+- Zig-side command and snapshot mapping are covered in `src/ios_shim.zig`.
+- `src/ios.zig` can run a configured shim command with one JSON request on
+  stdin and one JSON response on stdout.
+- `scripts/install-ios-shim.sh` writes an app-local `.zmr/ios-shim` command and
+  copies the XCTest source files into the app repo for inclusion in a UI test
+  target.
+- The generated command caches `xcodebuild build-for-testing` output and runs
+  selector commands through `test-without-building`, exchanging per-command
+  files under `.zmr/ios-shim-state/`. Set `ZMR_IOS_SHIM_FORCE_REBUILD=1` to
+  refresh the cached test bundle, or `ZMR_IOS_SHIM_ONESHOT=1` to force the
+  slower one-command XCTest fallback for debugging.
+- The iOS adapter still uses `xcrun simctl` for simulator install, launch,
+  terminate, open link, screenshots, and logs. It uses `xcrun devicectl` for
+  physical-device lifecycle where Apple exposes a supported local command, and
+  uses the XCTest shim for physical-device screenshot artifacts.
+- `.zmr/ensure-ios-shim-target.sh` can create or update the UI test target for
+  common Xcode project/workspace layouts through the Ruby `xcodeproj` gem.
+  Users can still add the generated Swift files manually when their project
+  layout needs custom handling.
+- ZMR uses the shim as a native selector fast path for single-field tap, type,
+  erase-text, and wait/assert queries. Compound selectors stay on the portable
+  Zig observe-and-match path.
+- Snapshot capture is intentionally bounded to common XCTest element families
+  and at most 256 nodes. This keeps traces fast and predictable for large
+  React Native, SwiftUI, and UIKit trees while preserving the controls agents
+  normally need for follow-up actions.
+
+Support target:
+
+- iOS simulators for full local artifacts: lifecycle, screenshots, logs,
+  selector actions, native selector waits, and bounded snapshots.
+- Physical iOS devices for lifecycle and selector-grade XCTest automation,
+  subject to local signing, provisioning, Developer Mode, and Apple
+  `devicectl` availability. Screenshots use the XCTest shim; log artifact
+  capture remains simulator-first.
+- XCTest/XCUIAutomation snapshots mapped into `UiNode`.
+- Selector actions: tap, type, erase text, hide keyboard, swipe, and home/back
+  equivalent navigation.
+- Clear state means best-effort app uninstall by bundle id.

package/shims/ios/ZMRShim.swift

@@ -0,0 +1,110 @@
+import Foundation
+import XCTest
+
+struct ZMRShimCommand: Decodable {
+    let cmd: String
+    let selector: String?
+    let text: String?
+    let x: Int?
+    let y: Int?
+    let x1: Int?
+    let y1: Int?
+    let x2: Int?
+    let y2: Int?
+    let durationMs: UInt?
+    let maxChars: UInt?
+}
+
+struct ZMRShimBounds: Encodable {
+    let x: Int
+    let y: Int
+    let width: Int
+    let height: Int
+}
+
+struct ZMRShimNode: Encodable {
+    let id: String
+    let type: String
+    let label: String
+    let value: String
+    let identifier: String
+    let bounds: ZMRShimBounds
+    let enabled: Bool
+    let visible: Bool
+    let selected: Bool
+}
+
+enum ZMRShim {
+    static func snapshot(app: XCUIApplication) -> [ZMRShimNode] {
+        let queries: [(XCUIElement.ElementType, XCUIElementQuery)] = [
+            (.button, app.buttons),
+            (.staticText, app.staticTexts),
+            (.textField, app.textFields),
+            (.secureTextField, app.secureTextFields),
+            (.textView, app.textViews),
+            (.image, app.images),
+            (.switch, app.switches),
+            (.cell, app.cells),
+            (.scrollView, app.scrollViews),
+            (.table, app.tables),
+            (.collectionView, app.collectionViews)
+        ]
+
+        var nodes: [ZMRShimNode] = []
+        nodes.reserveCapacity(128)
+
+        for (type, query) in queries {
+            for element in query.allElementsBoundByIndex {
+                guard nodes.count < 256 else {
+                    return nodes
+                }
+                nodes.append(node(index: nodes.count, type: type, element: element))
+            }
+        }
+
+        if nodes.isEmpty {
+            nodes.append(node(index: 0, type: .application, element: app))
+        }
+        return nodes
+    }
+
+    private static func node(index: Int, type: XCUIElement.ElementType, element: XCUIElement) -> ZMRShimNode {
+        let frame = element.frame
+        return ZMRShimNode(
+            id: stableId(index: index, element: element),
+            type: String(describing: type),
+            label: element.label,
+            value: elementValue(element),
+            identifier: element.identifier,
+            bounds: ZMRShimBounds(
+                x: Int(frame.origin.x),
+                y: Int(frame.origin.y),
+                width: Int(frame.size.width),
+                height: Int(frame.size.height)
+            ),
+            enabled: element.isEnabled,
+            visible: element.exists && !frame.isEmpty,
+            selected: element.isSelected
+        )
+    }
+
+    private static func elementValue(_ element: XCUIElement) -> String {
+        if let value = element.value as? String {
+            return value
+        }
+        if let value = element.value {
+            return String(describing: value)
+        }
+        return ""
+    }
+
+    private static func stableId(index: Int, element: XCUIElement) -> String {
+        if !element.identifier.isEmpty {
+            return "id:\(element.identifier)"
+        }
+        if !element.label.isEmpty {
+            return "label:\(element.label):\(index)"
+        }
+        return "index:\(index)"
+    }
+}

package/shims/ios/ZMRShimUITestCase.swift

@@ -0,0 +1,518 @@
+import Foundation
+import XCTest
+
+final class ZMRShimUITestCase: XCTestCase {
+    func testRunZMRCommand() throws {
+        let environment = ProcessInfo.processInfo.environment
+        let app = makeApplication(bundleIdentifier: shimRuntimeValue("ZMR_APP_BUNDLE_ID", environment: environment))
+
+        if shimRuntimeValue("ZMR_SHIM_MODE", environment: environment) == "server" {
+            guard let serverDir = shimRuntimeValue("ZMR_SHIM_SERVER_DIR", environment: environment) else {
+                throw ZMRShimError.missingEnvironment
+            }
+            try runServer(serverDir: serverDir, app: app)
+            return
+        }
+
+        guard let requestFile = shimRuntimeValue("ZMR_SHIM_REQUEST_FILE", environment: environment),
+              let responseFile = shimRuntimeValue("ZMR_SHIM_RESPONSE_FILE", environment: environment) else {
+            throw ZMRShimError.missingEnvironment
+        }
+
+        try process(requestAt: requestFile, responseAt: responseFile, app: app)
+    }
+
+    private func runServer(serverDir: String, app: XCUIApplication) throws {
+        let fileManager = FileManager.default
+        try fileManager.createDirectory(atPath: serverDir, withIntermediateDirectories: true)
+
+        let readyFile = path(in: serverDir, named: "ready")
+        let stopFile = path(in: serverDir, named: "stop")
+        _ = fileManager.createFile(atPath: readyFile, contents: Data(), attributes: nil)
+
+        var idleDeadline = Date().addingTimeInterval(900)
+        while Date() < idleDeadline {
+            if fileManager.fileExists(atPath: stopFile) {
+                break
+            }
+
+            let requestNames = try fileManager.contentsOfDirectory(atPath: serverDir)
+                .filter { $0.hasPrefix("request-") && $0.hasSuffix(".json") }
+                .sorted()
+
+            if requestNames.isEmpty {
+                Thread.sleep(forTimeInterval: 0.05)
+                continue
+            }
+
+            for requestName in requestNames {
+                let requestID = requestName
+                    .dropFirst("request-".count)
+                    .dropLast(".json".count)
+                let requestFile = path(in: serverDir, named: requestName)
+                let responseFile = path(in: serverDir, named: "response-\(requestID).json")
+
+                try process(requestAt: requestFile, responseAt: responseFile, app: app)
+                try? fileManager.removeItem(atPath: requestFile)
+                idleDeadline = Date().addingTimeInterval(900)
+            }
+        }
+    }
+
+    private func process(requestAt requestFile: String, responseAt responseFile: String, app: XCUIApplication) throws {
+        let response = responseFor(requestAt: requestFile, app: app)
+        let responseData = try JSONSerialization.data(withJSONObject: response, options: [.sortedKeys])
+        let responseURL = URL(fileURLWithPath: responseFile)
+        let temporaryURL = URL(fileURLWithPath: "\(responseFile).tmp")
+        try responseData.write(to: temporaryURL, options: [.atomic])
+        if FileManager.default.fileExists(atPath: responseFile) {
+            try FileManager.default.removeItem(at: responseURL)
+        }
+        try FileManager.default.moveItem(at: temporaryURL, to: responseURL)
+    }
+
+    private func responseFor(requestAt requestFile: String, app: XCUIApplication) -> [String: Any] {
+        do {
+            let requestData = try Data(contentsOf: URL(fileURLWithPath: requestFile))
+            let command = try JSONDecoder().decode(ZMRShimCommand.self, from: requestData)
+            return run(command: command, app: app)
+        } catch {
+            return self.error("invalid.request", "\(error)")
+        }
+    }
+
+    private func path(in directory: String, named name: String) -> String {
+        (directory as NSString).appendingPathComponent(name)
+    }
+
+    private func shimRuntimeValue(_ key: String, environment: [String: String]) -> String? {
+        if let value = environment[key], !value.isEmpty, !value.hasPrefix("$(") {
+            return value
+        }
+        if let value = Bundle(for: Self.self).object(forInfoDictionaryKey: key) as? String,
+           !value.isEmpty,
+           !value.hasPrefix("$(") {
+            return value
+        }
+        return nil
+    }
+
+    private func makeApplication(bundleIdentifier: String?) -> XCUIApplication {
+        if let bundleIdentifier, !bundleIdentifier.isEmpty {
+            return XCUIApplication(bundleIdentifier: bundleIdentifier)
+        }
+        return XCUIApplication()
+    }
+
+    private func run(command: ZMRShimCommand, app: XCUIApplication) -> [String: Any] {
+        switch command.cmd {
+        case "snapshot":
+            return [
+                "status": "ok",
+                "nodes": ZMRShim.snapshot(app: app).map { $0.json }
+            ]
+        case "screenshot":
+            let screenshot = XCUIScreen.main.screenshot()
+            return [
+                "status": "ok",
+                "format": "png",
+                "base64": screenshot.pngRepresentation.base64EncodedString()
+            ]
+        case "query":
+            guard let selector = command.selector else {
+                return error("invalid.query", "query requires selector")
+            }
+            guard let parts = selectorParts(selector) else {
+                return error("selector.unsupported", "unsupported selector: \(selector)")
+            }
+            guard isFastQueryable(parts: parts) else {
+                return error("selector.unsupported", "unsupported query selector: \(selector)")
+            }
+            let element = resolveFastElement(selector: selector, app: app, preferredTypes: [])
+            return [
+                "status": "ok",
+                "exists": element?.exists ?? false,
+                "hittable": element?.isHittable ?? false
+            ]
+        case "tap":
+            if let selector = command.selector {
+                return tap(selector: selector, app: app)
+            }
+            guard let x = command.x, let y = command.y else {
+                return error("invalid.tap", "tap requires x and y")
+            }
+            app.coordinate(withNormalizedOffset: CGVector(dx: 0, dy: 0))
+                .withOffset(CGVector(dx: x, dy: y))
+                .tap()
+            return ok()
+        case "type":
+            if let selector = command.selector {
+                return typeText(selector: selector, text: command.text ?? "", app: app)
+            }
+            app.typeText(command.text ?? "")
+            return ok()
+        case "eraseText":
+            let count = Int(command.maxChars ?? 0)
+            if let selector = command.selector {
+                return eraseText(selector: selector, count: count, app: app)
+            }
+            if count > 0 {
+                app.typeText(String(repeating: XCUIKeyboardKey.delete.rawValue, count: count))
+            }
+            return ok()
+        case "hideKeyboard":
+            return hideKeyboard(app: app)
+        case "swipe":
+            guard let x1 = command.x1, let y1 = command.y1, let x2 = command.x2, let y2 = command.y2 else {
+                return error("invalid.swipe", "swipe requires x1, y1, x2, and y2")
+            }
+            let start = app.coordinate(withNormalizedOffset: CGVector(dx: 0, dy: 0))
+                .withOffset(CGVector(dx: x1, dy: y1))
+            let end = app.coordinate(withNormalizedOffset: CGVector(dx: 0, dy: 0))
+                .withOffset(CGVector(dx: x2, dy: y2))
+            start.press(forDuration: 0.01, thenDragTo: end)
+            return ok()
+        case "pressBack":
+            XCUIDevice.shared.press(.home)
+            return ok()
+        case "settle":
+            let timeout = TimeInterval(command.durationMs ?? 1000) / 1000.0
+            _ = app.wait(for: app.state, timeout: timeout)
+            return ok()
+        case "appState":
+            return ["status": "ok", "state": app.state.rawValue]
+        case "acceptSystemAlert":
+            return acceptSystemAlert(buttonText: command.text ?? "Open")
+        default:
+            return error("unknown.command", "unsupported command: \(command.cmd)")
+        }
+    }
+
+    private func ok() -> [String: Any] {
+        ["status": "ok"]
+    }
+
+    private func error(_ code: String, _ message: String) -> [String: Any] {
+        ["status": "error", "code": code, "message": message]
+    }
+
+    private func acceptSystemAlert(buttonText: String) -> [String: Any] {
+        let springboard = XCUIApplication(bundleIdentifier: "com.apple.springboard")
+        var labels = [buttonText, "Open", "Allow", "OK", "Continue"]
+        labels = labels.reduce(into: [String]()) { unique, label in
+            if !label.isEmpty && !unique.contains(label) {
+                unique.append(label)
+            }
+        }
+
+        var acceptedCount = 0
+        var lastAcceptedLabel = ""
+        for _ in 0..<3 {
+            var tapped = false
+            for label in labels {
+                let button = springboard.buttons[label].firstMatch
+                if button.waitForExistence(timeout: 2), button.isHittable {
+                    button.tap()
+                    acceptedCount += 1
+                    lastAcceptedLabel = label
+                    tapped = true
+                    Thread.sleep(forTimeInterval: 1.0)
+                    break
+                }
+            }
+            if !tapped {
+                break
+            }
+        }
+
+        if acceptedCount > 0 {
+            return ["status": "ok", "accepted": true, "label": lastAcceptedLabel, "count": acceptedCount]
+        }
+        return ["status": "ok", "accepted": false, "count": 0]
+    }
+
+    private func hideKeyboard(app: XCUIApplication) -> [String: Any] {
+        guard app.keyboards.firstMatch.exists else {
+            return ok()
+        }
+
+        let keyboard = app.keyboards.firstMatch
+        let dismissKeyNames = [
+            "Done",
+            "done",
+            "Return",
+            "return",
+            "Go",
+            "go",
+            "Next",
+            "next",
+            "Search",
+            "search",
+            "Send",
+            "send"
+        ]
+
+        for keyName in dismissKeyNames {
+            if tapKeyboardElement(keyboard.buttons[keyName]) {
+                return ok()
+            }
+            if tapKeyboardElement(keyboard.keys[keyName]) {
+                return ok()
+            }
+        }
+
+        app.coordinate(withNormalizedOffset: CGVector(dx: 0.5, dy: 0.05)).tap()
+        if app.keyboards.firstMatch.waitForNonExistence(timeout: 1) {
+            return ok()
+        }
+
+        return error("keyboard.dismiss_failed", "keyboard did not expose a known dismiss key")
+    }
+
+    private func tapKeyboardElement(_ element: XCUIElement) -> Bool {
+        guard element.exists, element.isHittable else {
+            return false
+        }
+        element.tap()
+        return true
+    }
+
+    private func tap(selector: String, app: XCUIApplication) -> [String: Any] {
+        guard selectorParts(selector) != nil else {
+            return error("selector.unsupported", "unsupported selector: \(selector)")
+        }
+        guard let element = resolveElement(selector: selector, app: app, preferredTypes: [.button]) else {
+            return error("selector.not_found", "selector did not match: \(selector)")
+        }
+        if !element.isHittable {
+            return error("selector.not_hittable", "selector matched a non-hittable element: \(selector)")
+        }
+        element.tap()
+        return ok()
+    }
+
+    private func typeText(selector: String, text: String, app: XCUIApplication) -> [String: Any] {
+        guard selectorParts(selector) != nil else {
+            return error("selector.unsupported", "unsupported selector: \(selector)")
+        }
+        guard let element = resolveElement(selector: selector, app: app, preferredTypes: [.textField, .secureTextField, .textView]) else {
+            return error("selector.not_found", "selector did not match: \(selector)")
+        }
+        if !element.isHittable {
+            return error("selector.not_hittable", "selector matched a non-hittable element: \(selector)")
+        }
+        element.tap()
+        app.typeText(text)
+        return ok()
+    }
+
+    private func eraseText(selector: String, count: Int, app: XCUIApplication) -> [String: Any] {
+        guard selectorParts(selector) != nil else {
+            return error("selector.unsupported", "unsupported selector: \(selector)")
+        }
+        guard let element = resolveElement(selector: selector, app: app, preferredTypes: [.textField, .secureTextField, .textView]) else {
+            return error("selector.not_found", "selector did not match: \(selector)")
+        }
+        if !element.isHittable {
+            return error("selector.not_hittable", "selector matched a non-hittable element: \(selector)")
+        }
+        element.tap()
+        if count > 0 {
+            app.typeText(String(repeating: XCUIKeyboardKey.delete.rawValue, count: count))
+        }
+        return ok()
+    }
+
+    private func resolveElement(selector: String, app: XCUIApplication, preferredTypes: [XCUIElement.ElementType] = []) -> XCUIElement? {
+        if let fast = resolveFastElement(selector: selector, app: app, preferredTypes: preferredTypes), fast.exists {
+            return fast
+        }
+
+        let matchedElements = app.descendants(matching: .any).allElementsBoundByIndex.filter { element in
+            matches(selector: selector, element: element)
+        }
+        if let preferred = matchedElements.first(where: { preferredTypes.contains($0.elementType) && $0.isHittable }) {
+            return preferred
+        }
+        if let preferred = matchedElements.first(where: { preferredTypes.contains($0.elementType) }) {
+            return preferred
+        }
+        if let hittable = matchedElements.first(where: { $0.isHittable }) {
+            return hittable
+        }
+        return matchedElements.first
+    }
+
+    private func resolveFastElement(selector: String, app: XCUIApplication, preferredTypes: [XCUIElement.ElementType]) -> XCUIElement? {
+        guard let parts = selectorParts(selector) else {
+            return nil
+        }
+
+        switch parts.field {
+        case "text", "label":
+            let queries = fastTextQueries(app: app, preferredTypes: preferredTypes)
+            if parts.contains {
+                let predicate = NSPredicate(format: "label CONTAINS[c] %@", parts.value)
+                return firstExistingElement(queries: queries.map { $0.matching(predicate) })
+            }
+            let predicate = NSPredicate(format: "label == %@", parts.value)
+            return firstExistingElement(queries: queries.map { $0.matching(predicate) })
+        case "identifier", "resourceId":
+            let queries = fastIdentifierQueries(app: app, preferredTypes: preferredTypes, contains: parts.contains)
+            if parts.contains {
+                let predicate = NSPredicate(format: "identifier CONTAINS[c] %@", parts.value)
+                return firstExistingElement(queries: queries.map { $0.matching(predicate) })
+            }
+            return firstExistingElement(queries: queries.map { $0.matching(identifier: parts.value) })
+        case "value":
+            let queries = fastTextQueries(app: app, preferredTypes: preferredTypes)
+            let predicate = parts.contains
+                ? NSPredicate(format: "value CONTAINS[c] %@", parts.value)
+                : NSPredicate(format: "value == %@", parts.value)
+            return firstExistingElement(queries: queries.map { $0.matching(predicate) })
+        case "type", "id":
+            return nil
+        default:
+            return nil
+        }
+    }
+
+    private func fastTextQueries(app: XCUIApplication, preferredTypes: [XCUIElement.ElementType]) -> [XCUIElementQuery] {
+        var queries: [XCUIElementQuery] = []
+        if !preferredTypes.isEmpty {
+            queries.append(contentsOf: preferredTypes.map { app.descendants(matching: $0) })
+        }
+        queries.append(contentsOf: [
+            app.buttons,
+            app.staticTexts,
+            app.textFields,
+            app.secureTextFields,
+            app.textViews,
+            app.images
+        ])
+        return queries
+    }
+
+    private func fastIdentifierQueries(
+        app: XCUIApplication,
+        preferredTypes: [XCUIElement.ElementType],
+        contains: Bool
+    ) -> [XCUIElementQuery] {
+        var queries = fastTextQueries(app: app, preferredTypes: preferredTypes)
+        if !contains {
+            queries.append(app.otherElements)
+        }
+        return queries
+    }
+
+    private func firstExistingElement(queries: [XCUIElementQuery]) -> XCUIElement? {
+        for query in queries {
+            let element = query.firstMatch
+            if element.exists {
+                return element
+            }
+        }
+        return nil
+    }
+
+    private func isFastQueryable(parts: (field: String, value: String, contains: Bool)) -> Bool {
+        switch parts.field {
+        case "text", "label", "identifier", "resourceId", "value":
+            return true
+        default:
+            return false
+        }
+    }
+
+    private func matches(selector: String, element: XCUIElement) -> Bool {
+        guard element.exists, let parts = selectorParts(selector) else {
+            return false
+        }
+
+        let actual: String
+        switch parts.field {
+        case "text", "label":
+            actual = element.label
+        case "identifier", "resourceId":
+            actual = element.identifier
+        case "id":
+            actual = stableId(element: element)
+        case "value":
+            actual = element.value as? String ?? ""
+        case "type":
+            actual = String(describing: element.elementType)
+        default:
+            return false
+        }
+
+        if parts.contains {
+            return actual.localizedCaseInsensitiveContains(parts.value)
+        }
+        return actual == parts.value
+    }
+
+    private func selectorParts(_ selector: String) -> (field: String, value: String, contains: Bool)? {
+        let supportedPrefixes = [
+            ("textContains=", "text", true),
+            ("labelContains=", "label", true),
+            ("identifierContains=", "identifier", true),
+            ("resourceIdContains=", "resourceId", true),
+            ("valueContains=", "value", true),
+            ("type=", "type", false),
+            ("text=", "text", false),
+            ("label=", "label", false),
+            ("identifier=", "identifier", false),
+            ("resourceId=", "resourceId", false),
+            ("id=", "id", false),
+            ("value=", "value", false)
+        ]
+
+        for (prefix, field, contains) in supportedPrefixes {
+            if selector.hasPrefix(prefix) {
+                let value = String(selector.dropFirst(prefix.count))
+                return value.isEmpty ? nil : (field, value, contains)
+            }
+        }
+        return nil
+    }
+
+    private func stableId(element: XCUIElement) -> String {
+        if !element.identifier.isEmpty {
+            return "id:\(element.identifier)"
+        }
+        if !element.label.isEmpty {
+            return "label:\(element.label)"
+        }
+        return String(describing: element.elementType)
+    }
+}
+
+enum ZMRShimError: Error {
+    case missingEnvironment
+}
+
+private extension ZMRShimBounds {
+    var json: [String: Any] {
+        [
+            "x": x,
+            "y": y,
+            "width": width,
+            "height": height
+        ]
+    }
+}
+
+private extension ZMRShimNode {
+    var json: [String: Any] {
+        [
+            "id": id,
+            "type": type,
+            "label": label,
+            "identifier": identifier,
+            "bounds": bounds.json,
+            "enabled": enabled,
+            "visible": visible,
+            "selected": selected
+        ]
+    }
+}