serve-sim 0.1.44-beta.93.1 → 0.1.44-beta.99.3

package/Package.swift

@@ -0,0 +1,78 @@
+// swift-tools-version:6.0
+//
+// Builds serve-sim-native — the in-process N-API addon that replaces the
+// spawned serve-sim-bin helper. The JS bindings are written directly in Swift
+// with node-swift (NodeAPI), so there is no Objective-C++ glue: SimHID /
+// SimCapture are NodeClasses and the accessibility dumps are async NodeFunctions
+// (see Sources/SimNative/sim-module.swift). The reverse-engineered streaming
+// logic in SimStreamHelper is reused verbatim.
+//
+// The actual .node is produced by Sources/SimNative/build.sh, which drives
+// `swift build --arch arm64 --arch x86_64` for a universal binary and links
+// napi_* with `-undefined dynamic_lookup` (resolved against the host Node/Bun
+// at dlopen). `node-swift rebuild` is intentionally not used: it builds a single
+// arch only, and cross-compiling per-arch with `--triple` breaks the
+// `#NodeModule` macro.
+
+import PackageDescription
+
+let package = Package(
+    name: "serve-sim-native",
+    platforms: [.macOS(.v14)],
+    products: [
+        .library(
+            name: "serve-sim-native",
+            type: .dynamic,
+            targets: ["SimNativeModule"]
+        ),
+    ],
+    dependencies: [
+        .package(path: "node_modules/node-swift"),
+    ],
+    targets: [
+        .target(
+            name: "SimNativeModule",
+            dependencies: [
+                .product(name: "NodeAPI", package: "node-swift"),
+                .product(name: "NodeModuleSupport", package: "node-swift"),
+            ],
+            path: "Sources",
+            // Sibling targets built by their own build.sh (Obj-C/ObjC++), not
+            // part of this SwiftPM package.
+            exclude: [
+                "SimCameraInjector",
+                "SimCameraHelper",
+                "SimAXSettings",
+                "SimNative/build.sh",
+            ],
+            sources: [
+                "SimNative/sim-module.swift",
+                "SimNative/sim-capture.swift",
+                "SimStreamHelper/HIDInjector.swift",
+                "SimStreamHelper/FrameCapture.swift",
+                "SimStreamHelper/VideoEncoder.swift",
+                "SimStreamHelper/H264Encoder.swift",
+                "SimStreamHelper/StreamFormat.swift",
+                "SimStreamHelper/AccessibilityBridge.swift",
+                "SimStreamHelper/SimFrameworks.swift",
+                "SimStreamHelper/Xcode.swift",
+            ],
+            linkerSettings: [
+                .linkedFramework("CoreVideo"),
+                .linkedFramework("CoreMedia"),
+                .linkedFramework("IOSurface"),
+                .linkedFramework("CoreGraphics"),
+                .linkedFramework("VideoToolbox"),
+                .linkedFramework("ImageIO"),
+                .linkedFramework("AppKit"),
+            ]
+        ),
+    ],
+    // The reused SimStreamHelper logic was written against the standalone helper
+    // (plain swiftc, which defaults to Swift 5 mode). Build in Swift 5 mode so its
+    // benign cross-queue captures stay warnings rather than Swift 6 errors;
+    // node-swift's API works in v5. Declared at package level so the target's
+    // language mode is valid under stricter toolchains — CI rejects a target
+    // language mode that isn't among the package's declared modes.
+    swiftLanguageModes: [.v5]
+)

package/Sources/SimNative/build.sh

@@ -0,0 +1,51 @@
+#!/bin/bash
+# Builds serve-sim-native.node — the in-process N-API addon that replaces the
+# spawned serve-sim-bin helper. The JS bindings are written in Swift with
+# node-swift (see ../../Package.swift and sim-module.swift).
+#
+# Host-arch only (arm64 on Apple Silicon / CI). We use a plain `swift build`,
+# i.e. the NATIVE SwiftPM build system, because that is the only mode that
+# resolves node-swift's #NodeModule macro plugin: both `--arch X --arch Y`
+# (universal) and `--triple <other-arch>` force Xcode's XCBuild, which fails to
+# resolve the NodeAPIMacros plugin on stock toolchains ("missing target
+# NodeAPIMacros" / "unable to resolve module SwiftSyntax"). Cross-compiling the
+# x86_64 slice would therefore require an x86_64-native toolchain (Rosetta),
+# which isn't available everywhere; serve-sim targets Apple Silicon. napi_* stay
+# undefined and resolve against the host (Node/Bun) at dlopen via
+# `-undefined dynamic_lookup`.
+set -euo pipefail
+HERE="$(cd "$(dirname "$0")" && pwd)"
+PKG="$(cd "$HERE/../.." && pwd)"          # packages/serve-sim (Package.swift root)
+OUT_DIR="${1:-$PKG/dist/native}"
+BUILD_DIR="$PKG/.build"
+PRODUCT="serve-sim-native"
+mkdir -p "$OUT_DIR"
+
+if [ ! -d "$PKG/node_modules/node-swift" ]; then
+  echo "node-swift not found at $PKG/node_modules/node-swift (run: bun install)" >&2
+  exit 1
+fi
+
+swift build \
+  -c release \
+  --product "$PRODUCT" \
+  --package-path "$PKG" \
+  --build-path "$BUILD_DIR" \
+  -Xlinker -undefined -Xlinker dynamic_lookup
+
+# `-print -quit` (not `| head`): under `pipefail`, head closing the pipe early
+# would SIGPIPE find and fail the script. Match by name rather than a fixed
+# subpath: the product dir varies by toolchain (native SwiftPM emits
+# .build/release/, the Xcode build system emits .build/out/Products/Release/).
+DYLIB="$(find "$BUILD_DIR" -name "lib${PRODUCT}.dylib" -type f -not -path '*.dSYM*' -print -quit)"
+if [ -z "$DYLIB" ]; then
+  echo "Build succeeded but lib${PRODUCT}.dylib was not found under $BUILD_DIR" >&2
+  exit 1
+fi
+
+OUT="$OUT_DIR/${PRODUCT}.node"
+cp "$DYLIB" "$OUT"
+codesign -s - -f "$OUT" 2>/dev/null || true
+
+echo "Built: $OUT"
+lipo -info "$OUT"

package/Sources/SimNative/sim-capture.swift

@@ -0,0 +1,224 @@
+import Foundation
+import CoreVideo
+import CoreMedia
+
+// The capture + encode engine, reused verbatim from SimStreamHelper. Replicates
+// main.swift's frameHandler: MJPEG always encodes while clients exist; H.264 runs
+// only while AVCC is active. Encoded bytes (JPEG, or natively-framed AVCC
+// envelopes) are handed back through a Swift closure on a native encode thread;
+// the node-swift binding (sim-module.swift) marshals them onto the JS thread via
+// a NodeAsyncQueue (threadsafe function).
+
+/// (codec, data, width, height, flags) -> Void, invoked on a native encode
+/// thread. codec: 0 = MJPEG, 1 = AVCC. flags (AVCC): bit0 = description,
+/// bit1 = keyframe. `data` is a freshly-copied value safe to retain.
+typealias SimFrameCallback = (Int32, Data, Int32, Int32, Int32) -> Void
+
+final class CaptureEngine {
+    static let codecMJPEG: Int32 = 0
+    static let codecAVCC: Int32 = 1
+    static let flagDescription: Int32 = 1 << 0
+    static let flagKeyframe: Int32 = 1 << 1
+
+    private let deviceUDID: String
+    private let onFrame: SimFrameCallback
+
+    private let frameCapture = FrameCapture()
+    private let videoEncoder = VideoEncoder(quality: 0.7)
+    private let h264Encoder = H264Encoder(fps: 60)
+    private let encodeQueue = DispatchQueue(label: "napi.encode", qos: .userInteractive)
+    private let h264Queue = DispatchQueue(label: "napi.encode.h264", qos: .userInteractive)
+    private static let h264EncodeTimeoutMs = 500
+
+    // Mirrors main.swift's globals; mutated from the capture queue, read from the
+    // encode queues. Benign races (same pattern as the standalone helper).
+    private var screenWidth = 0
+    private var screenHeight = 0
+    private var encoderReady = false
+    private var encoding = false       // MJPEG backpressure
+    private var h264Encoding = false   // H.264 backpressure
+    private var forceKeyframe = false
+    private var avccActive = false
+    private var h264FrameToken: UInt64 = 0
+    private var started = false
+    private var stopped = false
+
+    init(deviceUDID: String, onFrame: @escaping SimFrameCallback) {
+        self.deviceUDID = deviceUDID
+        self.onFrame = onFrame
+
+        h264Encoder.onEncoded = { [weak self] encoded in
+            guard let self else { return }
+            if let description = encoded.description {
+                self.emit(codec: Self.codecAVCC,
+                          data: AVCCEnvelope.description(avcc: description),
+                          flags: Self.flagDescription)
+            }
+            switch encoded.kind {
+            case .keyframe:
+                self.emit(codec: Self.codecAVCC, data: AVCCEnvelope.keyframe(avcc: encoded.avcc),
+                          flags: Self.flagKeyframe)
+            case .delta:
+                self.emit(codec: Self.codecAVCC, data: AVCCEnvelope.delta(avcc: encoded.avcc), flags: 0)
+            }
+        }
+    }
+
+    /// Hand encoded bytes to the binding. Gated by `stopped` so no callback fires
+    /// once teardown has begun.
+    private func emit(codec: Int32, data: Data, flags: Int32) {
+        if stopped { return }
+        onFrame(codec, data, Int32(screenWidth), Int32(screenHeight), flags)
+    }
+
+    func start() throws {
+        guard !started else { return }
+        // Latch `started` only after capture actually begins: if start() throws
+        // (e.g. device not booted), a later retry should still be allowed.
+        try frameCapture.start(deviceUDID: deviceUDID) { [weak self] pixelBuffer, _ in
+            self?.handleFrame(pixelBuffer)
+        }
+        started = true
+    }
+
+    private func handleFrame(_ pixelBuffer: CVPixelBuffer) {
+        let w = CVPixelBufferGetWidth(pixelBuffer)
+        let h = CVPixelBufferGetHeight(pixelBuffer)
+
+        if !encoderReady || w != screenWidth || h != screenHeight {
+            screenWidth = w
+            screenHeight = h
+            videoEncoder.stop()
+            videoEncoder.setup(width: Int32(w), height: Int32(h), fps: 60) { [weak self] jpeg in
+                self?.emit(codec: Self.codecMJPEG, data: jpeg, flags: 0)
+            }
+            encoderReady = true
+        }
+
+        let h264Request = reserveH264EncodeIfNeeded()
+        let shouldEncodeJpeg = encoderReady && !encoding
+        if !shouldEncodeJpeg && h264Request == nil { return }
+
+        guard let stableFrame = copyPixelBuffer(pixelBuffer) else {
+            if let h264Request {
+                finishH264Encode(token: h264Request.token, restoreKeyframe: h264Request.forceKeyframe)
+            }
+            return
+        }
+
+        if shouldEncodeJpeg {
+            encoding = true
+            encodeQueue.async { [weak self] in
+                guard let self else { return }
+                self.videoEncoder.encode(pixelBuffer: stableFrame)
+                self.encoding = false
+            }
+        }
+
+        // H.264 runs only while a viewer wants AVCC, so an all-MJPEG session pays
+        // no VideoToolbox cost.
+        if let h264Request {
+            h264Queue.async { [weak self] in
+                guard let self else { return }
+                self.h264Encoder.encode(stableFrame, forceKeyframe: h264Request.forceKeyframe) {
+                    self.finishH264Encode(token: h264Request.token)
+                }
+                self.scheduleH264EncodeTimeout(token: h264Request.token)
+            }
+        }
+    }
+
+    /// Copy the live Simulator IOSurface immediately on the capture queue. The
+    /// encoders run later and SimulatorKit recycles/mutates that IOSurface in
+    /// place, so passing the wrapper CVPixelBuffer across queues can encode a
+    /// half-updated frame.
+    private func copyPixelBuffer(_ source: CVPixelBuffer) -> CVPixelBuffer? {
+        let width = CVPixelBufferGetWidth(source)
+        let height = CVPixelBufferGetHeight(source)
+        let pixelFormat = CVPixelBufferGetPixelFormatType(source)
+        let attrs: [String: Any] = [
+            kCVPixelBufferPixelFormatTypeKey as String: pixelFormat,
+            kCVPixelBufferWidthKey as String: width,
+            kCVPixelBufferHeightKey as String: height,
+            kCVPixelBufferCGImageCompatibilityKey as String: true,
+            kCVPixelBufferCGBitmapContextCompatibilityKey as String: true,
+        ]
+        var out: CVPixelBuffer?
+        guard CVPixelBufferCreate(
+            kCFAllocatorDefault, width, height, pixelFormat, attrs as CFDictionary, &out
+        ) == kCVReturnSuccess, let dst = out else { return nil }
+
+        CVPixelBufferLockBaseAddress(source, .readOnly)
+        CVPixelBufferLockBaseAddress(dst, [])
+        defer {
+            CVPixelBufferUnlockBaseAddress(dst, [])
+            CVPixelBufferUnlockBaseAddress(source, .readOnly)
+        }
+        guard let srcAddr = CVPixelBufferGetBaseAddress(source),
+              let dstAddr = CVPixelBufferGetBaseAddress(dst) else { return nil }
+        let srcStride = CVPixelBufferGetBytesPerRow(source)
+        let dstStride = CVPixelBufferGetBytesPerRow(dst)
+        let rows = CVPixelBufferGetHeight(source)
+        let copyBytes = min(srcStride, dstStride)
+        for row in 0..<rows {
+            memcpy(dstAddr + row * dstStride, srcAddr + row * srcStride, copyBytes)
+        }
+        return dst
+    }
+
+    private func reserveH264EncodeIfNeeded() -> (forceKeyframe: Bool, token: UInt64)? {
+        h264Queue.sync {
+            guard avccActive, !h264Encoding else { return nil }
+            h264Encoding = true
+            h264FrameToken &+= 1
+            let token = h264FrameToken
+            let force = forceKeyframe
+            forceKeyframe = false
+            return (forceKeyframe: force, token: token)
+        }
+    }
+
+    private func finishH264Encode(token: UInt64, restoreKeyframe: Bool = false) {
+        h264Queue.async { [weak self] in
+            guard let self, self.h264FrameToken == token else { return }
+            self.h264Encoding = false
+            if restoreKeyframe { self.forceKeyframe = true }
+        }
+    }
+
+    private func scheduleH264EncodeTimeout(token: UInt64) {
+        h264Queue.asyncAfter(deadline: .now().advanced(by: .milliseconds(Self.h264EncodeTimeoutMs))) { [weak self] in
+            guard let self, self.h264FrameToken == token else { return }
+            self.h264Encoding = false
+        }
+    }
+
+    /// Toggle H.264 encoding. Turning it on forces the next frame to an IDR so a
+    /// freshly-connected decoder has a keyframe to start from.
+    func setAvccActive(_ active: Bool) {
+        h264Queue.async { [weak self] in
+            guard let self else { return }
+            if active && !self.avccActive { self.forceKeyframe = true }
+            self.avccActive = active
+        }
+    }
+
+    func requestKeyframe() {
+        h264Queue.async { [weak self] in self?.forceKeyframe = true }
+    }
+
+    func screenSize() -> (Int, Int) { (screenWidth, screenHeight) }
+
+    /// Halt frame production and drain the encode queues so no callback can fire
+    /// after this returns — the N-API layer relies on that before releasing the
+    /// threadsafe function.
+    func stop() {
+        if stopped { return }
+        stopped = true
+        frameCapture.stop()
+        encodeQueue.sync {}
+        h264Queue.sync {}
+        videoEncoder.stop()
+        h264Encoder.stop()
+    }
+}

package/Sources/SimNative/sim-module.swift

@@ -0,0 +1,195 @@
+import Foundation
+import NodeAPI
+
+/// Safe Int→UInt32 for HID codes coming from JS. Plain `UInt32(x)` traps on
+/// negative or too-large values, which would crash the in-process server on
+/// malformed input; clamp out-of-range values to 0 (a harmless no-op code).
+private func u32(_ v: Int) -> UInt32 {
+    UInt32(exactly: v) ?? 0
+}
+
+// node-swift entrypoint for serve-sim-native — the in-process N-API addon that
+// replaces the spawned serve-sim-bin helper. The JS surface is expressed
+// directly in Swift (no Objective-C++ glue): HID and frame capture are
+// NodeClasses, and the accessibility dumps are async NodeFunctions. The
+// reverse-engineered logic (HIDInjector, the CaptureEngine + encoders,
+// AccessibilityBridge) is reused verbatim from SimStreamHelper.
+
+// MARK: - HID
+
+/// In-process HID injector for one simulator. Mirrors the WebSocket HID protocol
+/// the spawned helper used to handle, as direct native calls. The instance is
+/// released (freeing the injector) when its JS handle is garbage-collected.
+@NodeClass @NodeActor final class SimHID {
+    private let injector: HIDInjector
+    private let udid: String
+
+    @NodeConstructor init(_ udid: String) throws {
+        self.udid = udid
+        injector = HIDInjector()
+        try injector.setup(deviceUDID: udid)
+    }
+
+    @NodeMethod func touch(_ type: String, _ x: Double, _ y: Double,
+                           _ w: Int, _ h: Int, _ edge: Int) {
+        injector.sendTouch(type: type, x: x, y: y,
+                           screenWidth: w, screenHeight: h, edge: u32(edge))
+    }
+
+    @NodeMethod func multiTouch(_ type: String, _ x1: Double, _ y1: Double,
+                                _ x2: Double, _ y2: Double, _ w: Int, _ h: Int) {
+        injector.sendMultiTouch(type: type, x1: x1, y1: y1, x2: x2, y2: y2,
+                                screenWidth: w, screenHeight: h)
+    }
+
+    @NodeMethod func button(_ button: String) {
+        injector.sendButton(button: button, deviceUDID: udid)
+    }
+
+    @NodeMethod func buttonHid(_ page: Int, _ usage: Int, _ phase: String) {
+        injector.sendButtonHID(page: u32(page), usage: u32(usage), phase: phase)
+    }
+
+    @NodeMethod func key(_ type: String, _ usage: Int) {
+        injector.sendKey(type: type, usage: u32(usage))
+    }
+
+    /// NaN anchorX/anchorY mean "center" (the Swift API's nil).
+    @NodeMethod func scroll(_ dx: Double, _ dy: Double,
+                            _ anchorX: Double, _ anchorY: Double, _ w: Int, _ h: Int) {
+        injector.sendScroll(dx: dx, dy: dy,
+                            anchorX: anchorX.isNaN ? nil : anchorX,
+                            anchorY: anchorY.isNaN ? nil : anchorY,
+                            screenWidth: w, screenHeight: h)
+    }
+
+    @NodeMethod func digitalCrown(_ delta: Double) {
+        injector.sendDigitalCrown(delta: delta)
+    }
+
+    @NodeMethod func orientation(_ orientation: Int) -> Bool {
+        injector.sendOrientation(orientation: u32(orientation))
+    }
+
+    @NodeMethod func memoryWarning() {
+        injector.simulateMemoryWarning()
+    }
+
+    @NodeMethod func softwareKeyboard() {
+        injector.toggleSoftwareKeyboard()
+    }
+
+    @NodeMethod func caDebug(_ name: String, _ enabled: Bool) -> Bool {
+        injector.setCADebugOption(name: name, enabled: enabled)
+    }
+}
+
+// MARK: - Capture
+
+/// In-process frame capture + encode for one simulator. MJPEG frames are always
+/// produced; H.264/AVCC runs only while `setAvccActive(true)`. Encoded frames are
+/// produced on a native encode thread and marshalled onto the JS thread through a
+/// NodeAsyncQueue (threadsafe function), then handed to `onFrame` as
+/// (codec, Buffer, width, height, flags).
+@NodeClass @NodeActor final class SimCapture {
+    private let engine: CaptureEngine
+    private let onFrame: NodeFunction
+    private let queue: NodeAsyncQueue
+
+    @NodeConstructor init(_ udid: String, _ onFrame: NodeFunction) throws {
+        // unref'd by NodeAsyncQueue's init, so the frame pipeline alone won't
+        // keep the event loop alive. Bounded queue + blocking AVCC preserves
+        // inter-frame ordering; MJPEG is nonblocking and drops under backpressure.
+        let queue = try NodeAsyncQueue(label: "simCapture", maxQueueSize: 16)
+        self.onFrame = onFrame
+        self.queue = queue
+
+        // Capture the locals (not self) so the closure can be built before the
+        // engine property is initialized, and so it holds no strong ref to self.
+        engine = CaptureEngine(deviceUDID: udid) { codec, data, w, h, flags in
+            // Runs on a native encode thread. AVCC is inter-frame H.264 — dropping
+            // a delta corrupts the decoder until the next IDR — so deliver it
+            // blocking; MJPEG is stateless and safe to drop. We copy the bytes
+            // into a managed Buffer (NodeBuffer(copying:)) on the JS thread:
+            // external buffers crash Bun's GC under frame churn, and the
+            // production CLI is a bun-compiled binary.
+            let blocking = codec == CaptureEngine.codecAVCC
+            try? queue.run(blocking: blocking) {
+                _ = try? onFrame.call([
+                    Int(codec), try NodeBuffer(copying: data),
+                    Int(w), Int(h), Int(flags),
+                ])
+            }
+        }
+    }
+
+    @NodeMethod func start() throws {
+        try engine.start()
+    }
+
+    @NodeMethod func setAvccActive(_ active: Bool) {
+        engine.setAvccActive(active)
+    }
+
+    @NodeMethod func requestKeyframe() {
+        engine.requestKeyframe()
+    }
+
+    @NodeMethod func screenSize() -> [String: any NodePropertyConvertible] {
+        let (w, h) = engine.screenSize()
+        return ["width": w, "height": h]
+    }
+
+    @NodeMethod func stop() {
+        engine.stop()
+    }
+
+    deinit {
+        // Abort the queue first so any encode thread blocked in `run` unblocks
+        // (its call returns .closing and the frame is dropped); then drain the
+        // encoders so nothing can fire afterwards. The tsfn is released when
+        // `queue` deinitializes after this body.
+        try? queue.close()
+        engine.stop()
+    }
+}
+
+// MARK: - Accessibility
+
+/// Run a blocking accessibility query off the JS event loop (on a background
+/// queue) and resolve with its result, mirroring the old napi_async_work path.
+private func axQuery(
+    _ udid: String, _ body: @escaping @Sendable (String) throws -> String
+) async throws -> String {
+    try await withCheckedThrowingContinuation { cont in
+        DispatchQueue.global(qos: .userInitiated).async {
+            do {
+                cont.resume(returning: try body(udid))
+            } catch {
+                cont.resume(throwing: error)
+            }
+        }
+    }
+}
+
+#NodeModule(exports: [
+    "SimHID": SimHID.deferredConstructor,
+    "SimCapture": SimCapture.deferredConstructor,
+    // axDescribe(udid): Promise<string> — axe-shaped accessibility JSON.
+    "axDescribe": try NodeFunction { (udid: String) async throws -> String in
+        try await axQuery(udid) { udid in
+            SimFrameworks.load()  // /ax may be hit before capture/HID load them
+            let data = try AccessibilityBridge.shared.describeUI(udid: udid)
+            return String(decoding: data, as: UTF8.self)
+        }
+    },
+    // axFrontmost(udid): Promise<string> — JSON `{ bundleId, pid }`.
+    "axFrontmost": try NodeFunction { (udid: String) async throws -> String in
+        try await axQuery(udid) { udid in
+            SimFrameworks.load()
+            let info = try AccessibilityBridge.shared.frontmostApp(udid: udid)
+            let data = try JSONSerialization.data(withJSONObject: info)
+            return String(decoding: data, as: UTF8.self)
+        }
+    },
+])