serve-sim 0.1.49-beta.131.1 → 0.1.49-beta.133.1

package/README.md

@@ -111,6 +111,8 @@ serve-sim exec <command...> [-d udid] Run a host shell command through a server
                                       started with --allow-exec. One argument
                                       is a shell line (exec "xcrun simctl list");
                                       several are quoted as an argv
+serve-sim doctor [-d udid]            Collect redacted diagnostics for bug reports
+                                      (--json, --out <file>)
 
 serve-sim camera <bundle-id> [-d udid] [source-options]
                                       Inject a synthetic camera feed and (re)launch the app

package/Sources/SimNative/CaptureEngine.swift

@@ -33,8 +33,10 @@ actor CaptureConsumer<E: FrameEncoder>: CaptureConsuming {
 
     init(
         encoder: E,
+        diagnostics: EncoderDiagnostics? = nil,
         onFrame: @escaping @isolated(any) (E.Encoded) async -> Void
     ) {
+        let codec = String(describing: E.self).replacingOccurrences(of: "Encoder", with: "")
         let (stream, continuation) = AsyncStream.makeStream(
             of: Frame.self,
             // drop old frames if there's backpressure
@@ -46,9 +48,17 @@ actor CaptureConsumer<E: FrameEncoder>: CaptureConsuming {
             for await frame in stream {
                 do {
                     let encoded = try await encoder.encode(frame)
+                    diagnostics?.recordSuccess(codec: codec)
                     await onFrame(encoded)
                 } catch {
-                    print("error encoding \(E.self) frame: \(error)")
+                    // One line per distinct failure plus periodic counts; the
+                    // per-frame print used to flood the log (x20 per second).
+                    let size = frame.pixelBuffer.dimensions
+                    if let diagnostics {
+                        diagnostics.recordFailure(codec: codec, error: error, width: size.width, height: size.height)
+                    } else {
+                        print("error encoding \(E.self) frame: \(error)")
+                    }
                     continue
                 }
             }
@@ -76,6 +86,7 @@ actor CaptureEngine {
 
     // mjpeg is stateless so we can share a single encoder instance
     private let mjpegEncoder = MJPEGEncoder()
+    private let encoderDiagnostics = EncoderDiagnostics()
 
     private(set) var screenSize = Dimensions(width: 0, height: 0)
     private var consumers = [UUID: CaptureConsuming]()
@@ -88,6 +99,20 @@ actor CaptureEngine {
         await frameCapture.setPreferredScreenSize(width: width, height: height)
     }
 
+    /// JSON snapshot for `serve-sim doctor`: display selection + encoder state.
+    func diagnostics() async -> String {
+        let capture = await frameCapture.diagnosticsJSON()
+        let encoders = jsonString(encoderDiagnostics.snapshot())
+        let phaseName: String
+        switch phase {
+        case .unstarted: phaseName = "unstarted"
+        case .starting: phaseName = "starting"
+        case .running: phaseName = "running"
+        case .stopped: phaseName = "stopped"
+        }
+        return "{\"phase\":\"\(phaseName)\",\"consumers\":\(consumers.count),\"capture\":\(capture),\"encoders\":\(encoders)}"
+    }
+
     func start() async throws {
         guard phase == .unstarted else { return }
         phase = .starting
@@ -130,7 +155,7 @@ actor CaptureEngine {
         encoder: E,
         onFrame: sending @escaping @isolated(any) (E.Encoded) async -> Void
     ) -> (@Sendable () async -> Void) {
-        let consumer = CaptureConsumer(encoder: encoder) { [weak self] encoded in
+        let consumer = CaptureConsumer(encoder: encoder, diagnostics: encoderDiagnostics) { [weak self] encoded in
             guard let self, await self.phase == .running else { return }
             await onFrame(encoded)
         }
@@ -165,7 +190,7 @@ actor CaptureEngine {
     func addAVCCConsumer(
         onFrame: sending @escaping (Dimensions, Data, Int32) async -> Void
     ) -> (@Sendable () async -> Void) {
-        addConsumer(encoder: AVCCEncoder()) { [weak self] encoded in
+        addConsumer(encoder: AVCCEncoder(diagnostics: encoderDiagnostics)) { [weak self] encoded in
             let flagDescription: Int32 = 1 << 0
             let flagKeyframe: Int32 = 1 << 1
 
@@ -219,10 +244,12 @@ actor MJPEGEncoder: FrameEncoder {
 actor AVCCEncoder: FrameEncoder {
     private static let timeout: Duration = .milliseconds(500)
 
-    let h264Encoder = H264Encoder(fps: 60)
+    let h264Encoder: H264Encoder
     var forceKeyframe = true
 
-    init() {}
+    init(diagnostics: EncoderDiagnostics? = nil) {
+        h264Encoder = H264Encoder(fps: 60, diagnostics: diagnostics)
+    }
 
     func encode(_ frame: Frame) async throws -> H264Encoder.Encoded? {
         // TODO: cancel after timeout using TaskGroup

package/Sources/SimNative/EncoderDiagnostics.swift

@@ -0,0 +1,96 @@
+import Foundation
+import SimNativeSupport
+
+/// Per-capture-engine encoder counters for `serve-sim doctor`, plus the
+/// collapsed encoder-error log. Touched once per failed encode, per session
+/// rebuild, and (cheaply) per successful encode after a failure.
+final class EncoderDiagnostics: @unchecked Sendable {
+    private struct Count {
+        var count: Int
+        var first: Date
+        var last: Date
+    }
+
+    private let lock = NSLock()
+    // Per codec: MJPEG and AVCC consumers share this instance.
+    private var log = CodecEncoderErrorLogs()
+    private var errors: [EncoderErrorKey: Count] = [:]
+    private var sessions: [[String: Any]] = []
+
+    static func describe(_ error: Error) -> String {
+        if let h264 = error as? H264Encoder.Errors, case .encodingFailed(let status) = h264 {
+            return "encodingFailed: \(VideoToolboxStatus.describe(status))"
+        }
+        return String(describing: error)
+    }
+
+    func recordFailure(codec: String, error: Error, width: Int, height: Int) {
+        let key = EncoderErrorKey(codec: codec, error: Self.describe(error), width: width, height: height)
+        let now = Date()
+        lock.lock()
+        var entry = errors[key] ?? Count(count: 0, first: now, last: now)
+        entry.count += 1
+        entry.last = now
+        errors[key] = entry
+        let lines = log.record(key, at: now.timeIntervalSince1970)
+        lock.unlock()
+        for line in lines { print(line) }
+    }
+
+    func recordSuccess(codec: String) {
+        lock.lock()
+        guard log.isFailing(codec: codec) else { lock.unlock(); return }
+        let lines = log.recovered(codec: codec, at: Date().timeIntervalSince1970)
+        lock.unlock()
+        for line in lines { print(line) }
+    }
+
+    func recordSession(codec: String, width: Int, height: Int, status: Int32, lowLatency: Bool) {
+        lock.lock()
+        let session: [String: Any] = [
+            "at": isoTimestamp(Date()),
+            "codec": codec,
+            "width": width,
+            "height": height,
+            "status": Int(status),
+            "status_name": status == 0 ? "noErr" : VideoToolboxStatus.describe(status),
+            "low_latency": lowLatency,
+        ]
+        sessions.append(session)
+        if sessions.count > 10 { sessions.removeFirst(sessions.count - 10) }
+        lock.unlock()
+    }
+
+    func snapshot() -> [String: Any] {
+        lock.lock()
+        defer { lock.unlock() }
+        let errorList: [[String: Any]] = errors
+            .sorted { $0.value.last > $1.value.last }
+            .map { entry -> [String: Any] in
+                [
+                    "codec": entry.key.codec,
+                    "error": entry.key.error,
+                    "width": entry.key.width,
+                    "height": entry.key.height,
+                    "count": entry.value.count,
+                    "first_at": isoTimestamp(entry.value.first),
+                    "last_at": isoTimestamp(entry.value.last),
+                ]
+            }
+        return ["sessions": sessions, "errors": errorList]
+    }
+}
+
+func isoTimestamp(_ date: Date) -> String {
+    let formatter = ISO8601DateFormatter()
+    formatter.formatOptions = [.withInternetDateTime, .withFractionalSeconds]
+    return formatter.string(from: date)
+}
+
+func jsonString(_ object: Any) -> String {
+    guard JSONSerialization.isValidJSONObject(object),
+          let data = try? JSONSerialization.data(withJSONObject: object, options: [.sortedKeys]),
+          let string = String(data: data, encoding: .utf8)
+    else { return "{}" }
+    return string
+}

package/Sources/SimNative/FrameCapture.swift

@@ -45,6 +45,11 @@ actor FrameCapture {
     private var preferredScreenSize: FramebufferSurfaceSize?
     private var didLogRejectedPresentationSurface = false
     private var didLogMissingExpectedSurface = false
+    // Diagnostics for `serve-sim doctor`: what was live, what was chosen, why.
+    private var lastCandidates: [FramebufferSurfaceSize] = []
+    private var lastSelectedIndex: Int?
+    private var lastSelectionReason = "none"
+    private var sizeChanges: [[String: Any]] = []
 
     func start(deviceUDID: String, onFrame: @escaping @Sendable (CVPixelBuffer, CMTime) -> Void) throws {
         self.onFrame = onFrame
@@ -166,13 +171,20 @@ actor FrameCapture {
                 height: IOSurfaceGetHeight(surf)
             )
         }
+        lastCandidates = sizes
         guard let selection = FramebufferSurfaceSelector.select(
             from: sizes,
             expectedSizes: expectedScreenSizes,
             preferredSize: preferredScreenSize
         ) else {
+            lastSelectedIndex = nil
+            lastSelectionReason = "no_live_surface"
             return nil
         }
+        lastSelectedIndex = selection.index
+        lastSelectionReason = Self.selectionReason(
+            selection, selected: sizes[selection.index], preferred: preferredScreenSize
+        )
 
         if selection.matchedExpectedSize, !didLogRejectedPresentationSurface {
             let selected = sizes[selection.index]
@@ -180,10 +192,19 @@ actor FrameCapture {
             if let larger = sizes.filter(\.isLive).max(by: {
                 Int64($0.width) * Int64($0.height) < Int64($1.width) * Int64($1.height)
             }), Int64(larger.width) * Int64(larger.height) > selectedArea {
-                print(
-                    "[capture] Ignoring non-device framebuffer \(larger.width)x\(larger.height); "
-                    + "native screen is \(selected.width)x\(selected.height)"
-                )
+                // On a foldable the larger surface is usually the other native
+                // panel (Duo inner), not a presentation surface. Say which.
+                if expectedScreenSizes.contains(where: { Self.sameSize($0, larger) }) {
+                    print(
+                        "[capture] Selected native display \(selected.width)x\(selected.height) "
+                        + "(\(lastSelectionReason)); other native display \(larger.width)x\(larger.height) is also live"
+                    )
+                } else {
+                    print(
+                        "[capture] Ignoring non-device framebuffer \(larger.width)x\(larger.height); "
+                        + "native screen is \(selected.width)x\(selected.height)"
+                    )
+                }
                 didLogRejectedPresentationSurface = true
             }
         } else if
@@ -276,9 +297,18 @@ actor FrameCapture {
         guard w > 0, h > 0 else { return }
 
         if capturedWidth != w || capturedHeight != h {
+            let from = "\(capturedWidth)x\(capturedHeight)"
             capturedWidth = w
             capturedHeight = h
-            print("[capture] Surface size changed: \(w)x\(h)")
+            print("[capture] Surface size changed: \(from) -> \(w)x\(h) (\(lastSelectionReason))")
+            let change: [String: Any] = [
+                "at": isoTimestamp(Date()),
+                "from": from,
+                "to": "\(w)x\(h)",
+                "reason": lastSelectionReason,
+            ]
+            sizeChanges.append(change)
+            if sizeChanges.count > 20 { sizeChanges.removeFirst(sizeChanges.count - 20) }
         }
 
         var pixelBuffer: Unmanaged<CVPixelBuffer>?
@@ -326,6 +356,10 @@ actor FrameCapture {
         preferredScreenSize = nil
         didLogRejectedPresentationSurface = false
         didLogMissingExpectedSurface = false
+        lastCandidates = []
+        lastSelectedIndex = nil
+        lastSelectionReason = "none"
+        sizeChanges = []
     }
 
     func setPreferredScreenSize(width: Int, height: Int) {
@@ -339,6 +373,50 @@ actor FrameCapture {
         captureFrame(force: true)
     }
 
+    /// JSON snapshot of display selection for `serve-sim doctor`.
+    func diagnosticsJSON() -> String {
+        func size(_ s: FramebufferSurfaceSize) -> [String: Any] { ["width": s.width, "height": s.height] }
+        let candidates: [[String: Any]] = lastCandidates.enumerated().map { entry -> [String: Any] in
+            [
+                "width": entry.element.width,
+                "height": entry.element.height,
+                "live": entry.element.isLive,
+                "native": expectedScreenSizes.contains(where: { Self.sameSize($0, entry.element) }),
+                "selected": entry.offset == lastSelectedIndex,
+            ]
+        }
+        let age = ContinuousClock.now - lastCaptureTime
+        let ageMs = Int(age.components.seconds) * 1000
+            + Int(age.components.attoseconds / 1_000_000_000_000_000)
+        var object: [String: Any] = [
+            "native_displays": expectedScreenSizes.map(size),
+            "candidates": candidates,
+            "selection_reason": lastSelectionReason,
+            "captured": ["width": capturedWidth, "height": capturedHeight],
+            "frame_count": Int(frameCount),
+            "size_changes": sizeChanges,
+        ]
+        if frameCount > 0 { object["last_frame_age_ms"] = ageMs }
+        if let preferredScreenSize { object["preferred"] = size(preferredScreenSize) }
+        return jsonString(object)
+    }
+
+    private static func sameSize(_ a: FramebufferSurfaceSize, _ b: FramebufferSurfaceSize) -> Bool {
+        (a.width == b.width && a.height == b.height) || (a.width == b.height && a.height == b.width)
+    }
+
+    /// preferred: pinned by the Duo panel follower (or WS 0x0d).
+    /// native_smallest: no pin; smallest native display (foldable rest pose).
+    /// largest_fallback: no native-size metadata matched.
+    private static func selectionReason(
+        _ selection: FramebufferSurfaceSelection,
+        selected: FramebufferSurfaceSize,
+        preferred: FramebufferSurfaceSize?
+    ) -> String {
+        if let preferred, preferred.isLive, sameSize(preferred, selected) { return "preferred" }
+        return selection.matchedExpectedSize ? "native_smallest" : "largest_fallback"
+    }
+
     // MARK: - Helpers
 
     private func makeError(_ code: Int, _ msg: String) -> NSError {

package/Sources/SimNative/H264Encoder.swift

@@ -32,9 +32,12 @@ actor H264Encoder {
     private var emittedDescription = false
     private var frameCount: Int64 = 0
 
-    init(fps: Int = 60, bitrate: Int = 6_000_000) {
+    private let diagnostics: EncoderDiagnostics?
+
+    init(fps: Int = 60, bitrate: Int = 6_000_000, diagnostics: EncoderDiagnostics? = nil) {
         self.fps = Int32(fps)
         self.bitrate = bitrate
+        self.diagnostics = diagnostics
     }
 
     deinit {
@@ -125,10 +128,16 @@ actor H264Encoder {
             )
         }
         var status = create(spec: lowLatencySpec)
+        var lowLatency = true
         if status != noErr || sess == nil {
             sess = nil
+            lowLatency = false
             status = create(spec: nil)
         }
+        diagnostics?.recordSession(
+            codec: "AVCC", width: Int(width), height: Int(height),
+            status: status, lowLatency: lowLatency
+        )
         guard status == noErr, let sess else { return }
 
         let props: [(CFString, Any)] = [

package/Sources/SimNative/sim-module.swift

@@ -173,6 +173,12 @@ private func u32(_ v: Int) -> UInt32 {
         try await engine.start()
     }
 
+    /// JSON string: display selection, surface-size history, encoder sessions
+    /// and error counts. Read by `/api/diagnostics` (`serve-sim doctor`).
+    @NodeMethod func diagnostics() async -> String {
+        await engine.diagnostics()
+    }
+
     @NodeMethod func stop() async {
         await engine.stop()
     }

package/Sources/SimNativeSupport/EncoderErrorLog.swift

@@ -0,0 +1,123 @@
+import Foundation
+
+/// Names for the VideoToolbox OSStatus values an encoder session can return.
+/// Kept as a table (not VideoToolbox imports) so it stays testable anywhere.
+public enum VideoToolboxStatus {
+    private static let names: [Int32: String] = [
+        -12900: "kVTPropertyNotSupportedErr",
+        -12901: "kVTPropertyReadOnlyErr",
+        -12902: "kVTParameterErr",
+        -12903: "kVTInvalidSessionErr",
+        -12904: "kVTAllocationFailedErr",
+        -12905: "kVTPixelTransferNotSupportedErr",
+        -12906: "kVTCouldNotFindVideoDecoderErr",
+        -12907: "kVTCouldNotCreateInstanceErr",
+        -12908: "kVTCouldNotFindVideoEncoderErr",
+        -12912: "kVTVideoEncoderMalfunctionErr",
+        -12915: "kVTVideoEncoderNotAvailableNowErr",
+        -12916: "kVTFormatDescriptionChangeNotSupportedErr",
+        -12211: "kVTVideoEncoderAuthorizationErr",
+        -12218: "kVTPixelTransferNotPermittedErr",
+        -17691: "kVTSessionMalfunctionErr",
+        -17693: "kVTVideoEncoderNeedsRosettaErr",
+    ]
+
+    public static func name(_ status: Int32) -> String? { names[status] }
+
+    /// "kVTPixelTransferNotSupportedErr (-12905)", or "OSStatus -1" when unknown.
+    public static func describe(_ status: Int32) -> String {
+        if let name = names[status] { return "\(name) (\(status))" }
+        return "OSStatus \(status)"
+    }
+}
+
+public struct EncoderErrorKey: Hashable, Sendable {
+    public let codec: String
+    public let error: String
+    public let width: Int
+    public let height: Int
+
+    public init(codec: String, error: String, width: Int, height: Int) {
+        self.codec = codec
+        self.error = error
+        self.width = width
+        self.height = height
+    }
+
+    var line: String { "[capture] \(codec) encode failed at \(width)x\(height): \(error)" }
+}
+
+/// Collapses a per-frame encoder error into one line plus periodic counts.
+/// The first failure for a key prints immediately. Repeats are counted and
+/// summarized every `summaryInterval` seconds, when the key changes, or when
+/// encoding recovers. Time is injected so the policy is unit-testable.
+public struct EncoderErrorLog: Sendable {
+    public let summaryInterval: Double
+    private var key: EncoderErrorKey?
+    private var suppressed = 0
+    private var windowStart: Double = 0
+
+    public init(summaryInterval: Double = 10) {
+        self.summaryInterval = summaryInterval
+    }
+
+    /// True between a recorded failure and the next `recovered`.
+    public var isFailing: Bool { key != nil }
+
+    public mutating func record(_ next: EncoderErrorKey, at now: Double) -> [String] {
+        if key == next {
+            suppressed += 1
+            guard now - windowStart >= summaryInterval else { return [] }
+            return [summary(at: now)]
+        }
+        var lines: [String] = []
+        if key != nil, suppressed > 0 { lines.append(summary(at: now)) }
+        key = next
+        suppressed = 0
+        windowStart = now
+        lines.append(next.line)
+        return lines
+    }
+
+    /// A successful encode ends the run; report anything still uncounted.
+    public mutating func recovered(at now: Double) -> [String] {
+        guard key != nil else { return [] }
+        let lines = suppressed > 0 ? [summary(at: now, suffix: "; recovered")] : []
+        key = nil
+        suppressed = 0
+        return lines
+    }
+
+    private mutating func summary(at now: Double, suffix: String = "") -> String {
+        let seconds = max(0, now - windowStart)
+        let line = "\(key!.line) repeated \(suppressed)x in \(String(format: "%.1f", seconds))s\(suffix)"
+        suppressed = 0
+        windowStart = now
+        return line
+    }
+}
+
+/// One `EncoderErrorLog` per codec. The MJPEG and AVCC consumers share a
+/// capture engine, so a success on one codec must not end (and reset the
+/// suppression of) a failure run on the other.
+public struct CodecEncoderErrorLogs: Sendable {
+    public let summaryInterval: Double
+    private var logs: [String: EncoderErrorLog] = [:]
+
+    public init(summaryInterval: Double = 10) {
+        self.summaryInterval = summaryInterval
+    }
+
+    public func isFailing(codec: String) -> Bool {
+        logs[codec]?.isFailing ?? false
+    }
+
+    public mutating func record(_ next: EncoderErrorKey, at now: Double) -> [String] {
+        logs[next.codec, default: EncoderErrorLog(summaryInterval: summaryInterval)].record(next, at: now)
+    }
+
+    public mutating func recovered(codec: String, at now: Double) -> [String] {
+        guard isFailing(codec: codec) else { return [] }
+        return logs[codec]!.recovered(at: now)
+    }
+}

package/Tests/SimNativeSupportTests/EncoderErrorLogTests.swift

@@ -0,0 +1,55 @@
+import XCTest
+@testable import SimNativeSupport
+
+final class EncoderErrorLogTests: XCTestCase {
+    private let inner = EncoderErrorKey(
+        codec: "AVCC",
+        error: VideoToolboxStatus.describe(-12905),
+        width: 2007,
+        height: 2853
+    )
+
+    func testNamesPixelTransferStatus() {
+        XCTAssertEqual(VideoToolboxStatus.describe(-12905), "kVTPixelTransferNotSupportedErr (-12905)")
+        XCTAssertEqual(VideoToolboxStatus.describe(-1), "OSStatus -1")
+    }
+
+    func testFirstErrorPrintsOnceThenRepeatsAreCounted() {
+        var log = EncoderErrorLog(summaryInterval: 10)
+        XCTAssertEqual(log.record(inner, at: 0), [
+            "[capture] AVCC encode failed at 2007x2853: kVTPixelTransferNotSupportedErr (-12905)",
+        ])
+        for i in 1...19 { XCTAssertEqual(log.record(inner, at: Double(i) * 0.1), []) }
+        XCTAssertEqual(log.recovered(at: 2), [
+            "[capture] AVCC encode failed at 2007x2853: kVTPixelTransferNotSupportedErr (-12905) repeated 19x in 2.0s; recovered",
+        ])
+        XCTAssertEqual(log.recovered(at: 3), [])
+    }
+
+    func testPeriodicSummaryAndKeyChange() {
+        var log = EncoderErrorLog(summaryInterval: 10)
+        _ = log.record(inner, at: 0)
+        XCTAssertEqual(log.record(inner, at: 5), [])
+        XCTAssertEqual(log.record(inner, at: 10).count, 1)
+        let cover = EncoderErrorKey(codec: "AVCC", error: inner.error, width: 1398, height: 2034)
+        XCTAssertEqual(log.record(cover, at: 11), [
+            "[capture] AVCC encode failed at 1398x2034: kVTPixelTransferNotSupportedErr (-12905)",
+        ])
+    }
+
+    func testSuccessOnOneCodecDoesNotResetAnotherCodecsSuppression() {
+        var logs = CodecEncoderErrorLogs(summaryInterval: 10)
+        XCTAssertEqual(logs.record(inner, at: 0).count, 1)
+        // MJPEG keeps encoding fine while AVCC fails every frame.
+        XCTAssertEqual(logs.recovered(codec: "MJPEG", at: 0.05), [])
+        XCTAssertEqual(logs.record(inner, at: 0.1), [])
+        XCTAssertEqual(logs.recovered(codec: "MJPEG", at: 0.15), [])
+        XCTAssertEqual(logs.record(inner, at: 0.2), [])
+        XCTAssertTrue(logs.isFailing(codec: "AVCC"))
+        XCTAssertFalse(logs.isFailing(codec: "MJPEG"))
+        XCTAssertEqual(logs.recovered(codec: "AVCC", at: 1), [
+            "[capture] AVCC encode failed at 2007x2853: kVTPixelTransferNotSupportedErr (-12905) repeated 2x in 1.0s; recovered",
+        ])
+        XCTAssertFalse(logs.isFailing(codec: "AVCC"))
+    }
+}