serve-sim-sjchmiela 0.1.39 → 0.1.41

package/Sources/SimNative/sim-capture.swift

@@ -15,6 +15,13 @@ import CoreMedia
 typealias SimFrameCallback = (Int32, Data, Int32, Int32, Int32) -> Void
 typealias SimInputCallback = (Data) -> Void
 
+struct CaptureEngineOptions {
+    let mjpegFps: Int
+    let mjpegQuality: Double
+    let h264Fps: Int
+    let h264Bitrate: Int
+}
+
 final class CaptureEngine {
     static let codecMJPEG: Int32 = 0
     static let codecAVCC: Int32 = 1
@@ -26,8 +33,8 @@ final class CaptureEngine {
     private let webRTCPublisher = WebRTCPublisher()
 
     private let frameCapture = FrameCapture()
-    private let videoEncoder = VideoEncoder(quality: 0.7)
-    private let h264Encoder = H264Encoder(fps: 60)
+    private let videoEncoder: VideoEncoder
+    private let h264Encoder: H264Encoder
     private let encodeQueue = DispatchQueue(label: "napi.encode", qos: .userInteractive)
     private let h264Queue = DispatchQueue(label: "napi.encode.h264", qos: .userInteractive)
     private static let h264EncodeTimeoutMs = 500
@@ -42,26 +49,55 @@ final class CaptureEngine {
     private var forceKeyframe = false
     private var avccActive = false
     private var h264FrameToken: UInt64 = 0
+    private var h264ReservedCount: Int64 = 0
+    private var h264SubmittedCount: Int64 = 0
+    private var h264BackpressureSkips: Int64 = 0
+    private var mjpegThrottleSkips: Int64 = 0
+    private var h264ThrottleSkips: Int64 = 0
+    private var avccNativeEmitCount: Int64 = 0
+    private var lastMjpegReservedAtNs: UInt64 = 0
+    private var lastH264ReservedAtNs: UInt64 = 0
+    private let mjpegMinFrameIntervalNs: UInt64
+    private let h264MinFrameIntervalNs: UInt64
     private var started = false
     private var stopped = false
 
-    init(deviceUDID: String, onFrame: @escaping SimFrameCallback, onWebRTCInput: @escaping SimInputCallback) {
+    init(
+        deviceUDID: String,
+        options: CaptureEngineOptions,
+        onFrame: @escaping SimFrameCallback,
+        onWebRTCInput: @escaping SimInputCallback
+    ) {
         self.deviceUDID = deviceUDID
         self.onFrame = onFrame
+        let mjpegFps = max(1, options.mjpegFps)
+        let h264Fps = max(1, options.h264Fps)
+        let h264Bitrate = max(1, options.h264Bitrate)
+        videoEncoder = VideoEncoder(quality: CGFloat(min(max(options.mjpegQuality, 0.0), 1.0)))
+        h264Encoder = H264Encoder(fps: h264Fps, bitrate: h264Bitrate)
+        mjpegMinFrameIntervalNs = UInt64(1_000_000_000 / mjpegFps)
+        h264MinFrameIntervalNs = UInt64(1_000_000_000 / h264Fps)
         webRTCPublisher.onInput = onWebRTCInput
 
         h264Encoder.onEncoded = { [weak self] encoded in
             guard let self else { return }
+            self.avccNativeEmitCount += 1
+            let emitCount = self.avccNativeEmitCount
             if let description = encoded.description {
+                streamLog("[stream:avcc] native emit description bytes=\(description.count)")
                 self.emit(codec: Self.codecAVCC,
                           data: AVCCEnvelope.description(avcc: description),
                           flags: Self.flagDescription)
             }
             switch encoded.kind {
             case .keyframe:
+                streamLog("[stream:avcc] native emit keyframe bytes=\(encoded.avcc.count)")
                 self.emit(codec: Self.codecAVCC, data: AVCCEnvelope.keyframe(avcc: encoded.avcc),
                           flags: Self.flagKeyframe)
             case .delta:
+                if streamShouldLog(emitCount) {
+                    streamLog("[stream:avcc] native emit delta #\(emitCount) bytes=\(encoded.avcc.count)")
+                }
                 self.emit(codec: Self.codecAVCC, data: AVCCEnvelope.delta(avcc: encoded.avcc), flags: 0)
             }
         }
@@ -100,11 +136,12 @@ final class CaptureEngine {
 
         let h264Request = reserveH264EncodeIfNeeded()
         let shouldSendWebRTC = webRTCPublisher.isActive
-        let shouldEncodeJpeg = encoderReady && !encoding
+        let shouldEncodeJpeg = encoderReady && !encoding && reserveMjpegEncodeIfNeeded()
         if !shouldEncodeJpeg && h264Request == nil && !shouldSendWebRTC { return }
 
         guard let stableFrame = copyPixelBuffer(pixelBuffer) else {
             if let h264Request {
+                streamLog("[stream:avcc] failed to copy capture frame for H.264 token=\(h264Request.token)")
                 finishH264Encode(token: h264Request.token, restoreKeyframe: h264Request.forceKeyframe)
             }
             return
@@ -128,6 +165,14 @@ final class CaptureEngine {
         if let h264Request {
             h264Queue.async { [weak self] in
                 guard let self else { return }
+                self.h264SubmittedCount += 1
+                let submitted = self.h264SubmittedCount
+                if streamShouldLog(submitted) || h264Request.forceKeyframe {
+                    streamLog(
+                        "[stream:avcc] send frame to H264Encoder #\(submitted) token=\(h264Request.token) " +
+                        "forceKeyframe=\(h264Request.forceKeyframe)"
+                    )
+                }
                 self.h264Encoder.encode(stableFrame, forceKeyframe: h264Request.forceKeyframe) {
                     self.finishH264Encode(token: h264Request.token)
                 }
@@ -136,6 +181,19 @@ final class CaptureEngine {
         }
     }
 
+    private func reserveMjpegEncodeIfNeeded() -> Bool {
+        let now = DispatchTime.now().uptimeNanoseconds
+        if lastMjpegReservedAtNs != 0 && now - lastMjpegReservedAtNs < mjpegMinFrameIntervalNs {
+            mjpegThrottleSkips += 1
+            if streamShouldLog(mjpegThrottleSkips) {
+                streamLog("[stream:mjpeg] skip frame: fps throttle")
+            }
+            return false
+        }
+        lastMjpegReservedAtNs = now
+        return true
+    }
+
     /// Copy the live Simulator IOSurface immediately on the capture queue. The
     /// encoders run later and SimulatorKit recycles/mutates that IOSurface in
     /// place, so passing the wrapper CVPixelBuffer across queues can encode a
@@ -176,12 +234,32 @@ final class CaptureEngine {
 
     private func reserveH264EncodeIfNeeded() -> (forceKeyframe: Bool, token: UInt64)? {
         h264Queue.sync {
-            guard avccActive, !h264Encoding else { return nil }
+            guard avccActive else { return nil }
+            let now = DispatchTime.now().uptimeNanoseconds
+            if !forceKeyframe && lastH264ReservedAtNs != 0 && now - lastH264ReservedAtNs < h264MinFrameIntervalNs {
+                h264ThrottleSkips += 1
+                if streamShouldLog(h264ThrottleSkips) {
+                    streamLog("[stream:avcc] skip H.264 frame: fps throttle")
+                }
+                return nil
+            }
+            guard !h264Encoding else {
+                h264BackpressureSkips += 1
+                if streamShouldLog(h264BackpressureSkips) {
+                    streamLog("[stream:avcc] skip H.264 frame: encode pending token=\(h264FrameToken)")
+                }
+                return nil
+            }
             h264Encoding = true
             h264FrameToken &+= 1
             let token = h264FrameToken
             let force = forceKeyframe
             forceKeyframe = false
+            lastH264ReservedAtNs = now
+            h264ReservedCount += 1
+            if streamShouldLog(h264ReservedCount) || force {
+                streamLog("[stream:avcc] reserved H.264 frame #\(h264ReservedCount) token=\(token) forceKeyframe=\(force)")
+            }
             return (forceKeyframe: force, token: token)
         }
     }
@@ -196,8 +274,10 @@ final class CaptureEngine {
 
     private func scheduleH264EncodeTimeout(token: UInt64) {
         h264Queue.asyncAfter(deadline: .now().advanced(by: .milliseconds(Self.h264EncodeTimeoutMs))) { [weak self] in
-            guard let self, self.h264FrameToken == token else { return }
+            guard let self, self.h264FrameToken == token, self.h264Encoding else { return }
             self.h264Encoding = false
+            self.h264Encoder.handleEncodeTimeout()
+            streamLog("[stream:avcc] H.264 encode timed out token=\(token)")
         }
     }
 
@@ -207,12 +287,18 @@ final class CaptureEngine {
         h264Queue.async { [weak self] in
             guard let self else { return }
             if active && !self.avccActive { self.forceKeyframe = true }
+            if active != self.avccActive {
+                streamLog("[stream:avcc] active=\(active) forceKeyframe=\(self.forceKeyframe)")
+            }
             self.avccActive = active
         }
     }
 
     func requestKeyframe() {
-        h264Queue.async { [weak self] in self?.forceKeyframe = true }
+        h264Queue.async { [weak self] in
+            self?.forceKeyframe = true
+            streamLog("[stream:avcc] keyframe requested")
+        }
     }
 
     func handleWebRTCOffer(_ offerJson: String) throws -> String {

package/Sources/SimNative/sim-module.swift

@@ -99,7 +99,15 @@ private func u32(_ v: Int) -> UInt32 {
     private let queue: NodeAsyncQueue
     private let inputQueue: NodeAsyncQueue
 
-    @NodeConstructor init(_ udid: String, _ onFrame: NodeFunction, _ onWebRTCInput: NodeFunction) throws {
+    @NodeConstructor init(
+        _ udid: String,
+        _ onFrame: NodeFunction,
+        _ onWebRTCInput: NodeFunction,
+        _ mjpegFps: Int,
+        _ mjpegQuality: Double,
+        _ h264Fps: Int,
+        _ h264Bitrate: Int
+    ) throws {
         // unref'd by NodeAsyncQueue's init, so the frame pipeline alone won't
         // keep the event loop alive. Bounded queue + blocking AVCC preserves
         // inter-frame ordering; MJPEG is nonblocking and drops under backpressure.
@@ -112,7 +120,12 @@ private func u32(_ v: Int) -> UInt32 {
 
         // Capture the locals (not self) so the closure can be built before the
         // engine property is initialized, and so it holds no strong ref to self.
-        engine = CaptureEngine(deviceUDID: udid, onFrame: { codec, data, w, h, flags in
+        engine = CaptureEngine(deviceUDID: udid, options: CaptureEngineOptions(
+            mjpegFps: mjpegFps,
+            mjpegQuality: mjpegQuality,
+            h264Fps: h264Fps,
+            h264Bitrate: h264Bitrate
+        ), onFrame: { codec, data, w, h, flags in
             // Runs on a native encode thread. AVCC is inter-frame H.264 — dropping
             // a delta corrupts the decoder until the next IDR — so deliver it
             // blocking; MJPEG is stateless and safe to drop. We copy the bytes

package/Sources/SimStreamHelper/H264Encoder.swift

@@ -33,6 +33,10 @@ final class H264Encoder {
     private let stateQueue = DispatchQueue(label: "H264Encoder.state")
     private var emittedDescription = false
     private var frameCount: Int64 = 0
+    private var encodedCount: Int64 = 0
+    private var lowLatencyEnabled = true
+    private var forceKeyframeAfterReset = false
+    private var retiredSessions: [VTCompressionSession] = []
 
     init(fps: Int = 60, bitrate: Int = 6_000_000) {
         self.fps = Int32(fps)
@@ -41,6 +45,7 @@ final class H264Encoder {
 
     deinit {
         if let session { VTCompressionSessionInvalidate(session) }
+        for session in retiredSessions { VTCompressionSessionInvalidate(session) }
     }
 
     /// Submit a frame. Returns immediately; `onEncoded` fires on VT's queue.
@@ -53,17 +58,33 @@ final class H264Encoder {
             height = h
             rebuildSession()
         }
-        guard let session, let copy = copyBuffer(source) else {
+        guard let session else {
+            streamLog("[stream:h264] drop frame: VTCompressionSession unavailable size=\(w)x\(h)")
+            lock.unlock()
+            completion?()
+            return
+        }
+        guard let copy = copyBuffer(source) else {
+            streamLog("[stream:h264] drop frame: failed to copy pixel buffer size=\(w)x\(h)")
             lock.unlock()
             completion?()
             return
         }
 
         frameCount += 1
+        let submittedFrame = frameCount
         let pts = CMTime(value: frameCount, timescale: fps)
-        let frameProps: NSDictionary? = forceKeyframe
+        let effectiveForceKeyframe = forceKeyframe || forceKeyframeAfterReset
+        forceKeyframeAfterReset = false
+        let frameProps: NSDictionary? = effectiveForceKeyframe
             ? [kVTEncodeFrameOptionKey_ForceKeyFrame: kCFBooleanTrue!] as NSDictionary
             : nil
+        if streamShouldLog(submittedFrame) || effectiveForceKeyframe {
+            streamLog(
+                "[stream:h264] submit frame #\(submittedFrame) size=\(w)x\(h) " +
+                "forceKeyframe=\(effectiveForceKeyframe)"
+            )
+        }
         lock.unlock()
 
         let status = VTCompressionSessionEncodeFrame(
@@ -75,14 +96,42 @@ final class H264Encoder {
             infoFlagsOut: nil
         ) { [weak self] status, _, sampleBuffer in
             defer { completion?() }
-            guard let self, status == noErr, let sb = sampleBuffer else { return }
-            if let encoded = self.extract(from: sb) { self.onEncoded?(encoded) }
+            guard let self else { return }
+            guard status == noErr else {
+                streamLog("[stream:h264] encode callback failed frame #\(submittedFrame) status=\(status)")
+                self.fallbackFromLowLatency(reason: "callback status=\(status)")
+                return
+            }
+            guard let sb = sampleBuffer else {
+                streamLog("[stream:h264] encode callback missing sample frame #\(submittedFrame)")
+                self.fallbackFromLowLatency(reason: "callback missing sample")
+                return
+            }
+            guard let encoded = self.extract(from: sb) else {
+                streamLog("[stream:h264] encode callback produced unextractable sample frame #\(submittedFrame)")
+                return
+            }
+            let encodedFrame = self.nextEncodedCount()
+            if streamShouldLog(encodedFrame) || encoded.description != nil || encoded.kind == .keyframe {
+                let kind = encoded.kind == .keyframe ? "keyframe" : "delta"
+                streamLog(
+                    "[stream:h264] encoded #\(encodedFrame) kind=\(kind) bytes=\(encoded.avcc.count) " +
+                    "descriptionBytes=\(encoded.description?.count ?? 0)"
+                )
+            }
+            self.onEncoded?(encoded)
         }
         if status != noErr {
+            streamLog("[stream:h264] VTCompressionSessionEncodeFrame failed frame #\(submittedFrame) status=\(status)")
+            fallbackFromLowLatency(reason: "submit status=\(status)")
             completion?()
         }
     }
 
+    func handleEncodeTimeout() {
+        fallbackFromLowLatency(reason: "encode timeout")
+    }
+
     func stop() {
         lock.lock()
         defer { lock.unlock() }
@@ -90,6 +139,8 @@ final class H264Encoder {
             VTCompressionSessionInvalidate(session)
             self.session = nil
         }
+        for session in retiredSessions { VTCompressionSessionInvalidate(session) }
+        retiredSessions.removeAll()
         pool = nil
     }
 
@@ -150,17 +201,29 @@ final class H264Encoder {
                 compressionSessionOut: &sess
             )
         }
-        var status = create(spec: lowLatencySpec)
-        if status != noErr || sess == nil {
+        let preferredSpec: CFDictionary?
+        if lowLatencyEnabled {
+            preferredSpec = lowLatencySpec
+        } else {
+            preferredSpec = nil
+        }
+        var status = create(spec: preferredSpec)
+        if lowLatencyEnabled && (status != noErr || sess == nil) {
+            streamLog("[stream:h264] low-latency VT session create failed status=\(status); retrying default")
+            lowLatencyEnabled = false
+            forceKeyframeAfterReset = true
             sess = nil
             status = create(spec: nil)
         }
-        guard status == noErr, let sess else { return }
+        guard status == noErr, let sess else {
+            streamLog("[stream:h264] VT session create failed status=\(status) size=\(width)x\(height)")
+            return
+        }
 
         let props: [(CFString, Any)] = [
             (kVTCompressionPropertyKey_RealTime, kCFBooleanTrue!),
             (kVTCompressionPropertyKey_ProfileLevel, kVTProfileLevel_H264_High_AutoLevel),
-            (kVTCompressionPropertyKey_AllowFrameReordering, kCFBooleanFalse!),
+            (kVTCompressionPropertyKey_AllowFrameReordering, kCFBooleanTrue!),
             (kVTCompressionPropertyKey_AverageBitRate, NSNumber(value: bitrate)),
             (kVTCompressionPropertyKey_ExpectedFrameRate, NSNumber(value: fps)),
             // 5s keyframe interval: IDRs are far larger than P-frames, so
@@ -169,12 +232,19 @@ final class H264Encoder {
             (kVTCompressionPropertyKey_MaxKeyFrameInterval, NSNumber(value: fps * 5)),
         ]
         for (key, value) in props {
-            VTSessionSetProperty(sess, key: key, value: value as CFTypeRef)
+            let propStatus = VTSessionSetProperty(sess, key: key, value: value as CFTypeRef)
+            if propStatus != noErr {
+                streamLog("[stream:h264] VTSessionSetProperty failed key=\(key) status=\(propStatus)")
+            }
+        }
+        let prepareStatus = VTCompressionSessionPrepareToEncodeFrames(sess)
+        if prepareStatus != noErr {
+            streamLog("[stream:h264] VTCompressionSessionPrepareToEncodeFrames status=\(prepareStatus)")
         }
-        VTCompressionSessionPrepareToEncodeFrames(sess)
         session = sess
         stateQueue.sync {
             emittedDescription = false
+            encodedCount = 0
         }
 
         // Pool feeding the deep-copy; BGRA matches the framebuffer surface.
@@ -185,10 +255,41 @@ final class H264Encoder {
             kCVPixelBufferIOSurfacePropertiesKey as String: [:],
         ]
         var newPool: CVPixelBufferPool?
-        CVPixelBufferPoolCreate(kCFAllocatorDefault, nil, attrs as CFDictionary, &newPool)
+        let poolStatus = CVPixelBufferPoolCreate(kCFAllocatorDefault, nil, attrs as CFDictionary, &newPool)
+        if poolStatus != kCVReturnSuccess || newPool == nil {
+            streamLog("[stream:h264] pixel buffer pool create failed status=\(poolStatus) size=\(width)x\(height)")
+        } else {
+            let mode = lowLatencyEnabled ? "low-latency" : "default"
+            streamLog("[stream:h264] VT session ready mode=\(mode) size=\(width)x\(height) fps=\(fps) bitrate=\(bitrate)")
+        }
         pool = newPool
     }
 
+    private func fallbackFromLowLatency(reason: String) {
+        lock.lock()
+        defer { lock.unlock() }
+        guard lowLatencyEnabled else { return }
+        streamLog("[stream:h264] low-latency encoder failed (\(reason)); default VT session will be used")
+        lowLatencyEnabled = false
+        forceKeyframeAfterReset = true
+        if let session {
+            retiredSessions.append(session)
+            self.session = nil
+        }
+        pool = nil
+        stateQueue.sync {
+            emittedDescription = false
+            encodedCount = 0
+        }
+    }
+
+    private func nextEncodedCount() -> Int64 {
+        stateQueue.sync {
+            encodedCount += 1
+            return encodedCount
+        }
+    }
+
     private func extract(from sample: CMSampleBuffer) -> Encoded? {
         let isKeyframe = !notSync(sample)
         guard let dataBuf = CMSampleBufferGetDataBuffer(sample) else { return nil }

package/Sources/SimStreamHelper/StreamFormat.swift

@@ -1,5 +1,21 @@
 import Foundation
 
+/// Opt-in AVCC/H.264 diagnostics for staging stream failures. This path is hot
+/// during playback, so keep it off unless explicitly requested.
+let streamDebugEnabled =
+    ProcessInfo.processInfo.environment["SERVE_SIM_DEBUG_STREAM"] != nil ||
+    ProcessInfo.processInfo.environment["SERVE_SIM_DEBUG_AVCC"] != nil
+
+@inline(__always)
+func streamLog(_ message: @autoclosure () -> String) {
+    if streamDebugEnabled { print(message()) }
+}
+
+@inline(__always)
+func streamShouldLog(_ count: Int64, first: Int64 = 5, every: Int64 = 120) -> Bool {
+    count <= first || count % every == 0
+}
+
 /// Wire format a viewer can request for the screen stream.
 ///
 /// - `mjpeg`: stateless JPEG-per-frame inside a `multipart/x-mixed-replace`

package/Sources/SimStreamHelper/WebRTCPublisher.swift

@@ -12,6 +12,7 @@ struct WebRTCIceServerPayload: Codable {
 struct WebRTCOfferPayload: Codable {
     let type: String
     let sdp: String
+    let codec: String?
     let iceServers: [WebRTCIceServerPayload]?
 }
 
@@ -103,7 +104,11 @@ final class WebRTCPublisher {
             return
         }
 
-        _ = peerConnection.add(videoTrack, streamIds: ["stream0"])
+        if let transceiver = peerConnection.addTransceiver(with: videoTrack) {
+            applyVideoCodecPreference(request.codec, to: transceiver)
+        } else {
+            _ = peerConnection.add(videoTrack, streamIds: ["stream0"])
+        }
         let session = WebRTCSession(peerConnection: peerConnection, delegate: delegate)
         self.session?.close()
         self.session = session
@@ -154,6 +159,25 @@ final class WebRTCPublisher {
         }
     }
 
+    private func applyVideoCodecPreference(_ codec: String?, to transceiver: LKRTCRtpTransceiver) {
+        let preferredName = codec == "vp8" ? "VP8" : "H264"
+        let capabilities = factory.rtpSenderCapabilities(forKind: "video")
+        let preferredCodecs = capabilities.codecs.filter {
+            $0.name.caseInsensitiveCompare(preferredName) == .orderedSame ||
+                $0.mimeType.caseInsensitiveCompare("video/\(preferredName)") == .orderedSame
+        }
+        guard !preferredCodecs.isEmpty else {
+            print("[webrtc] No sender codec capability found for \(preferredName); using default order")
+            return
+        }
+        let remainingCodecs = capabilities.codecs.filter { capability in
+            !preferredCodecs.contains { $0 === capability }
+        }
+        let orderedCodecs = preferredCodecs + remainingCodecs
+        transceiver.codecPreferences = orderedCodecs
+        print("[webrtc] Preferred video codec: \(preferredName)")
+    }
+
     private func makeError(_ message: String) -> Error {
         NSError(domain: "serve-sim.webrtc", code: 1, userInfo: [NSLocalizedDescriptionKey: message])
     }