serve-sim-sjchmiela 0.1.52 → 0.1.53

package/Sources/SimNative/sim-capture.swift

@@ -39,6 +39,8 @@ final class CaptureEngine {
     private let h264Encoder: H264Encoder
     private let encodeQueue = DispatchQueue(label: "napi.encode", qos: .userInteractive)
     private let h264Queue = DispatchQueue(label: "napi.encode.h264", qos: .userInteractive)
+    private let framePreparationQueue = DispatchQueue(label: "napi.frame.prepare", qos: .userInteractive)
+    private let framePreparationLock = NSLock()
     private static let h264EncodeTimeoutMs = 500
 
     // Mirrors main.swift's globals; mutated from the capture queue, read from the
@@ -51,6 +53,7 @@ final class CaptureEngine {
     private var encoding = false       // MJPEG backpressure
     private var h264Encoding = false   // H.264 backpressure
     private var forceKeyframe = false
+    private var mjpegActive = false
     private var avccActive = false
     private var h264FrameToken: UInt64 = 0
     private var h264ReservedCount: Int64 = 0
@@ -60,9 +63,11 @@ final class CaptureEngine {
     private var h264ThrottleSkips: Int64 = 0
     private var webRTCReservedCount: Int64 = 0
     private var webRTCThrottleSkips: Int64 = 0
+    private var webRTCDirectCount: Int64 = 0
     private var webRTCCopyFailures: Int64 = 0
     private var avccNativeEmitCount: Int64 = 0
     private var pixelBufferCopyCount: Int64 = 0
+    private var scaledFramePreparationPending = false
     private let statsLock = NSLock()
     private let statsStartNs = DispatchTime.now().uptimeNanoseconds
     private var statsCaptureFrames: Int64 = 0
@@ -71,10 +76,17 @@ final class CaptureEngine {
     private var statsScaleTotalMs = 0.0
     private var statsScaleLastMs = 0.0
     private var statsScaleMaxMs = 0.0
+    private var statsScaleBackpressureSkips: Int64 = 0
     private var statsScaleInputWidth = 0
     private var statsScaleInputHeight = 0
     private var statsScaleOutputWidth = 0
     private var statsScaleOutputHeight = 0
+    private var statsCopyCount: Int64 = 0
+    private var statsCopyTotalMs = 0.0
+    private var statsCopyLastMs = 0.0
+    private var statsCopyMaxMs = 0.0
+    private var statsCopyWidth = 0
+    private var statsCopyHeight = 0
     private var statsMjpegReserved: Int64 = 0
     private var statsMjpegThrottleSkips: Int64 = 0
     private var statsH264Reserved: Int64 = 0
@@ -82,6 +94,7 @@ final class CaptureEngine {
     private var statsH264BackpressureSkips: Int64 = 0
     private var statsWebRTCReserved: Int64 = 0
     private var statsWebRTCThrottleSkips: Int64 = 0
+    private var statsWebRTCDirect: Int64 = 0
     private var statsWebRTCCopyFailures: Int64 = 0
     private var lastMjpegReservedAtNs: UInt64 = 0
     private var lastH264ReservedAtNs: UInt64 = 0
@@ -168,12 +181,19 @@ final class CaptureEngine {
         recordCapturedFrame()
         let encodeSize = encodedSize(width: w, height: h)
 
-        if !encoderReady || w != screenWidth || h != screenHeight || encodeSize.width != encodeWidth || encodeSize.height != encodeHeight {
+        let dimensionsChanged = w != screenWidth || h != screenHeight ||
+            encodeSize.width != encodeWidth || encodeSize.height != encodeHeight
+        if dimensionsChanged {
             screenWidth = w
             screenHeight = h
             encodeWidth = encodeSize.width
             encodeHeight = encodeSize.height
-            videoEncoder.stop()
+            if encoderReady {
+                videoEncoder.stop()
+                encoderReady = false
+            }
+        }
+        if mjpegActive && !encoderReady {
             videoEncoder.setup(width: Int32(encodeSize.width), height: Int32(encodeSize.height), fps: 60) { [weak self] jpeg in
                 self?.emit(codec: Self.codecMJPEG, data: jpeg, flags: 0)
             }
@@ -182,24 +202,79 @@ final class CaptureEngine {
 
         let h264Request = reserveH264EncodeIfNeeded()
         let shouldSendWebRTC = reserveWebRTCFrameIfNeeded()
-        let shouldEncodeJpeg = encoderReady && !encoding && reserveMjpegEncodeIfNeeded()
+        let shouldEncodeJpeg = mjpegActive && encoderReady && !encoding && reserveMjpegEncodeIfNeeded()
         if !shouldEncodeJpeg && h264Request == nil && !shouldSendWebRTC { return }
 
-        guard let stableFrame = copyPixelBuffer(pixelBuffer, targetWidth: encodeSize.width, targetHeight: encodeSize.height) else {
-            if let h264Request {
-                streamLog("[stream:avcc] failed to copy capture frame for H.264 token=\(h264Request.token)")
-                finishH264Encode(token: h264Request.token, restoreKeyframe: h264Request.forceKeyframe)
+        if shouldSendWebRTC && h264Request == nil && !shouldEncodeJpeg && encodeSize.width == w && encodeSize.height == h {
+            webRTCDirectCount += 1
+            recordWebRTCDirect()
+            if streamShouldLog(webRTCDirectCount) {
+                streamLog("[webrtc] send direct frame #\(webRTCDirectCount) \(w)x\(h)")
+            }
+            webRTCPublisher.sendFrameDirect(pixelBuffer, timestamp: timestamp)
+            return
+        }
+
+        if encodeSize.width != w || encodeSize.height != h {
+            guard reserveScaledFramePreparation() else {
+                recordScaleBackpressureSkip()
+                if shouldEncodeJpeg { lastMjpegReservedAtNs = 0 }
+                if let h264Request {
+                    finishH264Encode(token: h264Request.token, restoreKeyframe: h264Request.forceKeyframe)
+                }
+                return
+            }
+            if shouldEncodeJpeg { encoding = true }
+            guard let stableSourceFrame = copyPixelBuffer(pixelBuffer) else {
+                finishScaledFramePreparation()
+                if shouldEncodeJpeg { encoding = false }
+                handleStableFrameFailure(h264Request: h264Request, shouldSendWebRTC: shouldSendWebRTC)
+                return
             }
-            if shouldSendWebRTC {
-                webRTCCopyFailures += 1
-                recordWebRTCCopyFailure()
-                if streamShouldLog(webRTCCopyFailures) {
-                    streamLog("[webrtc] failed to copy capture frame for WebRTC")
+            framePreparationQueue.async { [weak self] in
+                guard let self else { return }
+                defer { self.finishScaledFramePreparation() }
+                guard let stableFrame = self.copyPixelBuffer(
+                    stableSourceFrame,
+                    targetWidth: encodeSize.width,
+                    targetHeight: encodeSize.height
+                ) else {
+                    if shouldEncodeJpeg { self.encoding = false }
+                    self.handleStableFrameFailure(h264Request: h264Request, shouldSendWebRTC: shouldSendWebRTC)
+                    return
                 }
+                self.deliverStableFrame(
+                    stableFrame,
+                    timestamp: timestamp,
+                    shouldSendWebRTC: shouldSendWebRTC,
+                    shouldEncodeJpeg: shouldEncodeJpeg,
+                    h264Request: h264Request
+                )
             }
             return
         }
 
+        guard let stableFrame = copyPixelBuffer(pixelBuffer, targetWidth: encodeSize.width, targetHeight: encodeSize.height) else {
+            handleStableFrameFailure(h264Request: h264Request, shouldSendWebRTC: shouldSendWebRTC)
+            return
+        }
+        if shouldEncodeJpeg { encoding = true }
+        deliverStableFrame(
+            stableFrame,
+            timestamp: timestamp,
+            shouldSendWebRTC: shouldSendWebRTC,
+            shouldEncodeJpeg: shouldEncodeJpeg,
+            h264Request: h264Request
+        )
+    }
+
+    private func deliverStableFrame(
+        _ stableFrame: CVPixelBuffer,
+        timestamp: CMTime,
+        shouldSendWebRTC: Bool,
+        shouldEncodeJpeg: Bool,
+        h264Request: (forceKeyframe: Bool, token: UInt64)?
+    ) {
         if shouldSendWebRTC {
             webRTCPublisher.sendFrame(stableFrame, timestamp: timestamp)
         }
@@ -234,6 +309,23 @@ final class CaptureEngine {
         }
     }
 
+    private func handleStableFrameFailure(
+        h264Request: (forceKeyframe: Bool, token: UInt64)?,
+        shouldSendWebRTC: Bool
+    ) {
+        if let h264Request {
+            streamLog("[stream:avcc] failed to prepare capture frame for H.264 token=\(h264Request.token)")
+            finishH264Encode(token: h264Request.token, restoreKeyframe: h264Request.forceKeyframe)
+        }
+        if shouldSendWebRTC {
+            webRTCCopyFailures += 1
+            recordWebRTCCopyFailure()
+            if streamShouldLog(webRTCCopyFailures) {
+                streamLog("[webrtc] failed to prepare capture frame for WebRTC")
+            }
+        }
+    }
+
     private func reserveMjpegEncodeIfNeeded() -> Bool {
         let now = DispatchTime.now().uptimeNanoseconds
         if lastMjpegReservedAtNs != 0 && now - lastMjpegReservedAtNs < mjpegMinFrameIntervalNs {
@@ -281,9 +373,22 @@ final class CaptureEngine {
         let dstStride = CVPixelBufferGetBytesPerRow(dst)
         let rows = CVPixelBufferGetHeight(source)
         let copyBytes = min(srcStride, dstStride)
+        let startNs = DispatchTime.now().uptimeNanoseconds
         for row in 0..<rows {
             memcpy(dstAddr + row * dstStride, srcAddr + row * srcStride, copyBytes)
         }
+        let durationMs = Double(DispatchTime.now().uptimeNanoseconds - startNs) / 1_000_000.0
+        recordCopiedFrame(
+            width: width,
+            height: height,
+            durationMs: durationMs
+        )
+        if streamShouldLog(statsCopyCount) {
+            streamLog(
+                "[stream] copied frame #\(statsCopyCount) \(width)x\(height) " +
+                "ms=\(String(format: "%.2f", durationMs))"
+            )
+        }
         return dst
     }
 
@@ -426,6 +531,7 @@ final class CaptureEngine {
         let uptimeSec = max(0.001, Double(nowNs - statsStartNs) / 1_000_000_000.0)
         let captureStats: [String: Any]
         let scaleStats: [String: Any]
+        let copyStats: [String: Any]
         let mjpegStats: [String: Any]
         let h264Stats: [String: Any]
         let webRTCStats: [String: Any]
@@ -449,11 +555,22 @@ final class CaptureEngine {
             "msLast": statsScaleLastMs,
             "msAvg": statsScaleCount > 0 ? statsScaleTotalMs / Double(statsScaleCount) : 0.0,
             "msMax": statsScaleMaxMs,
+            "backpressureSkips": statsScaleBackpressureSkips,
+        ]
+        copyStats = [
+            "frames": statsCopyCount,
+            "avgFps": Double(statsCopyCount) / uptimeSec,
+            "width": statsCopyWidth,
+            "height": statsCopyHeight,
+            "msLast": statsCopyLastMs,
+            "msAvg": statsCopyCount > 0 ? statsCopyTotalMs / Double(statsCopyCount) : 0.0,
+            "msMax": statsCopyMaxMs,
         ]
         mjpegStats = [
             "reserved": statsMjpegReserved,
             "reservedAvgFps": Double(statsMjpegReserved) / uptimeSec,
             "throttleSkips": statsMjpegThrottleSkips,
+            "active": mjpegActive,
         ]
         h264Stats = [
             "reserved": statsH264Reserved,
@@ -466,6 +583,7 @@ final class CaptureEngine {
             "reserved": statsWebRTCReserved,
             "reservedAvgFps": Double(statsWebRTCReserved) / uptimeSec,
             "throttleSkips": statsWebRTCThrottleSkips,
+            "direct": statsWebRTCDirect,
             "copyFailures": statsWebRTCCopyFailures,
             "minFrameIntervalNs": webRTCMinFrameIntervalNs,
             "maxFps": webRTCMaxFps,
@@ -478,6 +596,7 @@ final class CaptureEngine {
             "maxDimension": maxDimension,
             "capture": captureStats,
             "scale": scaleStats,
+            "copy": copyStats,
             "mjpeg": mjpegStats,
             "h264": h264Stats,
             "webrtc": webRTCStats,
@@ -497,6 +616,20 @@ final class CaptureEngine {
         statsLock.unlock()
     }
 
+    private func reserveScaledFramePreparation() -> Bool {
+        framePreparationLock.lock()
+        defer { framePreparationLock.unlock() }
+        if scaledFramePreparationPending { return false }
+        scaledFramePreparationPending = true
+        return true
+    }
+
+    private func finishScaledFramePreparation() {
+        framePreparationLock.lock()
+        scaledFramePreparationPending = false
+        framePreparationLock.unlock()
+    }
+
     private func recordCapturedFrame() {
         let nowNs = DispatchTime.now().uptimeNanoseconds
         withStatsLock {
@@ -524,6 +657,21 @@ final class CaptureEngine {
         }
     }
 
+    private func recordScaleBackpressureSkip() {
+        withStatsLock { statsScaleBackpressureSkips += 1 }
+    }
+
+    private func recordCopiedFrame(width: Int, height: Int, durationMs: Double) {
+        withStatsLock {
+            statsCopyCount += 1
+            statsCopyTotalMs += durationMs
+            statsCopyLastMs = durationMs
+            statsCopyMaxMs = max(statsCopyMaxMs, durationMs)
+            statsCopyWidth = width
+            statsCopyHeight = height
+        }
+    }
+
     private func recordMjpegReserved() {
         withStatsLock { statsMjpegReserved += 1 }
     }
@@ -552,6 +700,10 @@ final class CaptureEngine {
         withStatsLock { statsWebRTCThrottleSkips += 1 }
     }
 
+    private func recordWebRTCDirect() {
+        withStatsLock { statsWebRTCDirect += 1 }
+    }
+
     private func recordWebRTCCopyFailure() {
         withStatsLock { statsWebRTCCopyFailures += 1 }
     }
@@ -586,6 +738,19 @@ final class CaptureEngine {
         }
     }
 
+    func setMjpegActive(_ active: Bool) {
+        encodeQueue.async { [weak self] in
+            guard let self else { return }
+            if active != self.mjpegActive {
+                streamLog("[stream:mjpeg] active=\(active)")
+            }
+            self.mjpegActive = active
+            if active {
+                self.lastMjpegReservedAtNs = 0
+            }
+        }
+    }
+
     func requestKeyframe() {
         h264Queue.async { [weak self] in
             self?.forceKeyframe = true

package/Sources/SimNative/sim-module.swift

@@ -156,6 +156,10 @@ private func u32(_ v: Int) -> UInt32 {
         engine.setAvccActive(active)
     }
 
+    @NodeMethod func setMjpegActive(_ active: Bool) {
+        engine.setMjpegActive(active)
+    }
+
     @NodeMethod func requestKeyframe() {
         engine.requestKeyframe()
     }

package/Sources/SimStreamHelper/WebRTCPublisher.swift

@@ -34,6 +34,8 @@ final class WebRTCPublisher {
     private var lastOutputWidth = 0
     private var lastOutputHeight = 0
     private var sentFrameCount: Int64 = 0
+    private var queuedInputFrameCount: Int64 = 0
+    private var directInputFrameCount: Int64 = 0
     private var lastFrameTimestampNs: Int64 = 0
     private let statsStartNs = DispatchTime.now().uptimeNanoseconds
     private var lastFrameSentAtNs: UInt64 = 0
@@ -101,46 +103,16 @@ final class WebRTCPublisher {
     func sendFrame(_ pixelBuffer: CVPixelBuffer, timestamp: CMTime) {
         queue.async {
             guard self.session != nil else { return }
-            let width = CVPixelBufferGetWidth(pixelBuffer)
-            let height = CVPixelBufferGetHeight(pixelBuffer)
-            if width != self.lastOutputWidth || height != self.lastOutputHeight {
-                self.lastOutputWidth = width
-                self.lastOutputHeight = height
-                self.videoSource.adaptOutputFormat(
-                    toWidth: Int32(width),
-                    height: Int32(height),
-                    fps: Int32(self.maxFps)
-                )
-                print("[webrtc] Video source output format: \(width)x\(height) @ \(self.maxFps)fps")
-            }
-            if !self.loggedInputFormat {
-                self.loggedInputFormat = true
-                let pixelFormat = CVPixelBufferGetPixelFormatType(pixelBuffer)
-                let supported = LKRTCCVPixelBuffer.supportedPixelFormats()
-                    .contains(NSNumber(value: UInt32(pixelFormat)))
-                print("[webrtc] Input pixel format: \(pixelFormat) cvPixelBufferSupported=\(supported); forwarding as I420")
-            }
-            let timeNs = self.nextFrameTimestampNs(timestamp)
-            let cvFrame = LKRTCVideoFrame(
-                buffer: LKRTCCVPixelBuffer(pixelBuffer: pixelBuffer),
-                rotation: ._0,
-                timeStampNs: timeNs
-            )
-            let convertStartNs = DispatchTime.now().uptimeNanoseconds
-            let frame = cvFrame.newI420()
-            self.videoSource.capturer(self.capturer, didCapture: frame)
-            let convertDurationMs = Double(DispatchTime.now().uptimeNanoseconds - convertStartNs) / 1_000_000.0
-            self.sentFrameCount += 1
-            self.lastFrameSentAtNs = DispatchTime.now().uptimeNanoseconds
-            self.lastI420Ms = convertDurationMs
-            self.totalI420Ms += convertDurationMs
-            self.maxI420Ms = max(self.maxI420Ms, convertDurationMs)
-            if self.shouldLogFrame(self.sentFrameCount) {
-                print(
-                    "[webrtc] Sent video frame #\(self.sentFrameCount) size=\(width)x\(height) " +
-                    "timestampNs=\(timeNs) i420Ms=\(String(format: "%.2f", convertDurationMs))"
-                )
-            }
+            self.queuedInputFrameCount += 1
+            self.sendFrameOnQueue(pixelBuffer, timestamp: timestamp, mode: "queued")
+        }
+    }
+
+    func sendFrameDirect(_ pixelBuffer: CVPixelBuffer, timestamp: CMTime) {
+        queue.sync {
+            guard self.session != nil else { return }
+            self.directInputFrameCount += 1
+            self.sendFrameOnQueue(pixelBuffer, timestamp: timestamp, mode: "direct")
         }
     }
 
@@ -155,6 +127,8 @@ final class WebRTCPublisher {
                 "outputWidth": lastOutputWidth,
                 "outputHeight": lastOutputHeight,
                 "sentFrames": sentFrameCount,
+                "queuedInputFrames": queuedInputFrameCount,
+                "directInputFrames": directInputFrameCount,
                 "avgSentFps": Double(sentFrameCount) / uptimeSec,
                 "lastFrameAgeMs": lastFrameAgeMs,
                 "i420MsLast": lastI420Ms,
@@ -178,6 +152,49 @@ final class WebRTCPublisher {
         return timestampNs
     }
 
+    private func sendFrameOnQueue(_ pixelBuffer: CVPixelBuffer, timestamp: CMTime, mode: String) {
+        let width = CVPixelBufferGetWidth(pixelBuffer)
+        let height = CVPixelBufferGetHeight(pixelBuffer)
+        if width != lastOutputWidth || height != lastOutputHeight {
+            lastOutputWidth = width
+            lastOutputHeight = height
+            videoSource.adaptOutputFormat(
+                toWidth: Int32(width),
+                height: Int32(height),
+                fps: Int32(maxFps)
+            )
+            print("[webrtc] Video source output format: \(width)x\(height) @ \(maxFps)fps")
+        }
+        if !loggedInputFormat {
+            loggedInputFormat = true
+            let pixelFormat = CVPixelBufferGetPixelFormatType(pixelBuffer)
+            let supported = LKRTCCVPixelBuffer.supportedPixelFormats()
+                .contains(NSNumber(value: UInt32(pixelFormat)))
+            print("[webrtc] Input pixel format: \(pixelFormat) cvPixelBufferSupported=\(supported); forwarding as I420")
+        }
+        let timeNs = nextFrameTimestampNs(timestamp)
+        let cvFrame = LKRTCVideoFrame(
+            buffer: LKRTCCVPixelBuffer(pixelBuffer: pixelBuffer),
+            rotation: ._0,
+            timeStampNs: timeNs
+        )
+        let convertStartNs = DispatchTime.now().uptimeNanoseconds
+        let frame = cvFrame.newI420()
+        videoSource.capturer(capturer, didCapture: frame)
+        let convertDurationMs = Double(DispatchTime.now().uptimeNanoseconds - convertStartNs) / 1_000_000.0
+        sentFrameCount += 1
+        lastFrameSentAtNs = DispatchTime.now().uptimeNanoseconds
+        lastI420Ms = convertDurationMs
+        totalI420Ms += convertDurationMs
+        maxI420Ms = max(maxI420Ms, convertDurationMs)
+        if shouldLogFrame(sentFrameCount) {
+            print(
+                "[webrtc] Sent video frame #\(sentFrameCount) mode=\(mode) size=\(width)x\(height) " +
+                "timestampNs=\(timeNs) i420Ms=\(String(format: "%.2f", convertDurationMs))"
+            )
+        }
+    }
+
     func stop() {
         queue.sync {
             session?.close()