serve-sim-sjchmiela 0.1.49 → 0.1.51

package/Sources/SimNative/sim-capture.swift

@@ -58,11 +58,17 @@ final class CaptureEngine {
     private var h264BackpressureSkips: Int64 = 0
     private var mjpegThrottleSkips: Int64 = 0
     private var h264ThrottleSkips: Int64 = 0
+    private var webRTCReservedCount: Int64 = 0
+    private var webRTCThrottleSkips: Int64 = 0
+    private var webRTCCopyFailures: Int64 = 0
     private var avccNativeEmitCount: Int64 = 0
     private var lastMjpegReservedAtNs: UInt64 = 0
     private var lastH264ReservedAtNs: UInt64 = 0
+    private var lastWebRTCReservedAtNs: UInt64 = 0
     private var mjpegMinFrameIntervalNs: UInt64
     private var h264MinFrameIntervalNs: UInt64
+    private var webRTCMinFrameIntervalNs: UInt64
+    private var webRTCMaxFps: Int
     private var maxDimension: Int
     private var started = false
     private var stopped = false
@@ -82,8 +88,17 @@ final class CaptureEngine {
         h264Encoder = H264Encoder(fps: h264Fps, bitrate: h264Bitrate)
         mjpegMinFrameIntervalNs = UInt64(1_000_000_000 / mjpegFps)
         h264MinFrameIntervalNs = UInt64(1_000_000_000 / h264Fps)
+        webRTCMinFrameIntervalNs = UInt64(1_000_000_000 / h264Fps)
+        webRTCMaxFps = h264Fps
         maxDimension = max(0, options.maxDimension)
         webRTCPublisher.onInput = onWebRTCInput
+        webRTCPublisher.onActiveChanged = { [weak self] active in
+            guard let self else { return }
+            self.lastWebRTCReservedAtNs = 0
+            self.frameCapture.setIdleRefreshFps(active ? self.webRTCMaxFps : 5)
+            streamLog("[webrtc] active=\(active) idleRefreshFps=\(active ? self.webRTCMaxFps : 5)")
+        }
+        webRTCPublisher.updateSettings(maxFps: h264Fps, bitrate: h264Bitrate)
 
         h264Encoder.onEncoded = { [weak self] encoded in
             guard let self else { return }
@@ -144,7 +159,7 @@ final class CaptureEngine {
         }
 
         let h264Request = reserveH264EncodeIfNeeded()
-        let shouldSendWebRTC = webRTCPublisher.isActive
+        let shouldSendWebRTC = reserveWebRTCFrameIfNeeded()
         let shouldEncodeJpeg = encoderReady && !encoding && reserveMjpegEncodeIfNeeded()
         if !shouldEncodeJpeg && h264Request == nil && !shouldSendWebRTC { return }
 
@@ -153,6 +168,12 @@ final class CaptureEngine {
                 streamLog("[stream:avcc] failed to copy capture frame for H.264 token=\(h264Request.token)")
                 finishH264Encode(token: h264Request.token, restoreKeyframe: h264Request.forceKeyframe)
             }
+            if shouldSendWebRTC {
+                webRTCCopyFailures += 1
+                if streamShouldLog(webRTCCopyFailures) {
+                    streamLog("[webrtc] failed to copy capture frame for WebRTC")
+                }
+            }
             return
         }
 
@@ -336,6 +357,24 @@ final class CaptureEngine {
         }
     }
 
+    private func reserveWebRTCFrameIfNeeded() -> Bool {
+        guard webRTCPublisher.isActive else { return false }
+        let now = DispatchTime.now().uptimeNanoseconds
+        if lastWebRTCReservedAtNs != 0 && now - lastWebRTCReservedAtNs < webRTCMinFrameIntervalNs {
+            webRTCThrottleSkips += 1
+            if streamShouldLog(webRTCThrottleSkips) {
+                streamLog("[webrtc] skip frame: fps throttle")
+            }
+            return false
+        }
+        lastWebRTCReservedAtNs = now
+        webRTCReservedCount += 1
+        if streamShouldLog(webRTCReservedCount) {
+            streamLog("[webrtc] reserved frame #\(webRTCReservedCount)")
+        }
+        return true
+    }
+
     private func finishH264Encode(token: UInt64, restoreKeyframe: Bool = false) {
         h264Queue.async { [weak self] in
             guard let self, self.h264FrameToken == token else { return }
@@ -391,10 +430,17 @@ final class CaptureEngine {
         }
         h264Queue.sync {
             self.h264MinFrameIntervalNs = UInt64(1_000_000_000 / normalizedH264Fps)
+            self.webRTCMinFrameIntervalNs = UInt64(1_000_000_000 / normalizedH264Fps)
+            self.webRTCMaxFps = normalizedH264Fps
             self.h264Encoder.update(fps: normalizedH264Fps, bitrate: normalizedBitrate)
+            self.webRTCPublisher.updateSettings(maxFps: normalizedH264Fps, bitrate: normalizedBitrate)
+            if self.webRTCPublisher.isActive {
+                self.frameCapture.setIdleRefreshFps(normalizedH264Fps)
+            }
             self.forceKeyframe = true
             self.h264Encoding = false
             self.lastH264ReservedAtNs = 0
+            self.lastWebRTCReservedAtNs = 0
         }
         streamLog(
             "[stream] settings updated mjpegFps=\(normalizedMjpegFps) mjpegQuality=\(normalizedQuality) " +
@@ -405,6 +451,7 @@ final class CaptureEngine {
     func handleWebRTCOffer(_ offerJson: String) throws -> String {
         let request = try JSONDecoder().decode(WebRTCOfferPayload.self, from: Data(offerJson.utf8))
         let answer = try webRTCPublisher.handleOffer(request)
+        lastWebRTCReservedAtNs = 0
         let data = try JSONEncoder().encode(answer)
         return String(decoding: data, as: UTF8.self)
     }

package/Sources/SimStreamHelper/FrameCapture.swift

@@ -20,6 +20,7 @@ final class FrameCapture {
     private var idleTimer: DispatchSourceTimer?
     private let captureQueue = DispatchQueue(label: "frame-capture", qos: .userInteractive)
     private var lastCaptureTimeMs: UInt64 = 0
+    private var idleIntervalMs: UInt64 = 200
     private var lastSeeds: [ObjectIdentifier: UInt32] = [:]
     private var rewireTickCount: Int = 0
     /// Interval at which the idle timer re-emits the current frame even when
@@ -32,7 +33,8 @@ final class FrameCapture {
     ///    one subscriber is due for it — a late-joining relay subscriber on an
     ///    idle sim never gets a cached frame to show.
     /// Re-emitting at ~5 fps fixes both without meaningful CPU cost.
-    private static let idleIntervalMs: UInt64 = 200
+    private static let defaultIdleRefreshFps = 5
+    private static let idleTimerTickMs: UInt64 = 33
 
     private var descriptors: [NSObject] = []
     private var callbackUUIDs: [ObjectIdentifier: NSUUID] = [:]
@@ -61,6 +63,16 @@ final class FrameCapture {
         print("[capture] Frame callbacks registered (event-driven) + 5fps idle floor")
     }
 
+    func setIdleRefreshFps(_ fps: Int) {
+        let normalizedFps = max(Self.defaultIdleRefreshFps, min(120, fps))
+        let nextIntervalMs = max(1, UInt64(1_000 / normalizedFps))
+        captureQueue.async { [weak self] in
+            guard let self, self.idleIntervalMs != nextIntervalMs else { return }
+            self.idleIntervalMs = nextIntervalMs
+            streamLog("[capture] Idle refresh fps=\(normalizedFps) intervalMs=\(nextIntervalMs)")
+        }
+    }
+
     /// Find all framebuffer display descriptors, register callbacks on each,
     /// and cache them. Safe to re-call if the cached descriptors become stale.
     ///
@@ -191,12 +203,12 @@ final class FrameCapture {
 
     private func startIdleTimer() {
         let timer = DispatchSource.makeTimerSource(queue: captureQueue)
-        timer.schedule(deadline: .now().advanced(by: .milliseconds(Int(Self.idleIntervalMs))),
-                       repeating: .milliseconds(Int(Self.idleIntervalMs)))
+        timer.schedule(deadline: .now().advanced(by: .milliseconds(Int(Self.idleTimerTickMs))),
+                       repeating: .milliseconds(Int(Self.idleTimerTickMs)))
         timer.setEventHandler { [weak self] in
             guard let self else { return }
             let nowMs = DispatchTime.now().uptimeNanoseconds / 1_000_000
-            if (nowMs - self.lastCaptureTimeMs) >= Self.idleIntervalMs {
+            if (nowMs - self.lastCaptureTimeMs) >= self.idleIntervalMs {
                 self.captureFrame()
             }
             // Self-heal: if we've never captured a frame, the cached descriptor
@@ -236,7 +248,7 @@ final class FrameCapture {
         let nowMs = DispatchTime.now().uptimeNanoseconds / 1_000_000
         let sinceLastMs = nowMs &- lastCaptureTimeMs
         let seedChanged = lastSeeds[key] != seed
-        let idleRefreshDue = frameCount > 0 && sinceLastMs >= Self.idleIntervalMs
+        let idleRefreshDue = frameCount > 0 && sinceLastMs >= idleIntervalMs
         if frameCount > 0, !seedChanged, !idleRefreshDue { return }
         lastSeeds[key] = seed
 
@@ -260,7 +272,13 @@ final class FrameCapture {
 
         lastCaptureTimeMs = nowMs
         frameCount += 1
-        let timestamp = CMTime(value: CMTimeValue(frameCount), timescale: 60)
+        let nowNs = DispatchTime.now().uptimeNanoseconds
+        let timestamp = CMTime(value: CMTimeValue(nowNs), timescale: 1_000_000_000)
+        if streamShouldLog(Int64(frameCount)) {
+            let reason = seedChanged ? "changed" : "idle"
+            let interval = frameCount == 1 ? 0 : sinceLastMs
+            streamLog("[capture] frame #\(frameCount) reason=\(reason) intervalMs=\(interval) size=\(w)x\(h)")
+        }
         onFrame?(pb, timestamp)
     }
 

package/Sources/SimStreamHelper/WebRTCPublisher.swift

@@ -23,9 +23,10 @@ struct WebRTCAnswerPayload: Codable {
 
 final class WebRTCPublisher {
     var onInput: ((Data) -> Void)?
+    var onActiveChanged: ((Bool) -> Void)?
 
     private let queue = DispatchQueue(label: "webrtc-publisher")
-    private let factory = LKRTCPeerConnectionFactory()
+    private let factory: LKRTCPeerConnectionFactory
     private let videoSource: LKRTCVideoSource
     private let videoTrack: LKRTCVideoTrack
     private let capturer: LKRTCVideoCapturer
@@ -34,16 +35,49 @@ final class WebRTCPublisher {
     private var lastOutputHeight = 0
     private var sentFrameCount: Int64 = 0
     private var lastFrameTimestampNs: Int64 = 0
+    private var loggedInputFormat = false
+    private var maxFps = 30
+    private var targetBitrate = 6_000_000
     var isActive: Bool {
         queue.sync { session != nil }
     }
 
     init() {
+        let encoderFactory = LKRTCDefaultVideoEncoderFactory()
+        let decoderFactory = LKRTCDefaultVideoDecoderFactory()
+        factory = LKRTCPeerConnectionFactory(
+            encoderFactory: encoderFactory,
+            decoderFactory: decoderFactory
+        )
         videoSource = factory.videoSource(forScreenCast: true)
         videoTrack = factory.videoTrack(with: videoSource, trackId: "simulator-video")
         videoTrack.isEnabled = true
         capturer = LKRTCVideoCapturer(delegate: videoSource)
-        print("[webrtc] Publisher ready (factory + screen-cast video source)")
+        print(
+            "[webrtc] Publisher ready (default codec factory + screen-cast video source) " +
+            "senderCodecs=\(senderCodecSummary())"
+        )
+    }
+
+    func updateSettings(maxFps: Int, bitrate: Int) {
+        let normalizedFps = max(1, min(120, maxFps))
+        let normalizedBitrate = max(100_000, bitrate)
+        queue.async {
+            guard self.maxFps != normalizedFps || self.targetBitrate != normalizedBitrate else { return }
+            self.maxFps = normalizedFps
+            self.targetBitrate = normalizedBitrate
+            if self.lastOutputWidth > 0, self.lastOutputHeight > 0 {
+                self.videoSource.adaptOutputFormat(
+                    toWidth: Int32(self.lastOutputWidth),
+                    height: Int32(self.lastOutputHeight),
+                    fps: Int32(self.maxFps)
+                )
+            }
+            if let session = self.session {
+                self.applyBitrateSettings(to: session)
+            }
+            print("[webrtc] Settings updated fps=\(normalizedFps) bitrate=\(normalizedBitrate)")
+        }
     }
 
     func handleOffer(_ request: WebRTCOfferPayload) throws -> WebRTCAnswerPayload {
@@ -67,19 +101,36 @@ final class WebRTCPublisher {
             if width != self.lastOutputWidth || height != self.lastOutputHeight {
                 self.lastOutputWidth = width
                 self.lastOutputHeight = height
-                self.videoSource.adaptOutputFormat(toWidth: Int32(width), height: Int32(height), fps: 30)
-                print("[webrtc] Video source output format: \(width)x\(height) @ 30fps")
+                self.videoSource.adaptOutputFormat(
+                    toWidth: Int32(width),
+                    height: Int32(height),
+                    fps: Int32(self.maxFps)
+                )
+                print("[webrtc] Video source output format: \(width)x\(height) @ \(self.maxFps)fps")
+            }
+            if !self.loggedInputFormat {
+                self.loggedInputFormat = true
+                let pixelFormat = CVPixelBufferGetPixelFormatType(pixelBuffer)
+                let supported = LKRTCCVPixelBuffer.supportedPixelFormats()
+                    .contains(NSNumber(value: UInt32(pixelFormat)))
+                print("[webrtc] Input pixel format: \(pixelFormat) cvPixelBufferSupported=\(supported); forwarding as I420")
             }
             let timeNs = self.nextFrameTimestampNs(timestamp)
-            let frame = LKRTCVideoFrame(
+            let cvFrame = LKRTCVideoFrame(
                 buffer: LKRTCCVPixelBuffer(pixelBuffer: pixelBuffer),
                 rotation: ._0,
                 timeStampNs: timeNs
             )
+            let convertStartNs = DispatchTime.now().uptimeNanoseconds
+            let frame = cvFrame.newI420()
             self.videoSource.capturer(self.capturer, didCapture: frame)
+            let convertDurationMs = Double(DispatchTime.now().uptimeNanoseconds - convertStartNs) / 1_000_000.0
             self.sentFrameCount += 1
             if self.shouldLogFrame(self.sentFrameCount) {
-                print("[webrtc] Sent video frame #\(self.sentFrameCount) size=\(width)x\(height) timestampNs=\(timeNs)")
+                print(
+                    "[webrtc] Sent video frame #\(self.sentFrameCount) size=\(width)x\(height) " +
+                    "timestampNs=\(timeNs) i420Ms=\(String(format: "%.2f", convertDurationMs))"
+                )
             }
         }
     }
@@ -98,6 +149,7 @@ final class WebRTCPublisher {
         queue.sync {
             session?.close()
             session = nil
+            onActiveChanged?(false)
         }
     }
 
@@ -127,6 +179,8 @@ final class WebRTCPublisher {
         )
         let delegate = WebRTCSessionDelegate(onInput: { [weak self] data in
             self?.onInput?(data)
+        }, onClosed: { [weak self] peerConnection in
+            self?.clearSession(peerConnection)
         })
         guard let peerConnection = factory.peerConnection(
             with: config,
@@ -140,6 +194,7 @@ final class WebRTCPublisher {
         let session = WebRTCSession(peerConnection: peerConnection, delegate: delegate)
         self.session?.close()
         self.session = session
+        onActiveChanged?(true)
 
         let remoteDescription = LKRTCSessionDescription(type: .offer, sdp: request.sdp)
         peerConnection.setRemoteDescription(remoteDescription) { error in
@@ -207,6 +262,11 @@ final class WebRTCPublisher {
             print("[webrtc] Failed to set video transceiver direction: \(directionError.localizedDescription)")
         }
         applyVideoCodecPreference(codec, to: transceiver)
+        let session = self.session
+        session?.videoSender = transceiver.sender
+        if let session {
+            applyBitrateSettings(to: session)
+        }
     }
 
     private func createFallbackVideoTransceiver(on peerConnection: LKRTCPeerConnection) -> LKRTCRtpTransceiver? {
@@ -414,6 +474,51 @@ final class WebRTCPublisher {
         print("[webrtc] Preferred video codec: \(preferredName)")
     }
 
+    private func applyBitrateSettings(to session: WebRTCSession) {
+        guard let sender = session.videoSender else { return }
+        let parameters = sender.parameters
+        let encodings = parameters.encodings.isEmpty
+            ? [LKRTCRtpEncodingParameters()]
+            : parameters.encodings
+        let maxBitrate = NSNumber(value: targetBitrate)
+        let minBitrate = NSNumber(value: max(100_000, targetBitrate / 4))
+        let fps = NSNumber(value: maxFps)
+        for encoding in encodings {
+            encoding.isActive = true
+            encoding.maxBitrateBps = maxBitrate
+            encoding.minBitrateBps = minBitrate
+            encoding.maxFramerate = fps
+        }
+        parameters.encodings = encodings
+        sender.parameters = parameters
+        let bweUpdated = session.peerConnection.setBweMinBitrateBps(
+            minBitrate,
+            currentBitrateBps: maxBitrate,
+            maxBitrateBps: maxBitrate
+        )
+        print(
+            "[webrtc] Sender parameters fps=\(maxFps) minBitrate=\(minBitrate) " +
+            "maxBitrate=\(maxBitrate) bweUpdated=\(bweUpdated)"
+        )
+    }
+
+    private func clearSession(_ peerConnection: LKRTCPeerConnection?) {
+        queue.async {
+            guard let session = self.session, session.peerConnection === peerConnection else { return }
+            session.close()
+            self.session = nil
+            self.onActiveChanged?(false)
+            print("[webrtc] Peer connection closed; publisher inactive")
+        }
+    }
+
+    private func senderCodecSummary() -> String {
+        let names = factory.rtpSenderCapabilities(forKind: "video").codecs.map { capability in
+            capability.mimeType.isEmpty ? capability.name : capability.mimeType
+        }
+        return names.joined(separator: ",")
+    }
+
     private func makeError(_ message: String) -> Error {
         NSError(domain: "serve-sim.webrtc", code: 1, userInfo: [NSLocalizedDescriptionKey: message])
     }
@@ -426,6 +531,7 @@ final class WebRTCPublisher {
 private final class WebRTCSession {
     let peerConnection: LKRTCPeerConnection
     let delegate: WebRTCSessionDelegate
+    var videoSender: LKRTCRtpSender?
     private let iceGatheringTimeout: DispatchTimeInterval = .milliseconds(3_000)
 
     init(peerConnection: LKRTCPeerConnection, delegate: WebRTCSessionDelegate) {
@@ -466,14 +572,16 @@ private final class WebRTCSession {
 
 private final class WebRTCSessionDelegate: NSObject, LKRTCPeerConnectionDelegate, LKRTCDataChannelDelegate {
     private let onInput: (Data) -> Void
+    private let onClosed: (LKRTCPeerConnection) -> Void
     private let iceGatheringCompleteHandlerLock = NSLock()
     private var iceGatheringCompleteHandler: (() -> Void)?
     private let candidatesLock = NSLock()
     private var generatedCandidates: [LKRTCIceCandidate] = []
     private var statsScheduled = false
 
-    init(onInput: @escaping (Data) -> Void) {
+    init(onInput: @escaping (Data) -> Void, onClosed: @escaping (LKRTCPeerConnection) -> Void) {
         self.onInput = onInput
+        self.onClosed = onClosed
     }
 
     func peerConnection(_ peerConnection: LKRTCPeerConnection, didChange stateChanged: LKRTCSignalingState) {}
@@ -484,6 +592,14 @@ private final class WebRTCSessionDelegate: NSObject, LKRTCPeerConnectionDelegate
         print("[webrtc] ICE connection state: \(newState.rawValue)")
         if newState == .connected || newState == .completed {
             scheduleOutboundStats(peerConnection)
+        } else if newState == .failed || newState == .closed {
+            onClosed(peerConnection)
+        }
+    }
+    func peerConnection(_ peerConnection: LKRTCPeerConnection, didChange newState: LKRTCPeerConnectionState) {
+        print("[webrtc] Peer connection state: \(newState.rawValue)")
+        if newState == .failed || newState == .closed {
+            onClosed(peerConnection)
         }
     }
     func peerConnection(_ peerConnection: LKRTCPeerConnection, didChange newState: LKRTCIceGatheringState) {