serve-sim-sjchmiela 0.1.41 → 0.1.43

package/README.md

@@ -71,6 +71,10 @@ Options:
       --codec <codec> Stream codec for the preview UI: 'auto' (H.264 when the
                       browser can decode it) or 'mjpeg' (force software JPEG —
                       e.g. on VMs without H.264 encode)
+      --transport <http|webrtc>
+                      Stream transport (default: http)
+      --webrtc-codec <vp8|vp9|h264>
+                      WebRTC video codec (default: h264)
       --list [device] List running streams
       --kill [device] Kill running stream(s)
 

package/Sources/SimNative/sim-capture.swift

@@ -1,6 +1,7 @@
 import Foundation
 import CoreVideo
 import CoreMedia
+import CoreImage
 
 // The capture + encode engine, reused verbatim from SimStreamHelper. Replicates
 // main.swift's frameHandler: MJPEG always encodes while clients exist; H.264 runs
@@ -20,6 +21,7 @@ struct CaptureEngineOptions {
     let mjpegQuality: Double
     let h264Fps: Int
     let h264Bitrate: Int
+    let maxDimension: Int
 }
 
 final class CaptureEngine {
@@ -33,8 +35,9 @@ final class CaptureEngine {
     private let webRTCPublisher = WebRTCPublisher()
 
     private let frameCapture = FrameCapture()
-    private let videoEncoder: VideoEncoder
+    private var videoEncoder: VideoEncoder
     private let h264Encoder: H264Encoder
+    private let ciContext = CIContext(options: [.workingColorSpace: NSNull()])
     private let encodeQueue = DispatchQueue(label: "napi.encode", qos: .userInteractive)
     private let h264Queue = DispatchQueue(label: "napi.encode.h264", qos: .userInteractive)
     private static let h264EncodeTimeoutMs = 500
@@ -43,6 +46,8 @@ final class CaptureEngine {
     // encode queues. Benign races (same pattern as the standalone helper).
     private var screenWidth = 0
     private var screenHeight = 0
+    private var encodeWidth = 0
+    private var encodeHeight = 0
     private var encoderReady = false
     private var encoding = false       // MJPEG backpressure
     private var h264Encoding = false   // H.264 backpressure
@@ -57,8 +62,9 @@ final class CaptureEngine {
     private var avccNativeEmitCount: Int64 = 0
     private var lastMjpegReservedAtNs: UInt64 = 0
     private var lastH264ReservedAtNs: UInt64 = 0
-    private let mjpegMinFrameIntervalNs: UInt64
-    private let h264MinFrameIntervalNs: UInt64
+    private var mjpegMinFrameIntervalNs: UInt64
+    private var h264MinFrameIntervalNs: UInt64
+    private var maxDimension: Int
     private var started = false
     private var stopped = false
 
@@ -77,6 +83,7 @@ final class CaptureEngine {
         h264Encoder = H264Encoder(fps: h264Fps, bitrate: h264Bitrate)
         mjpegMinFrameIntervalNs = UInt64(1_000_000_000 / mjpegFps)
         h264MinFrameIntervalNs = UInt64(1_000_000_000 / h264Fps)
+        maxDimension = max(0, options.maxDimension)
         webRTCPublisher.onInput = onWebRTCInput
 
         h264Encoder.onEncoded = { [weak self] encoded in
@@ -123,12 +130,15 @@ final class CaptureEngine {
     private func handleFrame(_ pixelBuffer: CVPixelBuffer, timestamp: CMTime) {
         let w = CVPixelBufferGetWidth(pixelBuffer)
         let h = CVPixelBufferGetHeight(pixelBuffer)
+        let encodeSize = encodedSize(width: w, height: h)
 
-        if !encoderReady || w != screenWidth || h != screenHeight {
+        if !encoderReady || w != screenWidth || h != screenHeight || encodeSize.width != encodeWidth || encodeSize.height != encodeHeight {
             screenWidth = w
             screenHeight = h
+            encodeWidth = encodeSize.width
+            encodeHeight = encodeSize.height
             videoEncoder.stop()
-            videoEncoder.setup(width: Int32(w), height: Int32(h), fps: 60) { [weak self] jpeg in
+            videoEncoder.setup(width: Int32(encodeSize.width), height: Int32(encodeSize.height), fps: 60) { [weak self] jpeg in
                 self?.emit(codec: Self.codecMJPEG, data: jpeg, flags: 0)
             }
             encoderReady = true
@@ -139,7 +149,7 @@ final class CaptureEngine {
         let shouldEncodeJpeg = encoderReady && !encoding && reserveMjpegEncodeIfNeeded()
         if !shouldEncodeJpeg && h264Request == nil && !shouldSendWebRTC { return }
 
-        guard let stableFrame = copyPixelBuffer(pixelBuffer) else {
+        guard let stableFrame = copyPixelBuffer(pixelBuffer, targetWidth: encodeSize.width, targetHeight: encodeSize.height) else {
             if let h264Request {
                 streamLog("[stream:avcc] failed to copy capture frame for H.264 token=\(h264Request.token)")
                 finishH264Encode(token: h264Request.token, restoreKeyframe: h264Request.forceKeyframe)
@@ -232,6 +242,50 @@ final class CaptureEngine {
         return dst
     }
 
+    private func encodedSize(width: Int, height: Int) -> (width: Int, height: Int) {
+        guard maxDimension > 0, max(width, height) > maxDimension else {
+            return (width, height)
+        }
+        let scale = Double(maxDimension) / Double(max(width, height))
+        return (evenDimension(width, scale: scale), evenDimension(height, scale: scale))
+    }
+
+    private func evenDimension(_ value: Int, scale: Double) -> Int {
+        let scaled = max(2, Int((Double(value) * scale).rounded()))
+        return scaled.isMultiple(of: 2) ? scaled : max(2, scaled - 1)
+    }
+
+    private func copyPixelBuffer(_ source: CVPixelBuffer, targetWidth: Int, targetHeight: Int) -> CVPixelBuffer? {
+        let width = CVPixelBufferGetWidth(source)
+        let height = CVPixelBufferGetHeight(source)
+        if width == targetWidth && height == targetHeight {
+            return copyPixelBuffer(source)
+        }
+
+        let attrs: [String: Any] = [
+            kCVPixelBufferPixelFormatTypeKey as String: kCVPixelFormatType_32BGRA,
+            kCVPixelBufferWidthKey as String: targetWidth,
+            kCVPixelBufferHeightKey as String: targetHeight,
+            kCVPixelBufferCGImageCompatibilityKey as String: true,
+            kCVPixelBufferCGBitmapContextCompatibilityKey as String: true,
+        ]
+        var out: CVPixelBuffer?
+        guard CVPixelBufferCreate(
+            kCFAllocatorDefault, targetWidth, targetHeight, kCVPixelFormatType_32BGRA, attrs as CFDictionary, &out
+        ) == kCVReturnSuccess, let dst = out else { return nil }
+
+        let scaleX = CGFloat(targetWidth) / CGFloat(width)
+        let scaleY = CGFloat(targetHeight) / CGFloat(height)
+        let image = CIImage(cvPixelBuffer: source).transformed(by: CGAffineTransform(scaleX: scaleX, y: scaleY))
+        ciContext.render(
+            image,
+            to: dst,
+            bounds: CGRect(x: 0, y: 0, width: targetWidth, height: targetHeight),
+            colorSpace: CGColorSpaceCreateDeviceRGB()
+        )
+        return dst
+    }
+
     private func reserveH264EncodeIfNeeded() -> (forceKeyframe: Bool, token: UInt64)? {
         h264Queue.sync {
             guard avccActive else { return nil }
@@ -301,6 +355,35 @@ final class CaptureEngine {
         }
     }
 
+    func updateSettings(mjpegFps: Int, mjpegQuality: Double, h264Fps: Int, h264Bitrate: Int, maxDimension: Int) {
+        let normalizedMjpegFps = max(1, mjpegFps)
+        let normalizedQuality = CGFloat(min(max(mjpegQuality, 0.0), 1.0))
+        let normalizedH264Fps = max(1, h264Fps)
+        let normalizedBitrate = max(1, h264Bitrate)
+        let normalizedMaxDimension = max(0, maxDimension)
+
+        encodeQueue.sync {
+            self.mjpegMinFrameIntervalNs = UInt64(1_000_000_000 / normalizedMjpegFps)
+            self.maxDimension = normalizedMaxDimension
+            self.videoEncoder.stop()
+            self.videoEncoder = VideoEncoder(quality: normalizedQuality)
+            self.encoderReady = false
+            self.encoding = false
+            self.lastMjpegReservedAtNs = 0
+        }
+        h264Queue.sync {
+            self.h264MinFrameIntervalNs = UInt64(1_000_000_000 / normalizedH264Fps)
+            self.h264Encoder.update(fps: normalizedH264Fps, bitrate: normalizedBitrate)
+            self.forceKeyframe = true
+            self.h264Encoding = false
+            self.lastH264ReservedAtNs = 0
+        }
+        streamLog(
+            "[stream] settings updated mjpegFps=\(normalizedMjpegFps) mjpegQuality=\(normalizedQuality) " +
+            "h264Fps=\(normalizedH264Fps) h264Bitrate=\(normalizedBitrate) maxDimension=\(normalizedMaxDimension)"
+        )
+    }
+
     func handleWebRTCOffer(_ offerJson: String) throws -> String {
         let request = try JSONDecoder().decode(WebRTCOfferPayload.self, from: Data(offerJson.utf8))
         let answer = try webRTCPublisher.handleOffer(request)

package/Sources/SimNative/sim-module.swift

@@ -106,7 +106,8 @@ private func u32(_ v: Int) -> UInt32 {
         _ mjpegFps: Int,
         _ mjpegQuality: Double,
         _ h264Fps: Int,
-        _ h264Bitrate: Int
+        _ h264Bitrate: Int,
+        _ maxDimension: Int
     ) throws {
         // unref'd by NodeAsyncQueue's init, so the frame pipeline alone won't
         // keep the event loop alive. Bounded queue + blocking AVCC preserves
@@ -124,7 +125,8 @@ private func u32(_ v: Int) -> UInt32 {
             mjpegFps: mjpegFps,
             mjpegQuality: mjpegQuality,
             h264Fps: h264Fps,
-            h264Bitrate: h264Bitrate
+            h264Bitrate: h264Bitrate,
+            maxDimension: maxDimension
         ), onFrame: { codec, data, w, h, flags in
             // Runs on a native encode thread. AVCC is inter-frame H.264 — dropping
             // a delta corrupts the decoder until the next IDR — so deliver it
@@ -158,6 +160,22 @@ private func u32(_ v: Int) -> UInt32 {
         engine.requestKeyframe()
     }
 
+    @NodeMethod func updateStreamSettings(
+        _ mjpegFps: Int,
+        _ mjpegQuality: Double,
+        _ h264Fps: Int,
+        _ h264Bitrate: Int,
+        _ maxDimension: Int
+    ) {
+        engine.updateSettings(
+            mjpegFps: mjpegFps,
+            mjpegQuality: mjpegQuality,
+            h264Fps: h264Fps,
+            h264Bitrate: h264Bitrate,
+            maxDimension: maxDimension
+        )
+    }
+
     @NodeMethod func handleWebRTCOffer(_ offerJson: String) throws -> String {
         try engine.handleWebRTCOffer(offerJson)
     }

package/Sources/SimStreamHelper/H264Encoder.swift

@@ -28,7 +28,7 @@ final class H264Encoder {
     private var pool: CVPixelBufferPool?
     private var width: Int32 = 0
     private var height: Int32 = 0
-    private let fps: Int32
+    private var fps: Int32
     private var bitrate: Int
     private let stateQueue = DispatchQueue(label: "H264Encoder.state")
     private var emittedDescription = false
@@ -39,8 +39,8 @@ final class H264Encoder {
     private var retiredSessions: [VTCompressionSession] = []
 
     init(fps: Int = 60, bitrate: Int = 6_000_000) {
-        self.fps = Int32(fps)
-        self.bitrate = bitrate
+        self.fps = Int32(max(1, fps))
+        self.bitrate = max(1, bitrate)
     }
 
     deinit {
@@ -132,6 +132,28 @@ final class H264Encoder {
         fallbackFromLowLatency(reason: "encode timeout")
     }
 
+    func update(fps nextFps: Int, bitrate nextBitrate: Int) {
+        lock.lock()
+        defer { lock.unlock() }
+        let normalizedFps = Int32(max(1, nextFps))
+        let normalizedBitrate = max(1, nextBitrate)
+        guard fps != normalizedFps || bitrate != normalizedBitrate else { return }
+        fps = normalizedFps
+        bitrate = normalizedBitrate
+        forceKeyframeAfterReset = true
+        frameCount = 0
+        if let session {
+            retiredSessions.append(session)
+            self.session = nil
+        }
+        pool = nil
+        stateQueue.sync {
+            emittedDescription = false
+            encodedCount = 0
+        }
+        streamLog("[stream:h264] settings updated fps=\(fps) bitrate=\(bitrate)")
+    }
+
     func stop() {
         lock.lock()
         defer { lock.unlock() }

package/Sources/SimStreamHelper/WebRTCPublisher.swift

@@ -104,11 +104,6 @@ final class WebRTCPublisher {
             return
         }
 
-        if let transceiver = peerConnection.addTransceiver(with: videoTrack) {
-            applyVideoCodecPreference(request.codec, to: transceiver)
-        } else {
-            _ = peerConnection.add(videoTrack, streamIds: ["stream0"])
-        }
         let session = WebRTCSession(peerConnection: peerConnection, delegate: delegate)
         self.session?.close()
         self.session = session
@@ -119,6 +114,7 @@ final class WebRTCPublisher {
                 completion(.failure(error))
                 return
             }
+            self.attachVideoTrack(to: peerConnection, codec: request.codec)
             peerConnection.answer(for: constraints) { answer, error in
                 if let error {
                     completion(.failure(error))
@@ -145,6 +141,32 @@ final class WebRTCPublisher {
         }
     }
 
+    private func attachVideoTrack(to peerConnection: LKRTCPeerConnection, codec: String?) {
+        let transceiver = peerConnection.transceivers.first { $0.mediaType == .video }
+            ?? createFallbackVideoTransceiver(on: peerConnection)
+        guard let transceiver else {
+            _ = peerConnection.add(videoTrack, streamIds: ["stream0"])
+            print("[webrtc] Could not find or create video transceiver; fell back to addTrack")
+            return
+        }
+
+        transceiver.sender.track = videoTrack
+        transceiver.sender.streamIds = ["stream0"]
+        var directionError: NSError?
+        transceiver.setDirection(.sendOnly, error: &directionError)
+        if let directionError {
+            print("[webrtc] Failed to set video transceiver direction: \(directionError.localizedDescription)")
+        }
+        applyVideoCodecPreference(codec, to: transceiver)
+    }
+
+    private func createFallbackVideoTransceiver(on peerConnection: LKRTCPeerConnection) -> LKRTCRtpTransceiver? {
+        let initOptions = LKRTCRtpTransceiverInit()
+        initOptions.direction = .sendOnly
+        initOptions.streamIds = ["stream0"]
+        return peerConnection.addTransceiver(with: videoTrack, init: initOptions)
+    }
+
     private func iceServers(from payload: [WebRTCIceServerPayload]?) -> [LKRTCIceServer] {
         let servers = payload ?? [
             WebRTCIceServerPayload(urls: ["stun:stun.l.google.com:19302"], username: nil, credential: nil),
@@ -160,7 +182,15 @@ final class WebRTCPublisher {
     }
 
     private func applyVideoCodecPreference(_ codec: String?, to transceiver: LKRTCRtpTransceiver) {
-        let preferredName = codec == "vp8" ? "VP8" : "H264"
+        let preferredName: String
+        switch codec?.lowercased() {
+        case "vp8":
+            preferredName = "VP8"
+        case "vp9":
+            preferredName = "VP9"
+        default:
+            preferredName = "H264"
+        }
         let capabilities = factory.rtpSenderCapabilities(forKind: "video")
         let preferredCodecs = capabilities.codecs.filter {
             $0.name.caseInsensitiveCompare(preferredName) == .orderedSame ||