react-native-webrtc-nitro 1.0.0

package/ios/FramePipeWrapper.mm

@@ -0,0 +1,320 @@
+#import "FramePipeWrapper.h"
+#include "FramePipe.hpp"
+#include <string>
+
+@implementation FramePipeWrapper
+
++ (void)cameraPublishVideo:(CMSampleBufferRef)sampleBuffer
+                   pipeIds:(NSArray<NSString *> *)pipeIds
+{
+    if (pipeIds.count == 0)
+    {
+        return;
+    }
+
+    CVPixelBufferRef pixelBuffer = CMSampleBufferGetImageBuffer (sampleBuffer);
+    CVPixelBufferLockBaseAddress (pixelBuffer, kCVPixelBufferLock_ReadOnly);
+
+    size_t width = CVPixelBufferGetWidth (pixelBuffer);
+    size_t height = CVPixelBufferGetHeight (pixelBuffer);
+
+    uint8_t *srcY
+        = (uint8_t *)CVPixelBufferGetBaseAddressOfPlane (pixelBuffer, 0);
+    uint8_t *srcUV
+        = (uint8_t *)CVPixelBufferGetBaseAddressOfPlane (pixelBuffer, 1);
+    int srcYStride = (int)CVPixelBufferGetBytesPerRowOfPlane (pixelBuffer, 0);
+    int srcUVStride = (int)CVPixelBufferGetBytesPerRowOfPlane (pixelBuffer, 1);
+
+    FFmpeg::Frame frame (AV_PIX_FMT_NV12, width, height);
+    // Copy Y
+    for (size_t i = 0; i < height; ++i)
+    {
+        memcpy (frame->data[0] + i * frame->linesize[0], srcY + i * srcYStride,
+                width);
+    }
+    // Copy UV
+    for (size_t i = 0; i < height / 2; ++i)
+    {
+        memcpy (frame->data[1] + i * frame->linesize[1],
+                srcUV + i * srcUVStride, width);
+    }
+
+    CVPixelBufferUnlockBaseAddress (pixelBuffer, kCVPixelBufferLock_ReadOnly);
+
+    for (NSString *pipeId in pipeIds)
+    {
+        publish (std::string ([pipeId UTF8String]), frame);
+    }
+}
+
++ (void)microphonePublishAudio:(CMSampleBufferRef)sampleBuffer
+                       pipeIds:(NSArray<NSString *> *)pipeIds
+{
+    if (pipeIds.count == 0)
+    {
+        return;
+    }
+
+    CMFormatDescriptionRef formatDescription
+        = CMSampleBufferGetFormatDescription (sampleBuffer);
+    const AudioStreamBasicDescription *audioFormat
+        = CMAudioFormatDescriptionGetStreamBasicDescription (
+            formatDescription);
+
+    if (audioFormat == NULL)
+    {
+        return;
+    }
+
+    if (audioFormat->mFormatID != kAudioFormatLinearPCM)
+    {
+        NSLog (@"[Microphone] Unsupported format: %u, expected Linear PCM",
+               audioFormat->mFormatID);
+        return;
+    }
+
+    int sampleRate = (int)audioFormat->mSampleRate;
+    int channels = audioFormat->mChannelsPerFrame;
+    CMItemCount numSamples = CMSampleBufferGetNumSamples (sampleBuffer);
+
+    bool isInterleaved
+        = !(audioFormat->mFormatFlags & kAudioFormatFlagIsNonInterleaved);
+
+    int bitsPerChannel = audioFormat->mBitsPerChannel;
+
+    AVSampleFormat sampleFormat;
+    if (bitsPerChannel == 16)
+    {
+        sampleFormat = isInterleaved ? AV_SAMPLE_FMT_S16 : AV_SAMPLE_FMT_S16P;
+    }
+    else if (bitsPerChannel == 32
+             && (audioFormat->mFormatFlags & kAudioFormatFlagIsFloat))
+    {
+        sampleFormat = isInterleaved ? AV_SAMPLE_FMT_FLT : AV_SAMPLE_FMT_FLTP;
+    }
+    else
+    {
+        NSLog (@"[Microphone] Unsupported bit depth: %d bits", bitsPerChannel);
+        return;
+    }
+
+    FFmpeg::Frame frame (sampleFormat, sampleRate, channels, (int)numSamples);
+
+    AudioBufferList audioBufferList;
+    CMBlockBufferRef blockBufferOut = NULL;
+    OSStatus status = CMSampleBufferGetAudioBufferListWithRetainedBlockBuffer (
+        sampleBuffer,
+        NULL, // bufferListSizeNeededOut
+        &audioBufferList, sizeof (audioBufferList),
+        NULL, // blockBufferAllocator
+        NULL, // blockBufferMemoryAllocator
+        kCMSampleBufferFlag_AudioBufferList_Assure16ByteAlignment,
+        &blockBufferOut);
+    if (status != noErr || blockBufferOut == NULL)
+    {
+        NSLog (@"[Microphone] Failed to get audio buffer list: %d", status);
+        return;
+    }
+
+    UInt32 numBuffers = audioBufferList.mNumberBuffers;
+    for (UInt32 i = 0; i < numBuffers; i++)
+    {
+        AudioBuffer *srcBuffer = &audioBufferList.mBuffers[i];
+        memcpy (frame->data[i], srcBuffer->mData, srcBuffer->mDataByteSize);
+    }
+
+    CFRelease (blockBufferOut);
+
+    for (NSString *pipeId in pipeIds)
+    {
+        publish (std::string ([pipeId UTF8String]), frame);
+    }
+}
+
++ (int)viewSubscribeVideo:(AVSampleBufferDisplayLayer *)displayLayer
+                   pipeId:(NSString *)pipeId
+{
+
+    auto scaler = std::make_shared<FFmpeg::Scaler> ();
+    auto callback
+        = [displayLayer, scaler] (std::string pipeId, int subscriptionId,
+                                  const FFmpeg::Frame &raw)
+    {
+        FFmpeg::Frame frame
+            = scaler->scale (raw, AV_PIX_FMT_NV12, raw->width, raw->height);
+
+        int width = frame->width;
+        int height = frame->height;
+        CMVideoFormatDescriptionRef formatDescription = NULL;
+        OSStatus status = CMVideoFormatDescriptionCreate (
+            kCFAllocatorDefault,
+            kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange, // NV12
+            width, height, NULL, &formatDescription);
+
+        if (status != noErr)
+        {
+            NSLog (@"Failed to create format description: %d", status);
+            return;
+        }
+
+        CVPixelBufferRef pixelBuffer = NULL;
+        NSDictionary *pixelBufferAttributes = @{
+            (NSString *)kCVPixelBufferPixelFormatTypeKey :
+                @(kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange),
+            (NSString *)kCVPixelBufferWidthKey : @(width),
+            (NSString *)kCVPixelBufferHeightKey : @(height),
+            (NSString *)kCVPixelBufferIOSurfacePropertiesKey : @{}
+        };
+
+        CVReturn result = CVPixelBufferCreate (
+            kCFAllocatorDefault, width, height,
+            kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange,
+            (__bridge CFDictionaryRef)pixelBufferAttributes, &pixelBuffer);
+
+        if (result != kCVReturnSuccess)
+        {
+            NSLog (@"Failed to create pixel buffer: %d", result);
+            return;
+        }
+
+        CVPixelBufferLockBaseAddress (pixelBuffer, 0);
+
+        // COPY Y PLANE
+        uint8_t *yDestPlane
+            = (uint8_t *)CVPixelBufferGetBaseAddressOfPlane (pixelBuffer, 0);
+        size_t yDestBytesPerRow
+            = CVPixelBufferGetBytesPerRowOfPlane (pixelBuffer, 0);
+        uint8_t *ySrcPlane = frame->data[0];
+        size_t ySrcBytesPerRow = frame->linesize[0];
+
+        for (int row = 0; row < height; row++)
+        {
+            memcpy (yDestPlane + row * yDestBytesPerRow,
+                    ySrcPlane + row * ySrcBytesPerRow, width);
+        }
+
+        // COPY UV PLANE
+        uint8_t *uvDestPlane
+            = (uint8_t *)CVPixelBufferGetBaseAddressOfPlane (pixelBuffer, 1);
+        size_t uvDestBytesPerRow
+            = CVPixelBufferGetBytesPerRowOfPlane (pixelBuffer, 1);
+        uint8_t *uvSrcPlane = frame->data[1];
+        size_t uvSrcBytesPerRow = frame->linesize[1];
+
+        for (int row = 0; row < height / 2; row++)
+        {
+            memcpy (uvDestPlane + row * uvDestBytesPerRow,
+                    uvSrcPlane + row * uvSrcBytesPerRow, width);
+        }
+
+        CVPixelBufferUnlockBaseAddress (pixelBuffer, 0);
+
+        CMSampleBufferRef sampleBuffer = NULL;
+        CMSampleTimingInfo timingInfo
+            = { .duration = kCMTimeInvalid,
+                .presentationTimeStamp
+                = CMTimeMakeWithSeconds (CACurrentMediaTime (), 1000000000),
+                .decodeTimeStamp = kCMTimeInvalid };
+
+        status = CMSampleBufferCreateReadyWithImageBuffer (
+            kCFAllocatorDefault, pixelBuffer, formatDescription, &timingInfo,
+            &sampleBuffer);
+
+        CVPixelBufferRelease (pixelBuffer);
+        CFRelease (formatDescription);
+
+        if (status != noErr)
+        {
+            NSLog (@"Failed to create sample buffer: %d", status);
+            return;
+        }
+
+        dispatch_async (dispatch_get_main_queue (), ^{
+          if (displayLayer.status == AVQueuedSampleBufferRenderingStatusFailed)
+          {
+              NSLog (
+                  @"[FramePipeWrapper] DisplayLayer status failed, flushing. "
+                  @"Error: %@",
+                  displayLayer.error);
+              [displayLayer flush];
+          }
+
+          if (displayLayer.isReadyForMoreMediaData)
+          {
+              // NSLog(@"[FramePipeWrapper] Enqueuing sample buffer to
+              // displayLayer");
+              [displayLayer enqueueSampleBuffer:sampleBuffer];
+          }
+          else
+          {
+              NSLog (@"[FramePipeWrapper] DisplayLayer not ready for more "
+                     @"media data");
+          }
+
+          CFRelease (sampleBuffer);
+        });
+    };
+
+    return subscribe ({ std::string ([pipeId UTF8String]) }, callback);
+}
+
++ (int)speakerSubscribeAudio:(AVAudioPlayerNode *)audioPlayer
+                      pipeId:(NSString *)pipeId
+                       queue:(dispatch_queue_t)queue
+{
+    auto resampler = std::make_shared<FFmpeg::Resampler> ();
+    FrameCallback callback
+        = [audioPlayer, resampler, queue] (
+              std::string pipeId, int subscriptionId, const FFmpeg::Frame &raw)
+    {
+        AVAudioFormat *format = [audioPlayer outputFormatForBus:0];
+
+        AVSampleFormat avSampleFormat = AV_SAMPLE_FMT_NONE;
+        if (format.isInterleaved)
+        {
+            avSampleFormat = AV_SAMPLE_FMT_FLT;
+        }
+        else
+        {
+            avSampleFormat = AV_SAMPLE_FMT_FLTP;
+        }
+        std::optional<FFmpeg::Frame> frame = resampler->resample (
+            raw, avSampleFormat, format.sampleRate, format.channelCount);
+        if (!frame.has_value ())
+        {
+            return;
+        }
+
+        AVAudioFrameCount frameCount
+            = (AVAudioFrameCount)frame.value ()->nb_samples;
+        AVAudioPCMBuffer *buffer =
+            [[AVAudioPCMBuffer alloc] initWithPCMFormat:format
+                                          frameCapacity:frameCount];
+
+        if (!buffer)
+        {
+            return;
+        }
+
+        for (int i = 0; i < frame.value ()->ch_layout.nb_channels; i++)
+        {
+            memcpy (buffer.audioBufferList->mBuffers[i].mData,
+                    frame.value ()->data[i],
+                    frame.value ()->nb_samples * sizeof (float));
+        }
+        buffer.frameLength = frameCount;
+
+        dispatch_async (queue, ^{
+          [audioPlayer scheduleBuffer:buffer completionHandler:nil];
+        });
+    };
+
+    return subscribe ({ std::string ([pipeId UTF8String]) }, callback);
+}
+
++ (void)unsubscribe:(int)subscriptionId
+{
+    unsubscribe (subscriptionId);
+}
+
+@end

package/ios/HybridCamera.swift

@@ -0,0 +1,111 @@
+//
+//  HybridCamera.swift
+//  Pods
+//
+//  Created by kaizhi-singtown on 2025/11/27.
+//
+
+import AVFoundation
+import Foundation
+import NitroModules
+
+private class CameraManager: NSObject, AVCaptureVideoDataOutputSampleBufferDelegate {
+    static let shared = CameraManager()
+    
+    let captureSession = AVCaptureSession()
+    let videoOutput = AVCaptureVideoDataOutput()
+    let videoQueue = DispatchQueue(label: "com.camera.shared.video.queue")
+    
+    private var activePipeIds: Set<String> = []
+    private let pipeIdsLock = NSLock()
+    
+    private override init() {
+        super.init()
+        captureSession.sessionPreset = .high
+        
+        videoOutput.videoSettings = [
+            kCVPixelBufferPixelFormatTypeKey as String:
+                kCVPixelFormatType_420YpCbCr8BiPlanarFullRange
+        ]
+        videoOutput.setSampleBufferDelegate(self, queue: videoQueue)
+        if captureSession.canAddOutput(videoOutput) {
+            captureSession.addOutput(videoOutput)
+        }
+        
+    }
+    
+    func prepare() throws {
+        guard let videoDevice = AVCaptureDevice.default(
+            .builtInWideAngleCamera,
+            for: .video,
+            position: .back) else {
+            throw RuntimeError.error(withMessage: "Unable to access camera")
+        }
+        let deviceInput = try AVCaptureDeviceInput(device: videoDevice)
+        
+        let videoDeviceInput = deviceInput
+        if captureSession.canAddInput(videoDeviceInput) {
+            captureSession.addInput(videoDeviceInput)
+        }
+    }
+    
+    // AVCaptureVideoDataOutputSampleBufferDelegate
+    func captureOutput(
+        _ output: AVCaptureOutput,
+        didOutput sampleBuffer: CMSampleBuffer,
+        from connection: AVCaptureConnection
+    ) {
+        pipeIdsLock.lock()
+        let pipeIds = Array(activePipeIds)
+        pipeIdsLock.unlock()
+        
+        FramePipeWrapper.cameraPublishVideo(sampleBuffer, pipeIds: pipeIds)
+    }
+    
+    func addActivePipeId(_ pipeId: String) {
+        pipeIdsLock.lock()
+        activePipeIds.insert(pipeId)
+        pipeIdsLock.unlock()
+        
+        if !captureSession.isRunning {
+            DispatchQueue.global(qos: .userInitiated).async {
+                self.captureSession.startRunning()
+            }
+        }
+    }
+    
+    func removeActivePipeId(_ pipeId: String) {
+        pipeIdsLock.lock()
+        activePipeIds.remove(pipeId)
+        let hasActivePipes = !activePipeIds.isEmpty
+        pipeIdsLock.unlock()
+        
+        if !hasActivePipes && captureSession.isRunning {
+            captureSession.stopRunning()
+        }
+    }
+}
+
+public class HybridCamera: HybridCameraSpec {
+    private let cameraManager = CameraManager.shared
+    public var pipeId: String = ""
+    
+    public func open(pipeId: String) throws -> Promise<Void> {
+        guard !pipeId.isEmpty else {
+            throw RuntimeError.error(withMessage: "Pipe ID cannot be empty")
+        }
+        
+        try cameraManager.prepare()
+        
+        return Promise.async {
+            try await requestPermission(for: .video)
+            self.pipeId = pipeId
+            self.cameraManager.addActivePipeId(pipeId)
+        }
+    }
+
+    public func dispose() {
+        cameraManager.removeActivePipeId(pipeId)
+        pipeId = ""
+    }
+}

package/ios/HybridMicrophone.swift

@@ -0,0 +1,103 @@
+//
+//  HybridMicrophone.swift
+//  Pods
+//
+//  Created by kaizhi-singtown on 2025/11/27.
+//
+
+import AVFoundation
+import Foundation
+import NitroModules
+
+private class MicrophoneManager: NSObject, AVCaptureAudioDataOutputSampleBufferDelegate {
+    static let shared = MicrophoneManager()
+    
+    let audioSession = AVAudioSession.sharedInstance()
+    let captureSession = AVCaptureSession()
+    let audioOutput = AVCaptureAudioDataOutput()
+    let audioQueue = DispatchQueue(label: "com.microphone.shared.audio.queue")
+    
+    private var activePipeIds: Set<String> = []
+    private let pipeIdsLock = NSLock()
+    
+    func prepare() throws {
+        captureSession.sessionPreset = .high
+        try audioSession.setCategory(
+            .playAndRecord, mode: .videoChat, options: [.defaultToSpeaker])
+        try audioSession.setActive(true)
+        
+        guard let audioDevice = AVCaptureDevice.default(for: .audio) else {
+            throw RuntimeError.error(withMessage: "No audio device found")
+        }
+        let audioInput = try AVCaptureDeviceInput(device: audioDevice)
+        if captureSession.canAddInput(audioInput) {
+            captureSession.addInput(audioInput)
+        }
+        audioOutput.setSampleBufferDelegate(self, queue: audioQueue)
+        if captureSession.canAddOutput(audioOutput) {
+            captureSession.addOutput(audioOutput)
+        }
+    }
+    
+    // AVCaptureAudioDataOutputSampleBufferDelegate
+    func captureOutput(
+        _ output: AVCaptureOutput,
+        didOutput sampleBuffer: CMSampleBuffer,
+        from connection: AVCaptureConnection
+    ) {
+        pipeIdsLock.lock()
+        let pipeIds = Array(activePipeIds)
+        pipeIdsLock.unlock()
+        
+        FramePipeWrapper.microphonePublishAudio(sampleBuffer, pipeIds: pipeIds)
+    }
+    
+    func addActivePipeId(_ pipeId: String) {
+        pipeIdsLock.lock()
+        activePipeIds.insert(pipeId)
+        pipeIdsLock.unlock()
+        
+        if !captureSession.isRunning {
+            DispatchQueue.global(qos: .userInitiated).async {
+                self.captureSession.startRunning()
+            }
+        }
+    }
+    
+    func removeActivePipeId(_ pipeId: String) {
+        pipeIdsLock.lock()
+        activePipeIds.remove(pipeId)
+        let hasActivePipes = !activePipeIds.isEmpty
+        pipeIdsLock.unlock()
+        
+        if !hasActivePipes && captureSession.isRunning {
+            DispatchQueue.global(qos: .userInitiated).async {
+                self.captureSession.stopRunning()
+            }
+        }
+    }
+}
+
+public class HybridMicrophone: HybridMicrophoneSpec {
+    private let microphoneManager = MicrophoneManager.shared
+    public var pipeId: String = ""
+    
+    public func open(pipeId: String) throws -> Promise<Void> {
+        guard !pipeId.isEmpty else {
+            throw RuntimeError.error(withMessage: "Pipe ID cannot be empty")
+        }
+        
+        try microphoneManager.prepare()
+        
+        return Promise.async {
+            try await requestPermission(for: .audio)
+            self.pipeId = pipeId
+            self.microphoneManager.addActivePipeId(pipeId)
+        }
+    }
+    
+    public func dispose() {
+        microphoneManager.removeActivePipeId(pipeId)
+        pipeId = ""
+    }
+}

package/ios/HybridWebrtcView.swift

@@ -0,0 +1,123 @@
+//
+//  HybridWebrtcView.swift
+//  Pods
+//
+//  Created by kaizhi-singtown on 2025/11/27.
+//
+
+import AVFoundation
+import Foundation
+import UIKit
+
+class WebrtcDisplayView: UIView {
+    var displayLayer: AVSampleBufferDisplayLayer = AVSampleBufferDisplayLayer()
+    
+    override init(frame: CGRect) {
+        super.init(frame: frame)
+        displayLayer.videoGravity = .resizeAspectFill
+        displayLayer.frame = bounds
+        layer.addSublayer(displayLayer)
+    }
+    
+    required init?(coder: NSCoder) {
+        super.init(coder: coder)
+        displayLayer.videoGravity = .resizeAspectFill
+        displayLayer.frame = bounds
+        layer.addSublayer(displayLayer)
+    }
+    
+    override func layoutSubviews() {
+        super.layoutSubviews()
+        displayLayer.frame = bounds
+    }
+    
+    deinit {
+        displayLayer.flushAndRemoveImage()
+    }
+}
+
+class SpeakerManager {
+    private static var sharedAudioSession = AVAudioSession()
+    private static var sharedAudioEngine = AVAudioEngine()
+    private var audioPlayer = AVAudioPlayerNode()
+    private var speakerQueue = DispatchQueue(label: "com.speaker.manager.audio.queue")
+    let subscriptionId: Int32
+    static var instanceNumber: Int = 0
+    
+    init(pipeId: String) throws {
+        Self.instanceNumber += 1
+        
+        try Self.sharedAudioSession.setCategory(
+            .playAndRecord, mode: .videoChat, options: [.defaultToSpeaker])
+        try Self.sharedAudioSession.setActive(true)
+        
+        Self.sharedAudioEngine.attach(audioPlayer)
+        
+        let outputFormat = Self.sharedAudioEngine.mainMixerNode.outputFormat(forBus: 0)
+        
+        Self.sharedAudioEngine.connect(
+            audioPlayer, to: Self.sharedAudioEngine.mainMixerNode, format: outputFormat)
+        
+        try Self.sharedAudioEngine.start()
+        audioPlayer.play()
+        
+        subscriptionId = FramePipeWrapper.speakerSubscribeAudio(
+            audioPlayer, pipeId: pipeId, queue: speakerQueue
+        )
+    }
+    
+    deinit {
+        Self.instanceNumber -= 1
+        if Self.instanceNumber == 0 {
+            Self.sharedAudioEngine.stop()
+        }
+        audioPlayer.stop()
+        Self.sharedAudioEngine.disconnectNodeOutput(audioPlayer)
+        Self.sharedAudioEngine.detach(audioPlayer)
+        FramePipeWrapper.unsubscribe(subscriptionId)
+    }
+}
+
+class HybridWebrtcView: HybridWebrtcViewSpec {
+    var view: UIView = WebrtcDisplayView()
+    var speaker: SpeakerManager? = nil
+    
+    var videoSubscriptionId: Int32 = -1
+    fileprivate static let lock = NSLock()
+    
+    var audioPipeId: String? {
+        didSet {
+            Self.lock.lock()
+            defer { Self.lock.unlock() }
+            
+            speaker = nil
+            if audioPipeId != nil {
+                speaker = try? SpeakerManager(pipeId: audioPipeId!)
+            }
+        }
+    }
+    
+    var videoPipeId: String? {
+        didSet {
+            FramePipeWrapper.unsubscribe(videoSubscriptionId)
+            videoSubscriptionId = -1
+            guard let videoPipeId = videoPipeId else {
+                return
+            }
+            if let displayView = view as? WebrtcDisplayView {
+                videoSubscriptionId = FramePipeWrapper.viewSubscribeVideo(
+                    displayView.displayLayer, pipeId: videoPipeId)
+            }
+        }
+    }
+    
+    deinit {
+        Self.lock.lock()
+        defer { Self.lock.unlock() }
+        
+        if videoSubscriptionId != -1 {
+            FramePipeWrapper.unsubscribe(videoSubscriptionId)
+        }
+        
+    }
+}

package/ios/Permission.swift

@@ -0,0 +1,26 @@
+import AVFoundation
+import Foundation
+import NitroModules
+
+public func requestPermission(for mediaType: AVMediaType) async throws {
+    let status = AVCaptureDevice.authorizationStatus(for: mediaType)
+    switch status {
+    case .authorized:
+        return
+    case .notDetermined:
+        let granted = await withCheckedContinuation { (continuation) in
+            AVCaptureDevice.requestAccess(for: mediaType) { ok in
+                continuation.resume(returning: ok)
+            }
+        }
+        if granted {
+            return
+        } else {
+            throw RuntimeError.error(withMessage: "Permission not allowed")
+        }
+    case .denied, .restricted:
+        throw RuntimeError.error(withMessage: "Permission not allowed")
+    @unknown default:
+        throw RuntimeError.error(withMessage: "Permission not allowed")
+    }
+}