react-native-webrtc-nitro 1.0.0

package/android/src/main/java/com/webrtc/HybridWebrtcView.kt

@@ -0,0 +1,96 @@
+package com.webrtc
+
+import android.view.Surface
+import android.view.SurfaceView
+import android.view.SurfaceHolder
+import android.media.AudioFormat
+import android.media.AudioManager
+import android.media.AudioTrack
+import androidx.annotation.Keep
+import com.facebook.proguard.annotations.DoNotStrip
+import com.facebook.react.uimanager.ThemedReactContext
+import com.margelo.nitro.webrtc.HybridWebrtcViewSpec
+
+@Keep
+@DoNotStrip
+class HybridWebrtcView(val context: ThemedReactContext) : HybridWebrtcViewSpec() {
+    // View
+    override val view: SurfaceView = SurfaceView(context)
+    private var audioTrack = AudioTrack(
+        AudioManager.STREAM_MUSIC,
+        48000,
+        AudioFormat.CHANNEL_OUT_STEREO,
+        AudioFormat.ENCODING_PCM_16BIT,
+        AudioTrack.getMinBufferSize(
+            48000,
+            AudioFormat.CHANNEL_OUT_STEREO,
+            AudioFormat.ENCODING_PCM_16BIT
+        ),
+        AudioTrack.MODE_STREAM
+    )
+
+    external fun unsubscribe(subscriptionId: Int)
+    external fun subscribeAudio(pipeId: String, track: AudioTrack): Int
+    external fun subscribeVideo(pipeId: String, surface: Surface): Int
+
+    private var _audioPipeId: String? = null
+    private var _videoPipeId: String? = null
+    private var videoSubscriptionId: Int = -1
+    private var audioSubscriptionId: Int = -1
+
+    override var audioPipeId: String?
+        get() = _audioPipeId
+        set(value) {
+            if (this.audioSubscriptionId > 0) {
+                this.unsubscribe(this.audioSubscriptionId)
+            }
+            if (value.isNullOrEmpty()) {
+                return;
+            }
+            this.audioSubscriptionId = subscribeAudio(value, audioTrack)
+            this._audioPipeId = value
+            audioTrack.play()
+        }
+
+
+    init {
+        view.holder.addCallback(object : SurfaceHolder.Callback {
+            override fun surfaceCreated(holder: SurfaceHolder) {
+                updateVideoPipeId(_videoPipeId, holder.surface)
+            }
+
+            override fun surfaceChanged(
+                holder: SurfaceHolder,
+                format: Int,
+                width: Int,
+                height: Int
+            ) {
+                updateVideoPipeId(_videoPipeId, holder.surface)
+            }
+
+            override fun surfaceDestroyed(holder: SurfaceHolder) {
+                updateVideoPipeId(_videoPipeId, null)
+            }
+        })
+    }
+
+    private fun updateVideoPipeId(newVideoPipeId: String?, surface: Surface?) {
+        if (this.videoSubscriptionId > 0) {
+            this.unsubscribe(this.videoSubscriptionId)
+        }
+        if (surface == null) {
+            return;
+        }
+        if (newVideoPipeId.isNullOrEmpty()) {
+            return;
+        }
+        this.videoSubscriptionId = subscribeVideo(newVideoPipeId, surface)
+        this._videoPipeId = newVideoPipeId
+    }
+
+    override var videoPipeId: String?
+        get() = _videoPipeId
+        set(value) {
+            updateVideoPipeId(value, view.holder.surface)
+        }
+}

package/android/src/main/java/com/webrtc/Permission.kt

@@ -0,0 +1,58 @@
+package com.webrtc
+
+import android.content.pm.PackageManager
+import androidx.core.content.ContextCompat
+import com.facebook.react.modules.core.PermissionAwareActivity
+import com.facebook.react.modules.core.PermissionListener
+import kotlin.random.Random
+import kotlinx.coroutines.CompletableDeferred
+import com.margelo.nitro.NitroModules
+
+suspend fun requestPermission(permission: String) {
+    val context = NitroModules.applicationContext
+        ?: throw RuntimeException("ReactApplicationContext is not available")
+
+    val currentActivity = context.currentActivity
+        ?: throw RuntimeException("No current Activity")
+
+    if (ContextCompat.checkSelfPermission(context, permission)
+        == PackageManager.PERMISSION_GRANTED
+    ) {
+        return
+    }
+
+    if (currentActivity !is PermissionAwareActivity) {
+        throw RuntimeException("Current activity doesn't support permissions")
+    }
+
+    val code = Random.nextInt(0, 65535)
+    val deferred = CompletableDeferred<Unit>()
+
+    val listener = object : PermissionListener {
+        override fun onRequestPermissionsResult(
+            requestCode: Int,
+            permissions: Array<String>,
+            grantResults: IntArray
+        ): Boolean {
+            if (requestCode != code) return false
+
+            if (permissions.isEmpty() || grantResults.isEmpty() ||
+                grantResults[0] != PackageManager.PERMISSION_GRANTED
+            ) {
+                deferred.completeExceptionally(
+                    RuntimeException("Permission denied by user")
+                )
+            } else {
+                deferred.complete(Unit)
+            }
+            return true
+        }
+    }
+
+    currentActivity.requestPermissions(
+        arrayOf(permission),
+        code,
+        listener
+    )
+    deferred.await()
+}

package/android/src/main/java/com/webrtc/WebrtcPackage.kt

@@ -0,0 +1,31 @@
+package com.webrtc;
+
+import com.facebook.react.bridge.NativeModule;
+import com.facebook.react.bridge.ReactApplicationContext;
+import com.facebook.react.module.model.ReactModuleInfoProvider;
+import com.facebook.react.TurboReactPackage;
+import com.facebook.react.uimanager.ViewManager;
+import com.margelo.nitro.webrtc.*;
+import com.margelo.nitro.webrtc.views.*;
+
+
+public class WebrtcPackage : TurboReactPackage() {
+    override fun getModule(name: String, reactContext: ReactApplicationContext): NativeModule? =
+        null
+
+    override fun getReactModuleInfoProvider(): ReactModuleInfoProvider =
+        ReactModuleInfoProvider { emptyMap() }
+
+    override fun createViewManagers(reactContext: ReactApplicationContext): List<ViewManager<*, *>> {
+        val viewManagers = ArrayList<ViewManager<*, *>>()
+        viewManagers.add(HybridWebrtcViewManager())
+        return viewManagers
+    }
+
+    companion object {
+        init {
+            WebrtcOnLoad.initializeNative()
+        }
+    }
+}
+

package/cpp/FFmpeg/AudioFifo.cpp

@@ -0,0 +1,78 @@
+#include "AudioFifo.hpp"
+
+using namespace FFmpeg;
+
+AudioFifo::~AudioFifo ()
+{
+    if (fifo != nullptr)
+    {
+        av_audio_fifo_free (fifo);
+        fifo = nullptr;
+    }
+}
+
+void AudioFifo::write (const Frame &frame)
+{
+    std::lock_guard lock (mutex);
+    if (fifo == nullptr)
+    {
+        // init
+        format = (AVSampleFormat)frame->format;
+        channels = frame->ch_layout.nb_channels;
+        sample_rate = frame->sample_rate;
+        pts = frame->pts;
+        constexpr int AUDIO_FIFO_BUFFER_SIZE = 1024;
+        fifo = av_audio_fifo_alloc (format, channels, AUDIO_FIFO_BUFFER_SIZE);
+        if (fifo == nullptr)
+        {
+            throw std::runtime_error ("av_audio_fifo_alloc faield");
+        }
+    }
+
+    int ret
+        = av_audio_fifo_write (fifo, (void **)frame->data, frame->nb_samples);
+    checkError (ret, "av_audio_fifo_write");
+    if (ret < frame->nb_samples)
+    {
+        throw std::runtime_error ("Could not write to audio fifo");
+    }
+}
+
+auto AudioFifo::read (int nb_samples) -> std::optional<Frame>
+{
+    std::lock_guard lock (mutex);
+    if (fifo == nullptr)
+    {
+        return std::nullopt;
+    }
+    if (av_audio_fifo_size (fifo) == 0)
+    {
+        return std::nullopt;
+    }
+    if (nb_samples < 0)
+    {
+        nb_samples = av_audio_fifo_size (fifo);
+    }
+    if (av_audio_fifo_size (fifo) < nb_samples)
+    {
+        return std::nullopt;
+    }
+
+    Frame frame (format, sample_rate, channels, nb_samples, pts);
+
+    int ret = av_audio_fifo_read (fifo, (void **)frame->data, nb_samples);
+    checkError (ret, "av_audio_fifo_read");
+
+    pts += nb_samples;
+
+    return frame;
+}
+
+void AudioFifo::clear ()
+{
+    std::lock_guard lock (mutex);
+    if (fifo != nullptr)
+    {
+        av_audio_fifo_reset (fifo);
+    }
+}

package/cpp/FFmpeg/AudioFifo.hpp

@@ -0,0 +1,24 @@
+#pragma once
+#include "Frame.hpp"
+
+namespace FFmpeg
+{
+    class AudioFifo
+    {
+      private:
+        std::recursive_mutex mutex;
+        AVAudioFifo *fifo = nullptr;
+
+        AVSampleFormat format = AV_SAMPLE_FMT_NONE;
+        int channels = 0;
+        int sample_rate = 0;
+        int64_t pts = -1;
+
+      public:
+        ~AudioFifo ();
+
+        void write (const Frame &frame);
+        auto read (int nb_samples) -> std::optional<Frame>;
+        void clear ();
+    };
+} // namespace FFmpeg

package/cpp/FFmpeg/Common.cpp

@@ -0,0 +1,26 @@
+#include "Common.hpp"
+
+namespace FFmpeg
+{
+
+    void checkError (int ret, const std::string &message)
+    {
+        if (ret >= 0)
+        {
+            return;
+        }
+        constexpr int length = 256;
+        std::vector<char> err (length);
+        av_strerror (ret, err.data (), length);
+        throw std::runtime_error (message + " failed: " + err.data ());
+    }
+
+    auto currentPts (const AVRational &time_base) -> int64_t
+    {
+        static auto globalBaseTime = std::chrono::steady_clock::now ();
+        auto now = std::chrono::steady_clock::now ();
+        double seconds
+            = std::chrono::duration<double> (now - globalBaseTime).count ();
+        return seconds * time_base.den / time_base.num;
+    }
+} // namespace FFmpeg

package/cpp/FFmpeg/Common.hpp

@@ -0,0 +1,26 @@
+#pragma once
+#include <memory>
+#include <mutex>
+#include <optional>
+#include <stdexcept>
+#include <string>
+#include <vector>
+
+extern "C"
+{
+#define AVMediaType FFmpeg_AVMediaType
+#include <libavcodec/avcodec.h>
+#include <libavformat/avformat.h>
+#include <libavutil/audio_fifo.h>
+#include <libavutil/imgutils.h>
+#include <libavutil/log.h>
+#include <libswresample/swresample.h>
+#include <libswscale/swscale.h>
+#undef AVMediaType
+}
+
+namespace FFmpeg
+{
+    void checkError (int ret, const std::string &message);
+    auto currentPts (const AVRational &time_base) -> int64_t;
+} // namespace FFmpeg

package/cpp/FFmpeg/Decoder.cpp

@@ -0,0 +1,72 @@
+#include "Decoder.hpp"
+
+using namespace FFmpeg;
+
+Decoder::Decoder (AVCodecID codecId)
+{
+    std::lock_guard lock (mutex);
+    const AVCodec *decoder = avcodec_find_decoder (codecId);
+    if (decoder == nullptr)
+    {
+        throw std::runtime_error ("Could not find decoder");
+    }
+
+    ctx = avcodec_alloc_context3 (decoder);
+    if (ctx == nullptr)
+    {
+        throw std::runtime_error ("Could not allocate AVCodecContext");
+    }
+    if (decoder->id == AV_CODEC_ID_OPUS)
+    {
+        ctx->time_base = (AVRational){ 1, OPUS_SAMPLE_RATE };
+        ctx->sample_rate = OPUS_SAMPLE_RATE;
+        ctx->bit_rate = OPUS_BIT_RATE;
+        ctx->sample_fmt = AV_SAMPLE_FMT_FLT;
+        av_channel_layout_default (&ctx->ch_layout, 2);
+    }
+    ctx->flags |= AV_CODEC_FLAG_LOW_DELAY;
+    int ret = avcodec_open2 (ctx, decoder, nullptr) < 0;
+    checkError (ret, "avcodec_open2");
+}
+
+Decoder::~Decoder ()
+{
+    if (ctx != nullptr)
+    {
+        avcodec_free_context (&ctx);
+        ctx = nullptr;
+    }
+}
+
+void Decoder::send (const Packet &packet)
+{
+    std::lock_guard lock (mutex);
+    int ret = avcodec_send_packet (ctx, packet.get ());
+    checkError (ret, "avcodec_send_packet");
+    while (true)
+    {
+        Frame frame;
+        int ret = avcodec_receive_frame (ctx, frame.get ());
+        if (ret == AVERROR (EAGAIN) || ret == AVERROR_EOF)
+        {
+            break;
+        }
+        checkError (ret, "avcodec_receive_frame");
+        buffer.push_back (frame);
+    }
+}
+
+auto Decoder::receive () -> std::vector<Frame>
+{
+    std::lock_guard lock (mutex);
+    std::vector<Frame> result = std::move (buffer);
+    buffer.clear ();
+    return result;
+}
+
+void Decoder::flush ()
+{
+    std::lock_guard lock (mutex);
+    int ret = avcodec_send_packet (ctx, nullptr);
+    checkError (ret, "avcodec_send_packet");
+}

package/cpp/FFmpeg/Decoder.hpp

@@ -0,0 +1,25 @@
+#pragma once
+#include "Frame.hpp"
+#include "Packet.hpp"
+
+namespace FFmpeg
+{
+    class Decoder
+    {
+      private:
+        std::recursive_mutex mutex;
+        AVCodecContext *ctx = nullptr;
+        std::vector<Frame> buffer;
+
+        static constexpr int OPUS_BIT_RATE = 64000;
+        static constexpr int OPUS_SAMPLE_RATE = 48000;
+
+      public:
+        Decoder (AVCodecID codecId);
+        ~Decoder ();
+
+        void send (const Packet &packet);
+        auto receive () -> std::vector<Frame>;
+        void flush ();
+    };
+} // namespace FFmpeg

package/cpp/FFmpeg/Encoder.cpp

@@ -0,0 +1,203 @@
+#include "Encoder.hpp"
+
+using namespace FFmpeg;
+
+Encoder::Encoder (AVCodecID codecId)
+{
+    std::lock_guard lock (mutex);
+    if (codecId == AV_CODEC_ID_NONE)
+    {
+        return;
+    }
+    encoder = avcodec_find_encoder (codecId);
+    if (encoder == nullptr)
+    {
+        throw std::runtime_error ("Could not find encoder");
+    }
+}
+
+Encoder::~Encoder ()
+{
+    if (ctx != nullptr)
+    {
+        avcodec_free_context (&ctx);
+        ctx = nullptr;
+    }
+}
+
+void Encoder::_init (const Frame &frame)
+{
+    ctx = avcodec_alloc_context3 (encoder);
+    if (ctx == nullptr)
+    {
+        throw std::runtime_error ("Could not allocate AVCodecContext");
+    }
+    this->basePts = frame->pts;
+    if (encoder->id == AV_CODEC_ID_H264)
+    {
+        ctx->codec_id = AV_CODEC_ID_H264;
+        ctx->width = frame->width;
+        ctx->height = frame->height;
+        ctx->time_base = (AVRational){ 1, H264_SAMPLE_RATE };
+        ctx->framerate = (AVRational){ H264_FPS, 1 };
+        ctx->bit_rate = H264_BIT_RATE;
+        ctx->gop_size = H264_GOP_SIZE;
+        ctx->max_b_frames = 0;
+        ctx->pix_fmt = AV_PIX_FMT_YUV420P;
+        ctx->profile = FF_PROFILE_H264_CONSTRAINED_BASELINE;
+        ctx->color_range = AVCOL_RANGE_MPEG;
+        ctx->color_primaries = AVCOL_PRI_BT709;
+        ctx->color_trc = AVCOL_TRC_BT709;
+        ctx->colorspace = AVCOL_SPC_BT709;
+    }
+    else if (encoder->id == AV_CODEC_ID_H265)
+    {
+        ctx->codec_id = AV_CODEC_ID_H265;
+        ctx->width = frame->width;
+        ctx->height = frame->height;
+        ctx->time_base = (AVRational){ 1, H265_SAMPLE_RATE };
+        ctx->framerate = (AVRational){ H265_FPS, 1 };
+        ctx->bit_rate = H265_BIT_RATE;
+        ctx->gop_size = H265_GOP_SIZE;
+        ctx->max_b_frames = 0;
+        ctx->pix_fmt = AV_PIX_FMT_YUV420P;
+        ctx->profile = FF_PROFILE_HEVC_MAIN;
+        ctx->color_range = AVCOL_RANGE_MPEG;
+        ctx->color_primaries = AVCOL_PRI_BT709;
+        ctx->color_trc = AVCOL_TRC_BT709;
+        ctx->colorspace = AVCOL_SPC_BT709;
+    }
+    else if (encoder->id == AV_CODEC_ID_OPUS)
+    {
+        ctx->codec_id = AV_CODEC_ID_OPUS;
+        ctx->time_base = (AVRational){ 1, OPUS_SAMPLE_RATE };
+        ctx->sample_rate = OPUS_SAMPLE_RATE;
+        ctx->bit_rate = OPUS_BIT_RATE;
+        ctx->sample_fmt = AV_SAMPLE_FMT_FLT;
+        av_channel_layout_default (&ctx->ch_layout, 2);
+    }
+    else if (encoder->id == AV_CODEC_ID_AAC)
+    {
+        ctx->codec_id = AV_CODEC_ID_AAC;
+        ctx->time_base = (AVRational){ 1, AAC_SAMPLE_RATE };
+        ctx->sample_rate = AAC_SAMPLE_RATE;
+        ctx->bit_rate = AAC_BIT_RATE;
+        ctx->sample_fmt = AV_SAMPLE_FMT_FLTP;
+        av_channel_layout_default (&ctx->ch_layout, 2);
+    }
+    else if (encoder->id == AV_CODEC_ID_PNG)
+    {
+        ctx->codec_id = AV_CODEC_ID_PNG;
+        ctx->time_base = (AVRational){ 1, PNG_SAMPLE_RATE };
+        ctx->width = frame->width;
+        ctx->height = frame->height;
+        ctx->pix_fmt = AV_PIX_FMT_RGBA;
+    }
+    else
+    {
+        throw std::runtime_error ("Unsupported encoder"
+                                  + std::to_string (encoder->id));
+    }
+    ctx->flags |= AV_CODEC_FLAG_LOW_DELAY;
+    int ret = avcodec_open2 (ctx, encoder, nullptr);
+    checkError (ret, "avcodec_open2");
+}
+
+void Encoder::_receive ()
+{
+    while (true)
+    {
+        Packet packet;
+        int ret = avcodec_receive_packet (ctx, packet.get ());
+        if (ret == AVERROR (EAGAIN) || ret == AVERROR_EOF)
+        {
+            break;
+        }
+        checkError (ret, "avcodec_receive_packet");
+        buffer.push_back (packet);
+    }
+}
+
+void Encoder::send (const Frame &frame)
+{
+    std::lock_guard lock (mutex);
+    if (ctx == nullptr)
+    {
+        _init (frame);
+    }
+
+    std::vector<Frame> frames;
+
+    if (encoder->id == AV_CODEC_ID_OPUS)
+    {
+        Frame resampled_frame = resampler.resample (frame, AV_SAMPLE_FMT_FLT,
+                                                    AAC_SAMPLE_RATE, 2);
+        fifo.write (resampled_frame);
+
+        while (std::optional<Frame> frame = fifo.read (OPUS_NB_SAMPLES))
+        {
+            frames.push_back (frame.value ());
+        }
+    }
+    else if (encoder->id == AV_CODEC_ID_AAC)
+    {
+        Frame resampled_frame = resampler.resample (frame, AV_SAMPLE_FMT_FLTP,
+                                                    OPUS_SAMPLE_RATE, 2);
+        fifo.write (resampled_frame);
+        while (std::optional<Frame> frame = fifo.read (AAC_NB_SAMPLES))
+        {
+            frames.push_back (frame.value ());
+        }
+    }
+    else if (encoder->id == AV_CODEC_ID_H264
+             || encoder->id == AV_CODEC_ID_H265)
+    {
+        frames.push_back (scaler.scale (frame, AV_PIX_FMT_YUV420P,
+                                        frame->width, frame->height));
+    }
+    else if (encoder->id == AV_CODEC_ID_PNG)
+    {
+        frames.push_back (scaler.scale (frame, AV_PIX_FMT_RGBA, frame->width,
+                                        frame->height));
+    }
+    else
+    {
+        throw std::runtime_error ("Unsupported encoder"
+                                  + std::to_string (encoder->id));
+    }
+
+    for (Frame &frame : frames)
+    {
+        frame->pts -= this->basePts;
+        int ret = avcodec_send_frame (ctx, frame.get ());
+        checkError (ret, "avcodec_send_frame");
+        _receive ();
+    }
+}
+
+auto Encoder::receive () -> std::vector<Packet>
+{
+    std::lock_guard lock (mutex);
+    if (ctx == nullptr)
+    {
+        return {};
+    }
+
+    std::vector<Packet> result = std::move (buffer);
+    buffer.clear ();
+    return result;
+}
+
+void Encoder::flush ()
+{
+    std::lock_guard lock (mutex);
+    if (ctx == nullptr)
+    {
+        return;
+    }
+
+    int ret = avcodec_send_frame (ctx, nullptr);
+    checkError (ret, "avcodec_send_frame");
+
+    _receive ();
+}

package/cpp/FFmpeg/Encoder.hpp

@@ -0,0 +1,54 @@
+#pragma once
+#include "AudioFifo.hpp"
+#include "Frame.hpp"
+#include "Packet.hpp"
+#include "Resampler.hpp"
+#include "Scaler.hpp"
+
+namespace FFmpeg
+{
+    class Encoder
+    {
+      private:
+        std::recursive_mutex mutex;
+        Scaler scaler;
+        Resampler resampler;
+        AudioFifo fifo;
+        int64_t basePts = -1;
+        std::vector<Packet> buffer;
+
+        static constexpr int AAC_NB_SAMPLES = 1024;
+        static constexpr int AAC_SAMPLE_RATE = 48000;
+        static constexpr int AAC_BIT_RATE = 128000;
+
+        static constexpr int OPUS_NB_SAMPLES = 960;
+        static constexpr int OPUS_SAMPLE_RATE = 48000;
+        static constexpr int OPUS_BIT_RATE = 64000;
+
+        static constexpr int PNG_SAMPLE_RATE = 90000;
+
+        static constexpr int H264_SAMPLE_RATE = 90000;
+        static constexpr int H264_GOP_SIZE = 60;
+        static constexpr int H264_BIT_RATE = 1000000;
+        static constexpr int H264_FPS = 30;
+
+        static constexpr int H265_SAMPLE_RATE = 90000;
+        static constexpr int H265_GOP_SIZE = 60;
+        static constexpr int H265_BIT_RATE = 1000000;
+        static constexpr int H265_FPS = 30;
+
+        void _init (const Frame &frame);
+        void _receive ();
+
+      public:
+        const AVCodec *encoder = nullptr;
+        AVCodecContext *ctx = nullptr;
+
+        Encoder (AVCodecID codecId);
+        ~Encoder ();
+
+        void send (const Frame &frame);
+        auto receive () -> std::vector<Packet>;
+        void flush ();
+    };
+} // namespace FFmpeg