react-native-webrtc-nitro 1.0.0

package/cpp/FFmpeg/FFmpeg.hpp

@@ -0,0 +1,9 @@
+#pragma once
+#include "AudioFifo.hpp"
+#include "Decoder.hpp"
+#include "Encoder.hpp"
+#include "Frame.hpp"
+#include "Muxer.hpp"
+#include "Packet.hpp"
+#include "Resampler.hpp"
+#include "Scaler.hpp"

package/cpp/FFmpeg/Frame.cpp

@@ -0,0 +1,289 @@
+#include "Frame.hpp"
+#include <random>
+
+using namespace FFmpeg;
+
+Frame::Frame ()
+{
+    frame = std::shared_ptr<AVFrame> (av_frame_alloc (), [] (AVFrame *frame)
+                                      { av_frame_free (&frame); });
+    if (!frame)
+    {
+        throw std::runtime_error ("Could not allocate AVFrame");
+    }
+}
+
+Frame::Frame (AVSampleFormat format, int sampleRate, int channels,
+              int nb_samples, int64_t pts)
+    : Frame ()
+{
+    AVRational time_base = { 1, sampleRate };
+    if (pts == -1)
+    {
+        pts = currentPts (time_base);
+    }
+
+    frame->format = format;
+    frame->time_base = time_base;
+    frame->pts = pts;
+    frame->sample_rate = sampleRate;
+    frame->nb_samples = nb_samples;
+    av_channel_layout_default (&frame->ch_layout, channels);
+    if (frame->nb_samples > 0)
+    {
+        int ret = av_frame_get_buffer (frame.get (), FRAME_ALIGN);
+        checkError (ret, "av_frame_get_buffer");
+    }
+}
+
+Frame::Frame (AVPixelFormat format, int width, int height, int64_t pts)
+    : Frame ()
+{
+    AVRational time_base = { 1, VIDEO_SAMPLE_RATE };
+    if (pts == -1)
+    {
+        pts = currentPts (time_base);
+    }
+
+    frame->format = format;
+    frame->time_base = (AVRational){ 1, VIDEO_SAMPLE_RATE };
+    frame->pts = pts;
+    frame->width = width;
+    frame->height = height;
+
+    int ret = av_frame_get_buffer (frame.get (), 1);
+    checkError (ret, "av_frame_get_buffer");
+}
+
+auto Frame::isAudio () -> bool
+{
+    return frame->width == 0 && frame->height == 0 && frame->nb_samples > 0;
+}
+
+auto Frame::isVideo () -> bool
+{
+    return frame->width > 0 && frame->height > 0 && frame->nb_samples == 0;
+}
+
+void Frame::fillNoiseAudioFLT ()
+{
+    constexpr float MIN_NUMBER = -1.0F;
+    constexpr float MAX_NUMBER = 1.0F;
+    std::mt19937 rng (std::random_device{}());
+    static std::uniform_real_distribution<float> dist (MIN_NUMBER, MAX_NUMBER);
+
+    int channels = frame->ch_layout.nb_channels;
+    auto *inData = (float *)frame->data[0];
+    for (int i = 0; i < frame->nb_samples * channels; ++i)
+    {
+        inData[i] = dist (rng);
+    }
+}
+
+void Frame::fillNoiseAudioFLTP ()
+{
+    constexpr float MIN_NUMBER = -1.0F;
+    constexpr float MAX_NUMBER = 1.0F;
+    std::mt19937 rng (std::random_device{}());
+    static std::uniform_real_distribution<float> dist (MIN_NUMBER, MAX_NUMBER);
+
+    int channels = frame->ch_layout.nb_channels;
+    for (int ch = 0; ch < channels; ++ch)
+    {
+        auto *inData = (float *)frame->data[ch];
+        for (int i = 0; i < frame->nb_samples; ++i)
+        {
+            inData[i] = dist (rng);
+        }
+    }
+}
+
+void Frame::fillNoiseAudioS16 ()
+{
+    constexpr int16_t MIN_NUMBER = -32768;
+    constexpr int16_t MAX_NUMBER = 32767;
+    std::mt19937 rng (std::random_device{}());
+    static std::uniform_int_distribution<int16_t> dist (MIN_NUMBER,
+                                                        MAX_NUMBER);
+
+    int channels = frame->ch_layout.nb_channels;
+    auto *inData = (int16_t *)frame->data[0];
+    for (int i = 0; i < frame->nb_samples * channels; ++i)
+    {
+        inData[i] = dist (rng);
+    }
+}
+
+void Frame::fillNoiseAudioS16P ()
+{
+    constexpr int16_t MIN_NUMBER = -32768;
+    constexpr int16_t MAX_NUMBER = 32767;
+    std::mt19937 rng (std::random_device{}());
+    static std::uniform_int_distribution<int16_t> dist (MIN_NUMBER,
+                                                        MAX_NUMBER);
+
+    int channels = frame->ch_layout.nb_channels;
+    for (int ch = 0; ch < channels; ++ch)
+    {
+        auto *inData = (int16_t *)frame->data[ch];
+        for (int i = 0; i < frame->nb_samples; ++i)
+        {
+            inData[i] = dist (rng);
+        }
+    }
+}
+
+void Frame::fillNoiseVideoRGB ()
+{
+    constexpr uint8_t MIN_NUMBER = 0;
+    constexpr uint8_t MAX_NUMBER = 255;
+    std::mt19937 rng (std::random_device{}());
+    static std::uniform_int_distribution<uint8_t> dist (MIN_NUMBER,
+                                                        MAX_NUMBER);
+
+    int planeCount = av_pix_fmt_count_planes ((AVPixelFormat)frame->format);
+    for (int plane = 0; plane < planeCount; ++plane)
+    {
+        uint8_t *data = frame->data[plane];
+        int linesize = frame->linesize[plane];
+        for (int y = 0; y < frame->height; ++y)
+        {
+            uint8_t *line = data + y * linesize;
+            for (int x = 0; x < linesize; ++x)
+            {
+                line[x] = dist (rng);
+            }
+        }
+    }
+}
+
+void Frame::fillNoiseVideoNV12 ()
+{
+    constexpr uint8_t MIN_NUMBER = 0;
+    constexpr uint8_t MAX_NUMBER = 255;
+    std::mt19937 rng (std::random_device{}());
+    static std::uniform_int_distribution<uint8_t> dist (MIN_NUMBER,
+                                                        MAX_NUMBER);
+    // Y plane
+    {
+        uint8_t *data = frame->data[0];
+        int linesize = frame->linesize[0];
+        for (int y = 0; y < frame->height; ++y)
+        {
+            uint8_t *line = data + y * linesize;
+            for (int x = 0; x < linesize; ++x)
+            {
+                line[x] = dist (rng);
+            }
+        }
+    }
+    // UV plane
+    {
+        uint8_t *data = frame->data[1];
+        int linesize = frame->linesize[1];
+        for (int y = 0; y < frame->height / 2; ++y)
+        {
+            uint8_t *line = data + y * linesize;
+            for (int x = 0; x < linesize; ++x)
+            {
+                line[x] = dist (rng);
+            }
+        }
+    }
+}
+
+void Frame::fillNoiseVideoYUV420P ()
+{
+    constexpr uint8_t MIN_NUMBER = 0;
+    constexpr uint8_t MAX_NUMBER = 255;
+    std::mt19937 rng (std::random_device{}());
+    static std::uniform_int_distribution<uint8_t> dist (MIN_NUMBER,
+                                                        MAX_NUMBER);
+    // Y plane
+    {
+        uint8_t *data = frame->data[0];
+        int linesize = frame->linesize[0];
+        for (int y = 0; y < frame->height; ++y)
+        {
+            uint8_t *line = data + y * linesize;
+            for (int x = 0; x < linesize; ++x)
+            {
+                line[x] = dist (rng);
+            }
+        }
+    }
+    // U plane
+    {
+        uint8_t *data = frame->data[1];
+        int linesize = frame->linesize[1];
+        for (int y = 0; y < frame->height / 2; ++y)
+        {
+            uint8_t *line = data + y * linesize;
+            for (int x = 0; x < linesize; ++x)
+            {
+                line[x] = dist (rng);
+            }
+        }
+    }
+    // V plane
+    {
+        uint8_t *data = frame->data[2];
+        int linesize = frame->linesize[2];
+        for (int y = 0; y < frame->height / 2; ++y)
+        {
+            uint8_t *line = data + y * linesize;
+            for (int x = 0; x < linesize; ++x)
+            {
+                line[x] = dist (rng);
+            }
+        }
+    }
+}
+
+void Frame::fillNoise ()
+{
+    if (isAudio ())
+    {
+        switch (frame->format)
+        {
+        case AV_SAMPLE_FMT_FLT:
+            fillNoiseAudioFLT ();
+            return;
+        case AV_SAMPLE_FMT_FLTP:
+            fillNoiseAudioFLTP ();
+            return;
+        case AV_SAMPLE_FMT_S16:
+            fillNoiseAudioS16 ();
+            return;
+        case AV_SAMPLE_FMT_S16P:
+            fillNoiseAudioS16P ();
+            return;
+        default:
+            break;
+        }
+    }
+    else if (isVideo ())
+    {
+        switch (frame->format)
+        {
+        case AV_PIX_FMT_RGB24:
+        case AV_PIX_FMT_BGR24:
+        case AV_PIX_FMT_ARGB:
+        case AV_PIX_FMT_RGBA:
+        case AV_PIX_FMT_ABGR:
+        case AV_PIX_FMT_BGRA:
+            fillNoiseVideoRGB ();
+            return;
+        case AV_PIX_FMT_NV12:
+        case AV_PIX_FMT_NV21:
+            fillNoiseVideoNV12 ();
+            return;
+        case AV_PIX_FMT_YUV420P:
+            fillNoiseVideoYUV420P ();
+            return;
+        default:
+            break;
+        }
+    }
+    throw std::runtime_error ("Unsupported format in fillNoise");
+}

package/cpp/FFmpeg/Frame.hpp

@@ -0,0 +1,42 @@
+#pragma once
+#include "Common.hpp"
+
+namespace FFmpeg
+{
+    class Frame
+    {
+      private:
+        std::shared_ptr<AVFrame> frame;
+        static constexpr int VIDEO_SAMPLE_RATE = 90000;
+        static constexpr int FRAME_ALIGN = 32;
+        void fillNoiseAudioFLT ();
+        void fillNoiseAudioFLTP ();
+        void fillNoiseAudioS16 ();
+        void fillNoiseAudioS16P ();
+        void fillNoiseVideoRGB ();
+        void fillNoiseVideoNV12 ();
+        void fillNoiseVideoYUV420P ();
+
+      public:
+        Frame ();
+        Frame (AVPixelFormat format, int width, int height, int64_t pts = -1);
+        Frame (AVSampleFormat format, int sampleRate, int channels,
+               int nb_samples, int64_t pts = -1);
+
+        auto isAudio () -> bool;
+        auto isVideo () -> bool;
+
+        void fillNoise ();
+
+        auto operator->() -> AVFrame * { return frame.get (); }
+
+        auto operator->() const -> const AVFrame * { return frame.get (); }
+
+        auto get () -> AVFrame * { return frame.get (); }
+
+        [[nodiscard]] auto get () const -> const AVFrame *
+        {
+            return frame.get ();
+        }
+    };
+} // namespace FFmpeg

package/cpp/FFmpeg/Muxer.cpp

@@ -0,0 +1,162 @@
+#include "Muxer.hpp"
+
+using namespace FFmpeg;
+
+Muxer::Muxer (const std::string &path,
+              AVCodecID audioCodecId = AV_CODEC_ID_NONE,
+              AVCodecID videoCodecId = AV_CODEC_ID_NONE)
+    : audioEncoder (audioCodecId), videoEncoder (videoCodecId)
+{
+    std::lock_guard lock (mutex);
+
+    int ret = avformat_alloc_output_context2 (&fmt_ctx, nullptr, nullptr,
+                                              path.c_str ());
+    checkError (ret, "avformat_alloc_output_context2");
+
+    ret = avio_open (&fmt_ctx->pb, path.c_str (), AVIO_FLAG_WRITE);
+    checkError (ret, "avio_open");
+}
+
+Muxer::~Muxer ()
+{
+    if (avio_ctx != nullptr)
+    {
+        av_free (avio_ctx->buffer);
+        av_free (avio_ctx);
+    }
+    if (fmt_ctx != nullptr)
+    {
+        avformat_free_context (fmt_ctx);
+        fmt_ctx = nullptr;
+    }
+}
+
+auto Muxer::hasAudio () -> bool { return audioEncoder.encoder != nullptr; }
+
+auto Muxer::hasVideo () -> bool { return videoEncoder.encoder != nullptr; }
+
+auto Muxer::isAudioOpened () -> bool { return audioStream != nullptr; }
+
+auto Muxer::isVideoOpened () -> bool { return videoStream != nullptr; }
+
+void Muxer::tryWriteHeader ()
+{
+    if (hasAudio () && !isAudioOpened ())
+    {
+        return;
+    }
+    if (hasVideo () && !isVideoOpened ())
+    {
+        return;
+    }
+    if (avformat_write_header (fmt_ctx, nullptr) < 0)
+    {
+        return;
+    }
+    hasWroteHeader = true;
+}
+
+void Muxer::tryWriteAudio ()
+{
+    if (audioStream == nullptr)
+    {
+        return;
+    }
+    if (!hasWroteHeader)
+    {
+        return;
+    }
+    std::vector<Packet> packets = audioEncoder.receive ();
+    for (Packet &packet : packets)
+    {
+        packet->stream_index = audioStream->index;
+        int ret = av_interleaved_write_frame (fmt_ctx, packet.get ());
+        checkError (ret, "av_interleaved_write_frame");
+    }
+}
+
+void Muxer::tryWriteVideo ()
+{
+    if (videoStream == nullptr)
+    {
+        return;
+    }
+    if (!hasWroteHeader)
+    {
+        return;
+    }
+    std::vector<Packet> packets = audioEncoder.receive ();
+    for (Packet &packet : packets)
+    {
+        packet->stream_index = videoStream->index;
+        int ret = av_interleaved_write_frame (fmt_ctx, packet.get ());
+        checkError (ret, "av_interleaved_write_frame");
+    }
+}
+
+void Muxer::writeAudio (const Frame &frame)
+{
+    std::lock_guard lock (mutex);
+
+    audioEncoder.send (frame);
+
+    if (audioStream == nullptr)
+    {
+        audioStream = avformat_new_stream (fmt_ctx, audioEncoder.encoder);
+        if (audioStream == nullptr)
+        {
+            throw std::runtime_error ("Could not create audio stream");
+        }
+        int ret = avcodec_parameters_from_context (audioStream->codecpar,
+                                                   audioEncoder.ctx);
+        checkError (ret, "avcodec_parameters_from_context");
+        tryWriteHeader ();
+    }
+    tryWriteAudio ();
+}
+
+void Muxer::writeVideo (const Frame &frame)
+{
+    std::lock_guard lock (mutex);
+
+    videoEncoder.send (frame);
+
+    if (videoStream == nullptr)
+    {
+        videoStream = avformat_new_stream (fmt_ctx, videoEncoder.encoder);
+        if (videoStream == nullptr)
+        {
+            throw std::runtime_error ("Could not create video stream");
+        }
+        int ret = avcodec_parameters_from_context (videoStream->codecpar,
+                                                   videoEncoder.ctx);
+        checkError (ret, "avcodec_parameters_from_context");
+        tryWriteHeader ();
+    }
+    tryWriteVideo ();
+}
+
+void Muxer::stop ()
+{
+    std::lock_guard lock (mutex);
+
+    if (!hasWroteHeader)
+    {
+        return;
+    }
+    if (audioStream != nullptr)
+    {
+        audioEncoder.flush ();
+        tryWriteAudio ();
+    }
+    if (videoStream != nullptr)
+    {
+        videoEncoder.flush ();
+        tryWriteVideo ();
+    }
+
+    int ret = av_write_trailer (fmt_ctx);
+    checkError (ret, "av_write_trailer");
+    ret = avio_closep (&fmt_ctx->pb);
+    checkError (ret, "avio_closep");
+}

package/cpp/FFmpeg/Muxer.hpp

@@ -0,0 +1,38 @@
+#pragma once
+#include "Encoder.hpp"
+#include "Frame.hpp"
+
+namespace FFmpeg
+{
+    class Muxer
+    {
+      private:
+        std::recursive_mutex mutex;
+        AVFormatContext *fmt_ctx = nullptr;
+        AVIOContext *avio_ctx = nullptr;
+        Encoder audioEncoder;
+        Encoder videoEncoder;
+
+        AVStream *audioStream = nullptr;
+        AVStream *videoStream = nullptr;
+        bool hasWroteHeader = false;
+
+        auto hasAudio () -> bool;
+        auto hasVideo () -> bool;
+        auto isAudioOpened () -> bool;
+        auto isVideoOpened () -> bool;
+
+        void tryWriteHeader ();
+        void tryWriteAudio ();
+        void tryWriteVideo ();
+
+      public:
+        Muxer (const std::string &path, AVCodecID audioCodecId,
+               AVCodecID videoCodecId);
+        ~Muxer ();
+
+        void writeAudio (const Frame &frame);
+        void writeVideo (const Frame &frame);
+        void stop ();
+    };
+} // namespace FFmpeg

package/cpp/FFmpeg/Packet.cpp

@@ -0,0 +1,16 @@
+#include "Packet.hpp"
+
+using namespace FFmpeg;
+
+Packet::Packet ()
+{
+    packet
+        = std::shared_ptr<AVPacket> (av_packet_alloc (), [] (AVPacket *packet)
+                                     { av_packet_free (&packet); });
+}
+
+Packet::Packet (size_t size) : Packet ()
+{
+    int ret = av_new_packet (packet.get (), size);
+    checkError (ret, "av_new_packet");
+}

package/cpp/FFmpeg/Packet.hpp

@@ -0,0 +1,26 @@
+#pragma once
+#include "Common.hpp"
+
+namespace FFmpeg
+{
+    class Packet
+    {
+      private:
+        std::shared_ptr<AVPacket> packet;
+
+      public:
+        Packet ();
+        Packet (size_t size);
+
+        auto operator->() -> AVPacket * { return packet.get (); }
+
+        auto operator->() const -> const AVPacket * { return packet.get (); }
+
+        auto get () -> AVPacket * { return packet.get (); }
+
+        [[nodiscard]] auto get () const -> const AVPacket *
+        {
+            return packet.get ();
+        }
+    };
+} // namespace FFmpeg

package/cpp/FFmpeg/Resampler.cpp

@@ -0,0 +1,68 @@
+#include "Resampler.hpp"
+
+using namespace FFmpeg;
+
+Resampler::~Resampler ()
+{
+    if (swr_ctx != nullptr)
+    {
+        swr_free (&swr_ctx);
+        swr_ctx = nullptr;
+    }
+}
+
+auto Resampler::resample (const Frame &frame, AVSampleFormat outFormat,
+                          int outSampleRate, int outChannels) -> Frame
+{
+    std::lock_guard lock (mutex);
+    if (swr_ctx == nullptr)
+    {
+        // init
+        auto inFormat = (AVSampleFormat)frame->format;
+        int inSampleRate = frame->sample_rate;
+
+        this->outFormat = outFormat;
+        this->outChannels = outChannels;
+        this->outSampleRate = outSampleRate;
+        this->pts = frame->pts;
+
+        AVChannelLayout outLayout;
+        av_channel_layout_default (&outLayout, outChannels);
+        int ret = swr_alloc_set_opts2 (&swr_ctx, &outLayout, outFormat,
+                                       outSampleRate, &frame->ch_layout,
+                                       inFormat, inSampleRate, 0, nullptr);
+        checkError (ret, "swr_alloc_set_opts2");
+
+        ret = swr_init (swr_ctx);
+        checkError (ret, "swr_init");
+    }
+    int in_nb_samples = frame->nb_samples;
+    int outSamples = swr_get_out_samples (swr_ctx, in_nb_samples);
+    Frame dst (outFormat, outSampleRate, outChannels, outSamples, pts);
+
+    int ret = swr_convert (swr_ctx, dst->data, dst->nb_samples, frame->data,
+                           in_nb_samples);
+    checkError (ret, "swr_convert");
+    dst->nb_samples = ret;
+    dst->pts = pts;
+    pts += ret;
+    return dst;
+}
+
+auto Resampler::flush () -> Frame
+{
+    std::lock_guard lock (mutex);
+    if (swr_ctx == nullptr)
+    {
+        throw std::runtime_error ("Resampler not initialized");
+    }
+    int outSamples = swr_get_out_samples (swr_ctx, 0);
+    Frame dst (outFormat, outSampleRate, outChannels, outSamples, pts);
+
+    int ret = swr_convert (swr_ctx, dst->data, dst->nb_samples, nullptr, 0);
+    checkError (ret, "swr_convert");
+    dst->nb_samples = ret;
+    dst->pts = pts;
+    pts += ret;
+    return dst;
+}

package/cpp/FFmpeg/Resampler.hpp

@@ -0,0 +1,24 @@
+#pragma once
+#include "Frame.hpp"
+
+namespace FFmpeg
+{
+    class Resampler
+    {
+      private:
+        std::recursive_mutex mutex;
+        SwrContext *swr_ctx = nullptr;
+
+        AVSampleFormat outFormat = AV_SAMPLE_FMT_NONE;
+        int outChannels = 0;
+        int outSampleRate = 0;
+        int64_t pts = -1;
+
+      public:
+        ~Resampler ();
+
+        auto resample (const Frame &frame, AVSampleFormat outFormat,
+                       int outSampleRate, int outChannels) -> Frame;
+        auto flush () -> Frame;
+    };
+} // namespace FFmpeg