react-native-audio-api 0.8.0-nightly-5a12f24-20250901 → 0.8.0

package/common/cpp/audioapi/core/sources/StreamerNode.cpp

@@ -0,0 +1,299 @@
+/*
+ * This file links to the FFmpeg library, which is licensed under the
+ * GNU Lesser General Public License (LGPL) version 2.1 or later.
+ *
+ * Our own code in this file is licensed under the MIT License, but note
+ * that linking with FFmpeg means you must comply with the terms of the
+ * LGPL for FFmpeg itself.
+ */
+
+#include <audioapi/core/BaseAudioContext.h>
+#include <audioapi/core/sources/StreamerNode.h>
+#include <audioapi/core/utils/Locker.h>
+#include <audioapi/dsp/VectorMath.h>
+#include <audioapi/utils/AudioArray.h>
+#include <audioapi/utils/AudioBus.h>
+#include <chrono>
+
+namespace audioapi {
+StreamerNode::StreamerNode(BaseAudioContext *context)
+    : AudioScheduledSourceNode(context),
+      fmtCtx_(nullptr),
+      codecCtx_(nullptr),
+      decoder_(nullptr),
+      codecpar_(nullptr),
+      pkt_(nullptr),
+      frame_(nullptr),
+      pendingFrame_(nullptr),
+      bufferedBus_(nullptr),
+      bufferedBusIndex_(0),
+      maxBufferSize_(0),
+      audio_stream_index_(-1),
+      swrCtx_(nullptr),
+      resampledData_(nullptr),
+      maxResampledSamples_(0) {}
+
+StreamerNode::~StreamerNode() {
+  cleanup();
+}
+
+bool StreamerNode::initialize(const std::string &input_url) {
+  if (isInitialized_) {
+    cleanup();
+  }
+
+  if (!openInput(input_url)) {
+    return false;
+  }
+
+  if (!findAudioStream() || !setupDecoder() || !setupResampler()) {
+    cleanup();
+    return false;
+  }
+
+  pkt_ = av_packet_alloc();
+  frame_ = av_frame_alloc();
+
+  if (pkt_ == nullptr || frame_ == nullptr) {
+    cleanup();
+    return false;
+  }
+
+  maxBufferSize_ = BUFFER_LENGTH_SECONDS * codecCtx_->sample_rate;
+  // If decoding is faster than playing, we buffer few seconds of audio
+  bufferedBus_ = std::make_shared<AudioBus>(
+      maxBufferSize_, codecpar_->ch_layout.nb_channels, codecCtx_->sample_rate);
+
+  channelCount_ = codecpar_->ch_layout.nb_channels;
+  audioBus_ = std::make_shared<AudioBus>(
+      RENDER_QUANTUM_SIZE, channelCount_, context_->getSampleRate());
+
+  streamingThread_ = std::thread(&StreamerNode::streamAudio, this);
+  streamFlag.store(true);
+  isInitialized_ = true;
+  return true;
+}
+
+void StreamerNode::stop(double when) {
+  AudioScheduledSourceNode::stop(when);
+  streamFlag.store(false);
+}
+
+bool StreamerNode::setupResampler() {
+  // Allocate resampler context
+  swrCtx_ = swr_alloc();
+  if (swrCtx_ == nullptr) {
+    return false;
+  }
+
+  // Set input parameters (from codec)
+  av_opt_set_chlayout(swrCtx_, "in_chlayout", &codecCtx_->ch_layout, 0);
+  av_opt_set_int(swrCtx_, "in_sample_rate", codecCtx_->sample_rate, 0);
+  av_opt_set_sample_fmt(swrCtx_, "in_sample_fmt", codecCtx_->sample_fmt, 0);
+
+  // Set output parameters (float)
+  av_opt_set_chlayout(swrCtx_, "out_chlayout", &codecCtx_->ch_layout, 0);
+  av_opt_set_int(swrCtx_, "out_sample_rate", context_->getSampleRate(), 0);
+  av_opt_set_sample_fmt(swrCtx_, "out_sample_fmt", AV_SAMPLE_FMT_FLTP, 0);
+
+  // Initialize the resampler
+  if (swr_init(swrCtx_) < 0) {
+    return false;
+  }
+
+  // Allocate output buffer for resampled data
+  maxResampledSamples_ = INITIAL_MAX_RESAMPLED_SAMPLES;
+  int ret = av_samples_alloc_array_and_samples(
+      &resampledData_,
+      nullptr,
+      codecCtx_->ch_layout.nb_channels,
+      maxResampledSamples_,
+      AV_SAMPLE_FMT_FLTP,
+      0);
+
+  return ret >= 0;
+}
+
+void StreamerNode::streamAudio() {
+  while (streamFlag.load()) {
+    if (pendingFrame_ != nullptr) {
+      if (!processFrameWithResampler(pendingFrame_)) {
+        cleanup();
+        return;
+      }
+    } else {
+      if (av_read_frame(fmtCtx_, pkt_) < 0) {
+        cleanup();
+        return;
+      }
+      if (pkt_->stream_index == audio_stream_index_) {
+        if (avcodec_send_packet(codecCtx_, pkt_) != 0) {
+          cleanup();
+          return;
+        }
+        if (avcodec_receive_frame(codecCtx_, frame_) != 0) {
+          cleanup();
+          return;
+        }
+        if (!processFrameWithResampler(frame_)) {
+          cleanup();
+          return;
+        }
+        av_packet_unref(pkt_);
+      }
+    }
+    std::this_thread::sleep_for(std::chrono::milliseconds(10));
+  }
+}
+
+void StreamerNode::processNode(
+    const std::shared_ptr<AudioBus> &processingBus,
+    int framesToProcess) {
+  size_t startOffset = 0;
+  size_t offsetLength = 0;
+  updatePlaybackInfo(processingBus, framesToProcess, startOffset, offsetLength);
+  // If we have enough buffered data, copy to output bus
+  if (bufferedBusIndex_ >= framesToProcess) {
+    Locker locker(mutex_);
+    for (int ch = 0; ch < processingBus->getNumberOfChannels(); ch++) {
+      memcpy(
+          processingBus->getChannel(ch)->getData(),
+          bufferedBus_->getChannel(ch)->getData(),
+          offsetLength * sizeof(float));
+
+      memmove(
+          bufferedBus_->getChannel(ch)->getData(),
+          bufferedBus_->getChannel(ch)->getData() + offsetLength,
+          (maxBufferSize_ - offsetLength) * sizeof(float));
+    }
+    bufferedBusIndex_ -= offsetLength;
+  }
+}
+
+bool StreamerNode::processFrameWithResampler(AVFrame *frame) {
+  // Check if we need to reallocate the resampled buffer
+  int out_samples = swr_get_out_samples(swrCtx_, frame->nb_samples);
+  if (out_samples > maxResampledSamples_) {
+    av_freep(&resampledData_[0]);
+    av_freep(&resampledData_);
+
+    maxResampledSamples_ = out_samples;
+    int ret = av_samples_alloc_array_and_samples(
+        &resampledData_,
+        nullptr,
+        codecCtx_->ch_layout.nb_channels,
+        maxResampledSamples_,
+        AV_SAMPLE_FMT_FLTP,
+        0);
+
+    if (ret < 0) {
+      return false;
+    }
+  }
+
+  // Convert the frame
+  int converted_samples = swr_convert(
+      swrCtx_,
+      resampledData_,
+      maxResampledSamples_,
+      (const uint8_t **)frame->data,
+      frame->nb_samples);
+
+  if (converted_samples < 0) {
+    return false;
+  }
+
+  // Check if converted data fits in buffer
+  if (bufferedBusIndex_ + converted_samples > maxBufferSize_) {
+    pendingFrame_ = frame;
+    return true;
+  } else {
+    pendingFrame_ = nullptr;
+  }
+
+  // Copy converted data to our buffer
+  Locker locker(mutex_);
+  for (int ch = 0; ch < codecCtx_->ch_layout.nb_channels; ch++) {
+    auto *src = reinterpret_cast<float *>(resampledData_[ch]);
+    float *dst = bufferedBus_->getChannel(ch)->getData() + bufferedBusIndex_;
+    memcpy(dst, src, converted_samples * sizeof(float));
+  }
+  bufferedBusIndex_ += converted_samples;
+  return true;
+}
+
+bool StreamerNode::openInput(const std::string &input_url) {
+  if (avformat_open_input(&fmtCtx_, input_url.c_str(), nullptr, nullptr) < 0) {
+    return false;
+  }
+  return avformat_find_stream_info(fmtCtx_, nullptr) >= 0;
+}
+
+bool StreamerNode::findAudioStream() {
+  audio_stream_index_ = -1;
+  codecpar_ = nullptr;
+
+  for (int i = 0; i < fmtCtx_->nb_streams; ++i) {
+    if (fmtCtx_->streams[i]->codecpar->codec_type == AVMEDIA_TYPE_AUDIO) {
+      audio_stream_index_ = i;
+      codecpar_ = fmtCtx_->streams[i]->codecpar;
+      break;
+    }
+  }
+
+  return audio_stream_index_ >= 0 && codecpar_ != nullptr;
+}
+
+bool StreamerNode::setupDecoder() {
+  decoder_ = avcodec_find_decoder(codecpar_->codec_id);
+  if (decoder_ == nullptr) {
+    return false;
+  }
+
+  codecCtx_ = avcodec_alloc_context3(decoder_);
+  if (codecCtx_ == nullptr) {
+    return false;
+  }
+
+  if (avcodec_parameters_to_context(codecCtx_, codecpar_) < 0) {
+    return false;
+  }
+
+  return avcodec_open2(codecCtx_, decoder_, nullptr) >= 0;
+}
+
+void StreamerNode::cleanup() {
+  streamFlag.store(false);
+  streamingThread_.join();
+  if (swrCtx_ != nullptr) {
+    swr_free(&swrCtx_);
+  }
+
+  if (resampledData_ != nullptr) {
+    av_freep(&resampledData_[0]);
+    av_freep(&resampledData_);
+  }
+
+  if (frame_ != nullptr) {
+    av_frame_free(&frame_);
+  }
+
+  if (pkt_ != nullptr) {
+    av_packet_free(&pkt_);
+  }
+
+  if (codecCtx_ != nullptr) {
+    avcodec_free_context(&codecCtx_);
+  }
+
+  if (fmtCtx_ != nullptr) {
+    avformat_close_input(&fmtCtx_);
+  }
+
+  audio_stream_index_ = -1;
+  isInitialized_ = false;
+  decoder_ = nullptr;
+  codecpar_ = nullptr;
+  maxResampledSamples_ = 0;
+}
+} // namespace audioapi

package/common/cpp/audioapi/core/sources/StreamerNode.h

@@ -0,0 +1,123 @@
+/*
+ * This file links to the FFmpeg library, which is licensed under the
+ * GNU Lesser General Public License (LGPL) version 2.1 or later.
+ *
+ * Our own code in this file is licensed under the MIT License, but note
+ * that linking with FFmpeg means you must comply with the terms of the
+ * LGPL for FFmpeg itself.
+ */
+
+#pragma once
+
+#include <audioapi/core/sources/AudioScheduledSourceNode.h>
+
+#ifndef AUDIO_API_TEST_SUITE
+extern "C" {
+  #include <libavformat/avformat.h>
+  #include <libavcodec/avcodec.h>
+  #include <libavutil/samplefmt.h>
+  #include <libavutil/channel_layout.h>
+  #include <libavutil/opt.h>
+  #include <libswresample/swresample.h>
+}
+#endif
+
+#include <cmath>
+#include <memory>
+#include <string>
+#include <atomic>
+
+namespace audioapi {
+
+class AudioBus;
+
+class StreamerNode : public AudioScheduledSourceNode {
+ public:
+  explicit StreamerNode(BaseAudioContext *context);
+  ~StreamerNode() override;
+
+  /**
+   * @brief Initialize all necessary ffmpeg components for streaming audio
+  */
+  bool initialize(const std::string& inputUrl);
+  void stop(double when) override;
+
+ private:
+  static constexpr int SIZE = 4'000'000; // 4MB
+
+ public:
+  static constexpr int getEstimatedSize() {
+    return StreamerNode::SIZE;
+  } // in bytes
+
+ protected:
+  void processNode(const std::shared_ptr<AudioBus>& processingBus, int framesToProcess) override;
+
+ private:
+  #ifndef AUDIO_API_TEST_SUITE
+  AVFormatContext* fmtCtx_;
+  AVCodecContext* codecCtx_;
+  const AVCodec* decoder_;
+  AVCodecParameters* codecpar_;
+  AVPacket* pkt_;
+  AVFrame* frame_; // Frame that is currently being processed
+  AVFrame* pendingFrame_; // Frame that is saved if bufferedBus is full
+  std::shared_ptr<AudioBus> bufferedBus_; // audio bus for buffering hls frames
+  size_t bufferedBusIndex_; // index in the buffered bus where we write the next frame
+  size_t maxBufferSize_; // maximum size of the buffered bus
+  int audio_stream_index_; // index of the audio stream channel in the input
+  SwrContext* swrCtx_;
+  uint8_t** resampledData_; // weird ffmpeg way of using raw byte pointers for resampled data
+  int maxResampledSamples_;
+  std::mutex mutex_;
+  std::thread streamingThread_;
+  std::atomic<bool> streamFlag; // Flag to control the streaming thread
+  static constexpr float BUFFER_LENGTH_SECONDS = 5.0f; // Length of the buffer in seconds
+  static constexpr int INITIAL_MAX_RESAMPLED_SAMPLES = 8192; // Initial size for resampled data
+
+  /**
+   * @brief Setting up the resampler
+   * @return true if successful, false otherwise
+   */
+  bool setupResampler();
+
+  /**
+   * @brief Resample the audio frame, change its sample format and channel layout
+   * @param frame The AVFrame to resample
+   * @return true if successful, false otherwise
+   */
+  bool processFrameWithResampler(AVFrame* frame);
+
+  /**
+   * @brief Thread function to continuously read and process audio frames
+   * @details This function runs in a separate thread to avoid blocking the main audio processing thread
+   * @note It will read frames from the input stream, resample them, and store them in the buffered bus
+   * @note The thread will stop when streamFlag is set to false
+   */
+  void streamAudio();
+
+  /** @brief Clean up resources */
+  void cleanup();
+
+  /**
+   * @brief Open the input stream
+   * @param input_url The URL of the input stream
+   * @return true if successful, false otherwise
+   * @note This function initializes the FFmpeg libraries and opens the input stream
+   */
+  bool openInput(const std::string& inputUrl);
+
+  /**
+   * @brief Find the audio stream channel in the input
+   * @return true if audio stream was found, false otherwise
+   */
+  bool findAudioStream();
+
+  /**
+   * @brief Set up the decoder for the audio stream
+   * @return true if successful, false otherwise
+   */
+  bool setupDecoder();
+  #endif // AUDIO_API_TEST_SUITE
+};
+} // namespace audioapi

package/common/cpp/audioapi/core/utils/AudioDecoder.h

@@ -5,9 +5,23 @@
 #include <memory>
 #include <string>
 #include <vector>
+#include <cstring>
+#include <algorithm>
 
 namespace audioapi {
 
+enum class AudioFormat {
+  UNKNOWN,
+  WAV,
+  OGG,
+  FLAC,
+  AAC,
+  MP3,
+  M4A,
+  MP4,
+  MOV
+};
+
 class AudioBus;
 
 static constexpr int CHUNK_SIZE = 4096;
@@ -73,6 +87,53 @@ class AudioDecoder {
     stretch_deinit(stretcher);
   }
 
+  static AudioFormat detectAudioFormat(const void* data, size_t size) {
+    if (size < 12) return AudioFormat::UNKNOWN;
+    const auto* bytes = static_cast<const unsigned char*>(data);
+
+    // WAV/RIFF
+    if (std::memcmp(bytes, "RIFF", 4) == 0 && std::memcmp(bytes + 8, "WAVE", 4) == 0)
+        return AudioFormat::WAV;
+
+    // OGG
+    if (std::memcmp(bytes, "OggS", 4) == 0)
+        return AudioFormat::OGG;
+
+    // FLAC
+    if (std::memcmp(bytes, "fLaC", 4) == 0)
+        return AudioFormat::FLAC;
+
+    // AAC starts with 0xFF 0xF1 or 0xFF 0xF9
+    if (bytes[0] == 0xFF && (bytes[1] & 0xF6) == 0xF0)
+        return AudioFormat::AAC;
+
+    // MP3: "ID3" or 11-bit frame sync (0xFF 0xE0)
+    if (std::memcmp(bytes, "ID3", 3) == 0)
+        return AudioFormat::MP3;
+    if (bytes[0] == 0xFF && (bytes[1] & 0xE0) == 0xE0)
+        return AudioFormat::MP3;
+
+    if (std::memcmp(bytes + 4, "ftyp", 4) == 0) {
+        if (std::memcmp(bytes + 8, "M4A ", 4) == 0)
+            return AudioFormat::M4A;
+        else if (std::memcmp(bytes + 8, "qt  ", 4) == 0)
+            return AudioFormat::MOV;
+        return AudioFormat::MP4;
+    }
+    return AudioFormat::UNKNOWN;
+  }
+
+  static inline bool pathHasExtension(const std::string &path, const std::vector<std::string> &extensions) {
+    std::string pathLower = path;
+    std::transform(pathLower.begin(), pathLower.end(), pathLower.begin(), ::tolower);
+    for (const auto& ext : extensions) {
+        if (pathLower.ends_with(ext))
+            return true;
+    }
+    return false;
+  }
+
+
   [[nodiscard]] static inline int16_t floatToInt16(float sample) {
     return static_cast<int16_t>(sample * 32768.0f);
   }

package/common/cpp/audioapi/core/utils/AudioNodeManager.cpp

@@ -213,6 +213,18 @@ void AudioNodeManager::handleAddToDeconstructionEvent(
   }
 }
 
+template <typename U>
+inline bool AudioNodeManager::nodeCanBeDestructed(
+    std::shared_ptr<U> const &node) {
+  // If the node is an AudioScheduledSourceNode, we need to check if it is
+  // playing
+  if constexpr (std::is_base_of_v<AudioScheduledSourceNode, U>) {
+    return node.use_count() == 1 &&
+        (node->isUnscheduled() || node->isFinished());
+  }
+  return node.use_count() == 1;
+}
+
 template <typename U>
 void AudioNodeManager::prepareNodesForDestruction(
     std::vector<std::shared_ptr<U>> &vec) {
@@ -238,10 +250,10 @@ void AudioNodeManager::prepareNodesForDestruction(
   // nodes in range [begin, vec.size()) should be deleted
   // so new size of the vector will be `begin`
   while (begin <= end) {
-    while (begin < end && vec[end].use_count() == 1) {
+    while (begin < end && AudioNodeManager::nodeCanBeDestructed(vec[end])) {
       end--;
     }
-    if (vec[begin].use_count() == 1) {
+    if (AudioNodeManager::nodeCanBeDestructed(vec[begin])) {
       std::swap(vec[begin], vec[end]);
       end--;
     }

package/common/cpp/audioapi/core/utils/AudioNodeManager.h

@@ -126,6 +126,9 @@ class AudioNodeManager {
 
   template <typename U>
   void prepareNodesForDestruction(std::vector<std::shared_ptr<U>> &vec);
+
+  template <typename U>
+  inline static bool nodeCanBeDestructed(std::shared_ptr<U> const& node);
 };
 
 #undef AUDIO_NODE_MANAGER_SPSC_OPTIONS

package/common/cpp/audioapi/external/ffmpeg_include/libavcodec/ac3_parser.h

@@ -0,0 +1,36 @@
+/*
+ * AC-3 parser prototypes
+ * Copyright (c) 2003 Fabrice Bellard
+ * Copyright (c) 2003 Michael Niedermayer
+ *
+ * This file is part of FFmpeg.
+ *
+ * FFmpeg is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * FFmpeg is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with FFmpeg; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+ */
+
+#ifndef AVCODEC_AC3_PARSER_H
+#define AVCODEC_AC3_PARSER_H
+
+#include <stddef.h>
+#include <stdint.h>
+
+/**
+ * Extract the bitstream ID and the frame size from AC-3 data.
+ */
+int av_ac3_parse_header(const uint8_t *buf, size_t size,
+                        uint8_t *bitstream_id, uint16_t *frame_size);
+
+
+#endif /* AVCODEC_AC3_PARSER_H */

package/common/cpp/audioapi/external/ffmpeg_include/libavcodec/adts_parser.h

@@ -0,0 +1,37 @@
+/*
+ * This file is part of FFmpeg.
+ *
+ * FFmpeg is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * FFmpeg is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with FFmpeg; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+ */
+
+#ifndef AVCODEC_ADTS_PARSER_H
+#define AVCODEC_ADTS_PARSER_H
+
+#include <stddef.h>
+#include <stdint.h>
+
+#define AV_AAC_ADTS_HEADER_SIZE 7
+
+/**
+ * Extract the number of samples and frames from AAC data.
+ * @param[in]  buf     pointer to AAC data buffer
+ * @param[out] samples Pointer to where number of samples is written
+ * @param[out] frames  Pointer to where number of frames is written
+ * @return Returns 0 on success, error code on failure.
+ */
+int av_adts_header_parse(const uint8_t *buf, uint32_t *samples,
+                         uint8_t *frames);
+
+#endif /* AVCODEC_ADTS_PARSER_H */