react-native-audio-api 0.7.2 → 0.8.0-nightly-daaceff-20250902

package/RNAudioAPI.podspec

@@ -35,10 +35,20 @@ Pod::Spec.new do |s|
 
   s.compiler_flags = "#{folly_flags}"
 
-external_dir = File.join(__dir__, "common/cpp/audioapi/external")
-lib_dir = "#{external_dir}/$(PLATFORM_NAME)"
-
+  # s.prepare_command = <<-CMD TODO: re-add when we have prebuilt libs put somewhere public
+  #   ruby -r './scripts/download-audioapi-libs.rb'
+  # CMD
 
+  external_dir = File.join(__dir__, "common/cpp/audioapi/external")
+  lib_dir = "#{external_dir}/$(PLATFORM_NAME)"
+  
+  s.ios.vendored_frameworks = [
+    'common/cpp/audioapi/external/libavcodec.xcframework',
+    'common/cpp/audioapi/external/libavformat.xcframework',
+    'common/cpp/audioapi/external/libavutil.xcframework',
+    'common/cpp/audioapi/external/libswresample.xcframework'
+  ]
+  
 s.pod_target_xcconfig = {
   "USE_HEADERMAP" => "YES",
   "CLANG_CXX_LANGUAGE_STANDARD" => "c++20",
@@ -49,6 +59,7 @@ s.pod_target_xcconfig = {
     #{external_dir}/include
     #{external_dir}/include/opus
     #{external_dir}/include/vorbis
+    $(PODS_TARGET_SRCROOT)/common/cpp/audioapi/external/ffmpeg_include
   ].join(" "),
   'OTHER_CFLAGS' => "$(inherited) #{folly_flags} #{fabric_flags} #{version_flag}"
 }
@@ -67,6 +78,4 @@ s.user_target_xcconfig = {
   # Use install_modules_dependencies helper to install the dependencies if React Native version >=0.71.0.
   # See https://github.com/facebook/react-native/blob/febf6b7f33fdb4904669f99d795eba4c0f95d7bf/scripts/cocoapods/new_architecture.rb#L79.
   install_modules_dependencies(s)
-end
-
-
+end

package/android/build.gradle

@@ -1,6 +1,5 @@
 import com.android.Version
 import org.apache.tools.ant.taskdefs.condition.Os
-import com.android.build.gradle.tasks.ExternalNativeBuildJsonTask
 
 buildscript {
   repositories {
@@ -92,6 +91,11 @@ def REACT_NATIVE_MINOR_VERSION = REACT_NATIVE_VERSION.startsWith("0.0.0-") ? 100
 def IS_NEW_ARCHITECTURE_ENABLED = isNewArchitectureEnabled()
 
 android {
+  sourceSets {
+      main {
+          jniLibs.srcDirs = ['src/main/jniLibs']
+      }
+  }
   if (supportsNamespace()) {
     namespace "com.swmansion.audioapi"
 
@@ -125,6 +129,9 @@ android {
   }
 
   packagingOptions {
+    jniLibs {
+      useLegacyPackaging = false  // enables 16KB alignment
+    }
     excludes = [
       "META-INF",
       "META-INF/**",
@@ -230,8 +237,10 @@ def assertMinimalReactNativeVersion = task assertMinimalReactNativeVersionTask {
     }
 }
 
+// Make preBuild depend on the download task
 tasks.preBuild {
     dependsOn assertMinimalReactNativeVersion
+    // dependsOn downloadStaticPrebuiltLibraries TODO: re-add when we have prebuilt libs put somewhere public
 }
 
 task cleanCmakeCache() {

package/android/src/main/cpp/audioapi/CMakeLists.txt

@@ -3,17 +3,26 @@ cmake_minimum_required(VERSION 3.12.0)
 file(GLOB_RECURSE ANDROID_CPP_SOURCES CONFIGURE_DEPENDS "${ANDROID_CPP_DIR}/audioapi/*.cpp")
 file(GLOB_RECURSE COMMON_CPP_SOURCES CONFIGURE_DEPENDS "${COMMON_CPP_DIR}/audioapi/*.cpp" "${COMMON_CPP_DIR}/audioapi/*.c")
 
+set(INCLUDE_DIR ${COMMON_CPP_DIR}/audioapi/external/include)
+set(FFMPEG_INCLUDE_DIR ${COMMON_CPP_DIR}/audioapi/external/ffmpeg_include)
 set(EXTERNAL_DIR ${COMMON_CPP_DIR}/audioapi/external)
-set(ARCH_LIB_DIR ${EXTERNAL_DIR}/${ANDROID_ABI})
-set(INCLUDE_DIR ${EXTERNAL_DIR}/include)
+set(JNI_LIBS_DIR ${COMMON_CPP_DIR}/../../android/src/main/jniLibs)
+
+set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wl,-Bsymbolic")
+set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wl,-Bsymbolic")
 
 add_library(react-native-audio-api SHARED ${ANDROID_CPP_SOURCES} ${COMMON_CPP_SOURCES})
 
-foreach(lib IN ITEMS opus opusfile ogg vorbis vorbisenc vorbisfile)
+foreach(lib IN ITEMS opus opusfile ogg vorbis vorbisenc vorbisfile crypto ssl)
   add_library(${lib} STATIC IMPORTED)
-  set_target_properties(${lib} PROPERTIES IMPORTED_LOCATION ${ARCH_LIB_DIR}/lib${lib}.a)
+  set_target_properties(${lib} PROPERTIES IMPORTED_LOCATION ${EXTERNAL_DIR}/${ANDROID_ABI}/lib${lib}.a)
 endforeach()
 
+ foreach (lib IN ITEMS avcodec avformat avutil swresample)
+   add_library(${lib} SHARED IMPORTED)
+   set_target_properties(${lib} PROPERTIES IMPORTED_LOCATION ${JNI_LIBS_DIR}/${ANDROID_ABI}/lib${lib}.so)
+ endforeach()
+
 find_package(ReactAndroid REQUIRED CONFIG)
 find_package(fbjni REQUIRED CONFIG)
 find_package(oboe REQUIRED CONFIG)
@@ -25,7 +34,7 @@ target_include_directories(
   "${ANDROID_CPP_DIR}"
   "${INCLUDE_DIR}"
   "${INCLUDE_DIR}/opus"
-  "${INCLUDE_DIR}/vorbis"
+  "${FFMPEG_INCLUDE_DIR}"
   "${REACT_NATIVE_DIR}/ReactCommon"
   "${REACT_NATIVE_DIR}/ReactAndroid/src/main/jni/react/turbomodule"
   "${REACT_NATIVE_DIR}/ReactCommon/callinvoker"
@@ -61,4 +70,11 @@ target_link_libraries(react-native-audio-api
   vorbis
   vorbisenc
   vorbisfile
+  avcodec
+  avformat
+  avutil
+  swresample
+  crypto
+  ssl
+  z
 )

package/common/cpp/audioapi/HostObjects/BaseAudioContextHostObject.h

@@ -14,6 +14,7 @@
 #include <audioapi/HostObjects/StereoPannerNodeHostObject.h>
 #include <audioapi/HostObjects/AnalyserNodeHostObject.h>
 #include <audioapi/HostObjects/RecorderAdapterNodeHostObject.h>
+#include <audioapi/HostObjects/StreamerNodeHostObject.h>
 
 #include <jsi/jsi.h>
 #include <memory>
@@ -42,6 +43,7 @@ class BaseAudioContextHostObject : public JsiHostObject {
     addFunctions(
         JSI_EXPORT_FUNCTION(BaseAudioContextHostObject, createRecorderAdapter),
         JSI_EXPORT_FUNCTION(BaseAudioContextHostObject, createOscillator),
+        JSI_EXPORT_FUNCTION(BaseAudioContextHostObject, createStreamer),
         JSI_EXPORT_FUNCTION(BaseAudioContextHostObject, createGain),
         JSI_EXPORT_FUNCTION(BaseAudioContextHostObject, createStereoPanner),
         JSI_EXPORT_FUNCTION(BaseAudioContextHostObject, createBiquadFilter),
@@ -88,6 +90,15 @@ class BaseAudioContextHostObject : public JsiHostObject {
     return jsi::Object::createFromHostObject(runtime, oscillatorHostObject);
   }
 
+  JSI_HOST_FUNCTION(createStreamer) {
+    auto streamer = context_->createStreamer();
+    auto streamerHostObject =
+        std::make_shared<StreamerNodeHostObject>(streamer);
+    auto object = jsi::Object::createFromHostObject(runtime, streamerHostObject);
+    object.setExternalMemoryPressure(runtime, StreamerNode::getEstimatedSize());
+    return object;
+  }
+
   JSI_HOST_FUNCTION(createGain) {
     auto gain = context_->createGain();
     auto gainHostObject = std::make_shared<GainNodeHostObject>(gain);

package/common/cpp/audioapi/HostObjects/StreamerNodeHostObject.h

@@ -0,0 +1,30 @@
+#pragma once
+
+#include <audioapi/HostObjects/AudioScheduledSourceNodeHostObject.h>
+#include <audioapi/HostObjects/AudioParamHostObject.h>
+#include <audioapi/HostObjects/PeriodicWaveHostObject.h>
+#include <audioapi/core/sources/StreamerNode.h>
+
+#include <memory>
+#include <string>
+#include <vector>
+
+namespace audioapi {
+using namespace facebook;
+
+class StreamerNodeHostObject : public AudioScheduledSourceNodeHostObject {
+ public:
+  explicit StreamerNodeHostObject(
+          const std::shared_ptr<StreamerNode> &node)
+      : AudioScheduledSourceNodeHostObject(node) {
+    addFunctions(JSI_EXPORT_FUNCTION(StreamerNodeHostObject, initialize));
+  }
+
+  JSI_HOST_FUNCTION(initialize) {
+    auto streamerNode = std::static_pointer_cast<StreamerNode>(node_);
+    auto path = args[0].getString(runtime).utf8(runtime);
+    auto result = streamerNode->initialize(path);
+    return jsi::Value(result);
+  }
+};
+} // namespace audioapi

package/common/cpp/audioapi/core/BaseAudioContext.cpp

@@ -9,6 +9,7 @@
 #include <audioapi/core/sources/AudioBufferSourceNode.h>
 #include <audioapi/core/sources/OscillatorNode.h>
 #include <audioapi/core/sources/RecorderAdapterNode.h>
+#include <audioapi/core/sources/StreamerNode.h>
 #include <audioapi/core/utils/AudioDecoder.h>
 #include <audioapi/core/utils/AudioNodeManager.h>
 #include <audioapi/events/AudioEventHandlerRegistry.h>
@@ -69,6 +70,14 @@ std::shared_ptr<OscillatorNode> BaseAudioContext::createOscillator() {
   return oscillator;
 }
 
+#ifndef AUDIO_API_TEST_SUITE
+std::shared_ptr<StreamerNode> BaseAudioContext::createStreamer() {
+  auto streamer = std::make_shared<StreamerNode>(this);
+  nodeManager_->addSourceNode(streamer);
+  return streamer;
+}
+#endif
+
 std::shared_ptr<GainNode> BaseAudioContext::createGain() {
   auto gain = std::make_shared<GainNode>(this);
   nodeManager_->addProcessingNode(gain);

package/common/cpp/audioapi/core/BaseAudioContext.h

@@ -31,6 +31,7 @@ class AnalyserNode;
 class AudioEventHandlerRegistry;
 class IAudioEventHandlerRegistry;
 class RecorderAdapterNode;
+class StreamerNode;
 
 class BaseAudioContext {
  public:
@@ -45,6 +46,7 @@ class BaseAudioContext {
 
   std::shared_ptr<RecorderAdapterNode> createRecorderAdapter();
   std::shared_ptr<OscillatorNode> createOscillator();
+  std::shared_ptr<StreamerNode> createStreamer();
   std::shared_ptr<GainNode> createGain();
   std::shared_ptr<StereoPannerNode> createStereoPanner();
   std::shared_ptr<BiquadFilterNode> createBiquadFilter();

package/common/cpp/audioapi/core/sources/StreamerNode.cpp

@@ -0,0 +1,299 @@
+/*
+ * This file links to the FFmpeg library, which is licensed under the
+ * GNU Lesser General Public License (LGPL) version 2.1 or later.
+ *
+ * Our own code in this file is licensed under the MIT License, but note
+ * that linking with FFmpeg means you must comply with the terms of the
+ * LGPL for FFmpeg itself.
+ */
+
+#include <audioapi/core/BaseAudioContext.h>
+#include <audioapi/core/sources/StreamerNode.h>
+#include <audioapi/core/utils/Locker.h>
+#include <audioapi/dsp/VectorMath.h>
+#include <audioapi/utils/AudioArray.h>
+#include <audioapi/utils/AudioBus.h>
+#include <chrono>
+
+namespace audioapi {
+StreamerNode::StreamerNode(BaseAudioContext *context)
+    : AudioScheduledSourceNode(context),
+      fmtCtx_(nullptr),
+      codecCtx_(nullptr),
+      decoder_(nullptr),
+      codecpar_(nullptr),
+      pkt_(nullptr),
+      frame_(nullptr),
+      pendingFrame_(nullptr),
+      bufferedBus_(nullptr),
+      bufferedBusIndex_(0),
+      maxBufferSize_(0),
+      audio_stream_index_(-1),
+      swrCtx_(nullptr),
+      resampledData_(nullptr),
+      maxResampledSamples_(0) {}
+
+StreamerNode::~StreamerNode() {
+  cleanup();
+}
+
+bool StreamerNode::initialize(const std::string &input_url) {
+  if (isInitialized_) {
+    cleanup();
+  }
+
+  if (!openInput(input_url)) {
+    return false;
+  }
+
+  if (!findAudioStream() || !setupDecoder() || !setupResampler()) {
+    cleanup();
+    return false;
+  }
+
+  pkt_ = av_packet_alloc();
+  frame_ = av_frame_alloc();
+
+  if (pkt_ == nullptr || frame_ == nullptr) {
+    cleanup();
+    return false;
+  }
+
+  maxBufferSize_ = BUFFER_LENGTH_SECONDS * codecCtx_->sample_rate;
+  // If decoding is faster than playing, we buffer few seconds of audio
+  bufferedBus_ = std::make_shared<AudioBus>(
+      maxBufferSize_, codecpar_->ch_layout.nb_channels, codecCtx_->sample_rate);
+
+  channelCount_ = codecpar_->ch_layout.nb_channels;
+  audioBus_ = std::make_shared<AudioBus>(
+      RENDER_QUANTUM_SIZE, channelCount_, context_->getSampleRate());
+
+  streamingThread_ = std::thread(&StreamerNode::streamAudio, this);
+  streamFlag.store(true);
+  isInitialized_ = true;
+  return true;
+}
+
+void StreamerNode::stop(double when) {
+  AudioScheduledSourceNode::stop(when);
+  streamFlag.store(false);
+}
+
+bool StreamerNode::setupResampler() {
+  // Allocate resampler context
+  swrCtx_ = swr_alloc();
+  if (swrCtx_ == nullptr) {
+    return false;
+  }
+
+  // Set input parameters (from codec)
+  av_opt_set_chlayout(swrCtx_, "in_chlayout", &codecCtx_->ch_layout, 0);
+  av_opt_set_int(swrCtx_, "in_sample_rate", codecCtx_->sample_rate, 0);
+  av_opt_set_sample_fmt(swrCtx_, "in_sample_fmt", codecCtx_->sample_fmt, 0);
+
+  // Set output parameters (float)
+  av_opt_set_chlayout(swrCtx_, "out_chlayout", &codecCtx_->ch_layout, 0);
+  av_opt_set_int(swrCtx_, "out_sample_rate", context_->getSampleRate(), 0);
+  av_opt_set_sample_fmt(swrCtx_, "out_sample_fmt", AV_SAMPLE_FMT_FLTP, 0);
+
+  // Initialize the resampler
+  if (swr_init(swrCtx_) < 0) {
+    return false;
+  }
+
+  // Allocate output buffer for resampled data
+  maxResampledSamples_ = INITIAL_MAX_RESAMPLED_SAMPLES;
+  int ret = av_samples_alloc_array_and_samples(
+      &resampledData_,
+      nullptr,
+      codecCtx_->ch_layout.nb_channels,
+      maxResampledSamples_,
+      AV_SAMPLE_FMT_FLTP,
+      0);
+
+  return ret >= 0;
+}
+
+void StreamerNode::streamAudio() {
+  while (streamFlag.load()) {
+    if (pendingFrame_ != nullptr) {
+      if (!processFrameWithResampler(pendingFrame_)) {
+        cleanup();
+        return;
+      }
+    } else {
+      if (av_read_frame(fmtCtx_, pkt_) < 0) {
+        cleanup();
+        return;
+      }
+      if (pkt_->stream_index == audio_stream_index_) {
+        if (avcodec_send_packet(codecCtx_, pkt_) != 0) {
+          cleanup();
+          return;
+        }
+        if (avcodec_receive_frame(codecCtx_, frame_) != 0) {
+          cleanup();
+          return;
+        }
+        if (!processFrameWithResampler(frame_)) {
+          cleanup();
+          return;
+        }
+        av_packet_unref(pkt_);
+      }
+    }
+    std::this_thread::sleep_for(std::chrono::milliseconds(10));
+  }
+}
+
+void StreamerNode::processNode(
+    const std::shared_ptr<AudioBus> &processingBus,
+    int framesToProcess) {
+  size_t startOffset = 0;
+  size_t offsetLength = 0;
+  updatePlaybackInfo(processingBus, framesToProcess, startOffset, offsetLength);
+  // If we have enough buffered data, copy to output bus
+  if (bufferedBusIndex_ >= framesToProcess) {
+    Locker locker(mutex_);
+    for (int ch = 0; ch < processingBus->getNumberOfChannels(); ch++) {
+      memcpy(
+          processingBus->getChannel(ch)->getData(),
+          bufferedBus_->getChannel(ch)->getData(),
+          offsetLength * sizeof(float));
+
+      memmove(
+          bufferedBus_->getChannel(ch)->getData(),
+          bufferedBus_->getChannel(ch)->getData() + offsetLength,
+          (maxBufferSize_ - offsetLength) * sizeof(float));
+    }
+    bufferedBusIndex_ -= offsetLength;
+  }
+}
+
+bool StreamerNode::processFrameWithResampler(AVFrame *frame) {
+  // Check if we need to reallocate the resampled buffer
+  int out_samples = swr_get_out_samples(swrCtx_, frame->nb_samples);
+  if (out_samples > maxResampledSamples_) {
+    av_freep(&resampledData_[0]);
+    av_freep(&resampledData_);
+
+    maxResampledSamples_ = out_samples;
+    int ret = av_samples_alloc_array_and_samples(
+        &resampledData_,
+        nullptr,
+        codecCtx_->ch_layout.nb_channels,
+        maxResampledSamples_,
+        AV_SAMPLE_FMT_FLTP,
+        0);
+
+    if (ret < 0) {
+      return false;
+    }
+  }
+
+  // Convert the frame
+  int converted_samples = swr_convert(
+      swrCtx_,
+      resampledData_,
+      maxResampledSamples_,
+      (const uint8_t **)frame->data,
+      frame->nb_samples);
+
+  if (converted_samples < 0) {
+    return false;
+  }
+
+  // Check if converted data fits in buffer
+  if (bufferedBusIndex_ + converted_samples > maxBufferSize_) {
+    pendingFrame_ = frame;
+    return true;
+  } else {
+    pendingFrame_ = nullptr;
+  }
+
+  // Copy converted data to our buffer
+  Locker locker(mutex_);
+  for (int ch = 0; ch < codecCtx_->ch_layout.nb_channels; ch++) {
+    auto *src = reinterpret_cast<float *>(resampledData_[ch]);
+    float *dst = bufferedBus_->getChannel(ch)->getData() + bufferedBusIndex_;
+    memcpy(dst, src, converted_samples * sizeof(float));
+  }
+  bufferedBusIndex_ += converted_samples;
+  return true;
+}
+
+bool StreamerNode::openInput(const std::string &input_url) {
+  if (avformat_open_input(&fmtCtx_, input_url.c_str(), nullptr, nullptr) < 0) {
+    return false;
+  }
+  return avformat_find_stream_info(fmtCtx_, nullptr) >= 0;
+}
+
+bool StreamerNode::findAudioStream() {
+  audio_stream_index_ = -1;
+  codecpar_ = nullptr;
+
+  for (int i = 0; i < fmtCtx_->nb_streams; ++i) {
+    if (fmtCtx_->streams[i]->codecpar->codec_type == AVMEDIA_TYPE_AUDIO) {
+      audio_stream_index_ = i;
+      codecpar_ = fmtCtx_->streams[i]->codecpar;
+      break;
+    }
+  }
+
+  return audio_stream_index_ >= 0 && codecpar_ != nullptr;
+}
+
+bool StreamerNode::setupDecoder() {
+  decoder_ = avcodec_find_decoder(codecpar_->codec_id);
+  if (decoder_ == nullptr) {
+    return false;
+  }
+
+  codecCtx_ = avcodec_alloc_context3(decoder_);
+  if (codecCtx_ == nullptr) {
+    return false;
+  }
+
+  if (avcodec_parameters_to_context(codecCtx_, codecpar_) < 0) {
+    return false;
+  }
+
+  return avcodec_open2(codecCtx_, decoder_, nullptr) >= 0;
+}
+
+void StreamerNode::cleanup() {
+  streamFlag.store(false);
+  streamingThread_.join();
+  if (swrCtx_ != nullptr) {
+    swr_free(&swrCtx_);
+  }
+
+  if (resampledData_ != nullptr) {
+    av_freep(&resampledData_[0]);
+    av_freep(&resampledData_);
+  }
+
+  if (frame_ != nullptr) {
+    av_frame_free(&frame_);
+  }
+
+  if (pkt_ != nullptr) {
+    av_packet_free(&pkt_);
+  }
+
+  if (codecCtx_ != nullptr) {
+    avcodec_free_context(&codecCtx_);
+  }
+
+  if (fmtCtx_ != nullptr) {
+    avformat_close_input(&fmtCtx_);
+  }
+
+  audio_stream_index_ = -1;
+  isInitialized_ = false;
+  decoder_ = nullptr;
+  codecpar_ = nullptr;
+  maxResampledSamples_ = 0;
+}
+} // namespace audioapi