react-native-audio-api 0.11.0-alpha.1 → 0.11.0-nightly-9f40b78-20251108

package/common/cpp/audioapi/dsp/Convolver.cpp

@@ -0,0 +1,213 @@
+// implementation of linear convolution algorithm described in this paper:
+// https://publications.rwth-aachen.de/record/466561/files/466561.pdf page 110
+
+#if defined(__ARM_NEON)
+#include <arm_neon.h>
+#endif
+
+#include <audioapi/core/sources/AudioBuffer.h>
+#include <audioapi/dsp/Convolver.h>
+#include <audioapi/dsp/VectorMath.h>
+#include <audioapi/utils/AudioArray.h>
+#include <chrono>
+#include <iostream>
+
+namespace audioapi {
+
+Convolver::Convolver()
+    : _blockSize(0),
+      _segSize(0),
+      _segCount(0),
+      _fftComplexSize(0),
+      _segments(),
+      _segmentsIR(),
+      _fftBuffer(0),
+      _fft(nullptr),
+      _preMultiplied(),
+      _current(0),
+      _inputBuffer(0) {}
+
+void Convolver::reset() {
+  _blockSize = 0;
+  _segSize = 0;
+  _segCount = 0;
+  _fftComplexSize = 0;
+  _current = 0;
+  _fft = nullptr;
+  _segments.clear();
+  _segmentsIR.clear();
+  _preMultiplied.clear();
+  _fftBuffer.zero();
+  _inputBuffer.zero();
+}
+
+bool Convolver::init(
+    size_t blockSize,
+    const audioapi::AudioArray &ir,
+    size_t irLen) {
+  reset();
+  // blockSize must be a power of two
+  if ((blockSize & (blockSize - 1))) {
+    return false;
+  }
+
+  // Ignore zeros at the end of the impulse response because they only waste
+  // computation time
+  _blockSize = blockSize;
+  _trueSegmentCount = (size_t)(std::ceil((float)irLen / (float)_blockSize));
+  while (irLen > 0 && ::fabs(ir[irLen - 1]) < 10e-3) {
+    --irLen;
+  }
+
+  if (irLen == 0) {
+    return true;
+  }
+
+  // The length-N is split into P = N/B length-B sub filters
+  _segCount = (size_t)(std::ceil((float)irLen / (float)_blockSize));
+  _segSize = 2 * _blockSize;
+  // size of the FFT is 2B, so the complex size is B+1, due to the
+  // complex-conjugate symmetricity
+  _fftComplexSize = _segSize / 2 + 1;
+  _fft = std::make_shared<dsp::FFT>((int)_segSize);
+  _fftBuffer.resize(_segSize);
+
+  // segments preparation
+  for (int i = 0; i < _segCount; ++i) {
+    aligned_vec_complex vec(_fftComplexSize, std::complex<float>(0.0f, 0.0f));
+    _segments.push_back(vec);
+  }
+
+  // ir preparation
+  for (int i = 0; i < _segCount; ++i) {
+    aligned_vec_complex segment(_fftComplexSize);
+    const size_t remainingSamples = irLen - (i * _blockSize);
+    const size_t samplesToCopy = std::min(_blockSize, remainingSamples);
+
+    if (samplesToCopy > 0) {
+      memcpy(
+          _fftBuffer.getData(),
+          ir.getData() + i * _blockSize,
+          samplesToCopy * sizeof(float));
+    }
+    // Each sub filter is zero-padded to length 2B and transformed using a
+    // 2B-point real-to-complex FFT.
+    memset(_fftBuffer.getData() + _blockSize, 0, _blockSize * sizeof(float));
+    _fft->doFFT(_fftBuffer.getData(), segment);
+    segment.at(0).imag(0.0f); // ensure DC component is real
+    _segmentsIR.push_back(segment);
+  }
+
+  _preMultiplied = aligned_vec_complex(_fftComplexSize);
+  _inputBuffer.resize(_segSize);
+  _current = 0;
+
+  return true;
+}
+
+/// @brief Fast pairwise complex multiplication using ARM NEON intrinsics
+/// @param ir Impulse response
+/// @param audio Input audio signal
+/// @param pre Output buffer for pre-multiplied results
+/// @note IMPORTANT: ir, audio, and pre must be the same size and should be
+/// aligned to 16 bytes for optimal performance
+void pairwise_complex_multiply_fast(
+    const Convolver::aligned_vec_complex &ir,
+    const Convolver::aligned_vec_complex &audio,
+    Convolver::aligned_vec_complex &pre) {
+  size_t n = ir.size();
+
+/// @note Using ARM NEON intrinsics for SIMD optimization
+/// This implementation is on average 2x faster than the scalar version on ARM
+/// architectures With 16-byte alignment it can be even faster up to 2.5x
+#ifdef __ARM_NEON
+  size_t j = 0;
+
+  // Main vector loop: process 4 complex samples (8 floats) per iteration using
+  // vld2q/vst2q deinterleave
+  for (; j <= n - 4; j += 4) {
+    // load de-interleaved real/imag for 4 complex values
+    float32x4x2_t ir_de = vld2q_f32(reinterpret_cast<const float *>(&ir[j]));
+    float32x4x2_t a_de = vld2q_f32(reinterpret_cast<const float *>(&audio[j]));
+    float32x4x2_t pre_de = vld2q_f32(reinterpret_cast<float *>(&pre[j]));
+
+    float32x4_t ir_re = ir_de.val[0];
+    float32x4_t ir_im = ir_de.val[1];
+    float32x4_t a_re = a_de.val[0];
+    float32x4_t a_im = a_de.val[1];
+
+    // real = ir_re * a_re - ir_im * a_im
+    float32x4_t real = vmulq_f32(ir_re, a_re);
+    real = vmlsq_f32(real, ir_im, a_im);
+    // imag = ir_re * a_im + ir_im * a_re
+    float32x4_t imag = vmulq_f32(ir_re, a_im);
+    imag = vmlaq_f32(imag, ir_im, a_re);
+
+    // accumulate into pre
+    float32x4_t new_re = vaddq_f32(pre_de.val[0], real);
+    float32x4_t new_im = vaddq_f32(pre_de.val[1], imag);
+
+    float32x4x2_t out_de;
+    out_de.val[0] = new_re;
+    out_de.val[1] = new_im;
+
+    vst2q_f32(reinterpret_cast<float *>(&pre[j]), out_de);
+  }
+
+  // Tail
+  for (; j < n; ++j) {
+    pre[j] += ir[j] * audio[j];
+  }
+
+#else
+  // Fallback scalar implementation
+  for (size_t i = 0; i < n; ++i) {
+    pre[i] += ir[i] * audio[i];
+  }
+#endif
+}
+
+void Convolver::process(float *data, float *outputData) {
+  // The input buffer acts as a 2B-point sliding window of the input signal.
+  // With each new input block, the right half of the input buffer is shifted
+  // to the left and the new block is stored in the right half.
+  memmove(
+      _inputBuffer.getData(),
+      _inputBuffer.getData() + _blockSize,
+      _blockSize * sizeof(float));
+  memcpy(_inputBuffer.getData() + _blockSize, data, _blockSize * sizeof(float));
+
+  // All contents (DFT spectra) in the FDL are shifted up by one slot.
+  _current = (_current > 0) ? _current - 1 : _segCount - 1;
+  // A 2B-point real-to-complex FFT is computed from the input buffer,
+  // resulting in B+1 complex-conjugate symmetric DFT coefficients. The
+  // result is stored in the first FDL slot.
+  // _current marks first FDL slot, which is the current input block.
+  _fft->doFFT(_inputBuffer.getData(), _segments[_current]);
+  _segments[_current][0].imag(0.0f); // ensure DC component is real
+
+  // The P sub filter spectra are pairwisely multiplied with the input spectra
+  // in the FDL. The results are accumulated in the frequency-domain.
+  memset(
+      _preMultiplied.data(),
+      0,
+      _preMultiplied.size() * sizeof(std::complex<float>));
+  // this is a bottleneck of the algorithm
+  for (int i = 0; i < _segCount; ++i) {
+    const int indexAudio = (_current + i) % _segCount;
+    const auto &impulseResponseSegment = _segmentsIR[i];
+    const auto &audioSegment = _segments[indexAudio];
+    pairwise_complex_multiply_fast(
+        impulseResponseSegment, audioSegment, _preMultiplied);
+  }
+  // Of the accumulated spectral convolutions, an 2B-point complex-to-real
+  // IFFT is computed. From the resulting 2B samples, the left half is
+  // discarded and the right half is returned as the next output block.
+  _fft->doInverseFFT(_preMultiplied, _fftBuffer.getData());
+
+  memcpy(
+      outputData,
+      _fftBuffer.getData() + _blockSize,
+      _blockSize * sizeof(float));
+}
+} // namespace audioapi

package/common/cpp/audioapi/dsp/Convolver.h

@@ -0,0 +1,45 @@
+#pragma once
+
+#include <audioapi/utils/AudioArray.h>
+#include <audioapi/dsp/FFT.h>
+#include <vector>
+#include <cstring>
+#include <complex>
+#include <memory>
+#include <audioapi/utils/AlignedAllocator.hpp>
+
+namespace audioapi {
+
+class AudioBuffer;
+
+class Convolver {
+  using aligned_vec_complex =
+      std::vector<std::complex<float>, AlignedAllocator<std::complex<float>, 16>>;
+
+ public:
+    Convolver();
+    bool init(size_t blockSize, const AudioArray &ir, size_t irLen);
+    void process(float* inputData, float* outputData);
+    void reset();
+    inline size_t getSegCount() const { return _trueSegmentCount; }
+
+ private:
+    size_t _trueSegmentCount;
+    size_t _blockSize;
+    size_t _segSize;
+    size_t _segCount;
+    size_t _fftComplexSize;
+    std::vector<aligned_vec_complex> _segments;
+    std::vector<aligned_vec_complex> _segmentsIR;
+    AudioArray _fftBuffer;
+    std::shared_ptr<dsp::FFT> _fft;
+    aligned_vec_complex _preMultiplied;
+    size_t _current;
+    AudioArray _inputBuffer;
+
+  friend void pairwise_complex_multiply_fast(
+      const aligned_vec_complex& ir,
+      const aligned_vec_complex& audio,
+      aligned_vec_complex& pre);
+};
+} // namespace audioapi

package/common/cpp/audioapi/dsp/FFT.cpp

@@ -12,30 +12,4 @@ FFT::~FFT() {
   pffft_aligned_free(work_);
 }
 
-void FFT::doFFT(float *in, std::vector<std::complex<float>> &out) {
-  pffft_transform_ordered(
-      pffftSetup_,
-      in,
-      reinterpret_cast<float *>(&out[0]),
-      work_,
-      PFFFT_FORWARD);
-
-  dsp::multiplyByScalar(
-      reinterpret_cast<float *>(&out[0]),
-      0.5f,
-      reinterpret_cast<float *>(&out[0]),
-      size_ * 2);
-}
-
-void FFT::doInverseFFT(std::vector<std::complex<float>> &in, float *out) {
-  pffft_transform_ordered(
-      pffftSetup_,
-      reinterpret_cast<float *>(&in[0]),
-      out,
-      work_,
-      PFFFT_BACKWARD);
-
-  dsp::multiplyByScalar(out, 1.0f / static_cast<float>(size_), out, size_);
-}
-
 } // namespace audioapi::dsp

package/common/cpp/audioapi/dsp/FFT.h

@@ -16,8 +16,32 @@ class FFT {
   explicit FFT(int size);
   ~FFT();
 
-  void doFFT(float *in, std::vector<std::complex<float>> &out);
-  void doInverseFFT(std::vector<std::complex<float>> &in, float *out);
+  template<typename Allocator>
+  void doFFT(float *in, std::vector<std::complex<float>, Allocator> &out) {
+    pffft_transform_ordered(
+        pffftSetup_,
+        in,
+        reinterpret_cast<float *>(&out[0]),
+        work_,
+        PFFFT_FORWARD);
+    // this is a possible place for bugs and mistakes
+    // due to pffft implementation and how it stores results
+    // keep this information in mind
+    // out[0].real = DC component - should be pure real
+    // out[0].imag = Nyquist component - should be pure real
+  }
+
+  template<typename Allocator>
+  void doInverseFFT(std::vector<std::complex<float>, Allocator> &in, float *out) {
+    pffft_transform_ordered(
+        pffftSetup_,
+        reinterpret_cast<float *>(&in[0]),
+        out,
+        work_,
+        PFFFT_BACKWARD);
+
+    dsp::multiplyByScalar(out, 1.0f / static_cast<float>(size_), out, size_);
+  }
 
  private:
   int size_;

package/common/cpp/audioapi/events/AudioEventHandlerRegistry.h

@@ -20,7 +20,7 @@ class AudioEventHandlerRegistry : public IAudioEventHandlerRegistry {
   explicit AudioEventHandlerRegistry(
       jsi::Runtime *runtime,
       const std::shared_ptr<react::CallInvoker> &callInvoker);
-  ~AudioEventHandlerRegistry();
+  ~AudioEventHandlerRegistry() override;
 
   uint64_t registerHandler(const std::string &eventName, const std::shared_ptr<jsi::Function> &handler) override;
   void unregisterHandler(const std::string &eventName, uint64_t listenerId) override;

package/common/cpp/audioapi/external/ffmpeg_include/libavcodec/avcodec.h

@@ -225,7 +225,7 @@ typedef struct RcOverride{
 #define AV_CODEC_FLAG_QPEL            (1 <<  4)
 /**
  * Request the encoder to output reconstructed frames, i.e.\ frames that would
- * be produced by decoding the encoded bistream. These frames may be retrieved
+ * be produced by decoding the encoded bitstream. These frames may be retrieved
  * by calling avcodec_receive_frame() immediately after a successful call to
  * avcodec_receive_packet().
  *
@@ -436,7 +436,7 @@ typedef struct AVCodecContext {
     const AVClass *av_class;
     int log_level_offset;
 
-    enum AVMediaTypeFFMPEG codec_type; /* see AVMEDIA_TYPE_xxx */
+    enum AVMediaTypeFFmpeg codec_type; /* see AVMEDIA_TYPE_xxx */
     const struct AVCodec  *codec;
     enum AVCodecID     codec_id; /* see AV_CODEC_ID_xxx */
 
@@ -1946,7 +1946,7 @@ typedef struct AVHWAccel {
      *
      * See AVMEDIA_TYPE_xxx
      */
-    enum AVMediaTypeFFMPEG type;
+    enum AVMediaTypeFFmpeg type;
 
     /**
      * Codec implemented by the hardware accelerator.
@@ -2900,7 +2900,7 @@ int avcodec_fill_audio_frame(AVFrame *frame, int nb_channels,
  *
  * @note for encoders, this function will only do something if the encoder
  * declares support for AV_CODEC_CAP_ENCODER_FLUSH. When called, the encoder
- * will drain any remaining packets, and can then be re-used for a different
+ * will drain any remaining packets, and can then be reused for a different
  * stream (as opposed to sending a null frame which will leave the encoder
  * in a permanent EOF state after draining). This can be desirable if the
  * cost of tearing down and replacing the encoder instance is high.

package/common/cpp/audioapi/external/ffmpeg_include/libavcodec/codec.h

@@ -182,7 +182,7 @@ typedef struct AVCodec {
      * You should use the NULL_IF_CONFIG_SMALL() macro to define it.
      */
     const char *long_name;
-    enum AVMediaTypeFFMPEG type;
+    enum AVMediaTypeFFmpeg type;
     enum AVCodecID id;
     /**
      * Codec capabilities.

package/common/cpp/audioapi/external/ffmpeg_include/libavcodec/codec_desc.h

@@ -37,7 +37,7 @@
  */
 typedef struct AVCodecDescriptor {
     enum AVCodecID     id;
-    enum AVMediaTypeFFMPEG type;
+    enum AVMediaTypeFFmpeg type;
     /**
      * Name of the codec described by this descriptor. It is non-empty and
      * unique for each codec descriptor. It should contain alphanumeric

package/common/cpp/audioapi/external/ffmpeg_include/libavcodec/codec_id.h

@@ -330,6 +330,7 @@ enum AVCodecID {
     AV_CODEC_ID_RV60,
     AV_CODEC_ID_JPEGXL_ANIM,
     AV_CODEC_ID_APV,
+    AV_CODEC_ID_PRORES_RAW,
 
     /* various PCM "codecs" */
     AV_CODEC_ID_FIRST_AUDIO = 0x10000,     ///< A dummy id pointing at the start of audio codecs
@@ -602,6 +603,7 @@ enum AVCodecID {
     AV_CODEC_ID_BIN_DATA,
     AV_CODEC_ID_SMPTE_2038,
     AV_CODEC_ID_LCEVC,
+    AV_CODEC_ID_SMPTE_436M_ANC,
 
 
     AV_CODEC_ID_PROBE = 0x19000, ///< codec_id is not known (like AV_CODEC_ID_NONE) but lavf should attempt to identify it
@@ -627,7 +629,7 @@ enum AVCodecID {
 /**
  * Get the type of the given codec.
  */
-enum AVMediaTypeFFMPEG avcodec_get_type(enum AVCodecID codec_id);
+enum AVMediaTypeFFmpeg avcodec_get_type(enum AVCodecID codec_id);
 
 /**
  * Get the name of a codec.

package/common/cpp/audioapi/external/ffmpeg_include/libavcodec/codec_par.h

@@ -48,7 +48,7 @@ typedef struct AVCodecParameters {
     /**
      * General type of the encoded data.
      */
-    enum AVMediaTypeFFMPEG codec_type;
+    enum AVMediaTypeFFmpeg codec_type;
     /**
      * Specific type of the encoded data (the codec used).
      */
@@ -148,7 +148,7 @@ typedef struct AVCodecParameters {
      * durations. Should be set to { 0, 1 } when some frames have differing
      * durations or if the value is not known.
      *
-     * @note This field correponds to values that are stored in codec-level
+     * @note This field corresponds to values that are stored in codec-level
      * headers and is typically overridden by container/transport-layer
      * timestamps, when available. It should thus be used only as a last resort,
      * when no higher-level timing information is available.

package/common/cpp/audioapi/external/ffmpeg_include/libavcodec/defs.h

@@ -185,6 +185,9 @@
 #define AV_PROFILE_PRORES_4444      4
 #define AV_PROFILE_PRORES_XQ        5
 
+#define AV_PROFILE_PRORES_RAW       0
+#define AV_PROFILE_PRORES_RAW_HQ    1
+
 #define AV_PROFILE_ARIB_PROFILE_A 0
 #define AV_PROFILE_ARIB_PROFILE_C 1
 

package/common/cpp/audioapi/external/ffmpeg_include/libavcodec/packet.h

@@ -142,7 +142,7 @@ enum AVPacketSideDataType {
     AV_PKT_DATA_CPB_PROPERTIES,
 
     /**
-     * Recommmends skipping the specified number of samples
+     * Recommends skipping the specified number of samples
      * @code
      * u32le number of samples to skip from start of this packet
      * u32le number of samples to skip from end of this packet