react-native-audio-api 0.8.3 → 0.9.0

package/common/cpp/audioapi/core/effects/StereoPannerNode.cpp

@@ -1,6 +1,6 @@
 #include <audioapi/core/BaseAudioContext.h>
-#include <audioapi/core/Constants.h>
 #include <audioapi/core/effects/StereoPannerNode.h>
+#include <audioapi/core/utils/Constants.h>
 #include <audioapi/utils/AudioArray.h>
 #include <audioapi/utils/AudioBus.h>
 
@@ -19,18 +19,20 @@ std::shared_ptr<AudioParam> StereoPannerNode::getPanParam() const {
   return panParam_;
 }
 
-void StereoPannerNode::processNode(
+std::shared_ptr<AudioBus> StereoPannerNode::processNode(
     const std::shared_ptr<AudioBus> &processingBus,
     int framesToProcess) {
   double time = context_->getCurrentTime();
   double deltaTime = 1.0 / context_->getSampleRate();
 
-  AudioArray *left = processingBus->getChannelByType(AudioBus::ChannelLeft);
-  AudioArray *right = processingBus->getChannelByType(AudioBus::ChannelRight);
+  auto *inputLeft = processingBus->getChannelByType(AudioBus::ChannelLeft);
   auto panParamValues = panParam_->processARateParam(framesToProcess, time)
                             ->getChannel(0)
                             ->getData();
 
+  auto *outputLeft = audioBus_->getChannelByType(AudioBus::ChannelLeft);
+  auto *outputRight = audioBus_->getChannelByType(AudioBus::ChannelRight);
+
   // Input is mono
   if (processingBus->getNumberOfChannels() == 1) {
     for (int i = 0; i < framesToProcess; i++) {
@@ -40,13 +42,14 @@ void StereoPannerNode::processNode(
       auto gainL = static_cast<float>(cos(x * PI / 2));
       auto gainR = static_cast<float>(sin(x * PI / 2));
 
-      float input = (*left)[i];
+      float input = (*inputLeft)[i];
 
-      (*left)[i] = input * gainL;
-      (*right)[i] = input * gainR;
+      (*outputLeft)[i] = input * gainL;
+      (*outputRight)[i] = input * gainR;
       time += deltaTime;
     }
   } else { // Input is stereo
+    auto *inputRight = processingBus->getChannelByType(AudioBus::ChannelRight);
     for (int i = 0; i < framesToProcess; i++) {
       auto pan = std::clamp(panParamValues[i], -1.0f, 1.0f);
       auto x = (pan <= 0 ? pan + 1 : pan);
@@ -54,20 +57,22 @@ void StereoPannerNode::processNode(
       auto gainL = static_cast<float>(cos(x * PI / 2));
       auto gainR = static_cast<float>(sin(x * PI / 2));
 
-      float inputL = (*left)[i];
-      float inputR = (*right)[i];
+      float inputL = (*inputLeft)[i];
+      float inputR = (*inputRight)[i];
 
       if (pan <= 0) {
-        (*left)[i] = inputL + inputR * gainL;
-        (*right)[i] = inputR * gainR;
+        (*outputLeft)[i] = inputL + inputR * gainL;
+        (*outputRight)[i] = inputR * gainR;
       } else {
-        (*left)[i] = inputL * gainL;
-        (*right)[i] = inputR + inputL * gainR;
+        (*outputLeft)[i] = inputL * gainL;
+        (*outputRight)[i] = inputR + inputL * gainR;
       }
 
       time += deltaTime;
     }
   }
+
+  return audioBus_;
 }
 
 } // namespace audioapi

package/common/cpp/audioapi/core/effects/StereoPannerNode.h

@@ -18,7 +18,7 @@ class StereoPannerNode : public AudioNode {
   [[nodiscard]] std::shared_ptr<AudioParam> getPanParam() const;
 
  protected:
-  void processNode(const std::shared_ptr<AudioBus>& processingBus, int framesToProcess) override;
+  std::shared_ptr<AudioBus> processNode(const std::shared_ptr<AudioBus>& processingBus, int framesToProcess) override;
 
  private:
   std::shared_ptr<AudioParam> panParam_;

package/common/cpp/audioapi/core/effects/WorkletNode.cpp

@@ -0,0 +1,89 @@
+#include <audioapi/core/effects/WorkletNode.h>
+
+namespace audioapi {
+
+WorkletNode::WorkletNode(
+    BaseAudioContext *context,
+    std::shared_ptr<worklets::SerializableWorklet> &worklet,
+    std::weak_ptr<worklets::WorkletRuntime> runtime,
+    size_t bufferLength,
+    size_t inputChannelCount)
+    : AudioNode(context),
+      buffRealLength_(bufferLength * sizeof(float)),
+      bufferLength_(bufferLength),
+      workletRunner_(runtime),
+      shareableWorklet_(worklet),
+      inputChannelCount_(inputChannelCount),
+      curBuffIndex_(0) {
+  buffs_.reserve(inputChannelCount_);
+  for (size_t i = 0; i < inputChannelCount_; ++i) {
+    buffs_.emplace_back(new uint8_t[buffRealLength_]);
+  }
+  isInitialized_ = true;
+}
+
+WorkletNode::~WorkletNode() {
+  for (auto &buff : buffs_) {
+    delete[] buff;
+  }
+}
+
+std::shared_ptr<AudioBus> WorkletNode::processNode(
+    const std::shared_ptr<AudioBus> &processingBus,
+    int framesToProcess) {
+  size_t processed = 0;
+  size_t channelCount_ = std::min(
+      inputChannelCount_,
+      static_cast<size_t>(processingBus->getNumberOfChannels()));
+  while (processed < framesToProcess) {
+    size_t framesToWorkletInvoke = bufferLength_ - curBuffIndex_;
+    size_t needsToProcess = framesToProcess - processed;
+    size_t shouldProcess = std::min(framesToWorkletInvoke, needsToProcess);
+
+    for (size_t ch = 0; ch < channelCount_; ch++) {
+      /// here we copy
+      /// to   uint8_t* [curBuffIndex_, curBuffIndex_ + shouldProcess]
+      /// from float* [processed, processed + shouldProcess]
+      /// so as the we need to copy shouldProcess * sizeof(float) bytes
+      auto channelData = processingBus->getChannel(ch)->getData();
+      std::memcpy(
+          /* dest */ buffs_[ch] + curBuffIndex_ * sizeof(float),
+          /* src */ reinterpret_cast<const uint8_t *>(channelData + processed),
+          /* size */ shouldProcess * sizeof(float));
+    }
+    processed += shouldProcess;
+    curBuffIndex_ += shouldProcess;
+
+    /// If we filled the entire buffer, we need to execute the worklet
+    if (curBuffIndex_ == bufferLength_) {
+      // Reset buffer index, channel buffers and execute worklet
+      curBuffIndex_ = 0;
+      workletRunner_.executeOnRuntimeGuardedSync(
+          [this, channelCount_](jsi::Runtime &uiRuntimeRaw) {
+            /// Arguments preparation
+            auto jsArray = jsi::Array(uiRuntimeRaw, channelCount_);
+            for (size_t ch = 0; ch < channelCount_; ch++) {
+              uint8_t *buffPtr = buffs_[ch];
+              buffs_[ch] = new uint8_t[buffRealLength_];
+              auto sharedAudioArray =
+                  std::make_shared<AudioArrayBuffer>(buffPtr, buffRealLength_);
+              auto arrayBuffer =
+                  jsi::ArrayBuffer(uiRuntimeRaw, std::move(sharedAudioArray));
+              jsArray.setValueAtIndex(uiRuntimeRaw, ch, std::move(arrayBuffer));
+            }
+            jsArray.setExternalMemoryPressure(
+                uiRuntimeRaw, channelCount_ * buffRealLength_);
+
+            workletRunner_.executeWorklet(
+                shareableWorklet_,
+                std::move(jsArray),
+                jsi::Value(uiRuntimeRaw, static_cast<int>(channelCount_)));
+            return jsi::Value::undefined();
+          });
+    }
+  }
+
+  return processingBus;
+}
+
+} // namespace audioapi

package/common/cpp/audioapi/core/effects/WorkletNode.h

@@ -0,0 +1,65 @@
+#pragma once
+
+
+#include <jsi/jsi.h>
+#include <audioapi/core/utils/worklets/WorkletsRunner.h>
+#include <audioapi/core/AudioNode.h>
+#include <audioapi/core/BaseAudioContext.h>
+#include <audioapi/utils/AudioBus.h>
+#include <audioapi/utils/AudioArray.h>
+#include <audioapi/jsi/AudioArrayBuffer.h>
+
+#include <memory>
+#include <vector>
+
+namespace audioapi {
+
+#if RN_AUDIO_API_TEST
+class WorkletNode : public AudioNode {
+ public:
+  explicit WorkletNode(
+      BaseAudioContext *context,
+      std::shared_ptr<worklets::SerializableWorklet> &worklet,
+      std::weak_ptr<worklets::WorkletRuntime> runtime,
+      size_t bufferLength,
+      size_t inputChannelCount
+  ) : AudioNode(context) {}
+
+ protected:
+  std::shared_ptr<AudioBus> processNode(const std::shared_ptr<AudioBus>& processingBus, int framesToProcess) override { return processingBus; }
+};
+#else
+
+using namespace facebook;
+
+class WorkletNode : public AudioNode {
+ public:
+  explicit WorkletNode(
+      BaseAudioContext *context,
+      std::shared_ptr<worklets::SerializableWorklet> &worklet,
+      std::weak_ptr<worklets::WorkletRuntime> runtime,
+      size_t bufferLength,
+      size_t inputChannelCount
+  );
+
+  ~WorkletNode() override;
+
+ protected:
+  std::shared_ptr<AudioBus> processNode(const std::shared_ptr<AudioBus>& processingBus, int framesToProcess) override;
+
+
+ private:
+  WorkletsRunner workletRunner_;
+  std::shared_ptr<worklets::SerializableWorklet> shareableWorklet_;
+  std::vector<uint8_t*> buffs_;
+
+  /// @brief Length of the byte buffer that will be passed to the AudioArrayBuffer
+  size_t buffRealLength_;
+  size_t bufferLength_;
+  size_t inputChannelCount_;
+  size_t curBuffIndex_;
+};
+
+#endif // RN_AUDIO_API_TEST
+
+} // namespace audioapi

package/common/cpp/audioapi/core/effects/WorkletProcessingNode.cpp

@@ -0,0 +1,91 @@
+#include <audioapi/core/effects/WorkletProcessingNode.h>
+#include <audioapi/core/utils/Constants.h>
+
+namespace audioapi {
+
+WorkletProcessingNode::WorkletProcessingNode(
+    BaseAudioContext *context,
+    std::shared_ptr<worklets::SerializableWorklet> &worklet,
+    std::weak_ptr<worklets::WorkletRuntime> runtime)
+    : AudioNode(context), workletRunner_(runtime), shareableWorklet_(worklet) {
+  isInitialized_ = true;
+
+  // Pre-allocate buffers for max 128 frames and 2 channels (stereo)
+  size_t maxChannelCount = 2;
+  inputBuffsHandles_.resize(maxChannelCount);
+  outputBuffsHandles_.resize(maxChannelCount);
+
+  for (size_t i = 0; i < maxChannelCount; ++i) {
+    auto inputBuff = new uint8_t[RENDER_QUANTUM_SIZE * sizeof(float)];
+    inputBuffsHandles_[i] = std::make_shared<AudioArrayBuffer>(
+        inputBuff, RENDER_QUANTUM_SIZE * sizeof(float));
+
+    auto outputBuff = new uint8_t[RENDER_QUANTUM_SIZE * sizeof(float)];
+    outputBuffsHandles_[i] = std::make_shared<AudioArrayBuffer>(
+        outputBuff, RENDER_QUANTUM_SIZE * sizeof(float));
+  }
+}
+
+std::shared_ptr<AudioBus> WorkletProcessingNode::processNode(
+    const std::shared_ptr<AudioBus> &processingBus,
+    int framesToProcess) {
+  size_t channelCount = std::min(
+      static_cast<size_t>(2), // Fixed to stereo for now
+      static_cast<size_t>(processingBus->getNumberOfChannels()));
+
+  // Copy input data to pre-allocated input buffers
+  for (size_t ch = 0; ch < channelCount; ch++) {
+    auto channelData = processingBus->getChannel(ch)->getData();
+    std::memcpy(
+        /* dest */ inputBuffsHandles_[ch]->data(),
+        /* src */ reinterpret_cast<const uint8_t *>(channelData),
+        /* size */ framesToProcess * sizeof(float));
+  }
+
+  // Execute the worklet
+  auto result = workletRunner_.executeOnRuntimeGuardedSync(
+      [this, channelCount, framesToProcess](jsi::Runtime &rt) {
+        auto inputJsArray = jsi::Array(rt, channelCount);
+        auto outputJsArray = jsi::Array(rt, channelCount);
+
+        for (size_t ch = 0; ch < channelCount; ch++) {
+          // Create input array buffer
+          auto inputArrayBuffer = jsi::ArrayBuffer(rt, inputBuffsHandles_[ch]);
+          inputJsArray.setValueAtIndex(rt, ch, inputArrayBuffer);
+
+          // Create output array buffer
+          auto outputArrayBuffer =
+              jsi::ArrayBuffer(rt, outputBuffsHandles_[ch]);
+          outputJsArray.setValueAtIndex(rt, ch, outputArrayBuffer);
+        }
+
+        return workletRunner_
+            .executeWorklet(
+                shareableWorklet_,
+                inputJsArray,
+                outputJsArray,
+                jsi::Value(rt, static_cast<int>(framesToProcess)),
+                jsi::Value(rt, this->context_->getCurrentTime()))
+            .value_or(jsi::Value::undefined());
+      });
+
+  // Copy processed output data back to the processing bus or zero on failure
+  for (size_t ch = 0; ch < channelCount; ch++) {
+    auto channelData = processingBus->getChannel(ch)->getData();
+
+    if (result.has_value()) {
+      // Copy processed output data
+      std::memcpy(
+          /* dest */ reinterpret_cast<uint8_t *>(channelData),
+          /* src */ outputBuffsHandles_[ch]->data(),
+          /* size */ framesToProcess * sizeof(float));
+    } else {
+      // Zero the output on worklet execution failure
+      std::memset(channelData, 0, framesToProcess * sizeof(float));
+    }
+  }
+
+  return processingBus;
+}
+
+} // namespace audioapi

package/common/cpp/audioapi/core/effects/WorkletProcessingNode.h

@@ -0,0 +1,52 @@
+#pragma once
+
+#include <jsi/jsi.h>
+#include <audioapi/core/utils/worklets/WorkletsRunner.h>
+#include <audioapi/core/AudioNode.h>
+#include <audioapi/core/BaseAudioContext.h>
+#include <audioapi/utils/AudioBus.h>
+#include <audioapi/utils/AudioArray.h>
+#include <audioapi/jsi/AudioArrayBuffer.h>
+
+#include <memory>
+#include <vector>
+
+namespace audioapi {
+
+#if RN_AUDIO_API_TEST
+class WorkletProcessingNode : public AudioNode {
+ public:
+  explicit WorkletProcessingNode(
+      BaseAudioContext *context,
+      std::shared_ptr<worklets::SerializableWorklet> &worklet,
+      std::weak_ptr<worklets::WorkletRuntime> runtime
+  ) : AudioNode(context) {}
+
+ protected:
+  std::shared_ptr<AudioBus> processNode(const std::shared_ptr<AudioBus>& processingBus, int framesToProcess) override { return processingBus; }
+};
+#else
+
+using namespace facebook;
+
+class WorkletProcessingNode : public AudioNode {
+ public:
+  explicit WorkletProcessingNode(
+      BaseAudioContext *context,
+      std::shared_ptr<worklets::SerializableWorklet> &worklet,
+      std::weak_ptr<worklets::WorkletRuntime> runtime
+  );
+
+ protected:
+  std::shared_ptr<AudioBus> processNode(const std::shared_ptr<AudioBus>& processingBus, int framesToProcess) override;
+
+ private:
+  WorkletsRunner workletRunner_;
+  std::shared_ptr<worklets::SerializableWorklet> shareableWorklet_;
+  std::vector<std::shared_ptr<AudioArrayBuffer>> inputBuffsHandles_;
+  std::vector<std::shared_ptr<AudioArrayBuffer>> outputBuffsHandles_;
+};
+
+#endif // RN_AUDIO_API_TEST
+
+} // namespace audioapi

package/common/cpp/audioapi/core/inputs/AudioRecorder.cpp

@@ -1,4 +1,4 @@
-#include <audioapi/HostObjects/AudioBufferHostObject.h>
+#include <audioapi/HostObjects/sources/AudioBufferHostObject.h>
 #include <audioapi/core/inputs/AudioRecorder.h>
 #include <audioapi/core/sources/AudioBuffer.h>
 #include <audioapi/core/sources/RecorderAdapterNode.h>

package/common/cpp/audioapi/core/inputs/AudioRecorder.h

@@ -1,11 +1,11 @@
 #pragma once
 
-
 #include <memory>
-#include <mutex>
 #include <atomic>
+#include <mutex>
 
 namespace audioapi {
+
 class RecorderAdapterNode;
 class AudioBus;
 class CircularAudioArray;

package/common/cpp/audioapi/core/sources/AudioBufferBaseSourceNode.cpp

@@ -1,14 +1,18 @@
 #include <audioapi/core/AudioParam.h>
 #include <audioapi/core/BaseAudioContext.h>
-#include <audioapi/core/Constants.h>
 #include <audioapi/core/sources/AudioBufferBaseSourceNode.h>
+#include <audioapi/core/utils/Constants.h>
 #include <audioapi/events/AudioEventHandlerRegistry.h>
 #include <audioapi/utils/AudioArray.h>
 #include <audioapi/utils/AudioBus.h>
 
 namespace audioapi {
-AudioBufferBaseSourceNode::AudioBufferBaseSourceNode(BaseAudioContext *context)
-    : AudioScheduledSourceNode(context), vReadIndex_(0.0) {
+AudioBufferBaseSourceNode::AudioBufferBaseSourceNode(
+    BaseAudioContext *context,
+    bool pitchCorrection)
+    : AudioScheduledSourceNode(context),
+      pitchCorrection_(pitchCorrection),
+      vReadIndex_(0.0) {
   onPositionChangedInterval_ = static_cast<int>(context->getSampleRate() * 0.1);
 
   detuneParam_ = std::make_shared<AudioParam>(
@@ -24,12 +28,7 @@ AudioBufferBaseSourceNode::AudioBufferBaseSourceNode(BaseAudioContext *context)
 }
 
 AudioBufferBaseSourceNode::~AudioBufferBaseSourceNode() {
-  if (onPositionChangedCallbackId_ != 0 &&
-      context_->audioEventHandlerRegistry_ != nullptr) {
-    context_->audioEventHandlerRegistry_->unregisterHandler(
-        "positionChanged", onPositionChangedCallbackId_);
-    onPositionChangedCallbackId_ = 0;
-  }
+  clearOnPositionChangedCallback();
 }
 
 std::shared_ptr<AudioParam> AudioBufferBaseSourceNode::getDetuneParam() const {
@@ -62,7 +61,7 @@ void AudioBufferBaseSourceNode::setOnPositionChangedInterval(int interval) {
       context_->getSampleRate() * static_cast<float>(interval) / 1000);
 }
 
-int AudioBufferBaseSourceNode::getOnPositionChangedInterval() {
+int AudioBufferBaseSourceNode::getOnPositionChangedInterval() const {
   return onPositionChangedInterval_;
 }
 
@@ -125,7 +124,45 @@ void AudioBufferBaseSourceNode::processWithPitchCorrection(
   if (detune != 0.0f) {
     stretch_->setTransposeSemitones(detune);
   }
+
   sendOnPositionChangedEvent();
 }
 
+void AudioBufferBaseSourceNode::processWithoutPitchCorrection(
+    const std::shared_ptr<AudioBus> &processingBus,
+    int framesToProcess) {
+  size_t startOffset = 0;
+  size_t offsetLength = 0;
+
+  auto computedPlaybackRate = getComputedPlaybackRateValue(framesToProcess);
+  updatePlaybackInfo(processingBus, framesToProcess, startOffset, offsetLength);
+
+  if (computedPlaybackRate == 0.0f || (!isPlaying() && !isStopScheduled())) {
+    processingBus->zero();
+    return;
+  }
+
+  if (std::fabs(computedPlaybackRate) == 1.0) {
+    processWithoutInterpolation(
+        processingBus, startOffset, offsetLength, computedPlaybackRate);
+  } else {
+    processWithInterpolation(
+        processingBus, startOffset, offsetLength, computedPlaybackRate);
+  }
+
+  sendOnPositionChangedEvent();
+}
+
+float AudioBufferBaseSourceNode::getComputedPlaybackRateValue(
+    int framesToProcess) {
+  auto time = context_->getCurrentTime();
+
+  auto playbackRate =
+      playbackRateParam_->processKRateParam(framesToProcess, time);
+  auto detune = std::pow(
+      2.0f, detuneParam_->processKRateParam(framesToProcess, time) / 1200.0f);
+
+  return playbackRate * detune;
+}
+
 } // namespace audioapi

package/common/cpp/audioapi/core/sources/AudioBufferBaseSourceNode.h

@@ -14,8 +14,8 @@ class AudioParam;
 
 class AudioBufferBaseSourceNode : public AudioScheduledSourceNode {
  public:
-  explicit AudioBufferBaseSourceNode(BaseAudioContext *context);
-  virtual ~AudioBufferBaseSourceNode();
+  explicit AudioBufferBaseSourceNode(BaseAudioContext *context, bool pitchCorrection);
+  ~AudioBufferBaseSourceNode() override;
 
   [[nodiscard]] std::shared_ptr<AudioParam> getDetuneParam() const;
   [[nodiscard]] std::shared_ptr<AudioParam> getPlaybackRateParam() const;
@@ -23,9 +23,12 @@ class AudioBufferBaseSourceNode : public AudioScheduledSourceNode {
   void clearOnPositionChangedCallback();
   void setOnPositionChangedCallbackId(uint64_t callbackId);
   void setOnPositionChangedInterval(int interval);
-  [[nodiscard]] int getOnPositionChangedInterval();
+  [[nodiscard]] int getOnPositionChangedInterval() const;
 
  protected:
+    // pitch correction
+    bool pitchCorrection_;
+
     std::mutex bufferLock_;
 
     // pitch correction
@@ -47,13 +50,25 @@ class AudioBufferBaseSourceNode : public AudioScheduledSourceNode {
     virtual double getCurrentPosition() const = 0;
 
     void sendOnPositionChangedEvent();
+
     void processWithPitchCorrection(const std::shared_ptr<AudioBus> &processingBus,
                                     int framesToProcess);
+    void processWithoutPitchCorrection(const std::shared_ptr<AudioBus> &processingBus,
+                                       int framesToProcess);
+
+    float getComputedPlaybackRateValue(int framesToProcess);
+
     virtual void processWithoutInterpolation(
             const std::shared_ptr<AudioBus>& processingBus,
             size_t startOffset,
             size_t offsetLength,
             float playbackRate) = 0;
+
+    virtual void processWithInterpolation(
+            const std::shared_ptr<AudioBus>& processingBus,
+            size_t startOffset,
+            size_t offsetLength,
+            float playbackRate) = 0;
 };
 
 } // namespace audioapi