react-native-nitro-onnx 0.1.0 → 0.1.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (47) hide show
  1. package/NitroOnnxSpeech.podspec +3 -1
  2. package/README.md +146 -41
  3. package/android/CMakeLists.txt +54 -1
  4. package/android/build.gradle +6 -0
  5. package/android/src/main/AndroidManifest.xml +0 -2
  6. package/android/src/main/cpp/cpp-adapter.cpp +2 -2
  7. package/android/src/main/java/com/margelo/nitro/onnx/speech/OnnxSpeechPackage.kt +26 -17
  8. package/cpp/AsrEngine.cpp +154 -59
  9. package/cpp/AsrEngine.hpp +19 -6
  10. package/cpp/AudioFileReader.cpp +4 -0
  11. package/cpp/ModelSingleton.hpp +14 -0
  12. package/cpp/NitroOnnxSpeech.cpp +37 -18
  13. package/cpp/NitroOnnxSpeech.hpp +1 -7
  14. package/cpp/OfflineAsr.cpp +17 -8
  15. package/cpp/OfflineAsr.hpp +1 -1
  16. package/cpp/ResourceDir.cpp +5 -5
  17. package/cpp/ResourceDir.hpp +5 -4
  18. package/cpp/SpeakerEngine.cpp +38 -28
  19. package/cpp/SpeakerEngine.hpp +14 -13
  20. package/cpp/SpeakerManager.cpp +16 -14
  21. package/cpp/SpeakerManager.hpp +2 -1
  22. package/cpp/SpeakerRecord.cpp +125 -0
  23. package/cpp/SpeakerRecord.hpp +42 -0
  24. package/cpp/StreamingAsr.cpp +20 -9
  25. package/cpp/StreamingAsr.hpp +1 -1
  26. package/cpp/Tts.cpp +44 -10
  27. package/cpp/Tts.hpp +2 -1
  28. package/cpp/TtsEngine.cpp +17 -9
  29. package/cpp/TtsEngine.hpp +24 -5
  30. package/cpp/Vad.cpp +12 -11
  31. package/cpp/Vad.hpp +1 -1
  32. package/cpp/VadEngine.cpp +27 -33
  33. package/cpp/VadEngine.hpp +9 -10
  34. package/cpp/Version.hpp +7 -0
  35. package/ios/OnnxSpeechInitializer.mm +18 -6
  36. package/lib/specs/OnnxSpeech.nitro.d.ts +43 -6
  37. package/lib/specs/OnnxSpeech.nitro.d.ts.map +1 -1
  38. package/nitrogen/generated/shared/c++/AsrModelConfig.hpp +10 -2
  39. package/nitrogen/generated/shared/c++/HybridOnnxSpeechSpec.cpp +1 -1
  40. package/nitrogen/generated/shared/c++/HybridOnnxSpeechSpec.hpp +1 -1
  41. package/nitrogen/generated/shared/c++/TtsModelConfig.hpp +10 -2
  42. package/nitrogen/generated/shared/c++/VadConfig.hpp +6 -2
  43. package/package.json +4 -3
  44. package/scripts/generate-version.js +22 -0
  45. package/src/specs/OnnxSpeech.nitro.ts +43 -6
  46. package/cpp/ThreadPool.cpp +0 -41
  47. package/cpp/ThreadPool.hpp +0 -62
package/cpp/Tts.cpp CHANGED
@@ -4,8 +4,10 @@
4
4
  #include "Tts.hpp"
5
5
 
6
6
  #include "AudioUtils.hpp"
7
+ #include "SpeakerEngine.hpp"
7
8
 
8
9
  #include <NitroModules/ArrayBuffer.hpp>
10
+ #include <bit>
9
11
  #include <fstream>
10
12
  #include <stdexcept>
11
13
 
@@ -13,6 +15,8 @@ namespace margelo::nitro::onnx::speech {
13
15
 
14
16
  namespace {
15
17
 
18
+ static_assert(std::endian::native == std::endian::little, "WAV writer assumes little-endian targets");
19
+
16
20
  TtsResult toTtsResult(const TtsEngineResult& native) {
17
21
  std::vector<uint8_t> bytes = floatVectorToBytes(native.samples);
18
22
  return TtsResult(
@@ -23,15 +27,14 @@ TtsResult toTtsResult(const TtsEngineResult& native) {
23
27
 
24
28
  } // namespace
25
29
 
26
- Tts::Tts(std::shared_ptr<ThreadPool> threadPool)
27
- : HybridObject(TAG), engine_(std::move(threadPool)) {}
30
+ Tts::Tts() : HybridObject(TAG) {}
28
31
 
29
32
  Tts::~Tts() {
30
33
  engine_.unload();
31
34
  }
32
35
 
33
36
  std::shared_ptr<Promise<void>> Tts::load(const TtsModelConfig& config) {
34
- return Promise<void>::async([this, config]() {
37
+ return Promise<void>::async([self = shared_cast<Tts>(), config]() {
35
38
  TtsEngineConfig native;
36
39
  native.type = static_cast<TtsModelType>(config.type);
37
40
  native.modelDir = config.modelDir;
@@ -57,7 +60,13 @@ std::shared_ptr<Promise<void>> Tts::load(const TtsModelConfig& config) {
57
60
  native.outputSampleRate = static_cast<int32_t>(config.outputSampleRate.value_or(16000.0));
58
61
  native.speakerId = static_cast<int32_t>(config.speakerId.value_or(0.0));
59
62
  native.speed = static_cast<float>(config.speed.value_or(1.0));
60
- engine_.load(native);
63
+ native.debug = config.debug.value_or(false);
64
+ #ifdef __APPLE__
65
+ native.provider = config.provider.value_or("coreml");
66
+ #else
67
+ native.provider = config.provider.value_or("cpu");
68
+ #endif
69
+ self->engine_.load(native);
61
70
  });
62
71
  }
63
72
 
@@ -66,9 +75,9 @@ bool Tts::isLoaded() {
66
75
  }
67
76
 
68
77
  std::shared_ptr<Promise<TtsResult>> Tts::synthesize(const std::string& text, std::optional<double> speed) {
69
- return Promise<TtsResult>::async([this, text, speed]() {
78
+ return Promise<TtsResult>::async([self = shared_cast<Tts>(), text, speed]() {
70
79
  float spd = speed.has_value() ? static_cast<float>(speed.value()) : -1.0f;
71
- return toTtsResult(engine_.synthesize(text, -1, spd));
80
+ return toTtsResult(self->engine_.synthesize(text, -1, spd));
72
81
  });
73
82
  }
74
83
 
@@ -76,10 +85,35 @@ std::shared_ptr<Promise<TtsResult>> Tts::synthesizeWithSpeaker(
76
85
  const std::string& text,
77
86
  const std::string& speakerId,
78
87
  std::optional<double> speed) {
79
- return Promise<TtsResult>::async([this, text, speakerId, speed]() {
80
- int32_t sid = std::stoi(speakerId);
88
+ return Promise<TtsResult>::async([self = shared_cast<Tts>(), text, speakerId, speed]() {
89
+ if (text.empty()) {
90
+ throw std::invalid_argument("TTS text must not be empty");
91
+ }
92
+ if (speakerId.empty()) {
93
+ throw std::invalid_argument("speakerId must not be empty");
94
+ }
81
95
  float spd = speed.has_value() ? static_cast<float>(speed.value()) : -1.0f;
82
- return toTtsResult(engine_.synthesize(text, sid, spd));
96
+
97
+ // Numeric IDs are model-internal speaker indices (Kokoro / VITS multi-speaker).
98
+ int32_t sid = -1;
99
+ if (tryParseSpeakerIndex(speakerId, sid)) {
100
+ return toTtsResult(self->engine_.synthesize(text, sid, spd));
101
+ }
102
+
103
+ // String IDs refer to registered speakers (voice-cloning path).
104
+ const SpeakerEngineRegisteredSpeaker record =
105
+ readSpeakerRecord(speakerId, speakerFilePath(speakerId));
106
+ if (!record.referenceAudio.empty()) {
107
+ TtsReferenceAudio reference;
108
+ reference.samples = record.referenceAudio;
109
+ reference.sampleRate = record.referenceSampleRate > 0 ? record.referenceSampleRate : 16000;
110
+ return toTtsResult(self->engine_.synthesize(text, -1, spd, &reference));
111
+ }
112
+
113
+ throw std::invalid_argument(
114
+ "Speaker \"" + speakerId + "\" was registered from an embedding only. "
115
+ "Zero-shot TTS needs reference audio — use registerSpeakerFromFile(). "
116
+ "For multi-speaker models (kokoro/vits), pass a numeric speaker index instead.");
83
117
  });
84
118
  }
85
119
 
@@ -126,7 +160,7 @@ std::shared_ptr<Promise<void>> Tts::saveWav(const TtsResult& result, const std::
126
160
  }
127
161
 
128
162
  std::shared_ptr<Promise<void>> Tts::unload() {
129
- return Promise<void>::async([this]() { engine_.unload(); });
163
+ return Promise<void>::async([self = shared_cast<Tts>()]() { self->engine_.unload(); });
130
164
  }
131
165
 
132
166
  } // namespace margelo::nitro::onnx::speech
package/cpp/Tts.hpp CHANGED
@@ -11,6 +11,7 @@
11
11
 
12
12
  #include <memory>
13
13
  #include <optional>
14
+ #include <string>
14
15
 
15
16
  namespace margelo::nitro::onnx::speech {
16
17
 
@@ -18,7 +19,7 @@ class Tts : public HybridTtsSpec {
18
19
  public:
19
20
  static constexpr auto TAG = "Tts";
20
21
 
21
- explicit Tts(std::shared_ptr<ThreadPool> threadPool);
22
+ Tts();
22
23
  ~Tts() override;
23
24
 
24
25
  std::shared_ptr<Promise<void>> load(const TtsModelConfig& config) override;
package/cpp/TtsEngine.cpp CHANGED
@@ -51,9 +51,6 @@ ModelSingleton<const SherpaOnnxOfflineTts> gTtsCache;
51
51
 
52
52
  } // namespace
53
53
 
54
- TtsEngine::TtsEngine(std::shared_ptr<ThreadPool> threadPool)
55
- : threadPool_(std::move(threadPool)) {}
56
-
57
54
  TtsEngine::~TtsEngine() {
58
55
  unload();
59
56
  }
@@ -62,8 +59,7 @@ void TtsEngine::load(const TtsEngineConfig& config) {
62
59
  unload();
63
60
  config_ = config;
64
61
 
65
- const std::string key = config_.modelDir + "|" + std::to_string(static_cast<int>(config_.type));
66
- auto cached = gTtsCache.getOrCreate(key, [this](const std::string&) {
62
+ auto cached = gTtsCache.getOrCreate(config_.cacheSignature(), [this](const std::string&) {
67
63
  SherpaOnnxOfflineTtsConfig c;
68
64
  std::memset(&c, 0, sizeof(c));
69
65
 
@@ -75,7 +71,6 @@ void TtsEngine::load(const TtsEngineConfig& config) {
75
71
  std::string voices = joinPath(config_.modelDir, config_.voices);
76
72
  std::string espeakNgData = joinPath(config_.modelDir, config_.espeakNgData);
77
73
  std::string dictDir = joinPath(config_.modelDir, config_.dictDir);
78
- std::string configPath = joinPath(config_.modelDir, config_.config);
79
74
 
80
75
  std::string lmMain = joinPath(config_.modelDir, config_.lmMain);
81
76
  std::string lmFlow = joinPath(config_.modelDir, config_.lmFlow);
@@ -146,8 +141,8 @@ void TtsEngine::load(const TtsEngineConfig& config) {
146
141
  }
147
142
 
148
143
  c.model.num_threads = config_.numThreads;
149
- c.model.debug = 1;
150
- c.model.provider = "cpu";
144
+ c.model.debug = config_.debug ? 1 : 0;
145
+ c.model.provider = config_.provider.c_str();
151
146
  c.rule_fsts = "";
152
147
  c.rule_fars = "";
153
148
  c.max_num_sentences = 1;
@@ -171,10 +166,17 @@ bool TtsEngine::isLoaded() const {
171
166
  return tts_ != nullptr;
172
167
  }
173
168
 
174
- TtsEngineResult TtsEngine::synthesize(const std::string& text, int32_t speakerId, float speed) {
169
+ TtsEngineResult TtsEngine::synthesize(
170
+ const std::string& text,
171
+ int32_t speakerId,
172
+ float speed,
173
+ const TtsReferenceAudio* referenceAudio) {
175
174
  if (tts_ == nullptr) {
176
175
  throw std::runtime_error("TTS not loaded");
177
176
  }
177
+ if (text.empty()) {
178
+ throw std::invalid_argument("TTS text must not be empty");
179
+ }
178
180
 
179
181
  const int32_t sid = speakerId >= 0 ? speakerId : config_.speakerId;
180
182
  const float playbackSpeed = speed > 0.0f ? speed : config_.speed;
@@ -183,6 +185,12 @@ TtsEngineResult TtsEngine::synthesize(const std::string& text, int32_t speakerId
183
185
  std::memset(&genConfig, 0, sizeof(genConfig));
184
186
  genConfig.sid = sid;
185
187
  genConfig.speed = playbackSpeed;
188
+ if (referenceAudio != nullptr && !referenceAudio->samples.empty()) {
189
+ genConfig.reference_audio = referenceAudio->samples.data();
190
+ genConfig.reference_audio_len = static_cast<int32_t>(referenceAudio->samples.size());
191
+ genConfig.reference_sample_rate =
192
+ referenceAudio->sampleRate > 0 ? referenceAudio->sampleRate : 16000;
193
+ }
186
194
 
187
195
  const SherpaOnnxGeneratedAudio* audio = SherpaOnnxOfflineTtsGenerateWithConfig(
188
196
  tts_.get(), text.c_str(), &genConfig, nullptr, nullptr);
package/cpp/TtsEngine.hpp CHANGED
@@ -1,13 +1,12 @@
1
1
  // ------------------------------------------------------------------------------
2
2
  // TtsEngine.hpp
3
3
  // Text-to-speech wrapper backed by sherpa-onnx OfflineTts.
4
- // Supports Kokoro, VITS, Matcha, Pocket and ZipVoice (placeholder) models.
4
+ // Supports Kokoro, VITS, Matcha, Pocket and ZipVoice models.
5
5
  // ------------------------------------------------------------------------------
6
6
  #pragma once
7
7
 
8
8
  #include "AudioUtils.hpp"
9
9
  #include "ModelSingleton.hpp"
10
- #include "ThreadPool.hpp"
11
10
 
12
11
  #include "TtsModelType.hpp"
13
12
 
@@ -45,6 +44,17 @@ struct TtsEngineConfig {
45
44
  int32_t outputSampleRate = 16000;
46
45
  int32_t speakerId = 0;
47
46
  float speed = 1.0f;
47
+ bool debug = false;
48
+ #ifdef __APPLE__
49
+ std::string provider = "coreml";
50
+ #else
51
+ std::string provider = "cpu";
52
+ #endif
53
+
54
+ std::string cacheSignature() const {
55
+ return modelDir + "|" + std::to_string(static_cast<int>(type)) + "|" + provider + "|" +
56
+ std::to_string(numThreads) + "|" + std::to_string(outputSampleRate);
57
+ }
48
58
  };
49
59
 
50
60
  /** Native synthesis result (samples are kept as a float vector). */
@@ -54,10 +64,16 @@ struct TtsEngineResult {
54
64
  double durationMs = 0.0;
55
65
  };
56
66
 
67
+ /** Optional zero-shot reference audio for voice-cloning models (e.g. Pocket). */
68
+ struct TtsReferenceAudio {
69
+ std::vector<float> samples;
70
+ int32_t sampleRate = 16000;
71
+ };
72
+
57
73
  /** Text-to-speech engine. */
58
74
  class TtsEngine final {
59
75
  public:
60
- explicit TtsEngine(std::shared_ptr<ThreadPool> threadPool);
76
+ TtsEngine() = default;
61
77
  ~TtsEngine();
62
78
 
63
79
  TtsEngine(const TtsEngine&) = delete;
@@ -65,11 +81,14 @@ class TtsEngine final {
65
81
 
66
82
  void load(const TtsEngineConfig& config);
67
83
  bool isLoaded() const;
68
- TtsEngineResult synthesize(const std::string& text, int32_t speakerId, float speed);
84
+ TtsEngineResult synthesize(
85
+ const std::string& text,
86
+ int32_t speakerId,
87
+ float speed,
88
+ const TtsReferenceAudio* referenceAudio = nullptr);
69
89
  void unload();
70
90
 
71
91
  private:
72
- std::shared_ptr<ThreadPool> threadPool_;
73
92
  TtsEngineConfig config_;
74
93
  std::shared_ptr<const SherpaOnnxOfflineTts> tts_;
75
94
  };
package/cpp/Vad.cpp CHANGED
@@ -32,8 +32,7 @@ std::vector<VadSegment> toVadSegments(const std::vector<VadEngineSegment>& nativ
32
32
 
33
33
  } // namespace
34
34
 
35
- Vad::Vad(std::shared_ptr<ThreadPool> threadPool)
36
- : HybridObject(TAG), engine_(std::move(threadPool)) {}
35
+ Vad::Vad() : HybridObject(TAG) {}
37
36
 
38
37
  Vad::~Vad() {
39
38
  engine_.dispose();
@@ -41,20 +40,21 @@ Vad::~Vad() {
41
40
 
42
41
  std::shared_ptr<Promise<void>> Vad::initialize(const VadConfig& config) {
43
42
  currentConfig_ = config;
44
- return Promise<void>::async([this, config]() {
43
+ return Promise<void>::async([self = shared_cast<Vad>(), config]() {
45
44
  // Use the bundled silero_vad.onnx when no custom path is provided.
46
45
  const std::string& resDir = getResourceDir();
47
46
  const std::string modelPath = config.modelPath.value_or(
48
47
  resDir.empty() ? "silero_vad.onnx" : resDir + "/silero_vad.onnx");
49
- engine_.initialize(
48
+ self->engine_.initialize(
50
49
  {
51
50
  .modelPath = modelPath,
52
51
  .threshold = static_cast<float>(config.threshold.value_or(0.5)),
53
52
  .minSilenceDuration = static_cast<float>(config.minSilenceDurationMs.value_or(500.0)) / 1000.0f,
54
53
  .minSpeechDuration = static_cast<float>(config.minSpeechDurationMs.value_or(250.0)) / 1000.0f,
55
54
  .preBufferMs = static_cast<int32_t>(config.preBufferMs.value_or(300.0)),
55
+ .debug = config.debug.value_or(false),
56
56
  },
57
- shared_cast<Vad>());
57
+ self);
58
58
  });
59
59
  }
60
60
 
@@ -63,19 +63,20 @@ bool Vad::isInitialized() {
63
63
  }
64
64
 
65
65
  std::shared_ptr<Promise<void>> Vad::process(const std::shared_ptr<ArrayBuffer>& samples) {
66
- return Promise<void>::async([this, samples]() {
67
- const auto data = samples->data();
68
- auto floatSamples = bytesToFloatVector(data, samples->size());
69
- engine_.acceptWaveform(floatSamples);
66
+ return Promise<void>::async([self = shared_cast<Vad>(), samples]() {
67
+ auto floatSamples = bytesToFloatVector(samples->data(), samples->size());
68
+ self->engine_.acceptWaveform(std::move(floatSamples));
70
69
  });
71
70
  }
72
71
 
73
72
  std::shared_ptr<Promise<std::vector<VadSegment>>> Vad::pullSegments() {
74
- return Promise<std::vector<VadSegment>>::async([this]() { return toVadSegments(engine_.pullSegments()); });
73
+ return Promise<std::vector<VadSegment>>::async([self = shared_cast<Vad>()]() {
74
+ return toVadSegments(self->engine_.pullSegments());
75
+ });
75
76
  }
76
77
 
77
78
  std::shared_ptr<Promise<void>> Vad::reset() {
78
- return Promise<void>::async([this]() { engine_.reset(); });
79
+ return Promise<void>::async([self = shared_cast<Vad>()]() { self->engine_.reset(); });
79
80
  }
80
81
 
81
82
  std::optional<std::function<void(const VadSegment& /* segment */)>> Vad::getOnSpeechStart() {
package/cpp/Vad.hpp CHANGED
@@ -25,7 +25,7 @@ class Vad : public HybridVadSpec,
25
25
  public:
26
26
  static constexpr auto TAG = "Vad";
27
27
 
28
- explicit Vad(std::shared_ptr<ThreadPool> threadPool);
28
+ Vad();
29
29
  ~Vad() override;
30
30
 
31
31
  std::shared_ptr<Promise<void>> initialize(const VadConfig& config) override;
package/cpp/VadEngine.cpp CHANGED
@@ -5,7 +5,6 @@
5
5
 
6
6
  #include "sherpa-onnx/c-api/c-api.h"
7
7
 
8
- #include <algorithm>
9
8
  #include <cstring>
10
9
  #include <stdexcept>
11
10
 
@@ -17,9 +16,6 @@ constexpr int32_t kSampleRate = 16000;
17
16
 
18
17
  } // namespace
19
18
 
20
- VadEngine::VadEngine(std::shared_ptr<ThreadPool> threadPool)
21
- : threadPool_(std::move(threadPool)) {}
22
-
23
19
  VadEngine::~VadEngine() {
24
20
  dispose();
25
21
  }
@@ -30,7 +26,7 @@ void VadEngine::initialize(const VadEngineConfig& config, std::shared_ptr<VadLis
30
26
  config_ = config;
31
27
  listener_ = std::move(listener);
32
28
  preBufferCapacity_ = static_cast<size_t>(msToSamples(config_.preBufferMs));
33
- preBuffer_.reserve(preBufferCapacity_);
29
+ preBuffer_.clear();
34
30
  streamMs_ = 0.0f;
35
31
  inSpeech_ = false;
36
32
 
@@ -42,6 +38,7 @@ void VadEngine::initialize(const VadEngineConfig& config, std::shared_ptr<VadLis
42
38
  vadConfig.silero_vad.min_speech_duration = config_.minSpeechDuration;
43
39
  vadConfig.sample_rate = kSampleRate;
44
40
  vadConfig.num_threads = 1;
41
+ vadConfig.debug = config_.debug ? 1 : 0;
45
42
 
46
43
  // The second argument is the buffer size in milliseconds used internally by
47
44
  // sherpa-onnx. We reuse the configured pre-buffer duration.
@@ -52,6 +49,7 @@ void VadEngine::initialize(const VadEngineConfig& config, std::shared_ptr<VadLis
52
49
 
53
50
  initialized_ = true;
54
51
  stop_ = false;
52
+ resetRequested_ = false;
55
53
  processorThread_ = std::thread(&VadEngine::processLoop, this);
56
54
  }
57
55
 
@@ -59,10 +57,10 @@ bool VadEngine::isInitialized() const {
59
57
  return initialized_.load();
60
58
  }
61
59
 
62
- void VadEngine::acceptWaveform(const std::vector<float>& samples) {
60
+ void VadEngine::acceptWaveform(std::vector<float> samples) {
63
61
  {
64
62
  std::lock_guard<std::mutex> lock(inputMutex_);
65
- inputQueue_.emplace(samples);
63
+ inputQueue_.emplace(std::move(samples));
66
64
  }
67
65
  inputCv_.notify_one();
68
66
  }
@@ -89,12 +87,9 @@ void VadEngine::reset() {
89
87
  pendingSegments_.clear();
90
88
  }
91
89
 
92
- if (vad_) {
93
- SherpaOnnxVoiceActivityDetectorClear(vad_);
94
- }
95
-
96
- // Defer speech-state clearing to the processor thread to avoid a data race.
90
+ // Clear the sherpa VAD on the processor thread to avoid racing with accept.
97
91
  resetRequested_ = true;
92
+ inputCv_.notify_one();
98
93
  }
99
94
 
100
95
  void VadEngine::dispose() {
@@ -116,10 +111,21 @@ void VadEngine::processLoop() {
116
111
  std::vector<float> chunk;
117
112
  {
118
113
  std::unique_lock<std::mutex> lock(inputMutex_);
119
- inputCv_.wait(lock, [this]() { return stop_ || !inputQueue_.empty(); });
114
+ inputCv_.wait(lock, [this]() { return stop_ || resetRequested_ || !inputQueue_.empty(); });
120
115
  if (stop_) {
121
116
  break;
122
117
  }
118
+ if (resetRequested_.exchange(false)) {
119
+ if (vad_) {
120
+ SherpaOnnxVoiceActivityDetectorClear(vad_);
121
+ }
122
+ inSpeech_ = false;
123
+ currentSpeechSamples_.clear();
124
+ streamMs_ = 0.0f;
125
+ }
126
+ if (inputQueue_.empty()) {
127
+ continue;
128
+ }
123
129
  chunk = std::move(inputQueue_.front());
124
130
  inputQueue_.pop();
125
131
  }
@@ -128,20 +134,13 @@ void VadEngine::processLoop() {
128
134
  continue;
129
135
  }
130
136
 
131
- if (resetRequested_.exchange(false)) {
132
- inSpeech_ = false;
133
- currentSpeechSamples_.clear();
134
- streamMs_ = 0.0f;
135
- }
136
-
137
137
  // Update the sliding pre-buffer with the newest chunk. This is the key
138
138
  // mechanism that preserves leading audio for onSpeechStart/onSpeechEnd.
139
139
  {
140
140
  std::lock_guard<std::mutex> lock(preBufferMutex_);
141
141
  preBuffer_.insert(preBuffer_.end(), chunk.begin(), chunk.end());
142
- if (preBuffer_.size() > preBufferCapacity_) {
143
- const size_t excess = preBuffer_.size() - preBufferCapacity_;
144
- preBuffer_.erase(preBuffer_.begin(), preBuffer_.begin() + excess);
142
+ while (preBuffer_.size() > preBufferCapacity_) {
143
+ preBuffer_.pop_front();
145
144
  }
146
145
  }
147
146
 
@@ -150,6 +149,7 @@ void VadEngine::processLoop() {
150
149
  streamMs_ += samplesToMs(static_cast<int32_t>(chunk.size()));
151
150
 
152
151
  const bool speechDetected = SherpaOnnxVoiceActivityDetectorDetected(vad_) != 0;
152
+ std::shared_ptr<VadListener> listener = listener_.lock();
153
153
 
154
154
  if (speechDetected && !inSpeech_) {
155
155
  // Transition to speech: capture the sliding pre-buffer as the start of
@@ -159,15 +159,15 @@ void VadEngine::processLoop() {
159
159
  currentSpeechStartMs_ = streamMs_ - samplesToMs(static_cast<int32_t>(chunk.size()));
160
160
  {
161
161
  std::lock_guard<std::mutex> lock(preBufferMutex_);
162
- currentSpeechSamples_ = preBuffer_;
162
+ currentSpeechSamples_.assign(preBuffer_.begin(), preBuffer_.end());
163
163
  }
164
164
 
165
165
  VadEngineSegment startSegment;
166
166
  startSegment.startMs = currentSpeechStartMs_ - samplesToMs(static_cast<int32_t>(currentSpeechSamples_.size()));
167
167
  startSegment.endMs = currentSpeechStartMs_;
168
168
  startSegment.samples = currentSpeechSamples_;
169
- if (listener_) {
170
- listener_->onSpeechStart(startSegment);
169
+ if (listener) {
170
+ listener->onSpeechStart(startSegment);
171
171
  }
172
172
  }
173
173
 
@@ -199,8 +199,8 @@ void VadEngine::processLoop() {
199
199
  std::lock_guard<std::mutex> lock(outputMutex_);
200
200
  pendingSegments_.push_back(endSegment);
201
201
  }
202
- if (listener_) {
203
- listener_->onSpeechEnd(endSegment);
202
+ if (listener) {
203
+ listener->onSpeechEnd(endSegment);
204
204
  }
205
205
 
206
206
  inSpeech_ = false;
@@ -208,10 +208,4 @@ void VadEngine::processLoop() {
208
208
  }
209
209
  }
210
210
 
211
- void VadEngine::flushPreBuffer(std::vector<float>& target) {
212
- std::lock_guard<std::mutex> lock(preBufferMutex_);
213
- target.insert(target.end(), preBuffer_.begin(), preBuffer_.end());
214
- preBuffer_.clear();
215
- }
216
-
217
211
  } // namespace margelo::nitro::onnx::speech
package/cpp/VadEngine.hpp CHANGED
@@ -12,9 +12,10 @@
12
12
  #pragma once
13
13
 
14
14
  #include "AudioUtils.hpp"
15
- #include "ThreadPool.hpp"
16
15
 
16
+ #include <atomic>
17
17
  #include <condition_variable>
18
+ #include <deque>
18
19
  #include <memory>
19
20
  #include <mutex>
20
21
  #include <optional>
@@ -34,6 +35,7 @@ struct VadEngineConfig {
34
35
  float minSilenceDuration = 0.5f;
35
36
  float minSpeechDuration = 0.25f;
36
37
  int32_t preBufferMs = 300;
38
+ bool debug = false;
37
39
  };
38
40
 
39
41
  /** One buffered or emitted speech segment with raw float samples. */
@@ -61,7 +63,7 @@ class VadListener {
61
63
  */
62
64
  class VadEngine final {
63
65
  public:
64
- explicit VadEngine(std::shared_ptr<ThreadPool> threadPool);
66
+ VadEngine() = default;
65
67
  ~VadEngine();
66
68
 
67
69
  VadEngine(const VadEngine&) = delete;
@@ -73,8 +75,8 @@ class VadEngine final {
73
75
  /** Return true if the VAD has been initialized. */
74
76
  bool isInitialized() const;
75
77
 
76
- /** Feed a chunk of 16 kHz mono f32 PCM audio. Non-blocking. */
77
- void acceptWaveform(const std::vector<float>& samples);
78
+ /** Feed a chunk of 16 kHz mono f32 PCM audio. Non-blocking. Takes ownership. */
79
+ void acceptWaveform(std::vector<float> samples);
78
80
 
79
81
  /** Return any buffered segments without waiting for speech end. */
80
82
  std::vector<VadEngineSegment> pullSegments();
@@ -87,11 +89,8 @@ class VadEngine final {
87
89
 
88
90
  private:
89
91
  void processLoop();
90
- void flushPreBuffer(std::vector<float>& target);
91
- void emitSegment();
92
92
 
93
- std::shared_ptr<ThreadPool> threadPool_;
94
- std::shared_ptr<VadListener> listener_;
93
+ std::weak_ptr<VadListener> listener_;
95
94
 
96
95
  const SherpaOnnxVoiceActivityDetector* vad_ = nullptr;
97
96
  VadEngineConfig config_;
@@ -106,7 +105,7 @@ class VadEngine final {
106
105
 
107
106
  // Sliding pre-buffer. Always holds the most recent N milliseconds of audio.
108
107
  std::mutex preBufferMutex_;
109
- std::vector<float> preBuffer_;
108
+ std::deque<float> preBuffer_;
110
109
  size_t preBufferCapacity_ = 0;
111
110
 
112
111
  // Output segments waiting to be pulled.
@@ -116,7 +115,7 @@ class VadEngine final {
116
115
  // Processor thread.
117
116
  std::thread processorThread_;
118
117
 
119
- // Stream time tracking.
118
+ // Stream time tracking. Only touched by the processor thread.
120
119
  float streamMs_ = 0.0f;
121
120
  bool inSpeech_ = false;
122
121
  float currentSpeechStartMs_ = 0.0f;
@@ -0,0 +1,7 @@
1
+ // ------------------------------------------------------------------------------
2
+ // Version.hpp
3
+ // Generated from package.json by scripts/generate-version.js. DO NOT EDIT.
4
+ // ------------------------------------------------------------------------------
5
+ #pragma once
6
+
7
+ #define NITRO_ONNX_SPEECH_VERSION "0.1.2"
@@ -1,7 +1,7 @@
1
1
  // ------------------------------------------------------------------------------
2
2
  // OnnxSpeechInitializer.mm
3
3
  // ------------------------------------------------------------------------------
4
- // Sets up resource and cache directories for the C++ layer on iOS.
4
+ // Sets up resource and document directories for the C++ layer on iOS.
5
5
  // Runs at +load time so paths are available before any HybridObject is created.
6
6
  // ------------------------------------------------------------------------------
7
7
  #import <Foundation/Foundation.h>
@@ -26,12 +26,24 @@
26
26
  margelo::nitro::onnx::speech::setResourceDir([[mainBundle resourcePath] UTF8String]);
27
27
  }
28
28
 
29
- NSArray<NSString*>* cachePaths = NSSearchPathForDirectoriesInDomains(
30
- NSCachesDirectory, NSUserDomainMask, YES);
31
- NSString* cacheDir = cachePaths.firstObject;
32
- if (cacheDir) {
33
- margelo::nitro::onnx::speech::setCacheDir([cacheDir UTF8String]);
29
+ // Application Support (not Documents): speaker embeddings are derived data
30
+ // and must not be backed up to iCloud.
31
+ NSArray<NSString*>* supportPaths = NSSearchPathForDirectoriesInDomains(
32
+ NSApplicationSupportDirectory, NSUserDomainMask, YES);
33
+ NSString* documentDir = supportPaths.firstObject;
34
+ if (documentDir == nil) {
35
+ return;
34
36
  }
37
+
38
+ NSError* error = nil;
39
+ [[NSFileManager defaultManager] createDirectoryAtPath:documentDir
40
+ withIntermediateDirectories:YES
41
+ attributes:nil
42
+ error:&error];
43
+ NSURL* dirURL = [NSURL fileURLWithPath:documentDir isDirectory:YES];
44
+ [dirURL setResourceValue:@YES forKey:NSURLIsExcludedFromBackupKey error:nil];
45
+
46
+ margelo::nitro::onnx::speech::setDocumentDir([documentDir UTF8String]);
35
47
  }
36
48
 
37
49
  @end