react-native-nitro-onnx 0.1.0 → 0.1.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/NitroOnnxSpeech.podspec +3 -1
- package/README.md +146 -41
- package/android/CMakeLists.txt +54 -1
- package/android/build.gradle +6 -0
- package/android/src/main/AndroidManifest.xml +0 -2
- package/android/src/main/cpp/cpp-adapter.cpp +2 -2
- package/android/src/main/java/com/margelo/nitro/onnx/speech/OnnxSpeechPackage.kt +26 -17
- package/cpp/AsrEngine.cpp +154 -59
- package/cpp/AsrEngine.hpp +19 -6
- package/cpp/AudioFileReader.cpp +4 -0
- package/cpp/ModelSingleton.hpp +14 -0
- package/cpp/NitroOnnxSpeech.cpp +37 -18
- package/cpp/NitroOnnxSpeech.hpp +1 -7
- package/cpp/OfflineAsr.cpp +17 -8
- package/cpp/OfflineAsr.hpp +1 -1
- package/cpp/ResourceDir.cpp +5 -5
- package/cpp/ResourceDir.hpp +5 -4
- package/cpp/SpeakerEngine.cpp +38 -28
- package/cpp/SpeakerEngine.hpp +14 -13
- package/cpp/SpeakerManager.cpp +16 -14
- package/cpp/SpeakerManager.hpp +2 -1
- package/cpp/SpeakerRecord.cpp +125 -0
- package/cpp/SpeakerRecord.hpp +42 -0
- package/cpp/StreamingAsr.cpp +20 -9
- package/cpp/StreamingAsr.hpp +1 -1
- package/cpp/Tts.cpp +44 -10
- package/cpp/Tts.hpp +2 -1
- package/cpp/TtsEngine.cpp +17 -9
- package/cpp/TtsEngine.hpp +24 -5
- package/cpp/Vad.cpp +12 -11
- package/cpp/Vad.hpp +1 -1
- package/cpp/VadEngine.cpp +27 -33
- package/cpp/VadEngine.hpp +9 -10
- package/cpp/Version.hpp +7 -0
- package/ios/OnnxSpeechInitializer.mm +18 -6
- package/lib/specs/OnnxSpeech.nitro.d.ts +43 -6
- package/lib/specs/OnnxSpeech.nitro.d.ts.map +1 -1
- package/nitrogen/generated/shared/c++/AsrModelConfig.hpp +10 -2
- package/nitrogen/generated/shared/c++/HybridOnnxSpeechSpec.cpp +1 -1
- package/nitrogen/generated/shared/c++/HybridOnnxSpeechSpec.hpp +1 -1
- package/nitrogen/generated/shared/c++/TtsModelConfig.hpp +10 -2
- package/nitrogen/generated/shared/c++/VadConfig.hpp +6 -2
- package/package.json +4 -3
- package/scripts/generate-version.js +22 -0
- package/src/specs/OnnxSpeech.nitro.ts +43 -6
- package/cpp/ThreadPool.cpp +0 -41
- package/cpp/ThreadPool.hpp +0 -62
package/cpp/Tts.cpp
CHANGED
|
@@ -4,8 +4,10 @@
|
|
|
4
4
|
#include "Tts.hpp"
|
|
5
5
|
|
|
6
6
|
#include "AudioUtils.hpp"
|
|
7
|
+
#include "SpeakerEngine.hpp"
|
|
7
8
|
|
|
8
9
|
#include <NitroModules/ArrayBuffer.hpp>
|
|
10
|
+
#include <bit>
|
|
9
11
|
#include <fstream>
|
|
10
12
|
#include <stdexcept>
|
|
11
13
|
|
|
@@ -13,6 +15,8 @@ namespace margelo::nitro::onnx::speech {
|
|
|
13
15
|
|
|
14
16
|
namespace {
|
|
15
17
|
|
|
18
|
+
static_assert(std::endian::native == std::endian::little, "WAV writer assumes little-endian targets");
|
|
19
|
+
|
|
16
20
|
TtsResult toTtsResult(const TtsEngineResult& native) {
|
|
17
21
|
std::vector<uint8_t> bytes = floatVectorToBytes(native.samples);
|
|
18
22
|
return TtsResult(
|
|
@@ -23,15 +27,14 @@ TtsResult toTtsResult(const TtsEngineResult& native) {
|
|
|
23
27
|
|
|
24
28
|
} // namespace
|
|
25
29
|
|
|
26
|
-
Tts::Tts(
|
|
27
|
-
: HybridObject(TAG), engine_(std::move(threadPool)) {}
|
|
30
|
+
Tts::Tts() : HybridObject(TAG) {}
|
|
28
31
|
|
|
29
32
|
Tts::~Tts() {
|
|
30
33
|
engine_.unload();
|
|
31
34
|
}
|
|
32
35
|
|
|
33
36
|
std::shared_ptr<Promise<void>> Tts::load(const TtsModelConfig& config) {
|
|
34
|
-
return Promise<void>::async([
|
|
37
|
+
return Promise<void>::async([self = shared_cast<Tts>(), config]() {
|
|
35
38
|
TtsEngineConfig native;
|
|
36
39
|
native.type = static_cast<TtsModelType>(config.type);
|
|
37
40
|
native.modelDir = config.modelDir;
|
|
@@ -57,7 +60,13 @@ std::shared_ptr<Promise<void>> Tts::load(const TtsModelConfig& config) {
|
|
|
57
60
|
native.outputSampleRate = static_cast<int32_t>(config.outputSampleRate.value_or(16000.0));
|
|
58
61
|
native.speakerId = static_cast<int32_t>(config.speakerId.value_or(0.0));
|
|
59
62
|
native.speed = static_cast<float>(config.speed.value_or(1.0));
|
|
60
|
-
|
|
63
|
+
native.debug = config.debug.value_or(false);
|
|
64
|
+
#ifdef __APPLE__
|
|
65
|
+
native.provider = config.provider.value_or("coreml");
|
|
66
|
+
#else
|
|
67
|
+
native.provider = config.provider.value_or("cpu");
|
|
68
|
+
#endif
|
|
69
|
+
self->engine_.load(native);
|
|
61
70
|
});
|
|
62
71
|
}
|
|
63
72
|
|
|
@@ -66,9 +75,9 @@ bool Tts::isLoaded() {
|
|
|
66
75
|
}
|
|
67
76
|
|
|
68
77
|
std::shared_ptr<Promise<TtsResult>> Tts::synthesize(const std::string& text, std::optional<double> speed) {
|
|
69
|
-
return Promise<TtsResult>::async([
|
|
78
|
+
return Promise<TtsResult>::async([self = shared_cast<Tts>(), text, speed]() {
|
|
70
79
|
float spd = speed.has_value() ? static_cast<float>(speed.value()) : -1.0f;
|
|
71
|
-
return toTtsResult(engine_.synthesize(text, -1, spd));
|
|
80
|
+
return toTtsResult(self->engine_.synthesize(text, -1, spd));
|
|
72
81
|
});
|
|
73
82
|
}
|
|
74
83
|
|
|
@@ -76,10 +85,35 @@ std::shared_ptr<Promise<TtsResult>> Tts::synthesizeWithSpeaker(
|
|
|
76
85
|
const std::string& text,
|
|
77
86
|
const std::string& speakerId,
|
|
78
87
|
std::optional<double> speed) {
|
|
79
|
-
return Promise<TtsResult>::async([
|
|
80
|
-
|
|
88
|
+
return Promise<TtsResult>::async([self = shared_cast<Tts>(), text, speakerId, speed]() {
|
|
89
|
+
if (text.empty()) {
|
|
90
|
+
throw std::invalid_argument("TTS text must not be empty");
|
|
91
|
+
}
|
|
92
|
+
if (speakerId.empty()) {
|
|
93
|
+
throw std::invalid_argument("speakerId must not be empty");
|
|
94
|
+
}
|
|
81
95
|
float spd = speed.has_value() ? static_cast<float>(speed.value()) : -1.0f;
|
|
82
|
-
|
|
96
|
+
|
|
97
|
+
// Numeric IDs are model-internal speaker indices (Kokoro / VITS multi-speaker).
|
|
98
|
+
int32_t sid = -1;
|
|
99
|
+
if (tryParseSpeakerIndex(speakerId, sid)) {
|
|
100
|
+
return toTtsResult(self->engine_.synthesize(text, sid, spd));
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
// String IDs refer to registered speakers (voice-cloning path).
|
|
104
|
+
const SpeakerEngineRegisteredSpeaker record =
|
|
105
|
+
readSpeakerRecord(speakerId, speakerFilePath(speakerId));
|
|
106
|
+
if (!record.referenceAudio.empty()) {
|
|
107
|
+
TtsReferenceAudio reference;
|
|
108
|
+
reference.samples = record.referenceAudio;
|
|
109
|
+
reference.sampleRate = record.referenceSampleRate > 0 ? record.referenceSampleRate : 16000;
|
|
110
|
+
return toTtsResult(self->engine_.synthesize(text, -1, spd, &reference));
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
throw std::invalid_argument(
|
|
114
|
+
"Speaker \"" + speakerId + "\" was registered from an embedding only. "
|
|
115
|
+
"Zero-shot TTS needs reference audio — use registerSpeakerFromFile(). "
|
|
116
|
+
"For multi-speaker models (kokoro/vits), pass a numeric speaker index instead.");
|
|
83
117
|
});
|
|
84
118
|
}
|
|
85
119
|
|
|
@@ -126,7 +160,7 @@ std::shared_ptr<Promise<void>> Tts::saveWav(const TtsResult& result, const std::
|
|
|
126
160
|
}
|
|
127
161
|
|
|
128
162
|
std::shared_ptr<Promise<void>> Tts::unload() {
|
|
129
|
-
return Promise<void>::async([
|
|
163
|
+
return Promise<void>::async([self = shared_cast<Tts>()]() { self->engine_.unload(); });
|
|
130
164
|
}
|
|
131
165
|
|
|
132
166
|
} // namespace margelo::nitro::onnx::speech
|
package/cpp/Tts.hpp
CHANGED
|
@@ -11,6 +11,7 @@
|
|
|
11
11
|
|
|
12
12
|
#include <memory>
|
|
13
13
|
#include <optional>
|
|
14
|
+
#include <string>
|
|
14
15
|
|
|
15
16
|
namespace margelo::nitro::onnx::speech {
|
|
16
17
|
|
|
@@ -18,7 +19,7 @@ class Tts : public HybridTtsSpec {
|
|
|
18
19
|
public:
|
|
19
20
|
static constexpr auto TAG = "Tts";
|
|
20
21
|
|
|
21
|
-
|
|
22
|
+
Tts();
|
|
22
23
|
~Tts() override;
|
|
23
24
|
|
|
24
25
|
std::shared_ptr<Promise<void>> load(const TtsModelConfig& config) override;
|
package/cpp/TtsEngine.cpp
CHANGED
|
@@ -51,9 +51,6 @@ ModelSingleton<const SherpaOnnxOfflineTts> gTtsCache;
|
|
|
51
51
|
|
|
52
52
|
} // namespace
|
|
53
53
|
|
|
54
|
-
TtsEngine::TtsEngine(std::shared_ptr<ThreadPool> threadPool)
|
|
55
|
-
: threadPool_(std::move(threadPool)) {}
|
|
56
|
-
|
|
57
54
|
TtsEngine::~TtsEngine() {
|
|
58
55
|
unload();
|
|
59
56
|
}
|
|
@@ -62,8 +59,7 @@ void TtsEngine::load(const TtsEngineConfig& config) {
|
|
|
62
59
|
unload();
|
|
63
60
|
config_ = config;
|
|
64
61
|
|
|
65
|
-
|
|
66
|
-
auto cached = gTtsCache.getOrCreate(key, [this](const std::string&) {
|
|
62
|
+
auto cached = gTtsCache.getOrCreate(config_.cacheSignature(), [this](const std::string&) {
|
|
67
63
|
SherpaOnnxOfflineTtsConfig c;
|
|
68
64
|
std::memset(&c, 0, sizeof(c));
|
|
69
65
|
|
|
@@ -75,7 +71,6 @@ void TtsEngine::load(const TtsEngineConfig& config) {
|
|
|
75
71
|
std::string voices = joinPath(config_.modelDir, config_.voices);
|
|
76
72
|
std::string espeakNgData = joinPath(config_.modelDir, config_.espeakNgData);
|
|
77
73
|
std::string dictDir = joinPath(config_.modelDir, config_.dictDir);
|
|
78
|
-
std::string configPath = joinPath(config_.modelDir, config_.config);
|
|
79
74
|
|
|
80
75
|
std::string lmMain = joinPath(config_.modelDir, config_.lmMain);
|
|
81
76
|
std::string lmFlow = joinPath(config_.modelDir, config_.lmFlow);
|
|
@@ -146,8 +141,8 @@ void TtsEngine::load(const TtsEngineConfig& config) {
|
|
|
146
141
|
}
|
|
147
142
|
|
|
148
143
|
c.model.num_threads = config_.numThreads;
|
|
149
|
-
c.model.debug = 1;
|
|
150
|
-
c.model.provider =
|
|
144
|
+
c.model.debug = config_.debug ? 1 : 0;
|
|
145
|
+
c.model.provider = config_.provider.c_str();
|
|
151
146
|
c.rule_fsts = "";
|
|
152
147
|
c.rule_fars = "";
|
|
153
148
|
c.max_num_sentences = 1;
|
|
@@ -171,10 +166,17 @@ bool TtsEngine::isLoaded() const {
|
|
|
171
166
|
return tts_ != nullptr;
|
|
172
167
|
}
|
|
173
168
|
|
|
174
|
-
TtsEngineResult TtsEngine::synthesize(
|
|
169
|
+
TtsEngineResult TtsEngine::synthesize(
|
|
170
|
+
const std::string& text,
|
|
171
|
+
int32_t speakerId,
|
|
172
|
+
float speed,
|
|
173
|
+
const TtsReferenceAudio* referenceAudio) {
|
|
175
174
|
if (tts_ == nullptr) {
|
|
176
175
|
throw std::runtime_error("TTS not loaded");
|
|
177
176
|
}
|
|
177
|
+
if (text.empty()) {
|
|
178
|
+
throw std::invalid_argument("TTS text must not be empty");
|
|
179
|
+
}
|
|
178
180
|
|
|
179
181
|
const int32_t sid = speakerId >= 0 ? speakerId : config_.speakerId;
|
|
180
182
|
const float playbackSpeed = speed > 0.0f ? speed : config_.speed;
|
|
@@ -183,6 +185,12 @@ TtsEngineResult TtsEngine::synthesize(const std::string& text, int32_t speakerId
|
|
|
183
185
|
std::memset(&genConfig, 0, sizeof(genConfig));
|
|
184
186
|
genConfig.sid = sid;
|
|
185
187
|
genConfig.speed = playbackSpeed;
|
|
188
|
+
if (referenceAudio != nullptr && !referenceAudio->samples.empty()) {
|
|
189
|
+
genConfig.reference_audio = referenceAudio->samples.data();
|
|
190
|
+
genConfig.reference_audio_len = static_cast<int32_t>(referenceAudio->samples.size());
|
|
191
|
+
genConfig.reference_sample_rate =
|
|
192
|
+
referenceAudio->sampleRate > 0 ? referenceAudio->sampleRate : 16000;
|
|
193
|
+
}
|
|
186
194
|
|
|
187
195
|
const SherpaOnnxGeneratedAudio* audio = SherpaOnnxOfflineTtsGenerateWithConfig(
|
|
188
196
|
tts_.get(), text.c_str(), &genConfig, nullptr, nullptr);
|
package/cpp/TtsEngine.hpp
CHANGED
|
@@ -1,13 +1,12 @@
|
|
|
1
1
|
// ------------------------------------------------------------------------------
|
|
2
2
|
// TtsEngine.hpp
|
|
3
3
|
// Text-to-speech wrapper backed by sherpa-onnx OfflineTts.
|
|
4
|
-
// Supports Kokoro, VITS, Matcha, Pocket and ZipVoice
|
|
4
|
+
// Supports Kokoro, VITS, Matcha, Pocket and ZipVoice models.
|
|
5
5
|
// ------------------------------------------------------------------------------
|
|
6
6
|
#pragma once
|
|
7
7
|
|
|
8
8
|
#include "AudioUtils.hpp"
|
|
9
9
|
#include "ModelSingleton.hpp"
|
|
10
|
-
#include "ThreadPool.hpp"
|
|
11
10
|
|
|
12
11
|
#include "TtsModelType.hpp"
|
|
13
12
|
|
|
@@ -45,6 +44,17 @@ struct TtsEngineConfig {
|
|
|
45
44
|
int32_t outputSampleRate = 16000;
|
|
46
45
|
int32_t speakerId = 0;
|
|
47
46
|
float speed = 1.0f;
|
|
47
|
+
bool debug = false;
|
|
48
|
+
#ifdef __APPLE__
|
|
49
|
+
std::string provider = "coreml";
|
|
50
|
+
#else
|
|
51
|
+
std::string provider = "cpu";
|
|
52
|
+
#endif
|
|
53
|
+
|
|
54
|
+
std::string cacheSignature() const {
|
|
55
|
+
return modelDir + "|" + std::to_string(static_cast<int>(type)) + "|" + provider + "|" +
|
|
56
|
+
std::to_string(numThreads) + "|" + std::to_string(outputSampleRate);
|
|
57
|
+
}
|
|
48
58
|
};
|
|
49
59
|
|
|
50
60
|
/** Native synthesis result (samples are kept as a float vector). */
|
|
@@ -54,10 +64,16 @@ struct TtsEngineResult {
|
|
|
54
64
|
double durationMs = 0.0;
|
|
55
65
|
};
|
|
56
66
|
|
|
67
|
+
/** Optional zero-shot reference audio for voice-cloning models (e.g. Pocket). */
|
|
68
|
+
struct TtsReferenceAudio {
|
|
69
|
+
std::vector<float> samples;
|
|
70
|
+
int32_t sampleRate = 16000;
|
|
71
|
+
};
|
|
72
|
+
|
|
57
73
|
/** Text-to-speech engine. */
|
|
58
74
|
class TtsEngine final {
|
|
59
75
|
public:
|
|
60
|
-
|
|
76
|
+
TtsEngine() = default;
|
|
61
77
|
~TtsEngine();
|
|
62
78
|
|
|
63
79
|
TtsEngine(const TtsEngine&) = delete;
|
|
@@ -65,11 +81,14 @@ class TtsEngine final {
|
|
|
65
81
|
|
|
66
82
|
void load(const TtsEngineConfig& config);
|
|
67
83
|
bool isLoaded() const;
|
|
68
|
-
TtsEngineResult synthesize(
|
|
84
|
+
TtsEngineResult synthesize(
|
|
85
|
+
const std::string& text,
|
|
86
|
+
int32_t speakerId,
|
|
87
|
+
float speed,
|
|
88
|
+
const TtsReferenceAudio* referenceAudio = nullptr);
|
|
69
89
|
void unload();
|
|
70
90
|
|
|
71
91
|
private:
|
|
72
|
-
std::shared_ptr<ThreadPool> threadPool_;
|
|
73
92
|
TtsEngineConfig config_;
|
|
74
93
|
std::shared_ptr<const SherpaOnnxOfflineTts> tts_;
|
|
75
94
|
};
|
package/cpp/Vad.cpp
CHANGED
|
@@ -32,8 +32,7 @@ std::vector<VadSegment> toVadSegments(const std::vector<VadEngineSegment>& nativ
|
|
|
32
32
|
|
|
33
33
|
} // namespace
|
|
34
34
|
|
|
35
|
-
Vad::Vad(
|
|
36
|
-
: HybridObject(TAG), engine_(std::move(threadPool)) {}
|
|
35
|
+
Vad::Vad() : HybridObject(TAG) {}
|
|
37
36
|
|
|
38
37
|
Vad::~Vad() {
|
|
39
38
|
engine_.dispose();
|
|
@@ -41,20 +40,21 @@ Vad::~Vad() {
|
|
|
41
40
|
|
|
42
41
|
std::shared_ptr<Promise<void>> Vad::initialize(const VadConfig& config) {
|
|
43
42
|
currentConfig_ = config;
|
|
44
|
-
return Promise<void>::async([
|
|
43
|
+
return Promise<void>::async([self = shared_cast<Vad>(), config]() {
|
|
45
44
|
// Use the bundled silero_vad.onnx when no custom path is provided.
|
|
46
45
|
const std::string& resDir = getResourceDir();
|
|
47
46
|
const std::string modelPath = config.modelPath.value_or(
|
|
48
47
|
resDir.empty() ? "silero_vad.onnx" : resDir + "/silero_vad.onnx");
|
|
49
|
-
engine_.initialize(
|
|
48
|
+
self->engine_.initialize(
|
|
50
49
|
{
|
|
51
50
|
.modelPath = modelPath,
|
|
52
51
|
.threshold = static_cast<float>(config.threshold.value_or(0.5)),
|
|
53
52
|
.minSilenceDuration = static_cast<float>(config.minSilenceDurationMs.value_or(500.0)) / 1000.0f,
|
|
54
53
|
.minSpeechDuration = static_cast<float>(config.minSpeechDurationMs.value_or(250.0)) / 1000.0f,
|
|
55
54
|
.preBufferMs = static_cast<int32_t>(config.preBufferMs.value_or(300.0)),
|
|
55
|
+
.debug = config.debug.value_or(false),
|
|
56
56
|
},
|
|
57
|
-
|
|
57
|
+
self);
|
|
58
58
|
});
|
|
59
59
|
}
|
|
60
60
|
|
|
@@ -63,19 +63,20 @@ bool Vad::isInitialized() {
|
|
|
63
63
|
}
|
|
64
64
|
|
|
65
65
|
std::shared_ptr<Promise<void>> Vad::process(const std::shared_ptr<ArrayBuffer>& samples) {
|
|
66
|
-
return Promise<void>::async([
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
engine_.acceptWaveform(floatSamples);
|
|
66
|
+
return Promise<void>::async([self = shared_cast<Vad>(), samples]() {
|
|
67
|
+
auto floatSamples = bytesToFloatVector(samples->data(), samples->size());
|
|
68
|
+
self->engine_.acceptWaveform(std::move(floatSamples));
|
|
70
69
|
});
|
|
71
70
|
}
|
|
72
71
|
|
|
73
72
|
std::shared_ptr<Promise<std::vector<VadSegment>>> Vad::pullSegments() {
|
|
74
|
-
return Promise<std::vector<VadSegment>>::async([
|
|
73
|
+
return Promise<std::vector<VadSegment>>::async([self = shared_cast<Vad>()]() {
|
|
74
|
+
return toVadSegments(self->engine_.pullSegments());
|
|
75
|
+
});
|
|
75
76
|
}
|
|
76
77
|
|
|
77
78
|
std::shared_ptr<Promise<void>> Vad::reset() {
|
|
78
|
-
return Promise<void>::async([
|
|
79
|
+
return Promise<void>::async([self = shared_cast<Vad>()]() { self->engine_.reset(); });
|
|
79
80
|
}
|
|
80
81
|
|
|
81
82
|
std::optional<std::function<void(const VadSegment& /* segment */)>> Vad::getOnSpeechStart() {
|
package/cpp/Vad.hpp
CHANGED
package/cpp/VadEngine.cpp
CHANGED
|
@@ -5,7 +5,6 @@
|
|
|
5
5
|
|
|
6
6
|
#include "sherpa-onnx/c-api/c-api.h"
|
|
7
7
|
|
|
8
|
-
#include <algorithm>
|
|
9
8
|
#include <cstring>
|
|
10
9
|
#include <stdexcept>
|
|
11
10
|
|
|
@@ -17,9 +16,6 @@ constexpr int32_t kSampleRate = 16000;
|
|
|
17
16
|
|
|
18
17
|
} // namespace
|
|
19
18
|
|
|
20
|
-
VadEngine::VadEngine(std::shared_ptr<ThreadPool> threadPool)
|
|
21
|
-
: threadPool_(std::move(threadPool)) {}
|
|
22
|
-
|
|
23
19
|
VadEngine::~VadEngine() {
|
|
24
20
|
dispose();
|
|
25
21
|
}
|
|
@@ -30,7 +26,7 @@ void VadEngine::initialize(const VadEngineConfig& config, std::shared_ptr<VadLis
|
|
|
30
26
|
config_ = config;
|
|
31
27
|
listener_ = std::move(listener);
|
|
32
28
|
preBufferCapacity_ = static_cast<size_t>(msToSamples(config_.preBufferMs));
|
|
33
|
-
preBuffer_.
|
|
29
|
+
preBuffer_.clear();
|
|
34
30
|
streamMs_ = 0.0f;
|
|
35
31
|
inSpeech_ = false;
|
|
36
32
|
|
|
@@ -42,6 +38,7 @@ void VadEngine::initialize(const VadEngineConfig& config, std::shared_ptr<VadLis
|
|
|
42
38
|
vadConfig.silero_vad.min_speech_duration = config_.minSpeechDuration;
|
|
43
39
|
vadConfig.sample_rate = kSampleRate;
|
|
44
40
|
vadConfig.num_threads = 1;
|
|
41
|
+
vadConfig.debug = config_.debug ? 1 : 0;
|
|
45
42
|
|
|
46
43
|
// The second argument is the buffer size in milliseconds used internally by
|
|
47
44
|
// sherpa-onnx. We reuse the configured pre-buffer duration.
|
|
@@ -52,6 +49,7 @@ void VadEngine::initialize(const VadEngineConfig& config, std::shared_ptr<VadLis
|
|
|
52
49
|
|
|
53
50
|
initialized_ = true;
|
|
54
51
|
stop_ = false;
|
|
52
|
+
resetRequested_ = false;
|
|
55
53
|
processorThread_ = std::thread(&VadEngine::processLoop, this);
|
|
56
54
|
}
|
|
57
55
|
|
|
@@ -59,10 +57,10 @@ bool VadEngine::isInitialized() const {
|
|
|
59
57
|
return initialized_.load();
|
|
60
58
|
}
|
|
61
59
|
|
|
62
|
-
void VadEngine::acceptWaveform(
|
|
60
|
+
void VadEngine::acceptWaveform(std::vector<float> samples) {
|
|
63
61
|
{
|
|
64
62
|
std::lock_guard<std::mutex> lock(inputMutex_);
|
|
65
|
-
inputQueue_.emplace(samples);
|
|
63
|
+
inputQueue_.emplace(std::move(samples));
|
|
66
64
|
}
|
|
67
65
|
inputCv_.notify_one();
|
|
68
66
|
}
|
|
@@ -89,12 +87,9 @@ void VadEngine::reset() {
|
|
|
89
87
|
pendingSegments_.clear();
|
|
90
88
|
}
|
|
91
89
|
|
|
92
|
-
|
|
93
|
-
SherpaOnnxVoiceActivityDetectorClear(vad_);
|
|
94
|
-
}
|
|
95
|
-
|
|
96
|
-
// Defer speech-state clearing to the processor thread to avoid a data race.
|
|
90
|
+
// Clear the sherpa VAD on the processor thread to avoid racing with accept.
|
|
97
91
|
resetRequested_ = true;
|
|
92
|
+
inputCv_.notify_one();
|
|
98
93
|
}
|
|
99
94
|
|
|
100
95
|
void VadEngine::dispose() {
|
|
@@ -116,10 +111,21 @@ void VadEngine::processLoop() {
|
|
|
116
111
|
std::vector<float> chunk;
|
|
117
112
|
{
|
|
118
113
|
std::unique_lock<std::mutex> lock(inputMutex_);
|
|
119
|
-
inputCv_.wait(lock, [this]() { return stop_ || !inputQueue_.empty(); });
|
|
114
|
+
inputCv_.wait(lock, [this]() { return stop_ || resetRequested_ || !inputQueue_.empty(); });
|
|
120
115
|
if (stop_) {
|
|
121
116
|
break;
|
|
122
117
|
}
|
|
118
|
+
if (resetRequested_.exchange(false)) {
|
|
119
|
+
if (vad_) {
|
|
120
|
+
SherpaOnnxVoiceActivityDetectorClear(vad_);
|
|
121
|
+
}
|
|
122
|
+
inSpeech_ = false;
|
|
123
|
+
currentSpeechSamples_.clear();
|
|
124
|
+
streamMs_ = 0.0f;
|
|
125
|
+
}
|
|
126
|
+
if (inputQueue_.empty()) {
|
|
127
|
+
continue;
|
|
128
|
+
}
|
|
123
129
|
chunk = std::move(inputQueue_.front());
|
|
124
130
|
inputQueue_.pop();
|
|
125
131
|
}
|
|
@@ -128,20 +134,13 @@ void VadEngine::processLoop() {
|
|
|
128
134
|
continue;
|
|
129
135
|
}
|
|
130
136
|
|
|
131
|
-
if (resetRequested_.exchange(false)) {
|
|
132
|
-
inSpeech_ = false;
|
|
133
|
-
currentSpeechSamples_.clear();
|
|
134
|
-
streamMs_ = 0.0f;
|
|
135
|
-
}
|
|
136
|
-
|
|
137
137
|
// Update the sliding pre-buffer with the newest chunk. This is the key
|
|
138
138
|
// mechanism that preserves leading audio for onSpeechStart/onSpeechEnd.
|
|
139
139
|
{
|
|
140
140
|
std::lock_guard<std::mutex> lock(preBufferMutex_);
|
|
141
141
|
preBuffer_.insert(preBuffer_.end(), chunk.begin(), chunk.end());
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
preBuffer_.erase(preBuffer_.begin(), preBuffer_.begin() + excess);
|
|
142
|
+
while (preBuffer_.size() > preBufferCapacity_) {
|
|
143
|
+
preBuffer_.pop_front();
|
|
145
144
|
}
|
|
146
145
|
}
|
|
147
146
|
|
|
@@ -150,6 +149,7 @@ void VadEngine::processLoop() {
|
|
|
150
149
|
streamMs_ += samplesToMs(static_cast<int32_t>(chunk.size()));
|
|
151
150
|
|
|
152
151
|
const bool speechDetected = SherpaOnnxVoiceActivityDetectorDetected(vad_) != 0;
|
|
152
|
+
std::shared_ptr<VadListener> listener = listener_.lock();
|
|
153
153
|
|
|
154
154
|
if (speechDetected && !inSpeech_) {
|
|
155
155
|
// Transition to speech: capture the sliding pre-buffer as the start of
|
|
@@ -159,15 +159,15 @@ void VadEngine::processLoop() {
|
|
|
159
159
|
currentSpeechStartMs_ = streamMs_ - samplesToMs(static_cast<int32_t>(chunk.size()));
|
|
160
160
|
{
|
|
161
161
|
std::lock_guard<std::mutex> lock(preBufferMutex_);
|
|
162
|
-
currentSpeechSamples_
|
|
162
|
+
currentSpeechSamples_.assign(preBuffer_.begin(), preBuffer_.end());
|
|
163
163
|
}
|
|
164
164
|
|
|
165
165
|
VadEngineSegment startSegment;
|
|
166
166
|
startSegment.startMs = currentSpeechStartMs_ - samplesToMs(static_cast<int32_t>(currentSpeechSamples_.size()));
|
|
167
167
|
startSegment.endMs = currentSpeechStartMs_;
|
|
168
168
|
startSegment.samples = currentSpeechSamples_;
|
|
169
|
-
if (
|
|
170
|
-
|
|
169
|
+
if (listener) {
|
|
170
|
+
listener->onSpeechStart(startSegment);
|
|
171
171
|
}
|
|
172
172
|
}
|
|
173
173
|
|
|
@@ -199,8 +199,8 @@ void VadEngine::processLoop() {
|
|
|
199
199
|
std::lock_guard<std::mutex> lock(outputMutex_);
|
|
200
200
|
pendingSegments_.push_back(endSegment);
|
|
201
201
|
}
|
|
202
|
-
if (
|
|
203
|
-
|
|
202
|
+
if (listener) {
|
|
203
|
+
listener->onSpeechEnd(endSegment);
|
|
204
204
|
}
|
|
205
205
|
|
|
206
206
|
inSpeech_ = false;
|
|
@@ -208,10 +208,4 @@ void VadEngine::processLoop() {
|
|
|
208
208
|
}
|
|
209
209
|
}
|
|
210
210
|
|
|
211
|
-
void VadEngine::flushPreBuffer(std::vector<float>& target) {
|
|
212
|
-
std::lock_guard<std::mutex> lock(preBufferMutex_);
|
|
213
|
-
target.insert(target.end(), preBuffer_.begin(), preBuffer_.end());
|
|
214
|
-
preBuffer_.clear();
|
|
215
|
-
}
|
|
216
|
-
|
|
217
211
|
} // namespace margelo::nitro::onnx::speech
|
package/cpp/VadEngine.hpp
CHANGED
|
@@ -12,9 +12,10 @@
|
|
|
12
12
|
#pragma once
|
|
13
13
|
|
|
14
14
|
#include "AudioUtils.hpp"
|
|
15
|
-
#include "ThreadPool.hpp"
|
|
16
15
|
|
|
16
|
+
#include <atomic>
|
|
17
17
|
#include <condition_variable>
|
|
18
|
+
#include <deque>
|
|
18
19
|
#include <memory>
|
|
19
20
|
#include <mutex>
|
|
20
21
|
#include <optional>
|
|
@@ -34,6 +35,7 @@ struct VadEngineConfig {
|
|
|
34
35
|
float minSilenceDuration = 0.5f;
|
|
35
36
|
float minSpeechDuration = 0.25f;
|
|
36
37
|
int32_t preBufferMs = 300;
|
|
38
|
+
bool debug = false;
|
|
37
39
|
};
|
|
38
40
|
|
|
39
41
|
/** One buffered or emitted speech segment with raw float samples. */
|
|
@@ -61,7 +63,7 @@ class VadListener {
|
|
|
61
63
|
*/
|
|
62
64
|
class VadEngine final {
|
|
63
65
|
public:
|
|
64
|
-
|
|
66
|
+
VadEngine() = default;
|
|
65
67
|
~VadEngine();
|
|
66
68
|
|
|
67
69
|
VadEngine(const VadEngine&) = delete;
|
|
@@ -73,8 +75,8 @@ class VadEngine final {
|
|
|
73
75
|
/** Return true if the VAD has been initialized. */
|
|
74
76
|
bool isInitialized() const;
|
|
75
77
|
|
|
76
|
-
/** Feed a chunk of 16 kHz mono f32 PCM audio. Non-blocking. */
|
|
77
|
-
void acceptWaveform(
|
|
78
|
+
/** Feed a chunk of 16 kHz mono f32 PCM audio. Non-blocking. Takes ownership. */
|
|
79
|
+
void acceptWaveform(std::vector<float> samples);
|
|
78
80
|
|
|
79
81
|
/** Return any buffered segments without waiting for speech end. */
|
|
80
82
|
std::vector<VadEngineSegment> pullSegments();
|
|
@@ -87,11 +89,8 @@ class VadEngine final {
|
|
|
87
89
|
|
|
88
90
|
private:
|
|
89
91
|
void processLoop();
|
|
90
|
-
void flushPreBuffer(std::vector<float>& target);
|
|
91
|
-
void emitSegment();
|
|
92
92
|
|
|
93
|
-
std::
|
|
94
|
-
std::shared_ptr<VadListener> listener_;
|
|
93
|
+
std::weak_ptr<VadListener> listener_;
|
|
95
94
|
|
|
96
95
|
const SherpaOnnxVoiceActivityDetector* vad_ = nullptr;
|
|
97
96
|
VadEngineConfig config_;
|
|
@@ -106,7 +105,7 @@ class VadEngine final {
|
|
|
106
105
|
|
|
107
106
|
// Sliding pre-buffer. Always holds the most recent N milliseconds of audio.
|
|
108
107
|
std::mutex preBufferMutex_;
|
|
109
|
-
std::
|
|
108
|
+
std::deque<float> preBuffer_;
|
|
110
109
|
size_t preBufferCapacity_ = 0;
|
|
111
110
|
|
|
112
111
|
// Output segments waiting to be pulled.
|
|
@@ -116,7 +115,7 @@ class VadEngine final {
|
|
|
116
115
|
// Processor thread.
|
|
117
116
|
std::thread processorThread_;
|
|
118
117
|
|
|
119
|
-
// Stream time tracking.
|
|
118
|
+
// Stream time tracking. Only touched by the processor thread.
|
|
120
119
|
float streamMs_ = 0.0f;
|
|
121
120
|
bool inSpeech_ = false;
|
|
122
121
|
float currentSpeechStartMs_ = 0.0f;
|
package/cpp/Version.hpp
ADDED
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
// ------------------------------------------------------------------------------
|
|
2
|
+
// Version.hpp
|
|
3
|
+
// Generated from package.json by scripts/generate-version.js. DO NOT EDIT.
|
|
4
|
+
// ------------------------------------------------------------------------------
|
|
5
|
+
#pragma once
|
|
6
|
+
|
|
7
|
+
#define NITRO_ONNX_SPEECH_VERSION "0.1.2"
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
// ------------------------------------------------------------------------------
|
|
2
2
|
// OnnxSpeechInitializer.mm
|
|
3
3
|
// ------------------------------------------------------------------------------
|
|
4
|
-
// Sets up resource and
|
|
4
|
+
// Sets up resource and document directories for the C++ layer on iOS.
|
|
5
5
|
// Runs at +load time so paths are available before any HybridObject is created.
|
|
6
6
|
// ------------------------------------------------------------------------------
|
|
7
7
|
#import <Foundation/Foundation.h>
|
|
@@ -26,12 +26,24 @@
|
|
|
26
26
|
margelo::nitro::onnx::speech::setResourceDir([[mainBundle resourcePath] UTF8String]);
|
|
27
27
|
}
|
|
28
28
|
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
NSString
|
|
32
|
-
|
|
33
|
-
|
|
29
|
+
// Application Support (not Documents): speaker embeddings are derived data
|
|
30
|
+
// and must not be backed up to iCloud.
|
|
31
|
+
NSArray<NSString*>* supportPaths = NSSearchPathForDirectoriesInDomains(
|
|
32
|
+
NSApplicationSupportDirectory, NSUserDomainMask, YES);
|
|
33
|
+
NSString* documentDir = supportPaths.firstObject;
|
|
34
|
+
if (documentDir == nil) {
|
|
35
|
+
return;
|
|
34
36
|
}
|
|
37
|
+
|
|
38
|
+
NSError* error = nil;
|
|
39
|
+
[[NSFileManager defaultManager] createDirectoryAtPath:documentDir
|
|
40
|
+
withIntermediateDirectories:YES
|
|
41
|
+
attributes:nil
|
|
42
|
+
error:&error];
|
|
43
|
+
NSURL* dirURL = [NSURL fileURLWithPath:documentDir isDirectory:YES];
|
|
44
|
+
[dirURL setResourceValue:@YES forKey:NSURLIsExcludedFromBackupKey error:nil];
|
|
45
|
+
|
|
46
|
+
margelo::nitro::onnx::speech::setDocumentDir([documentDir UTF8String]);
|
|
35
47
|
}
|
|
36
48
|
|
|
37
49
|
@end
|