react-native-nitro-onnx 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/NOTICE +241 -0
- package/NitroOnnxSpeech.podspec +51 -0
- package/Package.swift +52 -0
- package/README.md +398 -0
- package/android/CMakeLists.txt +117 -0
- package/android/build.gradle +65 -0
- package/android/src/main/AndroidManifest.xml +6 -0
- package/android/src/main/assets/silero_vad.onnx +0 -0
- package/android/src/main/cpp/cpp-adapter.cpp +24 -0
- package/android/src/main/java/com/margelo/nitro/onnx/speech/OnnxSpeechPackage.kt +67 -0
- package/assets/silero_vad.onnx +0 -0
- package/cpp/AndroidPthreadCompat.cpp +16 -0
- package/cpp/AsrEngine.cpp +277 -0
- package/cpp/AsrEngine.hpp +110 -0
- package/cpp/AudioFileReader.cpp +177 -0
- package/cpp/AudioFileReader.hpp +26 -0
- package/cpp/AudioUtils.cpp +28 -0
- package/cpp/AudioUtils.hpp +38 -0
- package/cpp/ModelSingleton.hpp +51 -0
- package/cpp/NitroOnnxSpeech.cpp +68 -0
- package/cpp/NitroOnnxSpeech.hpp +42 -0
- package/cpp/OfflineAsr.cpp +81 -0
- package/cpp/OfflineAsr.hpp +33 -0
- package/cpp/ResourceDir.cpp +29 -0
- package/cpp/ResourceDir.hpp +18 -0
- package/cpp/SpeakerEngine.cpp +170 -0
- package/cpp/SpeakerEngine.hpp +70 -0
- package/cpp/SpeakerManager.cpp +93 -0
- package/cpp/SpeakerManager.hpp +43 -0
- package/cpp/StreamingAsr.cpp +123 -0
- package/cpp/StreamingAsr.hpp +55 -0
- package/cpp/ThreadPool.cpp +41 -0
- package/cpp/ThreadPool.hpp +62 -0
- package/cpp/Tts.cpp +132 -0
- package/cpp/Tts.hpp +38 -0
- package/cpp/TtsEngine.cpp +222 -0
- package/cpp/TtsEngine.hpp +77 -0
- package/cpp/Vad.cpp +126 -0
- package/cpp/Vad.hpp +62 -0
- package/cpp/VadEngine.cpp +217 -0
- package/cpp/VadEngine.hpp +126 -0
- package/ios/OnnxSpeechInitializer.mm +37 -0
- package/ios/PrivacyInfo.xcprivacy +14 -0
- package/lib/index.d.ts +11 -0
- package/lib/index.d.ts.map +1 -0
- package/lib/index.js +19 -0
- package/lib/index.js.map +1 -0
- package/lib/specs/OnnxSpeech.nitro.d.ts +264 -0
- package/lib/specs/OnnxSpeech.nitro.d.ts.map +1 -0
- package/lib/specs/OnnxSpeech.nitro.js +6 -0
- package/lib/specs/OnnxSpeech.nitro.js.map +1 -0
- package/nitro.json +19 -0
- package/nitrogen/generated/.gitattributes +1 -0
- package/nitrogen/generated/android/NitroOnnxSpeech+autolinking.cmake +86 -0
- package/nitrogen/generated/android/NitroOnnxSpeech+autolinking.gradle +27 -0
- package/nitrogen/generated/android/NitroOnnxSpeechOnLoad.cpp +49 -0
- package/nitrogen/generated/android/NitroOnnxSpeechOnLoad.hpp +34 -0
- package/nitrogen/generated/android/kotlin/com/margelo/nitro/onnx/speech/NitroOnnxSpeechOnLoad.kt +35 -0
- package/nitrogen/generated/ios/NitroOnnxSpeech+autolinking.rb +62 -0
- package/nitrogen/generated/ios/NitroOnnxSpeech-Swift-Cxx-Bridge.cpp +17 -0
- package/nitrogen/generated/ios/NitroOnnxSpeech-Swift-Cxx-Bridge.hpp +27 -0
- package/nitrogen/generated/ios/NitroOnnxSpeech-Swift-Cxx-Umbrella.hpp +38 -0
- package/nitrogen/generated/ios/NitroOnnxSpeechAutolinking.mm +35 -0
- package/nitrogen/generated/ios/NitroOnnxSpeechAutolinking.swift +16 -0
- package/nitrogen/generated/shared/c++/AsrModelConfig.hpp +142 -0
- package/nitrogen/generated/shared/c++/AsrModelType.hpp +112 -0
- package/nitrogen/generated/shared/c++/AsrResult.hpp +105 -0
- package/nitrogen/generated/shared/c++/HybridOfflineAsrSpec.cpp +25 -0
- package/nitrogen/generated/shared/c++/HybridOfflineAsrSpec.hpp +73 -0
- package/nitrogen/generated/shared/c++/HybridOnnxSpeechSpec.cpp +27 -0
- package/nitrogen/generated/shared/c++/HybridOnnxSpeechSpec.hpp +82 -0
- package/nitrogen/generated/shared/c++/HybridSpeakerManagerSpec.cpp +28 -0
- package/nitrogen/generated/shared/c++/HybridSpeakerManagerSpec.hpp +77 -0
- package/nitrogen/generated/shared/c++/HybridStreamingAsrSpec.cpp +32 -0
- package/nitrogen/generated/shared/c++/HybridStreamingAsrSpec.hpp +81 -0
- package/nitrogen/generated/shared/c++/HybridTtsSpec.cpp +26 -0
- package/nitrogen/generated/shared/c++/HybridTtsSpec.hpp +74 -0
- package/nitrogen/generated/shared/c++/HybridVadSpec.cpp +31 -0
- package/nitrogen/generated/shared/c++/HybridVadSpec.hpp +81 -0
- package/nitrogen/generated/shared/c++/RegisteredSpeaker.hpp +91 -0
- package/nitrogen/generated/shared/c++/SpeakerEmbeddingConfig.hpp +91 -0
- package/nitrogen/generated/shared/c++/TtsModelConfig.hpp +178 -0
- package/nitrogen/generated/shared/c++/TtsModelType.hpp +88 -0
- package/nitrogen/generated/shared/c++/TtsResult.hpp +91 -0
- package/nitrogen/generated/shared/c++/VadConfig.hpp +100 -0
- package/nitrogen/generated/shared/c++/VadSegment.hpp +91 -0
- package/package.json +62 -0
- package/scripts/prepare-sherpa-onnx.js +236 -0
- package/scripts/test-cpp.js +24 -0
- package/src/index.ts +45 -0
- package/src/specs/OnnxSpeech.nitro.ts +326 -0
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
// ------------------------------------------------------------------------------
|
|
2
|
+
// ModelSingleton.hpp
|
|
3
|
+
// Heavy AI models are expensive to initialize. This template keeps at most one
|
|
4
|
+
// instance of a model per configuration key so repeated create/load calls reuse
|
|
5
|
+
// the same native object.
|
|
6
|
+
// ------------------------------------------------------------------------------
|
|
7
|
+
#pragma once
|
|
8
|
+
|
|
9
|
+
#include <functional>
|
|
10
|
+
#include <memory>
|
|
11
|
+
#include <mutex>
|
|
12
|
+
#include <string>
|
|
13
|
+
#include <unordered_map>
|
|
14
|
+
|
|
15
|
+
namespace margelo::nitro::onnx::speech {
|
|
16
|
+
|
|
17
|
+
/**
|
|
18
|
+
* Thread-safe cache for heavy model instances.
|
|
19
|
+
* @tparam T The native model type (e.g. SherpaOnnxOfflineRecognizer).
|
|
20
|
+
*/
|
|
21
|
+
template <typename T>
|
|
22
|
+
class ModelSingleton final {
|
|
23
|
+
public:
|
|
24
|
+
using Factory = std::function<std::shared_ptr<T>(const std::string& key)>;
|
|
25
|
+
|
|
26
|
+
/** Return a cached instance or create one using factory. */
|
|
27
|
+
std::shared_ptr<T> getOrCreate(const std::string& key, const Factory& factory) {
|
|
28
|
+
std::lock_guard<std::mutex> lock(mutex_);
|
|
29
|
+
auto it = instances_.find(key);
|
|
30
|
+
if (it != instances_.end()) {
|
|
31
|
+
if (auto alive = it->second.lock()) {
|
|
32
|
+
return alive;
|
|
33
|
+
}
|
|
34
|
+
}
|
|
35
|
+
auto created = factory(key);
|
|
36
|
+
instances_[key] = created;
|
|
37
|
+
return created;
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
/** Invalidate all cached instances. */
|
|
41
|
+
void clear() {
|
|
42
|
+
std::lock_guard<std::mutex> lock(mutex_);
|
|
43
|
+
instances_.clear();
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
private:
|
|
47
|
+
std::mutex mutex_;
|
|
48
|
+
std::unordered_map<std::string, std::weak_ptr<T>> instances_;
|
|
49
|
+
};
|
|
50
|
+
|
|
51
|
+
} // namespace margelo::nitro::onnx::speech
|
|
@@ -0,0 +1,68 @@
|
|
|
1
|
+
// ------------------------------------------------------------------------------
|
|
2
|
+
// NitroOnnxSpeech.cpp
|
|
3
|
+
// ------------------------------------------------------------------------------
|
|
4
|
+
#include "NitroOnnxSpeech.hpp"
|
|
5
|
+
|
|
6
|
+
#include "OfflineAsr.hpp"
|
|
7
|
+
#include "ResourceDir.hpp"
|
|
8
|
+
#include "SpeakerManager.hpp"
|
|
9
|
+
#include "StreamingAsr.hpp"
|
|
10
|
+
#include "Tts.hpp"
|
|
11
|
+
#include "Vad.hpp"
|
|
12
|
+
|
|
13
|
+
#include <cstdio>
|
|
14
|
+
#include <cstring>
|
|
15
|
+
|
|
16
|
+
namespace margelo::nitro::onnx::speech {
|
|
17
|
+
|
|
18
|
+
NitroOnnxSpeech::NitroOnnxSpeech()
|
|
19
|
+
: HybridObject(TAG),
|
|
20
|
+
threadPool_(std::make_shared<ThreadPool>()),
|
|
21
|
+
cacheDir_(getCacheDir()) {}
|
|
22
|
+
|
|
23
|
+
NitroOnnxSpeech::~NitroOnnxSpeech() = default;
|
|
24
|
+
|
|
25
|
+
std::shared_ptr<HybridVadSpec> NitroOnnxSpeech::createVad() {
|
|
26
|
+
return std::make_shared<Vad>(threadPool_);
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
std::shared_ptr<HybridOfflineAsrSpec> NitroOnnxSpeech::createOfflineAsr() {
|
|
30
|
+
return std::make_shared<OfflineAsr>(threadPool_);
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
std::shared_ptr<HybridStreamingAsrSpec> NitroOnnxSpeech::createStreamingAsr() {
|
|
34
|
+
return std::make_shared<StreamingAsr>(threadPool_);
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
std::shared_ptr<HybridTtsSpec> NitroOnnxSpeech::createTts() {
|
|
38
|
+
return std::make_shared<Tts>(threadPool_);
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
std::shared_ptr<HybridSpeakerManagerSpec> NitroOnnxSpeech::createSpeakerManager() {
|
|
42
|
+
return std::make_shared<SpeakerManager>(threadPool_, cacheDir_);
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
std::string NitroOnnxSpeech::getVersion() {
|
|
46
|
+
return "0.1.0";
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
bool NitroOnnxSpeech::isQualcommCpu() {
|
|
50
|
+
#ifdef __ANDROID__
|
|
51
|
+
FILE* f = fopen("/proc/cpuinfo", "r");
|
|
52
|
+
if (!f) return false;
|
|
53
|
+
char line[256];
|
|
54
|
+
bool found = false;
|
|
55
|
+
while (fgets(line, sizeof(line), f)) {
|
|
56
|
+
if (strstr(line, "Qualcomm") != nullptr) {
|
|
57
|
+
found = true;
|
|
58
|
+
break;
|
|
59
|
+
}
|
|
60
|
+
}
|
|
61
|
+
fclose(f);
|
|
62
|
+
return found;
|
|
63
|
+
#else
|
|
64
|
+
return false;
|
|
65
|
+
#endif
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
} // namespace margelo::nitro::onnx::speech
|
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
// ------------------------------------------------------------------------------
|
|
2
|
+
// OnnxSpeech.hpp
|
|
3
|
+
// Factory for all Nitro hybrid objects in the module.
|
|
4
|
+
// ------------------------------------------------------------------------------
|
|
5
|
+
#pragma once
|
|
6
|
+
|
|
7
|
+
#include "ThreadPool.hpp"
|
|
8
|
+
|
|
9
|
+
#include <NitroModules/HybridObject.hpp>
|
|
10
|
+
#include <HybridOnnxSpeechSpec.hpp>
|
|
11
|
+
#include <HybridOfflineAsrSpec.hpp>
|
|
12
|
+
#include <HybridSpeakerManagerSpec.hpp>
|
|
13
|
+
#include <HybridStreamingAsrSpec.hpp>
|
|
14
|
+
#include <HybridTtsSpec.hpp>
|
|
15
|
+
#include <HybridVadSpec.hpp>
|
|
16
|
+
|
|
17
|
+
#include <memory>
|
|
18
|
+
#include <string>
|
|
19
|
+
|
|
20
|
+
namespace margelo::nitro::onnx::speech {
|
|
21
|
+
|
|
22
|
+
class NitroOnnxSpeech : public HybridOnnxSpeechSpec {
|
|
23
|
+
public:
|
|
24
|
+
static constexpr auto TAG = "NitroOnnxSpeech";
|
|
25
|
+
|
|
26
|
+
NitroOnnxSpeech();
|
|
27
|
+
~NitroOnnxSpeech() override;
|
|
28
|
+
|
|
29
|
+
std::shared_ptr<HybridVadSpec> createVad() override;
|
|
30
|
+
std::shared_ptr<HybridOfflineAsrSpec> createOfflineAsr() override;
|
|
31
|
+
std::shared_ptr<HybridStreamingAsrSpec> createStreamingAsr() override;
|
|
32
|
+
std::shared_ptr<HybridTtsSpec> createTts() override;
|
|
33
|
+
std::shared_ptr<HybridSpeakerManagerSpec> createSpeakerManager() override;
|
|
34
|
+
std::string getVersion() override;
|
|
35
|
+
bool isQualcommCpu() override;
|
|
36
|
+
|
|
37
|
+
private:
|
|
38
|
+
std::shared_ptr<ThreadPool> threadPool_;
|
|
39
|
+
std::string cacheDir_;
|
|
40
|
+
};
|
|
41
|
+
|
|
42
|
+
} // namespace margelo::nitro::onnx::speech
|
|
@@ -0,0 +1,81 @@
|
|
|
1
|
+
// ------------------------------------------------------------------------------
|
|
2
|
+
// OfflineAsr.cpp
|
|
3
|
+
// ------------------------------------------------------------------------------
|
|
4
|
+
#include "OfflineAsr.hpp"
|
|
5
|
+
|
|
6
|
+
#include "AudioUtils.hpp"
|
|
7
|
+
|
|
8
|
+
#include <optional>
|
|
9
|
+
|
|
10
|
+
namespace margelo::nitro::onnx::speech {
|
|
11
|
+
|
|
12
|
+
namespace {
|
|
13
|
+
|
|
14
|
+
AsrResult toAsrResult(const AsrEngineResult& native) {
|
|
15
|
+
std::vector<double> timestamps;
|
|
16
|
+
timestamps.reserve(native.timestamps.size());
|
|
17
|
+
for (float t : native.timestamps) {
|
|
18
|
+
timestamps.push_back(static_cast<double>(t));
|
|
19
|
+
}
|
|
20
|
+
return AsrResult(
|
|
21
|
+
native.text,
|
|
22
|
+
std::optional<double>(static_cast<double>(native.score)),
|
|
23
|
+
std::optional<double>(static_cast<double>(native.startMs)),
|
|
24
|
+
std::optional<double>(static_cast<double>(native.endMs)),
|
|
25
|
+
timestamps.empty() ? std::nullopt : std::optional<std::vector<double>>(timestamps),
|
|
26
|
+
native.json.empty() ? std::nullopt : std::optional<std::string>(native.json));
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
} // namespace
|
|
30
|
+
|
|
31
|
+
OfflineAsr::OfflineAsr(std::shared_ptr<ThreadPool> threadPool)
|
|
32
|
+
: HybridObject(TAG), engine_(std::move(threadPool)) {}
|
|
33
|
+
|
|
34
|
+
OfflineAsr::~OfflineAsr() {
|
|
35
|
+
engine_.unload();
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
std::shared_ptr<Promise<void>> OfflineAsr::load(const AsrModelConfig& config) {
|
|
39
|
+
return Promise<void>::async([this, config]() {
|
|
40
|
+
AsrEngineConfig native;
|
|
41
|
+
native.type = config.type;
|
|
42
|
+
native.modelDir = config.modelDir;
|
|
43
|
+
native.tokensPath = config.tokensPath;
|
|
44
|
+
native.whisperEncoder = config.whisperEncoder.value_or("");
|
|
45
|
+
native.whisperDecoder = config.whisperDecoder.value_or("");
|
|
46
|
+
native.encoder = config.encoder.value_or("");
|
|
47
|
+
native.decoder = config.decoder.value_or("");
|
|
48
|
+
native.joiner = config.joiner.value_or("");
|
|
49
|
+
native.model = config.model.value_or("");
|
|
50
|
+
native.config = config.config.value_or("");
|
|
51
|
+
native.numThreads = static_cast<int32_t>(config.numThreads.value_or(2));
|
|
52
|
+
native.decodingMethod = config.decodingMethod.value_or("greedy_search");
|
|
53
|
+
native.maxActivePaths = static_cast<int32_t>(config.maxActivePaths.value_or(4));
|
|
54
|
+
native.language = config.language.value_or("en");
|
|
55
|
+
native.useItn = config.useItn.value_or(true);
|
|
56
|
+
engine_.load(native);
|
|
57
|
+
});
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
bool OfflineAsr::isLoaded() {
|
|
61
|
+
return engine_.isLoaded();
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
std::shared_ptr<Promise<AsrResult>> OfflineAsr::recognize(
|
|
65
|
+
const std::shared_ptr<ArrayBuffer>& samples) {
|
|
66
|
+
return Promise<AsrResult>::async([this, samples]() {
|
|
67
|
+
auto floatSamples = bytesToFloatVector(samples->data(), samples->size());
|
|
68
|
+
return toAsrResult(engine_.recognize(floatSamples));
|
|
69
|
+
});
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
std::shared_ptr<Promise<AsrResult>> OfflineAsr::recognizeFile(const std::string& path) {
|
|
73
|
+
return Promise<AsrResult>::async(
|
|
74
|
+
[this, path]() { return toAsrResult(engine_.recognizeFile(path)); });
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
std::shared_ptr<Promise<void>> OfflineAsr::unload() {
|
|
78
|
+
return Promise<void>::async([this]() { engine_.unload(); });
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
} // namespace margelo::nitro::onnx::speech
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
// ------------------------------------------------------------------------------
|
|
2
|
+
// OfflineAsr.hpp
|
|
3
|
+
// ------------------------------------------------------------------------------
|
|
4
|
+
#pragma once
|
|
5
|
+
|
|
6
|
+
#include "AsrEngine.hpp"
|
|
7
|
+
|
|
8
|
+
#include <NitroModules/Promise.hpp>
|
|
9
|
+
#include <NitroModules/HybridObject.hpp>
|
|
10
|
+
#include <HybridOfflineAsrSpec.hpp>
|
|
11
|
+
|
|
12
|
+
#include <memory>
|
|
13
|
+
|
|
14
|
+
namespace margelo::nitro::onnx::speech {
|
|
15
|
+
|
|
16
|
+
class OfflineAsr : public HybridOfflineAsrSpec {
|
|
17
|
+
public:
|
|
18
|
+
static constexpr auto TAG = "OfflineAsr";
|
|
19
|
+
|
|
20
|
+
explicit OfflineAsr(std::shared_ptr<ThreadPool> threadPool);
|
|
21
|
+
~OfflineAsr() override;
|
|
22
|
+
|
|
23
|
+
std::shared_ptr<Promise<void>> load(const AsrModelConfig& config) override;
|
|
24
|
+
bool isLoaded() override;
|
|
25
|
+
std::shared_ptr<Promise<AsrResult>> recognize(const std::shared_ptr<ArrayBuffer>& samples) override;
|
|
26
|
+
std::shared_ptr<Promise<AsrResult>> recognizeFile(const std::string& path) override;
|
|
27
|
+
std::shared_ptr<Promise<void>> unload() override;
|
|
28
|
+
|
|
29
|
+
private:
|
|
30
|
+
OfflineAsrEngine engine_;
|
|
31
|
+
};
|
|
32
|
+
|
|
33
|
+
} // namespace margelo::nitro::onnx::speech
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
// ------------------------------------------------------------------------------
|
|
2
|
+
// ResourceDir.cpp
|
|
3
|
+
// ------------------------------------------------------------------------------
|
|
4
|
+
#include "ResourceDir.hpp"
|
|
5
|
+
|
|
6
|
+
namespace margelo::nitro::onnx::speech {
|
|
7
|
+
|
|
8
|
+
namespace {
|
|
9
|
+
std::string g_resourceDir;
|
|
10
|
+
std::string g_cacheDir;
|
|
11
|
+
}
|
|
12
|
+
|
|
13
|
+
const std::string& getResourceDir() {
|
|
14
|
+
return g_resourceDir;
|
|
15
|
+
}
|
|
16
|
+
|
|
17
|
+
void setResourceDir(const std::string& dir) {
|
|
18
|
+
g_resourceDir = dir;
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
const std::string& getCacheDir() {
|
|
22
|
+
return g_cacheDir;
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
void setCacheDir(const std::string& dir) {
|
|
26
|
+
g_cacheDir = dir;
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
} // namespace margelo::nitro::onnx::speech
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
// ------------------------------------------------------------------------------
|
|
2
|
+
// ResourceDir.hpp
|
|
3
|
+
// Process-wide directories for bundled model files and persistent cache.
|
|
4
|
+
// Set once from the platform layer at startup (JNI on Android, ObjC++ on iOS).
|
|
5
|
+
// ------------------------------------------------------------------------------
|
|
6
|
+
#pragma once
|
|
7
|
+
|
|
8
|
+
#include <string>
|
|
9
|
+
|
|
10
|
+
namespace margelo::nitro::onnx::speech {
|
|
11
|
+
|
|
12
|
+
const std::string& getResourceDir();
|
|
13
|
+
void setResourceDir(const std::string& dir);
|
|
14
|
+
|
|
15
|
+
const std::string& getCacheDir();
|
|
16
|
+
void setCacheDir(const std::string& dir);
|
|
17
|
+
|
|
18
|
+
} // namespace margelo::nitro::onnx::speech
|
|
@@ -0,0 +1,170 @@
|
|
|
1
|
+
// ------------------------------------------------------------------------------
|
|
2
|
+
// SpeakerEngine.cpp
|
|
3
|
+
// ------------------------------------------------------------------------------
|
|
4
|
+
#include "SpeakerEngine.hpp"
|
|
5
|
+
|
|
6
|
+
#include "AudioFileReader.hpp"
|
|
7
|
+
#include "sherpa-onnx/c-api/c-api.h"
|
|
8
|
+
|
|
9
|
+
#include <cstring>
|
|
10
|
+
#include <filesystem>
|
|
11
|
+
#include <fstream>
|
|
12
|
+
#include <sstream>
|
|
13
|
+
#include <stdexcept>
|
|
14
|
+
|
|
15
|
+
namespace margelo::nitro::onnx::speech {
|
|
16
|
+
|
|
17
|
+
namespace {
|
|
18
|
+
|
|
19
|
+
std::string joinPath(const std::string& dir, const std::string& file) {
|
|
20
|
+
if (file.empty()) {
|
|
21
|
+
return dir;
|
|
22
|
+
}
|
|
23
|
+
if (file[0] == '/') {
|
|
24
|
+
return file;
|
|
25
|
+
}
|
|
26
|
+
if (dir.empty()) {
|
|
27
|
+
return file;
|
|
28
|
+
}
|
|
29
|
+
if (dir.back() == '/') {
|
|
30
|
+
return dir + file;
|
|
31
|
+
}
|
|
32
|
+
return dir + "/" + file;
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
ModelSingleton<const SherpaOnnxSpeakerEmbeddingExtractor> gExtractorCache;
|
|
36
|
+
|
|
37
|
+
} // namespace
|
|
38
|
+
|
|
39
|
+
SpeakerEngine::SpeakerEngine(std::shared_ptr<ThreadPool> threadPool, std::string cacheDir)
|
|
40
|
+
: threadPool_(std::move(threadPool)), cacheDir_(std::move(cacheDir)) {}
|
|
41
|
+
|
|
42
|
+
SpeakerEngine::~SpeakerEngine() {
|
|
43
|
+
unload();
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
void SpeakerEngine::load(const SpeakerEngineConfig& config) {
|
|
47
|
+
unload();
|
|
48
|
+
config_ = config;
|
|
49
|
+
|
|
50
|
+
const std::string key = config_.modelDir + "|speaker";
|
|
51
|
+
auto cached = gExtractorCache.getOrCreate(key, [this](const std::string&) {
|
|
52
|
+
SherpaOnnxSpeakerEmbeddingExtractorConfig c;
|
|
53
|
+
std::memset(&c, 0, sizeof(c));
|
|
54
|
+
|
|
55
|
+
std::string model = joinPath(config_.modelDir, config_.model);
|
|
56
|
+
c.model = model.c_str();
|
|
57
|
+
c.num_threads = config_.numThreads;
|
|
58
|
+
c.debug = 0;
|
|
59
|
+
|
|
60
|
+
const SherpaOnnxSpeakerEmbeddingExtractor* ex = SherpaOnnxCreateSpeakerEmbeddingExtractor(&c);
|
|
61
|
+
if (ex == nullptr) {
|
|
62
|
+
throw std::runtime_error("Failed to create speaker embedding extractor");
|
|
63
|
+
}
|
|
64
|
+
return std::shared_ptr<const SherpaOnnxSpeakerEmbeddingExtractor>(
|
|
65
|
+
ex, SherpaOnnxDestroySpeakerEmbeddingExtractor);
|
|
66
|
+
});
|
|
67
|
+
|
|
68
|
+
extractor_ = cached;
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
bool SpeakerEngine::isLoaded() const {
|
|
72
|
+
return extractor_ != nullptr;
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
std::vector<float> SpeakerEngine::computeEmbedding(const std::vector<float>& samples) {
|
|
76
|
+
if (extractor_ == nullptr) {
|
|
77
|
+
throw std::runtime_error("Speaker extractor not loaded");
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
const SherpaOnnxOnlineStream* stream = SherpaOnnxSpeakerEmbeddingExtractorCreateStream(extractor_.get());
|
|
81
|
+
SherpaOnnxOnlineStreamAcceptWaveform(stream, 16000, samples.data(), static_cast<int32_t>(samples.size()));
|
|
82
|
+
SherpaOnnxOnlineStreamInputFinished(stream);
|
|
83
|
+
|
|
84
|
+
const float* embedding = SherpaOnnxSpeakerEmbeddingExtractorComputeEmbedding(extractor_.get(), stream);
|
|
85
|
+
if (embedding == nullptr) {
|
|
86
|
+
SherpaOnnxDestroyOnlineStream(stream);
|
|
87
|
+
throw std::runtime_error("Failed to compute speaker embedding");
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
int32_t dim = SherpaOnnxSpeakerEmbeddingExtractorDim(extractor_.get());
|
|
91
|
+
std::vector<float> result(embedding, embedding + dim);
|
|
92
|
+
SherpaOnnxSpeakerEmbeddingExtractorDestroyEmbedding(embedding);
|
|
93
|
+
SherpaOnnxDestroyOnlineStream(stream);
|
|
94
|
+
return result;
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
SpeakerEngineRegisteredSpeaker SpeakerEngine::registerSpeaker(
|
|
98
|
+
const std::string& id,
|
|
99
|
+
const std::string& name,
|
|
100
|
+
const std::vector<float>& embedding) {
|
|
101
|
+
const std::string speakersDir = joinPath(cacheDir_, "speakers");
|
|
102
|
+
std::filesystem::create_directories(speakersDir);
|
|
103
|
+
const std::string path = joinPath(speakersDir, id + ".bin");
|
|
104
|
+
std::ofstream file(path, std::ios::binary);
|
|
105
|
+
if (!file) {
|
|
106
|
+
throw std::runtime_error("Failed to write speaker embedding: " + path);
|
|
107
|
+
}
|
|
108
|
+
const auto nameLen = static_cast<uint32_t>(name.size());
|
|
109
|
+
file.write(reinterpret_cast<const char*>(&nameLen), sizeof(nameLen));
|
|
110
|
+
file.write(name.data(), name.size());
|
|
111
|
+
file.write(reinterpret_cast<const char*>(embedding.data()), embedding.size() * sizeof(float));
|
|
112
|
+
return {id, name, path};
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
SpeakerEngineRegisteredSpeaker SpeakerEngine::registerSpeakerFromFile(
|
|
116
|
+
const std::string& id,
|
|
117
|
+
const std::string& name,
|
|
118
|
+
const std::string& path) {
|
|
119
|
+
if (extractor_ == nullptr) {
|
|
120
|
+
throw std::runtime_error("Speaker extractor not loaded");
|
|
121
|
+
}
|
|
122
|
+
std::vector<float> samples;
|
|
123
|
+
if (path.size() >= 4 && path.substr(path.size() - 4) == ".wav") {
|
|
124
|
+
samples = readWavFile(path);
|
|
125
|
+
} else {
|
|
126
|
+
samples = readRawPcmFile(path);
|
|
127
|
+
}
|
|
128
|
+
auto embedding = computeEmbedding(samples);
|
|
129
|
+
return registerSpeaker(id, name, embedding);
|
|
130
|
+
}
|
|
131
|
+
|
|
132
|
+
std::vector<SpeakerEngineRegisteredSpeaker> SpeakerEngine::listSpeakers() {
|
|
133
|
+
const std::string speakersDir = joinPath(cacheDir_, "speakers");
|
|
134
|
+
std::vector<SpeakerEngineRegisteredSpeaker> result;
|
|
135
|
+
if (!std::filesystem::exists(speakersDir)) {
|
|
136
|
+
return result;
|
|
137
|
+
}
|
|
138
|
+
for (const auto& entry : std::filesystem::directory_iterator(speakersDir)) {
|
|
139
|
+
if (!entry.is_regular_file()) continue;
|
|
140
|
+
const auto& filePath = entry.path();
|
|
141
|
+
if (filePath.extension() != ".bin") continue;
|
|
142
|
+
const std::string id = filePath.stem().string();
|
|
143
|
+
std::ifstream file(filePath, std::ios::binary);
|
|
144
|
+
if (!file) continue;
|
|
145
|
+
uint32_t nameLen = 0;
|
|
146
|
+
file.read(reinterpret_cast<char*>(&nameLen), sizeof(nameLen));
|
|
147
|
+
std::string name;
|
|
148
|
+
if (nameLen > 0 && nameLen < 1024) {
|
|
149
|
+
name.resize(nameLen);
|
|
150
|
+
file.read(name.data(), nameLen);
|
|
151
|
+
}
|
|
152
|
+
result.push_back({id, name, filePath.string()});
|
|
153
|
+
}
|
|
154
|
+
return result;
|
|
155
|
+
}
|
|
156
|
+
|
|
157
|
+
void SpeakerEngine::removeSpeaker(const std::string& id) {
|
|
158
|
+
const std::string path = joinPath(cacheDir_, "speakers/" + id + ".bin");
|
|
159
|
+
std::error_code ec;
|
|
160
|
+
std::filesystem::remove(path, ec);
|
|
161
|
+
if (ec) {
|
|
162
|
+
throw std::runtime_error("Failed to remove speaker: " + id + " (" + ec.message() + ")");
|
|
163
|
+
}
|
|
164
|
+
}
|
|
165
|
+
|
|
166
|
+
void SpeakerEngine::unload() {
|
|
167
|
+
extractor_.reset();
|
|
168
|
+
}
|
|
169
|
+
|
|
170
|
+
} // namespace margelo::nitro::onnx::speech
|
|
@@ -0,0 +1,70 @@
|
|
|
1
|
+
// ------------------------------------------------------------------------------
|
|
2
|
+
// SpeakerEngine.hpp
|
|
3
|
+
// Speaker embedding extraction and voice-cloning registration.
|
|
4
|
+
// Supports two cloning paths:
|
|
5
|
+
// 1. Reference-audio embedding passed directly to a TTS model.
|
|
6
|
+
// 2. Explicit speaker registration via speaker embedding model, later
|
|
7
|
+
// referenced by speaker ID during TTS synthesis.
|
|
8
|
+
// ------------------------------------------------------------------------------
|
|
9
|
+
#pragma once
|
|
10
|
+
|
|
11
|
+
#include "AudioUtils.hpp"
|
|
12
|
+
#include "ModelSingleton.hpp"
|
|
13
|
+
#include "ThreadPool.hpp"
|
|
14
|
+
|
|
15
|
+
#include <memory>
|
|
16
|
+
#include <string>
|
|
17
|
+
#include <vector>
|
|
18
|
+
|
|
19
|
+
struct SherpaOnnxSpeakerEmbeddingExtractor;
|
|
20
|
+
struct SherpaOnnxSpeakerEmbeddingExtractorConfig;
|
|
21
|
+
|
|
22
|
+
namespace margelo::nitro::onnx::speech {
|
|
23
|
+
|
|
24
|
+
/** Native configuration for the speaker embedding extractor. */
|
|
25
|
+
struct SpeakerEngineConfig {
|
|
26
|
+
std::string modelDir;
|
|
27
|
+
std::string model;
|
|
28
|
+
int32_t numThreads = 4;
|
|
29
|
+
};
|
|
30
|
+
|
|
31
|
+
/** Native registered speaker metadata before conversion to the generated type. */
|
|
32
|
+
struct SpeakerEngineRegisteredSpeaker {
|
|
33
|
+
std::string id;
|
|
34
|
+
std::string name;
|
|
35
|
+
std::string embeddingPath;
|
|
36
|
+
};
|
|
37
|
+
|
|
38
|
+
/** Speaker manager engine. */
|
|
39
|
+
class SpeakerEngine final {
|
|
40
|
+
public:
|
|
41
|
+
explicit SpeakerEngine(std::shared_ptr<ThreadPool> threadPool, std::string cacheDir);
|
|
42
|
+
~SpeakerEngine();
|
|
43
|
+
|
|
44
|
+
SpeakerEngine(const SpeakerEngine&) = delete;
|
|
45
|
+
SpeakerEngine& operator=(const SpeakerEngine&) = delete;
|
|
46
|
+
|
|
47
|
+
void load(const SpeakerEngineConfig& config);
|
|
48
|
+
bool isLoaded() const;
|
|
49
|
+
|
|
50
|
+
/** Compute an embedding from 16 kHz mono f32 PCM audio. */
|
|
51
|
+
std::vector<float> computeEmbedding(const std::vector<float>& samples);
|
|
52
|
+
|
|
53
|
+
/** Register a speaker embedding for later TTS use. */
|
|
54
|
+
SpeakerEngineRegisteredSpeaker registerSpeaker(const std::string& id, const std::string& name, const std::vector<float>& embedding);
|
|
55
|
+
|
|
56
|
+
/** Register a speaker from a reference audio file. */
|
|
57
|
+
SpeakerEngineRegisteredSpeaker registerSpeakerFromFile(const std::string& id, const std::string& name, const std::string& path);
|
|
58
|
+
|
|
59
|
+
std::vector<SpeakerEngineRegisteredSpeaker> listSpeakers();
|
|
60
|
+
void removeSpeaker(const std::string& id);
|
|
61
|
+
void unload();
|
|
62
|
+
|
|
63
|
+
private:
|
|
64
|
+
std::shared_ptr<ThreadPool> threadPool_;
|
|
65
|
+
std::string cacheDir_;
|
|
66
|
+
SpeakerEngineConfig config_;
|
|
67
|
+
std::shared_ptr<const SherpaOnnxSpeakerEmbeddingExtractor> extractor_;
|
|
68
|
+
};
|
|
69
|
+
|
|
70
|
+
} // namespace margelo::nitro::onnx::speech
|
|
@@ -0,0 +1,93 @@
|
|
|
1
|
+
// ------------------------------------------------------------------------------
|
|
2
|
+
// SpeakerManager.cpp
|
|
3
|
+
// ------------------------------------------------------------------------------
|
|
4
|
+
#include "SpeakerManager.hpp"
|
|
5
|
+
|
|
6
|
+
#include "AudioUtils.hpp"
|
|
7
|
+
|
|
8
|
+
#include <NitroModules/ArrayBuffer.hpp>
|
|
9
|
+
|
|
10
|
+
namespace margelo::nitro::onnx::speech {
|
|
11
|
+
|
|
12
|
+
namespace {
|
|
13
|
+
|
|
14
|
+
RegisteredSpeaker toRegisteredSpeaker(const SpeakerEngineRegisteredSpeaker& native) {
|
|
15
|
+
return RegisteredSpeaker(native.id, native.name, native.embeddingPath);
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
std::vector<RegisteredSpeaker> toRegisteredSpeakers(const std::vector<SpeakerEngineRegisteredSpeaker>& natives) {
|
|
19
|
+
std::vector<RegisteredSpeaker> result;
|
|
20
|
+
result.reserve(natives.size());
|
|
21
|
+
for (const auto& native : natives) {
|
|
22
|
+
result.push_back(toRegisteredSpeaker(native));
|
|
23
|
+
}
|
|
24
|
+
return result;
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
} // namespace
|
|
28
|
+
|
|
29
|
+
SpeakerManager::SpeakerManager(std::shared_ptr<ThreadPool> threadPool, std::string cacheDir)
|
|
30
|
+
: HybridObject(TAG), engine_(std::move(threadPool), std::move(cacheDir)) {}
|
|
31
|
+
|
|
32
|
+
SpeakerManager::~SpeakerManager() {
|
|
33
|
+
engine_.unload();
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
std::shared_ptr<Promise<void>> SpeakerManager::load(const SpeakerEmbeddingConfig& config) {
|
|
37
|
+
return Promise<void>::async([this, config]() {
|
|
38
|
+
SpeakerEngineConfig native;
|
|
39
|
+
native.modelDir = config.modelDir;
|
|
40
|
+
native.model = config.model;
|
|
41
|
+
native.numThreads = config.numThreads;
|
|
42
|
+
engine_.load(native);
|
|
43
|
+
});
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
bool SpeakerManager::isLoaded() {
|
|
47
|
+
return engine_.isLoaded();
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
std::shared_ptr<Promise<std::shared_ptr<ArrayBuffer>>> SpeakerManager::computeEmbedding(
|
|
51
|
+
const std::shared_ptr<ArrayBuffer>& samples) {
|
|
52
|
+
return Promise<std::shared_ptr<ArrayBuffer>>::async([this, samples]() {
|
|
53
|
+
auto floatSamples = bytesToFloatVector(samples->data(), samples->size());
|
|
54
|
+
auto embedding = engine_.computeEmbedding(floatSamples);
|
|
55
|
+
auto bytes = floatVectorToBytes(embedding);
|
|
56
|
+
return ArrayBuffer::move(std::move(bytes));
|
|
57
|
+
});
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
std::shared_ptr<Promise<RegisteredSpeaker>> SpeakerManager::registerSpeaker(
|
|
61
|
+
const std::string& id,
|
|
62
|
+
const std::string& name,
|
|
63
|
+
const std::shared_ptr<ArrayBuffer>& embedding) {
|
|
64
|
+
return Promise<RegisteredSpeaker>::async([this, id, name, embedding]() {
|
|
65
|
+
auto floatEmbedding = bytesToFloatVector(embedding->data(), embedding->size());
|
|
66
|
+
return toRegisteredSpeaker(engine_.registerSpeaker(id, name, floatEmbedding));
|
|
67
|
+
});
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
std::shared_ptr<Promise<RegisteredSpeaker>> SpeakerManager::registerSpeakerFromFile(
|
|
71
|
+
const std::string& id,
|
|
72
|
+
const std::string& name,
|
|
73
|
+
const std::string& path) {
|
|
74
|
+
return Promise<RegisteredSpeaker>::async([this, id, name, path]() {
|
|
75
|
+
return toRegisteredSpeaker(engine_.registerSpeakerFromFile(id, name, path));
|
|
76
|
+
});
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
std::shared_ptr<Promise<std::vector<RegisteredSpeaker>>> SpeakerManager::listSpeakers() {
|
|
80
|
+
return Promise<std::vector<RegisteredSpeaker>>::async([this]() {
|
|
81
|
+
return toRegisteredSpeakers(engine_.listSpeakers());
|
|
82
|
+
});
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
std::shared_ptr<Promise<void>> SpeakerManager::removeSpeaker(const std::string& id) {
|
|
86
|
+
return Promise<void>::async([this, id]() { engine_.removeSpeaker(id); });
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
std::shared_ptr<Promise<void>> SpeakerManager::unload() {
|
|
90
|
+
return Promise<void>::async([this]() { engine_.unload(); });
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
} // namespace margelo::nitro::onnx::speech
|