react-native-nitro-onnx 0.1.0 → 0.1.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/NitroOnnxSpeech.podspec +3 -1
- package/README.md +146 -41
- package/android/CMakeLists.txt +54 -1
- package/android/build.gradle +6 -0
- package/android/src/main/AndroidManifest.xml +0 -2
- package/android/src/main/cpp/cpp-adapter.cpp +2 -2
- package/android/src/main/java/com/margelo/nitro/onnx/speech/OnnxSpeechPackage.kt +26 -17
- package/cpp/AsrEngine.cpp +154 -59
- package/cpp/AsrEngine.hpp +19 -6
- package/cpp/AudioFileReader.cpp +4 -0
- package/cpp/ModelSingleton.hpp +14 -0
- package/cpp/NitroOnnxSpeech.cpp +37 -18
- package/cpp/NitroOnnxSpeech.hpp +1 -7
- package/cpp/OfflineAsr.cpp +17 -8
- package/cpp/OfflineAsr.hpp +1 -1
- package/cpp/ResourceDir.cpp +5 -5
- package/cpp/ResourceDir.hpp +5 -4
- package/cpp/SpeakerEngine.cpp +38 -28
- package/cpp/SpeakerEngine.hpp +14 -13
- package/cpp/SpeakerManager.cpp +16 -14
- package/cpp/SpeakerManager.hpp +2 -1
- package/cpp/SpeakerRecord.cpp +125 -0
- package/cpp/SpeakerRecord.hpp +42 -0
- package/cpp/StreamingAsr.cpp +20 -9
- package/cpp/StreamingAsr.hpp +1 -1
- package/cpp/Tts.cpp +44 -10
- package/cpp/Tts.hpp +2 -1
- package/cpp/TtsEngine.cpp +17 -9
- package/cpp/TtsEngine.hpp +24 -5
- package/cpp/Vad.cpp +12 -11
- package/cpp/Vad.hpp +1 -1
- package/cpp/VadEngine.cpp +27 -33
- package/cpp/VadEngine.hpp +9 -10
- package/cpp/Version.hpp +7 -0
- package/ios/OnnxSpeechInitializer.mm +18 -6
- package/lib/specs/OnnxSpeech.nitro.d.ts +43 -6
- package/lib/specs/OnnxSpeech.nitro.d.ts.map +1 -1
- package/nitrogen/generated/shared/c++/AsrModelConfig.hpp +10 -2
- package/nitrogen/generated/shared/c++/HybridOnnxSpeechSpec.cpp +1 -1
- package/nitrogen/generated/shared/c++/HybridOnnxSpeechSpec.hpp +1 -1
- package/nitrogen/generated/shared/c++/TtsModelConfig.hpp +10 -2
- package/nitrogen/generated/shared/c++/VadConfig.hpp +6 -2
- package/package.json +4 -3
- package/scripts/generate-version.js +22 -0
- package/src/specs/OnnxSpeech.nitro.ts +43 -6
- package/cpp/ThreadPool.cpp +0 -41
- package/cpp/ThreadPool.hpp +0 -62
package/cpp/ResourceDir.cpp
CHANGED
|
@@ -7,7 +7,7 @@ namespace margelo::nitro::onnx::speech {
|
|
|
7
7
|
|
|
8
8
|
namespace {
|
|
9
9
|
std::string g_resourceDir;
|
|
10
|
-
std::string
|
|
10
|
+
std::string g_documentDir;
|
|
11
11
|
}
|
|
12
12
|
|
|
13
13
|
const std::string& getResourceDir() {
|
|
@@ -18,12 +18,12 @@ void setResourceDir(const std::string& dir) {
|
|
|
18
18
|
g_resourceDir = dir;
|
|
19
19
|
}
|
|
20
20
|
|
|
21
|
-
const std::string&
|
|
22
|
-
return
|
|
21
|
+
const std::string& getDocumentDir() {
|
|
22
|
+
return g_documentDir;
|
|
23
23
|
}
|
|
24
24
|
|
|
25
|
-
void
|
|
26
|
-
|
|
25
|
+
void setDocumentDir(const std::string& dir) {
|
|
26
|
+
g_documentDir = dir;
|
|
27
27
|
}
|
|
28
28
|
|
|
29
29
|
} // namespace margelo::nitro::onnx::speech
|
package/cpp/ResourceDir.hpp
CHANGED
|
@@ -1,7 +1,8 @@
|
|
|
1
1
|
// ------------------------------------------------------------------------------
|
|
2
2
|
// ResourceDir.hpp
|
|
3
|
-
// Process-wide directories
|
|
4
|
-
//
|
|
3
|
+
// Process-wide directories set once from the platform layer at startup.
|
|
4
|
+
// resourceDir — bundled read-only assets (e.g. silero_vad.onnx)
|
|
5
|
+
// documentDir — writable app documents (registered speakers)
|
|
5
6
|
// ------------------------------------------------------------------------------
|
|
6
7
|
#pragma once
|
|
7
8
|
|
|
@@ -12,7 +13,7 @@ namespace margelo::nitro::onnx::speech {
|
|
|
12
13
|
const std::string& getResourceDir();
|
|
13
14
|
void setResourceDir(const std::string& dir);
|
|
14
15
|
|
|
15
|
-
const std::string&
|
|
16
|
-
void
|
|
16
|
+
const std::string& getDocumentDir();
|
|
17
|
+
void setDocumentDir(const std::string& dir);
|
|
17
18
|
|
|
18
19
|
} // namespace margelo::nitro::onnx::speech
|
package/cpp/SpeakerEngine.cpp
CHANGED
|
@@ -4,12 +4,13 @@
|
|
|
4
4
|
#include "SpeakerEngine.hpp"
|
|
5
5
|
|
|
6
6
|
#include "AudioFileReader.hpp"
|
|
7
|
+
#include "ResourceDir.hpp"
|
|
8
|
+
#include "SpeakerRecord.hpp"
|
|
7
9
|
#include "sherpa-onnx/c-api/c-api.h"
|
|
8
10
|
|
|
9
11
|
#include <cstring>
|
|
10
12
|
#include <filesystem>
|
|
11
13
|
#include <fstream>
|
|
12
|
-
#include <sstream>
|
|
13
14
|
#include <stdexcept>
|
|
14
15
|
|
|
15
16
|
namespace margelo::nitro::onnx::speech {
|
|
@@ -36,8 +37,9 @@ ModelSingleton<const SherpaOnnxSpeakerEmbeddingExtractor> gExtractorCache;
|
|
|
36
37
|
|
|
37
38
|
} // namespace
|
|
38
39
|
|
|
39
|
-
|
|
40
|
-
|
|
40
|
+
std::string speakerFilePath(const std::string& id) {
|
|
41
|
+
return joinPath(joinPath(getDocumentDir(), "speakers"), id + ".bin");
|
|
42
|
+
}
|
|
41
43
|
|
|
42
44
|
SpeakerEngine::~SpeakerEngine() {
|
|
43
45
|
unload();
|
|
@@ -47,8 +49,7 @@ void SpeakerEngine::load(const SpeakerEngineConfig& config) {
|
|
|
47
49
|
unload();
|
|
48
50
|
config_ = config;
|
|
49
51
|
|
|
50
|
-
|
|
51
|
-
auto cached = gExtractorCache.getOrCreate(key, [this](const std::string&) {
|
|
52
|
+
auto cached = gExtractorCache.getOrCreate(config_.cacheSignature(), [this](const std::string&) {
|
|
52
53
|
SherpaOnnxSpeakerEmbeddingExtractorConfig c;
|
|
53
54
|
std::memset(&c, 0, sizeof(c));
|
|
54
55
|
|
|
@@ -98,18 +99,19 @@ SpeakerEngineRegisteredSpeaker SpeakerEngine::registerSpeaker(
|
|
|
98
99
|
const std::string& id,
|
|
99
100
|
const std::string& name,
|
|
100
101
|
const std::vector<float>& embedding) {
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
const std::string path = joinPath(speakersDir, id + ".bin");
|
|
104
|
-
std::ofstream file(path, std::ios::binary);
|
|
105
|
-
if (!file) {
|
|
106
|
-
throw std::runtime_error("Failed to write speaker embedding: " + path);
|
|
102
|
+
if (id.empty()) {
|
|
103
|
+
throw std::invalid_argument("Speaker id must not be empty");
|
|
107
104
|
}
|
|
108
|
-
const
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
105
|
+
const std::string speakersDir = joinPath(getDocumentDir(), "speakers");
|
|
106
|
+
std::filesystem::create_directories(speakersDir);
|
|
107
|
+
const std::string path = speakerFilePath(id);
|
|
108
|
+
writeSpeakerRecord(path, name, embedding, nullptr, 0);
|
|
109
|
+
SpeakerEngineRegisteredSpeaker result;
|
|
110
|
+
result.id = id;
|
|
111
|
+
result.name = name;
|
|
112
|
+
result.embeddingPath = path;
|
|
113
|
+
result.embedding = embedding;
|
|
114
|
+
return result;
|
|
113
115
|
}
|
|
114
116
|
|
|
115
117
|
SpeakerEngineRegisteredSpeaker SpeakerEngine::registerSpeakerFromFile(
|
|
@@ -126,11 +128,24 @@ SpeakerEngineRegisteredSpeaker SpeakerEngine::registerSpeakerFromFile(
|
|
|
126
128
|
samples = readRawPcmFile(path);
|
|
127
129
|
}
|
|
128
130
|
auto embedding = computeEmbedding(samples);
|
|
129
|
-
|
|
131
|
+
|
|
132
|
+
const std::string speakersDir = joinPath(getDocumentDir(), "speakers");
|
|
133
|
+
std::filesystem::create_directories(speakersDir);
|
|
134
|
+
const std::string outPath = speakerFilePath(id);
|
|
135
|
+
writeSpeakerRecord(outPath, name, embedding, &samples, 16000);
|
|
136
|
+
|
|
137
|
+
SpeakerEngineRegisteredSpeaker result;
|
|
138
|
+
result.id = id;
|
|
139
|
+
result.name = name;
|
|
140
|
+
result.embeddingPath = outPath;
|
|
141
|
+
result.embedding = std::move(embedding);
|
|
142
|
+
result.referenceAudio = std::move(samples);
|
|
143
|
+
result.referenceSampleRate = 16000;
|
|
144
|
+
return result;
|
|
130
145
|
}
|
|
131
146
|
|
|
132
147
|
std::vector<SpeakerEngineRegisteredSpeaker> SpeakerEngine::listSpeakers() {
|
|
133
|
-
const std::string speakersDir = joinPath(
|
|
148
|
+
const std::string speakersDir = joinPath(getDocumentDir(), "speakers");
|
|
134
149
|
std::vector<SpeakerEngineRegisteredSpeaker> result;
|
|
135
150
|
if (!std::filesystem::exists(speakersDir)) {
|
|
136
151
|
return result;
|
|
@@ -140,22 +155,17 @@ std::vector<SpeakerEngineRegisteredSpeaker> SpeakerEngine::listSpeakers() {
|
|
|
140
155
|
const auto& filePath = entry.path();
|
|
141
156
|
if (filePath.extension() != ".bin") continue;
|
|
142
157
|
const std::string id = filePath.stem().string();
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
std::string name;
|
|
148
|
-
if (nameLen > 0 && nameLen < 1024) {
|
|
149
|
-
name.resize(nameLen);
|
|
150
|
-
file.read(name.data(), nameLen);
|
|
158
|
+
try {
|
|
159
|
+
result.push_back(readSpeakerRecord(id, filePath.string()));
|
|
160
|
+
} catch (...) {
|
|
161
|
+
// Skip unreadable / legacy records.
|
|
151
162
|
}
|
|
152
|
-
result.push_back({id, name, filePath.string()});
|
|
153
163
|
}
|
|
154
164
|
return result;
|
|
155
165
|
}
|
|
156
166
|
|
|
157
167
|
void SpeakerEngine::removeSpeaker(const std::string& id) {
|
|
158
|
-
const std::string path =
|
|
168
|
+
const std::string path = speakerFilePath(id);
|
|
159
169
|
std::error_code ec;
|
|
160
170
|
std::filesystem::remove(path, ec);
|
|
161
171
|
if (ec) {
|
package/cpp/SpeakerEngine.hpp
CHANGED
|
@@ -10,7 +10,8 @@
|
|
|
10
10
|
|
|
11
11
|
#include "AudioUtils.hpp"
|
|
12
12
|
#include "ModelSingleton.hpp"
|
|
13
|
-
#include "
|
|
13
|
+
#include "ResourceDir.hpp"
|
|
14
|
+
#include "SpeakerRecord.hpp"
|
|
14
15
|
|
|
15
16
|
#include <memory>
|
|
16
17
|
#include <string>
|
|
@@ -26,19 +27,19 @@ struct SpeakerEngineConfig {
|
|
|
26
27
|
std::string modelDir;
|
|
27
28
|
std::string model;
|
|
28
29
|
int32_t numThreads = 4;
|
|
29
|
-
};
|
|
30
30
|
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
std::string name;
|
|
35
|
-
std::string embeddingPath;
|
|
31
|
+
std::string cacheSignature() const {
|
|
32
|
+
return modelDir + "|speaker|" + model + "|" + std::to_string(numThreads);
|
|
33
|
+
}
|
|
36
34
|
};
|
|
37
35
|
|
|
36
|
+
/** Return the on-disk path for a registered speaker id under the document dir. */
|
|
37
|
+
std::string speakerFilePath(const std::string& id);
|
|
38
|
+
|
|
38
39
|
/** Speaker manager engine. */
|
|
39
40
|
class SpeakerEngine final {
|
|
40
41
|
public:
|
|
41
|
-
|
|
42
|
+
SpeakerEngine() = default;
|
|
42
43
|
~SpeakerEngine();
|
|
43
44
|
|
|
44
45
|
SpeakerEngine(const SpeakerEngine&) = delete;
|
|
@@ -51,18 +52,18 @@ class SpeakerEngine final {
|
|
|
51
52
|
std::vector<float> computeEmbedding(const std::vector<float>& samples);
|
|
52
53
|
|
|
53
54
|
/** Register a speaker embedding for later TTS use. */
|
|
54
|
-
SpeakerEngineRegisteredSpeaker registerSpeaker(
|
|
55
|
+
SpeakerEngineRegisteredSpeaker registerSpeaker(
|
|
56
|
+
const std::string& id, const std::string& name, const std::vector<float>& embedding);
|
|
55
57
|
|
|
56
|
-
/** Register a speaker from a reference audio file. */
|
|
57
|
-
SpeakerEngineRegisteredSpeaker registerSpeakerFromFile(
|
|
58
|
+
/** Register a speaker from a reference audio file (stores embedding + reference audio). */
|
|
59
|
+
SpeakerEngineRegisteredSpeaker registerSpeakerFromFile(
|
|
60
|
+
const std::string& id, const std::string& name, const std::string& path);
|
|
58
61
|
|
|
59
62
|
std::vector<SpeakerEngineRegisteredSpeaker> listSpeakers();
|
|
60
63
|
void removeSpeaker(const std::string& id);
|
|
61
64
|
void unload();
|
|
62
65
|
|
|
63
66
|
private:
|
|
64
|
-
std::shared_ptr<ThreadPool> threadPool_;
|
|
65
|
-
std::string cacheDir_;
|
|
66
67
|
SpeakerEngineConfig config_;
|
|
67
68
|
std::shared_ptr<const SherpaOnnxSpeakerEmbeddingExtractor> extractor_;
|
|
68
69
|
};
|
package/cpp/SpeakerManager.cpp
CHANGED
|
@@ -26,20 +26,20 @@ std::vector<RegisteredSpeaker> toRegisteredSpeakers(const std::vector<SpeakerEng
|
|
|
26
26
|
|
|
27
27
|
} // namespace
|
|
28
28
|
|
|
29
|
-
SpeakerManager::SpeakerManager(
|
|
30
|
-
: HybridObject(TAG)
|
|
29
|
+
SpeakerManager::SpeakerManager()
|
|
30
|
+
: HybridObject(TAG) {}
|
|
31
31
|
|
|
32
32
|
SpeakerManager::~SpeakerManager() {
|
|
33
33
|
engine_.unload();
|
|
34
34
|
}
|
|
35
35
|
|
|
36
36
|
std::shared_ptr<Promise<void>> SpeakerManager::load(const SpeakerEmbeddingConfig& config) {
|
|
37
|
-
return Promise<void>::async([
|
|
37
|
+
return Promise<void>::async([self = shared_cast<SpeakerManager>(), config]() {
|
|
38
38
|
SpeakerEngineConfig native;
|
|
39
39
|
native.modelDir = config.modelDir;
|
|
40
40
|
native.model = config.model;
|
|
41
41
|
native.numThreads = config.numThreads;
|
|
42
|
-
engine_.load(native);
|
|
42
|
+
self->engine_.load(native);
|
|
43
43
|
});
|
|
44
44
|
}
|
|
45
45
|
|
|
@@ -49,9 +49,9 @@ bool SpeakerManager::isLoaded() {
|
|
|
49
49
|
|
|
50
50
|
std::shared_ptr<Promise<std::shared_ptr<ArrayBuffer>>> SpeakerManager::computeEmbedding(
|
|
51
51
|
const std::shared_ptr<ArrayBuffer>& samples) {
|
|
52
|
-
return Promise<std::shared_ptr<ArrayBuffer>>::async([
|
|
52
|
+
return Promise<std::shared_ptr<ArrayBuffer>>::async([self = shared_cast<SpeakerManager>(), samples]() {
|
|
53
53
|
auto floatSamples = bytesToFloatVector(samples->data(), samples->size());
|
|
54
|
-
auto embedding = engine_.computeEmbedding(floatSamples);
|
|
54
|
+
auto embedding = self->engine_.computeEmbedding(floatSamples);
|
|
55
55
|
auto bytes = floatVectorToBytes(embedding);
|
|
56
56
|
return ArrayBuffer::move(std::move(bytes));
|
|
57
57
|
});
|
|
@@ -61,9 +61,9 @@ std::shared_ptr<Promise<RegisteredSpeaker>> SpeakerManager::registerSpeaker(
|
|
|
61
61
|
const std::string& id,
|
|
62
62
|
const std::string& name,
|
|
63
63
|
const std::shared_ptr<ArrayBuffer>& embedding) {
|
|
64
|
-
return Promise<RegisteredSpeaker>::async([
|
|
64
|
+
return Promise<RegisteredSpeaker>::async([self = shared_cast<SpeakerManager>(), id, name, embedding]() {
|
|
65
65
|
auto floatEmbedding = bytesToFloatVector(embedding->data(), embedding->size());
|
|
66
|
-
return toRegisteredSpeaker(engine_.registerSpeaker(id, name, floatEmbedding));
|
|
66
|
+
return toRegisteredSpeaker(self->engine_.registerSpeaker(id, name, floatEmbedding));
|
|
67
67
|
});
|
|
68
68
|
}
|
|
69
69
|
|
|
@@ -71,23 +71,25 @@ std::shared_ptr<Promise<RegisteredSpeaker>> SpeakerManager::registerSpeakerFromF
|
|
|
71
71
|
const std::string& id,
|
|
72
72
|
const std::string& name,
|
|
73
73
|
const std::string& path) {
|
|
74
|
-
return Promise<RegisteredSpeaker>::async([
|
|
75
|
-
return toRegisteredSpeaker(engine_.registerSpeakerFromFile(id, name, path));
|
|
74
|
+
return Promise<RegisteredSpeaker>::async([self = shared_cast<SpeakerManager>(), id, name, path]() {
|
|
75
|
+
return toRegisteredSpeaker(self->engine_.registerSpeakerFromFile(id, name, path));
|
|
76
76
|
});
|
|
77
77
|
}
|
|
78
78
|
|
|
79
79
|
std::shared_ptr<Promise<std::vector<RegisteredSpeaker>>> SpeakerManager::listSpeakers() {
|
|
80
|
-
return Promise<std::vector<RegisteredSpeaker>>::async([
|
|
81
|
-
return toRegisteredSpeakers(engine_.listSpeakers());
|
|
80
|
+
return Promise<std::vector<RegisteredSpeaker>>::async([self = shared_cast<SpeakerManager>()]() {
|
|
81
|
+
return toRegisteredSpeakers(self->engine_.listSpeakers());
|
|
82
82
|
});
|
|
83
83
|
}
|
|
84
84
|
|
|
85
85
|
std::shared_ptr<Promise<void>> SpeakerManager::removeSpeaker(const std::string& id) {
|
|
86
|
-
return Promise<void>::async([
|
|
86
|
+
return Promise<void>::async([self = shared_cast<SpeakerManager>(), id]() {
|
|
87
|
+
self->engine_.removeSpeaker(id);
|
|
88
|
+
});
|
|
87
89
|
}
|
|
88
90
|
|
|
89
91
|
std::shared_ptr<Promise<void>> SpeakerManager::unload() {
|
|
90
|
-
return Promise<void>::async([
|
|
92
|
+
return Promise<void>::async([self = shared_cast<SpeakerManager>()]() { self->engine_.unload(); });
|
|
91
93
|
}
|
|
92
94
|
|
|
93
95
|
} // namespace margelo::nitro::onnx::speech
|
package/cpp/SpeakerManager.hpp
CHANGED
|
@@ -10,6 +10,7 @@
|
|
|
10
10
|
#include <HybridSpeakerManagerSpec.hpp>
|
|
11
11
|
|
|
12
12
|
#include <memory>
|
|
13
|
+
#include <string>
|
|
13
14
|
|
|
14
15
|
namespace margelo::nitro::onnx::speech {
|
|
15
16
|
|
|
@@ -17,7 +18,7 @@ class SpeakerManager : public HybridSpeakerManagerSpec {
|
|
|
17
18
|
public:
|
|
18
19
|
static constexpr auto TAG = "SpeakerManager";
|
|
19
20
|
|
|
20
|
-
SpeakerManager(
|
|
21
|
+
SpeakerManager();
|
|
21
22
|
~SpeakerManager() override;
|
|
22
23
|
|
|
23
24
|
std::shared_ptr<Promise<void>> load(const SpeakerEmbeddingConfig& config) override;
|
|
@@ -0,0 +1,125 @@
|
|
|
1
|
+
// ------------------------------------------------------------------------------
|
|
2
|
+
// SpeakerRecord.cpp
|
|
3
|
+
// ------------------------------------------------------------------------------
|
|
4
|
+
#include "SpeakerRecord.hpp"
|
|
5
|
+
|
|
6
|
+
#include <cctype>
|
|
7
|
+
#include <cstring>
|
|
8
|
+
#include <fstream>
|
|
9
|
+
#include <limits>
|
|
10
|
+
#include <stdexcept>
|
|
11
|
+
|
|
12
|
+
namespace margelo::nitro::onnx::speech {
|
|
13
|
+
|
|
14
|
+
namespace {
|
|
15
|
+
|
|
16
|
+
constexpr char kSpeakerMagic[4] = {'S', 'P', 'K', '2'};
|
|
17
|
+
constexpr uint32_t kFlagHasReferenceAudio = 1u;
|
|
18
|
+
|
|
19
|
+
} // namespace
|
|
20
|
+
|
|
21
|
+
bool tryParseSpeakerIndex(const std::string& speakerId, int32_t& outIndex) {
|
|
22
|
+
if (speakerId.empty() || speakerId.size() > 10) {
|
|
23
|
+
return false;
|
|
24
|
+
}
|
|
25
|
+
for (char ch : speakerId) {
|
|
26
|
+
if (!std::isdigit(static_cast<unsigned char>(ch))) {
|
|
27
|
+
return false;
|
|
28
|
+
}
|
|
29
|
+
}
|
|
30
|
+
try {
|
|
31
|
+
const long value = std::stol(speakerId);
|
|
32
|
+
if (value < 0 || value > std::numeric_limits<int32_t>::max()) {
|
|
33
|
+
return false;
|
|
34
|
+
}
|
|
35
|
+
outIndex = static_cast<int32_t>(value);
|
|
36
|
+
return true;
|
|
37
|
+
} catch (...) {
|
|
38
|
+
return false;
|
|
39
|
+
}
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
void writeSpeakerRecord(
|
|
43
|
+
const std::string& path,
|
|
44
|
+
const std::string& name,
|
|
45
|
+
const std::vector<float>& embedding,
|
|
46
|
+
const std::vector<float>* referenceAudio,
|
|
47
|
+
int32_t referenceSampleRate) {
|
|
48
|
+
std::ofstream file(path, std::ios::binary);
|
|
49
|
+
if (!file) {
|
|
50
|
+
throw std::runtime_error("Failed to write speaker embedding: " + path);
|
|
51
|
+
}
|
|
52
|
+
file.write(kSpeakerMagic, sizeof(kSpeakerMagic));
|
|
53
|
+
const auto nameLen = static_cast<uint32_t>(name.size());
|
|
54
|
+
file.write(reinterpret_cast<const char*>(&nameLen), sizeof(nameLen));
|
|
55
|
+
file.write(name.data(), name.size());
|
|
56
|
+
const auto dim = static_cast<uint32_t>(embedding.size());
|
|
57
|
+
file.write(reinterpret_cast<const char*>(&dim), sizeof(dim));
|
|
58
|
+
if (!embedding.empty()) {
|
|
59
|
+
file.write(reinterpret_cast<const char*>(embedding.data()), embedding.size() * sizeof(float));
|
|
60
|
+
}
|
|
61
|
+
const uint32_t flags =
|
|
62
|
+
(referenceAudio != nullptr && !referenceAudio->empty()) ? kFlagHasReferenceAudio : 0u;
|
|
63
|
+
file.write(reinterpret_cast<const char*>(&flags), sizeof(flags));
|
|
64
|
+
if (flags & kFlagHasReferenceAudio) {
|
|
65
|
+
const auto sampleRate = static_cast<uint32_t>(referenceSampleRate > 0 ? referenceSampleRate : 16000);
|
|
66
|
+
const auto count = static_cast<uint32_t>(referenceAudio->size());
|
|
67
|
+
file.write(reinterpret_cast<const char*>(&sampleRate), sizeof(sampleRate));
|
|
68
|
+
file.write(reinterpret_cast<const char*>(&count), sizeof(count));
|
|
69
|
+
file.write(reinterpret_cast<const char*>(referenceAudio->data()), count * sizeof(float));
|
|
70
|
+
}
|
|
71
|
+
if (!file) {
|
|
72
|
+
throw std::runtime_error("Failed to write speaker embedding: " + path);
|
|
73
|
+
}
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
SpeakerEngineRegisteredSpeaker readSpeakerRecord(const std::string& id, const std::string& path) {
|
|
77
|
+
std::ifstream file(path, std::ios::binary);
|
|
78
|
+
if (!file) {
|
|
79
|
+
throw std::runtime_error("Speaker not found: " + id);
|
|
80
|
+
}
|
|
81
|
+
char magic[4] = {0};
|
|
82
|
+
file.read(magic, sizeof(magic));
|
|
83
|
+
if (std::memcmp(magic, kSpeakerMagic, sizeof(kSpeakerMagic)) != 0) {
|
|
84
|
+
throw std::runtime_error("Unsupported speaker record format for id: " + id);
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
SpeakerEngineRegisteredSpeaker result;
|
|
88
|
+
result.id = id;
|
|
89
|
+
result.embeddingPath = path;
|
|
90
|
+
|
|
91
|
+
uint32_t nameLen = 0;
|
|
92
|
+
file.read(reinterpret_cast<char*>(&nameLen), sizeof(nameLen));
|
|
93
|
+
if (nameLen > 0 && nameLen < 1024) {
|
|
94
|
+
result.name.resize(nameLen);
|
|
95
|
+
file.read(result.name.data(), nameLen);
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
uint32_t dim = 0;
|
|
99
|
+
file.read(reinterpret_cast<char*>(&dim), sizeof(dim));
|
|
100
|
+
if (dim > 0 && dim < 100000) {
|
|
101
|
+
result.embedding.resize(dim);
|
|
102
|
+
file.read(reinterpret_cast<char*>(result.embedding.data()), dim * sizeof(float));
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
uint32_t flags = 0;
|
|
106
|
+
file.read(reinterpret_cast<char*>(&flags), sizeof(flags));
|
|
107
|
+
if (flags & kFlagHasReferenceAudio) {
|
|
108
|
+
uint32_t sampleRate = 0;
|
|
109
|
+
uint32_t count = 0;
|
|
110
|
+
file.read(reinterpret_cast<char*>(&sampleRate), sizeof(sampleRate));
|
|
111
|
+
file.read(reinterpret_cast<char*>(&count), sizeof(count));
|
|
112
|
+
if (count > 0 && count < 16000u * 60u * 10u) {
|
|
113
|
+
result.referenceSampleRate = static_cast<int32_t>(sampleRate);
|
|
114
|
+
result.referenceAudio.resize(count);
|
|
115
|
+
file.read(reinterpret_cast<char*>(result.referenceAudio.data()), count * sizeof(float));
|
|
116
|
+
}
|
|
117
|
+
}
|
|
118
|
+
|
|
119
|
+
if (!file) {
|
|
120
|
+
throw std::runtime_error("Corrupt speaker record for id: " + id);
|
|
121
|
+
}
|
|
122
|
+
return result;
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
} // namespace margelo::nitro::onnx::speech
|
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
// ------------------------------------------------------------------------------
|
|
2
|
+
// SpeakerRecord.hpp
|
|
3
|
+
// On-disk speaker record I/O (embedding + optional reference audio).
|
|
4
|
+
// Pure file-format helpers with no model / sherpa-onnx dependency.
|
|
5
|
+
// ------------------------------------------------------------------------------
|
|
6
|
+
#pragma once
|
|
7
|
+
|
|
8
|
+
#include <cstdint>
|
|
9
|
+
#include <string>
|
|
10
|
+
#include <vector>
|
|
11
|
+
|
|
12
|
+
namespace margelo::nitro::onnx::speech {
|
|
13
|
+
|
|
14
|
+
/** On-disk speaker record (embedding + optional reference audio). */
|
|
15
|
+
struct SpeakerEngineRegisteredSpeaker {
|
|
16
|
+
std::string id;
|
|
17
|
+
std::string name;
|
|
18
|
+
std::string embeddingPath;
|
|
19
|
+
std::vector<float> embedding;
|
|
20
|
+
std::vector<float> referenceAudio;
|
|
21
|
+
int32_t referenceSampleRate = 0;
|
|
22
|
+
};
|
|
23
|
+
|
|
24
|
+
/** True when speakerId is a non-negative integer (model speaker index). */
|
|
25
|
+
bool tryParseSpeakerIndex(const std::string& speakerId, int32_t& outIndex);
|
|
26
|
+
|
|
27
|
+
/**
|
|
28
|
+
* Persist a speaker record. Format v2:
|
|
29
|
+
* magic "SPK2" | nameLen | name | dim | embedding[dim] | flags |
|
|
30
|
+
* [refSampleRate | refCount | refSamples[refCount]]
|
|
31
|
+
*/
|
|
32
|
+
void writeSpeakerRecord(
|
|
33
|
+
const std::string& path,
|
|
34
|
+
const std::string& name,
|
|
35
|
+
const std::vector<float>& embedding,
|
|
36
|
+
const std::vector<float>* referenceAudio,
|
|
37
|
+
int32_t referenceSampleRate);
|
|
38
|
+
|
|
39
|
+
/** Load a speaker record. Throws if the file is missing or corrupt. */
|
|
40
|
+
SpeakerEngineRegisteredSpeaker readSpeakerRecord(const std::string& id, const std::string& path);
|
|
41
|
+
|
|
42
|
+
} // namespace margelo::nitro::onnx::speech
|
package/cpp/StreamingAsr.cpp
CHANGED
|
@@ -28,15 +28,14 @@ AsrResult toAsrResult(const AsrEngineResult& native) {
|
|
|
28
28
|
|
|
29
29
|
} // namespace
|
|
30
30
|
|
|
31
|
-
StreamingAsr::StreamingAsr(
|
|
32
|
-
: HybridObject(TAG), engine_(std::move(threadPool)) {}
|
|
31
|
+
StreamingAsr::StreamingAsr() : HybridObject(TAG) {}
|
|
33
32
|
|
|
34
33
|
StreamingAsr::~StreamingAsr() {
|
|
35
34
|
engine_.unload();
|
|
36
35
|
}
|
|
37
36
|
|
|
38
37
|
std::shared_ptr<Promise<void>> StreamingAsr::load(const AsrModelConfig& config) {
|
|
39
|
-
return Promise<void>::async([
|
|
38
|
+
return Promise<void>::async([self = shared_cast<StreamingAsr>(), config]() {
|
|
40
39
|
AsrEngineConfig native;
|
|
41
40
|
native.type = config.type;
|
|
42
41
|
native.modelDir = config.modelDir;
|
|
@@ -47,7 +46,17 @@ std::shared_ptr<Promise<void>> StreamingAsr::load(const AsrModelConfig& config)
|
|
|
47
46
|
native.numThreads = static_cast<int32_t>(config.numThreads.value_or(2));
|
|
48
47
|
native.decodingMethod = config.decodingMethod.value_or("greedy_search");
|
|
49
48
|
native.maxActivePaths = static_cast<int32_t>(config.maxActivePaths.value_or(4));
|
|
50
|
-
|
|
49
|
+
native.debug = config.debug.value_or(false);
|
|
50
|
+
#if defined(__ANDROID__) && defined(SHERPA_ONNX_ENABLE_QNN)
|
|
51
|
+
native.provider = config.provider.value_or("qnn");
|
|
52
|
+
#elif defined(__ANDROID__)
|
|
53
|
+
native.provider = config.provider.value_or("nnapi");
|
|
54
|
+
#elif defined(__APPLE__)
|
|
55
|
+
native.provider = config.provider.value_or("coreml");
|
|
56
|
+
#else
|
|
57
|
+
native.provider = config.provider.value_or("cpu");
|
|
58
|
+
#endif
|
|
59
|
+
self->engine_.load(native, self);
|
|
51
60
|
});
|
|
52
61
|
}
|
|
53
62
|
|
|
@@ -57,22 +66,24 @@ bool StreamingAsr::isLoaded() {
|
|
|
57
66
|
|
|
58
67
|
std::shared_ptr<Promise<void>> StreamingAsr::acceptWaveform(
|
|
59
68
|
const std::shared_ptr<ArrayBuffer>& samples) {
|
|
60
|
-
return Promise<void>::async([
|
|
69
|
+
return Promise<void>::async([self = shared_cast<StreamingAsr>(), samples]() {
|
|
61
70
|
auto floatSamples = bytesToFloatVector(samples->data(), samples->size());
|
|
62
|
-
engine_.acceptWaveform(floatSamples);
|
|
71
|
+
self->engine_.acceptWaveform(floatSamples);
|
|
63
72
|
});
|
|
64
73
|
}
|
|
65
74
|
|
|
66
75
|
std::shared_ptr<Promise<AsrResult>> StreamingAsr::finalize() {
|
|
67
|
-
return Promise<AsrResult>::async([
|
|
76
|
+
return Promise<AsrResult>::async([self = shared_cast<StreamingAsr>()]() {
|
|
77
|
+
return toAsrResult(self->engine_.finalize());
|
|
78
|
+
});
|
|
68
79
|
}
|
|
69
80
|
|
|
70
81
|
std::shared_ptr<Promise<void>> StreamingAsr::reset() {
|
|
71
|
-
return Promise<void>::async([
|
|
82
|
+
return Promise<void>::async([self = shared_cast<StreamingAsr>()]() { self->engine_.reset(); });
|
|
72
83
|
}
|
|
73
84
|
|
|
74
85
|
std::shared_ptr<Promise<void>> StreamingAsr::unload() {
|
|
75
|
-
return Promise<void>::async([
|
|
86
|
+
return Promise<void>::async([self = shared_cast<StreamingAsr>()]() { self->engine_.unload(); });
|
|
76
87
|
}
|
|
77
88
|
|
|
78
89
|
std::optional<std::function<void(const AsrResult& /* result */)>> StreamingAsr::getOnPartialResult() {
|
package/cpp/StreamingAsr.hpp
CHANGED
|
@@ -20,7 +20,7 @@ class StreamingAsr : public HybridStreamingAsrSpec,
|
|
|
20
20
|
public:
|
|
21
21
|
static constexpr auto TAG = "StreamingAsr";
|
|
22
22
|
|
|
23
|
-
|
|
23
|
+
StreamingAsr();
|
|
24
24
|
~StreamingAsr() override;
|
|
25
25
|
|
|
26
26
|
std::shared_ptr<Promise<void>> load(const AsrModelConfig& config) override;
|