speechflow 2.0.4 → 2.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +15 -0
- package/README.md +34 -5
- package/etc/speechflow.yaml +20 -48
- package/etc/stx.conf +2 -2
- package/package.json +3 -3
- package/speechflow-cli/dst/speechflow-node-a2a-gtcrn-wt.d.ts +1 -0
- package/speechflow-cli/dst/speechflow-node-a2a-gtcrn-wt.js +60 -0
- package/speechflow-cli/dst/speechflow-node-a2a-gtcrn-wt.js.map +1 -0
- package/speechflow-cli/dst/speechflow-node-a2a-gtcrn.d.ts +15 -0
- package/speechflow-cli/dst/speechflow-node-a2a-gtcrn.js +234 -0
- package/speechflow-cli/dst/speechflow-node-a2a-gtcrn.js.map +1 -0
- package/speechflow-cli/dst/speechflow-node-a2a-meter.js +2 -2
- package/speechflow-cli/dst/speechflow-node-a2a-meter.js.map +1 -1
- package/speechflow-cli/dst/speechflow-node-a2t-assemblyai.d.ts +16 -0
- package/speechflow-cli/dst/speechflow-node-a2t-assemblyai.js +275 -0
- package/speechflow-cli/dst/speechflow-node-a2t-assemblyai.js.map +1 -0
- package/speechflow-cli/dst/speechflow-node-a2t-deepgram.js +32 -15
- package/speechflow-cli/dst/speechflow-node-a2t-deepgram.js.map +1 -1
- package/speechflow-cli/dst/speechflow-node-t2t-deepl.js +1 -1
- package/speechflow-cli/dst/speechflow-node-t2t-deepl.js.map +1 -1
- package/speechflow-cli/dst/speechflow-node-t2t-profanity.js +26 -6
- package/speechflow-cli/dst/speechflow-node-t2t-profanity.js.map +1 -1
- package/speechflow-cli/dst/speechflow-node-t2t-sentence.d.ts +1 -0
- package/speechflow-cli/dst/speechflow-node-t2t-sentence.js +72 -5
- package/speechflow-cli/dst/speechflow-node-t2t-sentence.js.map +1 -1
- package/speechflow-cli/etc/oxlint.jsonc +1 -0
- package/speechflow-cli/package.d/sherpa-onnx+1.12.23.patch +12 -0
- package/speechflow-cli/package.json +20 -17
- package/speechflow-cli/src/lib.d.ts +30 -4
- package/speechflow-cli/src/speechflow-node-a2a-gtcrn-wt.ts +68 -0
- package/speechflow-cli/src/speechflow-node-a2a-gtcrn.ts +219 -0
- package/speechflow-cli/src/speechflow-node-a2a-meter.ts +2 -2
- package/speechflow-cli/src/speechflow-node-a2t-deepgram.ts +33 -15
- package/speechflow-cli/src/speechflow-node-t2t-deepl.ts +1 -1
- package/speechflow-cli/src/speechflow-node-t2t-profanity.ts +30 -11
- package/speechflow-cli/src/speechflow-node-t2t-sentence.ts +86 -10
- package/speechflow-ui-db/dst/index.css +1 -1
- package/speechflow-ui-db/dst/index.js +13 -13
- package/speechflow-ui-db/package.json +12 -11
- package/speechflow-ui-db/src/app.vue +62 -17
- package/speechflow-ui-st/dst/index.css +1 -1
- package/speechflow-ui-st/dst/index.js +32 -32
- package/speechflow-ui-st/package.json +13 -12
- package/speechflow-ui-st/src/app.vue +9 -8
|
@@ -59,11 +59,14 @@ class SpeechFlowNodeT2TSentence extends speechflow_node_1.default {
|
|
|
59
59
|
queueSend = this.queue.pointerUse("send");
|
|
60
60
|
closing = false;
|
|
61
61
|
workingOffTimer = null;
|
|
62
|
+
previewTimer = null;
|
|
62
63
|
/* construct node */
|
|
63
64
|
constructor(id, cfg, opts, args) {
|
|
64
65
|
super(id, cfg, opts, args);
|
|
65
66
|
/* declare node configuration parameters */
|
|
66
|
-
this.configure({
|
|
67
|
+
this.configure({
|
|
68
|
+
timeout: { type: "number", pos: 0, val: 3 * 1000 }
|
|
69
|
+
});
|
|
67
70
|
/* declare node input/output format */
|
|
68
71
|
this.input = "text";
|
|
69
72
|
this.output = "text";
|
|
@@ -95,6 +98,7 @@ class SpeechFlowNodeT2TSentence extends speechflow_node_1.default {
|
|
|
95
98
|
this.queueSplit.walk(+1);
|
|
96
99
|
break;
|
|
97
100
|
}
|
|
101
|
+
/* perform sentence splitting on input chunk */
|
|
98
102
|
const chunk = element.chunk;
|
|
99
103
|
const payload = chunk.payload;
|
|
100
104
|
const m = payload.match(/^((?:.|\r?\n)+?[.;?!])\s*((?:.|\r?\n)*)$/);
|
|
@@ -131,20 +135,31 @@ class SpeechFlowNodeT2TSentence extends speechflow_node_1.default {
|
|
|
131
135
|
if (element2 === undefined)
|
|
132
136
|
break;
|
|
133
137
|
if (element2.type === "text-eof") {
|
|
138
|
+
/* no more chunks: output as final
|
|
139
|
+
(perhaps incomplete sentence at end of stream) */
|
|
134
140
|
element.complete = true;
|
|
135
141
|
this.queueSplit.touch();
|
|
136
142
|
this.queueSplit.walk(+1);
|
|
137
143
|
break;
|
|
138
144
|
}
|
|
145
|
+
/* merge into following chunk */
|
|
139
146
|
element2.chunk.timestampStart = element.chunk.timestampStart;
|
|
140
147
|
element2.chunk.payload =
|
|
141
148
|
element.chunk.payload + " " +
|
|
142
149
|
element2.chunk.payload;
|
|
150
|
+
/* reset preview state (merged content needs new preview) */
|
|
151
|
+
element2.preview = undefined;
|
|
143
152
|
this.queueSplit.delete();
|
|
144
153
|
this.queueSplit.touch();
|
|
145
154
|
}
|
|
146
|
-
else
|
|
155
|
+
else {
|
|
156
|
+
/* no following chunk yet: mark for intermediate preview output */
|
|
157
|
+
if (element.preview !== "sent") {
|
|
158
|
+
element.preview = "pending";
|
|
159
|
+
this.queueSplit.touch();
|
|
160
|
+
}
|
|
147
161
|
break;
|
|
162
|
+
}
|
|
148
163
|
}
|
|
149
164
|
}
|
|
150
165
|
/* re-initiate working off round (if still not destroyed) */
|
|
@@ -170,8 +185,21 @@ class SpeechFlowNodeT2TSentence extends speechflow_node_1.default {
|
|
|
170
185
|
callback(new Error("expected text input as string chunks"));
|
|
171
186
|
else if (chunk.payload.length === 0)
|
|
172
187
|
callback();
|
|
188
|
+
else if (chunk.kind === "intermediate") {
|
|
189
|
+
/* intermediate chunks: pass through immediately (bypass queue) */
|
|
190
|
+
self.log("info", `received text (${chunk.kind}): ${JSON.stringify(chunk.payload)}`);
|
|
191
|
+
self.log("info", `send text (intermediate pass-through): ${JSON.stringify(chunk.payload)}`);
|
|
192
|
+
this.push(chunk);
|
|
193
|
+
callback();
|
|
194
|
+
}
|
|
173
195
|
else {
|
|
174
|
-
|
|
196
|
+
/* final chunks: queue for sentence splitting */
|
|
197
|
+
self.log("info", `received text (${chunk.kind}): ${JSON.stringify(chunk.payload)}`);
|
|
198
|
+
/* cancel any pending preview timeout */
|
|
199
|
+
if (self.previewTimer !== null) {
|
|
200
|
+
clearTimeout(self.previewTimer);
|
|
201
|
+
self.previewTimer = null;
|
|
202
|
+
}
|
|
175
203
|
self.queueRecv.append({ type: "text-frame", chunk });
|
|
176
204
|
callback();
|
|
177
205
|
}
|
|
@@ -203,6 +231,7 @@ class SpeechFlowNodeT2TSentence extends speechflow_node_1.default {
|
|
|
203
231
|
else if (element !== undefined
|
|
204
232
|
&& element.type === "text-frame"
|
|
205
233
|
&& element.complete === true) {
|
|
234
|
+
/* send all consecutive complete chunks */
|
|
206
235
|
while (true) {
|
|
207
236
|
const nextElement = self.queueSend.peek();
|
|
208
237
|
if (nextElement === undefined)
|
|
@@ -215,12 +244,46 @@ class SpeechFlowNodeT2TSentence extends speechflow_node_1.default {
|
|
|
215
244
|
else if (nextElement.type === "text-frame"
|
|
216
245
|
&& nextElement.complete !== true)
|
|
217
246
|
break;
|
|
218
|
-
self.log("info", `send text: ${JSON.stringify(nextElement.chunk.payload)}`);
|
|
247
|
+
self.log("info", `send text (${nextElement.chunk.kind}): ${JSON.stringify(nextElement.chunk.payload)}`);
|
|
219
248
|
this.push(nextElement.chunk);
|
|
220
249
|
self.queueSend.walk(+1);
|
|
221
250
|
self.queue.trim();
|
|
222
251
|
}
|
|
223
252
|
}
|
|
253
|
+
else if (element !== undefined
|
|
254
|
+
&& element.type === "text-frame"
|
|
255
|
+
&& element.preview === "pending") {
|
|
256
|
+
/* send intermediate preview (without advancing pointer) */
|
|
257
|
+
const previewChunk = element.chunk.clone();
|
|
258
|
+
previewChunk.kind = "intermediate";
|
|
259
|
+
self.log("info", `send text (intermediate preview): ${JSON.stringify(previewChunk.payload)}`);
|
|
260
|
+
this.push(previewChunk);
|
|
261
|
+
element.preview = "sent";
|
|
262
|
+
self.queueSend.touch();
|
|
263
|
+
/* start preview timeout (if configured) */
|
|
264
|
+
const timeout = self.params.timeout;
|
|
265
|
+
if (timeout > 0 && self.previewTimer === null) {
|
|
266
|
+
self.previewTimer = setTimeout(() => {
|
|
267
|
+
self.previewTimer = null;
|
|
268
|
+
if (self.closing)
|
|
269
|
+
return;
|
|
270
|
+
/* promote preview to final chunk */
|
|
271
|
+
const el = self.queueSend.peek();
|
|
272
|
+
if (el !== undefined
|
|
273
|
+
&& el.type === "text-frame"
|
|
274
|
+
&& el.preview === "sent"
|
|
275
|
+
&& el.complete !== true) {
|
|
276
|
+
self.log("info", `timeout: promoting intermediate to final: ${JSON.stringify(el.chunk.payload)}`);
|
|
277
|
+
el.complete = true;
|
|
278
|
+
self.queueSend.touch();
|
|
279
|
+
self.queue.emit("write");
|
|
280
|
+
}
|
|
281
|
+
}, timeout);
|
|
282
|
+
}
|
|
283
|
+
/* wait for more data */
|
|
284
|
+
if (!self.closing)
|
|
285
|
+
self.queue.once("write", flushPendingChunks);
|
|
286
|
+
}
|
|
224
287
|
else if (!self.closing)
|
|
225
288
|
self.queue.once("write", flushPendingChunks);
|
|
226
289
|
};
|
|
@@ -232,11 +295,15 @@ class SpeechFlowNodeT2TSentence extends speechflow_node_1.default {
|
|
|
232
295
|
async close() {
|
|
233
296
|
/* indicate closing */
|
|
234
297
|
this.closing = true;
|
|
235
|
-
/* clean up
|
|
298
|
+
/* clean up timers */
|
|
236
299
|
if (this.workingOffTimer !== null) {
|
|
237
300
|
clearTimeout(this.workingOffTimer);
|
|
238
301
|
this.workingOffTimer = null;
|
|
239
302
|
}
|
|
303
|
+
if (this.previewTimer !== null) {
|
|
304
|
+
clearTimeout(this.previewTimer);
|
|
305
|
+
this.previewTimer = null;
|
|
306
|
+
}
|
|
240
307
|
/* remove any pending event listeners */
|
|
241
308
|
this.queue.removeAllListeners("write");
|
|
242
309
|
/* shutdown stream */
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"speechflow-node-t2t-sentence.js","sourceRoot":"","sources":["../src/speechflow-node-t2t-sentence.ts"],"names":[],"mappings":";AAAA;;;;EAIE;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAEF,6BAA6B;AAC7B,8DAA4C;AAE5C,6BAA6B;AAC7B,iCAAsC;AAEtC,6BAA6B;AAC7B,wEAAmE;AACnE,wDAAmE;
|
|
1
|
+
{"version":3,"file":"speechflow-node-t2t-sentence.js","sourceRoot":"","sources":["../src/speechflow-node-t2t-sentence.ts"],"names":[],"mappings":";AAAA;;;;EAIE;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAEF,6BAA6B;AAC7B,8DAA4C;AAE5C,6BAA6B;AAC7B,iCAAsC;AAEtC,6BAA6B;AAC7B,wEAAmE;AACnE,wDAAmE;AAYnE,8CAA8C;AAC9C,MAAqB,yBAA0B,SAAQ,yBAAc;IACjE,kCAAkC;IAC3B,MAAM,CAAC,IAAI,GAAG,cAAc,CAAA;IAEnC,sBAAsB;IACd,KAAK,GAAQ,IAAI,IAAI,CAAC,KAAK,EAAoB,CAAA;IAC/C,SAAS,GAAI,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,MAAM,CAAC,CAAA;IAC1C,UAAU,GAAG,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,OAAO,CAAC,CAAA;IAC3C,SAAS,GAAI,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,MAAM,CAAC,CAAA;IAC1C,OAAO,GAAI,KAAK,CAAA;IAChB,eAAe,GAAyC,IAAI,CAAA;IAC5D,YAAY,GAA4C,IAAI,CAAA;IAEpE,sBAAsB;IACtB,YAAa,EAAU,EAAE,GAA4B,EAAE,IAA6B,EAAE,IAAW;QAC7F,KAAK,CAAC,EAAE,EAAE,GAAG,EAAE,IAAI,EAAE,IAAI,CAAC,CAAA;QAE1B,6CAA6C;QAC7C,IAAI,CAAC,SAAS,CAAC;YACX,OAAO,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,GAAG,EAAE,CAAC,EAAE,GAAG,EAAE,CAAC,GAAG,IAAI,EAAE;SACrD,CAAC,CAAA;QAEF,wCAAwC;QACxC,IAAI,CAAC,KAAK,GAAI,MAAM,CAAA;QACpB,IAAI,CAAC,MAAM,GAAG,MAAM,CAAA;IACxB,CAAC;IAED,iBAAiB;IACjB,KAAK,CAAC,IAAI;QACN,8BAA8B;QAC9B,IAAI,CAAC,OAAO,GAAG,KAAK,CAAA;QAEpB,mCAAmC;QACnC,IAAI,UAAU,GAAG,KAAK,CAAA;QACtB,MAAM,YAAY,GAAG,KAAK,IAAI,EAAE;YAC5B,IAAI,IAAI,CAAC,OAAO;gBACZ,OAAM;YAEV,iCAAiC;YACjC,IAAI,UAAU;gBACV,OAAM;YACV,UAAU,GAAG,IAAI,CAAA;YACjB,IAAI,IAAI,CAAC,eAAe,KAAK,IAAI,EAAE,CAAC;gBAChC,YAAY,CAAC,IAAI,CAAC,eAAe,CAAC,CAAA;gBAClC,IAAI,CAAC,eAAe,GAAG,IAAI,CAAA;YAC/B,CAAC;YACD,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,OAAO,EAAE,YAAY,CAAC,CAAA;YAErC,0CAA0C;YAC1C,OAAO,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC;gBACnB,MAAM,OAAO,GAAG,IAAI,CAAC,UAAU,CAAC,IAAI,EAAE,CAAA;gBACtC,IAAI,OAAO,KAAK,SAAS;oBACrB,MAAK;gBACT,IAAI,OAAO,CAAC,IAAI,KAAK,UAAU,EAAE,CAAC;oBAC9B,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAA;oBACxB,MAAK;gBACT,CAAC;gBAED,iDAAiD;gBACjD,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,CAAA;gBAC3B,MAAM,OAAO,GAAG,KAAK,CAAC,OAAiB,CAAA;gBACvC,MAAM,CAAC,GAAG,OAAO,CAAC,KAAK,CAAC,0CAA0C,CAAC,CAAA;gBACnE,IAAI,CAAC,KAAK,IAAI,EAAE,CAAC;oBACb,2BAA2B;oBAC3B,MAAM,CAAE,AAAD,EAAG,QAAQ,EAAE,IAAI,CAAE,GAAG,CAAC,CAAA;oBAC9B,IAAI,IAAI,KAAK,EAAE,EAAE,CAAC;wBACd,qCAAqC;wBACrC,MAAM,MAAM,GAAG,KAAK,CAAC,KAAK,EAAE,CAAA;wBAC5B,MAAM,QAAQ,GAAG,gBAAQ,CAAC,UAAU,CAChC,KAAK,CAAC,YAAY,CAAC,KAAK,CAAC,KAAK,CAAC,cAAc,CAAC,CAAC,QAAQ,EAAE;4BACzD,CAAC,QAAQ,CAAC,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAA;wBACvC,MAAM,CAAC,cAAc,GAAG,KAAK,CAAC,cAAc,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAA;wBAC3D,KAAK,CAAC,YAAY,GAAM,MAAM,CAAC,cAAc,CAAA;wBAC7C,KAAK,CAAC,OAAO,GAAI,QAAQ,CAAA;wBACzB,MAAM,CAAC,OAAO,GAAG,IAAI,CAAA;wBACrB,OAAO,CAAC,QAAQ,GAAG,IAAI,CAAA;wBACvB,IAAI,CAAC,UAAU,CAAC,KAAK,EAAE,CAAA;wBACvB,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAA;wBACxB,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,EAAE,IAAI,EAAE,YAAY,EAAE,KAAK,EAAE,MAAM,EAAE,CAAC,CAAA;oBACjE,CAAC;yBACI,CAAC;wBACF,kCAAkC;wBAClC,OAAO,CAAC,QAAQ,GAAG,IAAI,CAAA;wBACvB,IAAI,CAAC,UAAU,CAAC,KAAK,EAAE,CAAA;wBACvB,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAA;oBAC5B,CAAC;gBACL,CAAC;qBACI,CAAC;oBACF,qCAAqC;oBACrC,MAAM,QAAQ,GAAG,IAAI,CAAC,UAAU,CAAC,QAAQ,EAAE,CAAA;oBAC3C,IAAI,QAAQ,GAAG,IAAI,CAAC,UAAU,CAAC,WAAW,EAAE,GAAG,CAAC,EAAE,CAAC;wBAC/C,kCAAkC;wBAClC,MAAM,QAAQ,GAAG,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,QAAQ,GAAG,CAAC,CAAC,CAAA;wBACnD,IAAI,QAAQ,KAAK,SAAS;4BACtB,MAAK;wBACT,IAAI,QAAQ,CAAC,IAAI,KAAK,UAAU,EAAE,CAAC;4BAC/B;kFACsD;4BACtD,OAAO,CAAC,QAAQ,GAAG,IAAI,CAAA;4BACvB,IAAI,CAAC,UAAU,CAAC,KAAK,EAAE,CAAA;4BACvB,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAA;4BACxB,MAAK;wBACT,CAAC;wBAED,kCAAkC;wBAClC,QAAQ,CAAC,KAAK,CAAC,cAAc,GAAG,OAAO,CAAC,KAAK,CAAC,cAAc,CAAA;wBAC5D,QAAQ,CAAC,KAAK,CAAC,OAAO;4BACjB,OAAO,CAAC,KAAK,CAAC,OAAmB,GAAG,GAAG;gCACvC,QAAQ,CAAC,KAAK,CAAC,OAAkB,CAAA;wBAEtC,8DAA8D;wBAC9D,QAAQ,CAAC,OAAO,GAAG,SAAS,CAAA;wBAC5B,IAAI,CAAC,UAAU,CAAC,MAAM,EAAE,CAAA;wBACxB,IAAI,CAAC,UAAU,CAAC,KAAK,EAAE,CAAA;oBAC3B,CAAC;yBACI,CAAC;wBACF,oEAAoE;wBACpE,IAAI,OAAO,CAAC,OAAO,KAAK,MAAM,EAAE,CAAC;4BAC7B,OAAO,CAAC,OAAO,GAAG,SAAS,CAAA;4BAC3B,IAAI,CAAC,UAAU,CAAC,KAAK,EAAE,CAAA;wBAC3B,CAAC;wBACD,MAAK;oBACT,CAAC;gBACL,CAAC;YACL,CAAC;YAED,8DAA8D;YAC9D,UAAU,GAAG,KAAK,CAAA;YAClB,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC;gBAChB,IAAI,CAAC,eAAe,GAAG,UAAU,CAAC,YAAY,EAAE,GAAG,CAAC,CAAA;gBACpD,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,OAAO,EAAE,YAAY,CAAC,CAAA;YAC1C,CAAC;QACL,CAAC,CAAA;QACD,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,OAAO,EAAE,YAAY,CAAC,CAAA;QAEtC,iEAAiE;QACjE,MAAM,IAAI,GAAG,IAAI,CAAA;QACjB,IAAI,CAAC,MAAM,GAAG,IAAI,qBAAM,CAAC,MAAM,CAAC;YAC5B,kBAAkB,EAAE,IAAI;YACxB,kBAAkB,EAAE,IAAI;YACxB,aAAa,EAAO,KAAK;YACzB,aAAa,EAAO,CAAC;YAErB,oDAAoD;YACpD,KAAK,CAAE,KAAsB,EAAE,QAAQ,EAAE,QAAQ;gBAC7C,IAAI,IAAI,CAAC,OAAO;oBACZ,QAAQ,CAAC,IAAI,KAAK,CAAC,0BAA0B,CAAC,CAAC,CAAA;qBAC9C,IAAI,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,OAAO,CAAC;oBACnC,QAAQ,CAAC,IAAI,KAAK,CAAC,sCAAsC,CAAC,CAAC,CAAA;qBAC1D,IAAI,KAAK,CAAC,OAAO,CAAC,MAAM,KAAK,CAAC;oBAC/B,QAAQ,EAAE,CAAA;qBACT,IAAI,KAAK,CAAC,IAAI,KAAK,cAAc,EAAE,CAAC;oBACrC,oEAAoE;oBACpE,IAAI,CAAC,GAAG,CAAC,MAAM,EAAE,kBAAkB,KAAK,CAAC,IAAI,MAAM,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,OAAO,CAAC,EAAE,CAAC,CAAA;oBACnF,IAAI,CAAC,GAAG,CAAC,MAAM,EAAE,0CAA0C,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,OAAO,CAAC,EAAE,CAAC,CAAA;oBAC3F,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;oBAChB,QAAQ,EAAE,CAAA;gBACd,CAAC;qBACI,CAAC;oBACF,kDAAkD;oBAClD,IAAI,CAAC,GAAG,CAAC,MAAM,EAAE,kBAAkB,KAAK,CAAC,IAAI,MAAM,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,OAAO,CAAC,EAAE,CAAC,CAAA;oBAEnF,0CAA0C;oBAC1C,IAAI,IAAI,CAAC,YAAY,KAAK,IAAI,EAAE,CAAC;wBAC7B,YAAY,CAAC,IAAI,CAAC,YAAY,CAAC,CAAA;wBAC/B,IAAI,CAAC,YAAY,GAAG,IAAI,CAAA;oBAC5B,CAAC;oBAED,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,EAAE,IAAI,EAAE,YAAY,EAAE,KAAK,EAAE,CAAC,CAAA;oBACpD,QAAQ,EAAE,CAAA;gBACd,CAAC;YACL,CAAC;YAED,6DAA6D;YAC7D,KAAK,CAAE,QAAQ;gBACX,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;oBACf,QAAQ,EAAE,CAAA;oBACV,OAAM;gBACV,CAAC;gBACD,0BAA0B;gBAC1B,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,EAAE,IAAI,EAAE,UAAU,EAAE,CAAC,CAAA;gBAC3C,QAAQ,EAAE,CAAA;YACd,CAAC;YAED,oDAAoD;YACpD,IAAI,CAAE,KAAK;gBACP,iCAAiC;gBACjC,MAAM,kBAAkB,GAAG,GAAG,EAAE;oBAC5B,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;wBACf,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;wBACf,OAAM;oBACV,CAAC;oBACD,MAAM,OAAO,GAAG,IAAI,CAAC,SAAS,CAAC,IAAI,EAAE,CAAA;oBACrC,IAAI,OAAO,KAAK,SAAS;2BAClB,OAAO,CAAC,IAAI,KAAK,UAAU,EAAE,CAAC;wBACjC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;wBACf,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAA;oBAC3B,CAAC;yBACI,IAAI,OAAO,KAAK,SAAS;2BACvB,OAAO,CAAC,IAAI,KAAK,YAAY;2BAC7B,OAAO,CAAC,QAAQ,KAAK,IAAI,EAAE,CAAC;wBAC/B,4CAA4C;wBAC5C,OAAO,IAAI,EAAE,CAAC;4BACV,MAAM,WAAW,GAAG,IAAI,CAAC,SAAS,CAAC,IAAI,EAAE,CAAA;4BACzC,IAAI,WAAW,KAAK,SAAS;gCACzB,MAAK;iCACJ,IAAI,WAAW,CAAC,IAAI,KAAK,UAAU,EAAE,CAAC;gCACvC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;gCACf,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAA;gCACvB,MAAK;4BACT,CAAC;iCACI,IAAI,WAAW,CAAC,IAAI,KAAK,YAAY;mCACnC,WAAW,CAAC,QAAQ,KAAK,IAAI;gCAChC,MAAK;4BACT,IAAI,CAAC,GAAG,CAAC,MAAM,EAAE,cAAc,WAAW,CAAC,KAAK,CAAC,IAAI,MAAM,IAAI,CAAC,SAAS,CAAC,WAAW,CAAC,KAAK,CAAC,OAAO,CAAC,EAAE,CAAC,CAAA;4BACvG,IAAI,CAAC,IAAI,CAAC,WAAW,CAAC,KAAK,CAAC,CAAA;4BAC5B,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAA;4BACvB,IAAI,CAAC,KAAK,CAAC,IAAI,EAAE,CAAA;wBACrB,CAAC;oBACL,CAAC;yBACI,IAAI,OAAO,KAAK,SAAS;2BACvB,OAAO,CAAC,IAAI,KAAK,YAAY;2BAC7B,OAAO,CAAC,OAAO,KAAK,SAAS,EAAE,CAAC;wBACnC,6DAA6D;wBAC7D,MAAM,YAAY,GAAG,OAAO,CAAC,KAAK,CAAC,KAAK,EAAE,CAAA;wBAC1C,YAAY,CAAC,IAAI,GAAG,cAAc,CAAA;wBAClC,IAAI,CAAC,GAAG,CAAC,MAAM,EAAE,qCAAqC,IAAI,CAAC,SAAS,CAAC,YAAY,CAAC,OAAO,CAAC,EAAE,CAAC,CAAA;wBAC7F,IAAI,CAAC,IAAI,CAAC,YAAY,CAAC,CAAA;wBACvB,OAAO,CAAC,OAAO,GAAG,MAAM,CAAA;wBACxB,IAAI,CAAC,SAAS,CAAC,KAAK,EAAE,CAAA;wBAEtB,6CAA6C;wBAC7C,MAAM,OAAO,GAAG,IAAI,CAAC,MAAM,CAAC,OAAiB,CAAA;wBAC7C,IAAI,OAAO,GAAG,CAAC,IAAI,IAAI,CAAC,YAAY,KAAK,IAAI,EAAE,CAAC;4BAC5C,IAAI,CAAC,YAAY,GAAG,UAAU,CAAC,GAAG,EAAE;gCAChC,IAAI,CAAC,YAAY,GAAG,IAAI,CAAA;gCACxB,IAAI,IAAI,CAAC,OAAO;oCACZ,OAAM;gCAEV,sCAAsC;gCACtC,MAAM,EAAE,GAAG,IAAI,CAAC,SAAS,CAAC,IAAI,EAAE,CAAA;gCAChC,IAAI,EAAE,KAAK,SAAS;uCACb,EAAE,CAAC,IAAI,KAAK,YAAY;uCACxB,EAAE,CAAC,OAAO,KAAK,MAAM;uCACrB,EAAE,CAAC,QAAQ,KAAK,IAAI,EAAE,CAAC;oCAC1B,IAAI,CAAC,GAAG,CAAC,MAAM,EAAE,6CAA6C,IAAI,CAAC,SAAS,CAAC,EAAE,CAAC,KAAK,CAAC,OAAO,CAAC,EAAE,CAAC,CAAA;oCACjG,EAAE,CAAC,QAAQ,GAAG,IAAI,CAAA;oCAClB,IAAI,CAAC,SAAS,CAAC,KAAK,EAAE,CAAA;oCACtB,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,CAAA;gCAC5B,CAAC;4BACL,CAAC,EAAE,OAAO,CAAC,CAAA;wBACf,CAAC;wBAED,0BAA0B;wBAC1B,IAAI,CAAC,IAAI,CAAC,OAAO;4BACb,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,OAAO,EAAE,kBAAkB,CAAC,CAAA;oBACpD,CAAC;yBACI,IAAI,CAAC,IAAI,CAAC,OAAO;wBAClB,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,OAAO,EAAE,kBAAkB,CAAC,CAAA;gBACpD,CAAC,CAAA;gBACD,kBAAkB,EAAE,CAAA;YACxB,CAAC;SACJ,CAAC,CAAA;IACN,CAAC;IAED,kBAAkB;IAClB,KAAK,CAAC,KAAK;QACP,wBAAwB;QACxB,IAAI,CAAC,OAAO,GAAG,IAAI,CAAA;QAEnB,uBAAuB;QACvB,IAAI,IAAI,CAAC,eAAe,KAAK,IAAI,EAAE,CAAC;YAChC,YAAY,CAAC,IAAI,CAAC,eAAe,CAAC,CAAA;YAClC,IAAI,CAAC,eAAe,GAAG,IAAI,CAAA;QAC/B,CAAC;QACD,IAAI,IAAI,CAAC,YAAY,KAAK,IAAI,EAAE,CAAC;YAC7B,YAAY,CAAC,IAAI,CAAC,YAAY,CAAC,CAAA;YAC/B,IAAI,CAAC,YAAY,GAAG,IAAI,CAAA;QAC5B,CAAC;QAED,0CAA0C;QAC1C,IAAI,CAAC,KAAK,CAAC,kBAAkB,CAAC,OAAO,CAAC,CAAA;QAEtC,uBAAuB;QACvB,IAAI,IAAI,CAAC,MAAM,KAAK,IAAI,EAAE,CAAC;YACvB,MAAM,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,MAAM,CAAC,CAAA;YACrC,IAAI,CAAC,MAAM,GAAG,IAAI,CAAA;QACtB,CAAC;IACL,CAAC;;AAhSL,4CAiSC"}
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
diff --git a/node_modules/sherpa-onnx/sherpa-onnx-speech-enhancement.js b/node_modules/sherpa-onnx/sherpa-onnx-speech-enhancement.js
|
|
2
|
+
index 3ee5a77..2aab4a2 100644
|
|
3
|
+
--- a/node_modules/sherpa-onnx/sherpa-onnx-speech-enhancement.js
|
|
4
|
+
+++ b/node_modules/sherpa-onnx/sherpa-onnx-speech-enhancement.js
|
|
5
|
+
@@ -101,7 +101,6 @@ function initSherpaOnnxOfflineSpeechDenoiserConfig(config, Module) {
|
|
6
|
+
|
|
7
|
+
class OfflineSpeechDenoiser {
|
|
8
|
+
constructor(configObj, Module) {
|
|
9
|
+
- console.log(configObj)
|
|
10
|
+
const config = initSherpaOnnxOfflineSpeechDenoiserConfig(configObj, Module)
|
|
11
|
+
// Module._MyPrint(config.ptr);
|
|
12
|
+
const handle = Module._SherpaOnnxCreateOfflineSpeechDenoiser(config.ptr);
|
|
@@ -24,15 +24,17 @@
|
|
|
24
24
|
"shell-parser": "1.0.0",
|
|
25
25
|
"@deepgram/sdk": "4.11.3",
|
|
26
26
|
"deepl-node": "1.24.0",
|
|
27
|
-
"@elevenlabs/elevenlabs-js": "2.
|
|
27
|
+
"@elevenlabs/elevenlabs-js": "2.33.0",
|
|
28
28
|
"get-stream": "9.0.1",
|
|
29
|
-
"@dotenvx/dotenvx": "1.
|
|
29
|
+
"@dotenvx/dotenvx": "1.52.0",
|
|
30
30
|
"speex-resampler": "3.0.1",
|
|
31
31
|
"@sapphi-red/speex-preprocess-wasm": "0.4.0",
|
|
32
32
|
"@shiguredo/rnnoise-wasm": "2025.1.5",
|
|
33
|
-
"
|
|
34
|
-
"
|
|
35
|
-
"@aws-sdk/client-
|
|
33
|
+
"sherpa-onnx": "1.12.23",
|
|
34
|
+
"axios": "1.13.3",
|
|
35
|
+
"@aws-sdk/client-transcribe-streaming": "3.975.0",
|
|
36
|
+
"@aws-sdk/client-translate": "3.975.0",
|
|
37
|
+
"@aws-sdk/client-polly": "3.975.0",
|
|
36
38
|
"@google-cloud/translate": "9.3.0",
|
|
37
39
|
"@google-cloud/speech": "7.2.1",
|
|
38
40
|
"@google-cloud/text-to-speech": "6.4.0",
|
|
@@ -51,7 +53,7 @@
|
|
|
51
53
|
"ollama": "0.6.3",
|
|
52
54
|
"openai": "6.16.0",
|
|
53
55
|
"@anthropic-ai/sdk": "0.71.2",
|
|
54
|
-
"@google/genai": "1.
|
|
56
|
+
"@google/genai": "1.38.0",
|
|
55
57
|
"@rse/ffmpeg": "1.4.2",
|
|
56
58
|
"ffmpeg-stream": "1.0.1",
|
|
57
59
|
"installed-packages": "1.0.13",
|
|
@@ -59,14 +61,13 @@
|
|
|
59
61
|
"wav": "1.0.2",
|
|
60
62
|
"mqtt": "5.14.1",
|
|
61
63
|
"vban": "1.5.6",
|
|
62
|
-
"cbor2": "2.
|
|
64
|
+
"cbor2": "2.2.1",
|
|
63
65
|
"arktype": "2.1.29",
|
|
64
66
|
"pure-uuid": "2.0.0",
|
|
65
67
|
"wavefile": "11.0.0",
|
|
66
68
|
"audio-inspect": "0.0.4",
|
|
67
69
|
"@huggingface/transformers": "3.8.1",
|
|
68
|
-
"@huggingface/hub": "2.
|
|
69
|
-
"onnxruntime-node": "1.20.1",
|
|
70
|
+
"@huggingface/hub": "2.8.0",
|
|
70
71
|
"kokoro-js": "1.2.1",
|
|
71
72
|
"@ericedouard/vad-node-realtime": "0.2.0",
|
|
72
73
|
"osc-js": "2.4.1",
|
|
@@ -74,10 +75,11 @@
|
|
|
74
75
|
"node-interval-tree": "2.1.2",
|
|
75
76
|
"wrap-text": "1.0.10",
|
|
76
77
|
"bad-words-next": "3.2.0",
|
|
78
|
+
"@2toad/profanity": "3.2.0",
|
|
77
79
|
"cli-table3": "0.6.5",
|
|
78
80
|
"mkdirp": "3.0.1",
|
|
79
81
|
"@soundtouchjs/audio-worklet": "0.2.1",
|
|
80
|
-
"werift": "0.22.
|
|
82
|
+
"werift": "0.22.3",
|
|
81
83
|
"@discordjs/opus": "0.10.0",
|
|
82
84
|
"@rse/stx": "1.1.4"
|
|
83
85
|
},
|
|
@@ -88,12 +90,12 @@
|
|
|
88
90
|
"eslint-plugin-promise": "7.2.1",
|
|
89
91
|
"eslint-plugin-import": "2.32.0",
|
|
90
92
|
"eslint-plugin-node": "11.1.0",
|
|
91
|
-
"typescript-eslint": "8.
|
|
92
|
-
"@typescript-eslint/eslint-plugin": "8.
|
|
93
|
-
"@typescript-eslint/parser": "8.
|
|
94
|
-
"oxlint": "1.
|
|
95
|
-
"oxlint-plugin-complexity": "0.
|
|
96
|
-
"eslint-plugin-oxlint": "1.
|
|
93
|
+
"typescript-eslint": "8.54.0",
|
|
94
|
+
"@typescript-eslint/eslint-plugin": "8.54.0",
|
|
95
|
+
"@typescript-eslint/parser": "8.54.0",
|
|
96
|
+
"oxlint": "1.42.0",
|
|
97
|
+
"oxlint-plugin-complexity": "0.3.0",
|
|
98
|
+
"eslint-plugin-oxlint": "1.42.0",
|
|
97
99
|
|
|
98
100
|
"@types/node": "24.10.1",
|
|
99
101
|
"@types/yargs": "17.0.35",
|
|
@@ -116,7 +118,8 @@
|
|
|
116
118
|
"cross-env": "10.1.0"
|
|
117
119
|
},
|
|
118
120
|
"overrides": {
|
|
119
|
-
"@huggingface/transformers": { "onnxruntime-node": "1.22.0" }
|
|
121
|
+
"@huggingface/transformers": { "onnxruntime-node": "1.22.0", "onnxruntime-web": "1.22.0" },
|
|
122
|
+
"@ericedouard/vad-node-realtime": { "onnxruntime-node": "1.22.0", "onnxruntime-web": "1.22.0" }
|
|
120
123
|
},
|
|
121
124
|
"upd": [
|
|
122
125
|
"!onnxruntime-node",
|
|
@@ -13,7 +13,7 @@ declare module "node:stream" {
|
|
|
13
13
|
}
|
|
14
14
|
|
|
15
15
|
/* type definitions for AudioWorkletProcessor */
|
|
16
|
-
declare interface
|
|
16
|
+
declare interface AudioWorkletProcessorType {
|
|
17
17
|
readonly port: MessagePort
|
|
18
18
|
process(
|
|
19
19
|
inputs: Float32Array[][],
|
|
@@ -22,8 +22,8 @@ declare interface AudioWorkletProcessor {
|
|
|
22
22
|
): boolean
|
|
23
23
|
}
|
|
24
24
|
declare const AudioWorkletProcessor: {
|
|
25
|
-
prototype:
|
|
26
|
-
new():
|
|
25
|
+
prototype: AudioWorkletProcessorType
|
|
26
|
+
new(): AudioWorkletProcessorType
|
|
27
27
|
}
|
|
28
28
|
declare interface AudioParamDescriptor {
|
|
29
29
|
name: string
|
|
@@ -34,7 +34,7 @@ declare interface AudioParamDescriptor {
|
|
|
34
34
|
}
|
|
35
35
|
declare function registerProcessor (
|
|
36
36
|
name: string,
|
|
37
|
-
processorCtor: new (...args: any[]) =>
|
|
37
|
+
processorCtor: new (...args: any[]) => AudioWorkletProcessorType
|
|
38
38
|
): void
|
|
39
39
|
|
|
40
40
|
/* type definition for "shell-parser" */
|
|
@@ -42,5 +42,31 @@ declare module "shell-parser" {
|
|
|
42
42
|
export default function shellParser (command: string): string[]
|
|
43
43
|
}
|
|
44
44
|
|
|
45
|
+
/* type definition for "sherpa-onnx" */
|
|
46
|
+
declare module "sherpa-onnx" {
|
|
47
|
+
export interface SherpaOnnxDenoiserConfig {
|
|
48
|
+
model: {
|
|
49
|
+
gtcrn: {
|
|
50
|
+
model: string
|
|
51
|
+
}
|
|
52
|
+
},
|
|
53
|
+
numThreads?: number,
|
|
54
|
+
provider?: string,
|
|
55
|
+
debug?: number
|
|
56
|
+
}
|
|
57
|
+
export interface SherpaOnnxAudioOutput {
|
|
58
|
+
samples: Float32Array
|
|
59
|
+
sampleRate: number
|
|
60
|
+
}
|
|
61
|
+
export interface SherpaOnnxOfflineSpeechDenoiser {
|
|
62
|
+
run(samples: Float32Array, sampleRate: number): SherpaOnnxAudioOutput
|
|
63
|
+
}
|
|
64
|
+
export interface SherpaOnnxModule {
|
|
65
|
+
createOfflineSpeechDenoiser(config: SherpaOnnxDenoiserConfig): SherpaOnnxOfflineSpeechDenoiser
|
|
66
|
+
}
|
|
67
|
+
const SherpaOnnx: SherpaOnnxModule
|
|
68
|
+
export default SherpaOnnx
|
|
69
|
+
}
|
|
70
|
+
|
|
45
71
|
/* eslint-enable no-unused-vars */
|
|
46
72
|
|
|
@@ -0,0 +1,68 @@
|
|
|
1
|
+
/*
|
|
2
|
+
** SpeechFlow - Speech Processing Flow Graph
|
|
3
|
+
** Copyright (c) 2024-2025 Dr. Ralf S. Engelschall <rse@engelschall.com>
|
|
4
|
+
** Licensed under GPL 3.0 <https://spdx.org/licenses/GPL-3.0-only>
|
|
5
|
+
*/
|
|
6
|
+
|
|
7
|
+
/* standard dependencies */
|
|
8
|
+
import { parentPort, workerData } from "node:worker_threads"
|
|
9
|
+
|
|
10
|
+
/* external dependencies */
|
|
11
|
+
import SherpaOnnx from "sherpa-onnx"
|
|
12
|
+
import type {
|
|
13
|
+
SherpaOnnxDenoiserConfig,
|
|
14
|
+
SherpaOnnxOfflineSpeechDenoiser
|
|
15
|
+
} from "sherpa-onnx"
|
|
16
|
+
|
|
17
|
+
/* receive model path from parent thread */
|
|
18
|
+
const modelPath: string = workerData.modelPath
|
|
19
|
+
|
|
20
|
+
/* GTCRN state */
|
|
21
|
+
let denoiser: SherpaOnnxOfflineSpeechDenoiser
|
|
22
|
+
|
|
23
|
+
/* helper: log message to parent */
|
|
24
|
+
const log = (level: string, message: string) => {
|
|
25
|
+
parentPort!.postMessage({ type: "log", level, message })
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
/* initialize globals */
|
|
29
|
+
;(async () => {
|
|
30
|
+
try {
|
|
31
|
+
/* create denoiser */
|
|
32
|
+
const config: SherpaOnnxDenoiserConfig = {
|
|
33
|
+
model: {
|
|
34
|
+
gtcrn: {
|
|
35
|
+
model: modelPath
|
|
36
|
+
}
|
|
37
|
+
},
|
|
38
|
+
numThreads: 1
|
|
39
|
+
}
|
|
40
|
+
denoiser = SherpaOnnx.createOfflineSpeechDenoiser(config)
|
|
41
|
+
log("info", "GTCRN denoiser initialized")
|
|
42
|
+
parentPort!.postMessage({ type: "ready" })
|
|
43
|
+
}
|
|
44
|
+
catch (err) {
|
|
45
|
+
parentPort!.postMessage({ type: "failed", message: `failed to initialize GTCRN: ${err}` })
|
|
46
|
+
process.exit(1)
|
|
47
|
+
}
|
|
48
|
+
})()
|
|
49
|
+
|
|
50
|
+
/* receive messages */
|
|
51
|
+
parentPort!.on("message", (msg) => {
|
|
52
|
+
if (msg.type === "process") {
|
|
53
|
+
const { id, samples } = msg
|
|
54
|
+
|
|
55
|
+
/* process with GTCRN denoiser
|
|
56
|
+
NOTICE: GTCRN can also resample out input, but will always
|
|
57
|
+
produces 16KHz output, so we already fixate 16KHz input here! */
|
|
58
|
+
const result = denoiser.run(samples, 16000)
|
|
59
|
+
|
|
60
|
+
/* copy to transferable ArrayBuffer and send back to parent */
|
|
61
|
+
const samplesDenoised = new Float32Array(result.samples)
|
|
62
|
+
parentPort!.postMessage({ type: "process-done", id, data: samplesDenoised }, [ samplesDenoised.buffer ])
|
|
63
|
+
}
|
|
64
|
+
else if (msg.type === "close") {
|
|
65
|
+
/* shutdown this process */
|
|
66
|
+
process.exit(0)
|
|
67
|
+
}
|
|
68
|
+
})
|
|
@@ -0,0 +1,219 @@
|
|
|
1
|
+
/*
|
|
2
|
+
** SpeechFlow - Speech Processing Flow Graph
|
|
3
|
+
** Copyright (c) 2024-2025 Dr. Ralf S. Engelschall <rse@engelschall.com>
|
|
4
|
+
** Licensed under GPL 3.0 <https://spdx.org/licenses/GPL-3.0-only>
|
|
5
|
+
*/
|
|
6
|
+
|
|
7
|
+
/* standard dependencies */
|
|
8
|
+
import fs from "node:fs"
|
|
9
|
+
import path from "node:path"
|
|
10
|
+
import Stream from "node:stream"
|
|
11
|
+
import { Worker } from "node:worker_threads"
|
|
12
|
+
|
|
13
|
+
/* external dependencies */
|
|
14
|
+
import axios from "axios"
|
|
15
|
+
import SpeexResampler from "speex-resampler"
|
|
16
|
+
|
|
17
|
+
/* internal dependencies */
|
|
18
|
+
import SpeechFlowNode, { SpeechFlowChunk } from "./speechflow-node"
|
|
19
|
+
import * as util from "./speechflow-util"
|
|
20
|
+
|
|
21
|
+
/* SpeechFlow node for GTCRN based noise suppression in audio-to-audio passing */
|
|
22
|
+
export default class SpeechFlowNodeA2AGTCRN extends SpeechFlowNode {
|
|
23
|
+
/* declare official node name */
|
|
24
|
+
public static name = "a2a-gtcrn"
|
|
25
|
+
|
|
26
|
+
/* internal state */
|
|
27
|
+
private closing = false
|
|
28
|
+
private worker: Worker | null = null
|
|
29
|
+
private resamplerDown: SpeexResampler | null = null
|
|
30
|
+
private resamplerUp: SpeexResampler | null = null
|
|
31
|
+
|
|
32
|
+
/* construct node */
|
|
33
|
+
constructor (id: string, cfg: { [ id: string ]: any }, opts: { [ id: string ]: any }, args: any[]) {
|
|
34
|
+
super(id, cfg, opts, args)
|
|
35
|
+
|
|
36
|
+
/* declare node configuration parameters */
|
|
37
|
+
this.configure({})
|
|
38
|
+
|
|
39
|
+
/* declare node input/output format */
|
|
40
|
+
this.input = "audio"
|
|
41
|
+
this.output = "audio"
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
/* open node */
|
|
45
|
+
async open () {
|
|
46
|
+
/* clear destruction flag */
|
|
47
|
+
this.closing = false
|
|
48
|
+
|
|
49
|
+
/* ensure GTCRN ONNX model is available */
|
|
50
|
+
const modelUrl = "https://github.com/k2-fsa/sherpa-onnx/" +
|
|
51
|
+
"releases/download/speech-enhancement-models/gtcrn_simple.onnx"
|
|
52
|
+
const modelDir = path.join(this.config.cacheDir, "gtcrn")
|
|
53
|
+
const modelPath = path.resolve(modelDir, "gtcrn_simple.onnx")
|
|
54
|
+
const stat = await fs.promises.stat(modelPath).catch(() => null)
|
|
55
|
+
if (stat === null) {
|
|
56
|
+
this.log("info", `GTCRN model downloading from "${modelUrl}"`)
|
|
57
|
+
await fs.promises.mkdir(modelDir, { recursive: true })
|
|
58
|
+
const response = await axios.get(modelUrl, {
|
|
59
|
+
responseType: "arraybuffer",
|
|
60
|
+
onDownloadProgress: (progressEvent) => {
|
|
61
|
+
if (progressEvent.total) {
|
|
62
|
+
const percent = (progressEvent.loaded / progressEvent.total) * 100
|
|
63
|
+
this.log("info", `GTCRN model download: ${percent.toFixed(1)}%`)
|
|
64
|
+
}
|
|
65
|
+
}
|
|
66
|
+
})
|
|
67
|
+
await fs.promises.writeFile(modelPath, Buffer.from(response.data))
|
|
68
|
+
this.log("info", `GTCRN model downloaded to "${modelPath}"`)
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
/* establish resamplers from SpeechFlow's internal 48KHz
|
|
72
|
+
to GTCRN's required 16KHz format and back */
|
|
73
|
+
this.resamplerDown = new SpeexResampler(1, this.config.audioSampleRate, 16000, 7)
|
|
74
|
+
this.resamplerUp = new SpeexResampler(1, 16000, this.config.audioSampleRate, 7)
|
|
75
|
+
|
|
76
|
+
/* initialize worker */
|
|
77
|
+
this.worker = new Worker(path.resolve(__dirname, "speechflow-node-a2a-gtcrn-wt.js"), {
|
|
78
|
+
workerData: { modelPath }
|
|
79
|
+
})
|
|
80
|
+
this.worker.on("error", (err) => {
|
|
81
|
+
this.log("error", `GTCRN worker thread error: ${err}`)
|
|
82
|
+
this.stream?.emit("error", err)
|
|
83
|
+
})
|
|
84
|
+
this.worker.on("exit", (code) => {
|
|
85
|
+
if (code !== 0)
|
|
86
|
+
this.log("error", `GTCRN worker thread exited with error code ${code}`)
|
|
87
|
+
else
|
|
88
|
+
this.log("info", `GTCRN worker thread exited with regular code ${code}`)
|
|
89
|
+
})
|
|
90
|
+
|
|
91
|
+
/* wait for worker to be ready */
|
|
92
|
+
await new Promise<void>((resolve, reject) => {
|
|
93
|
+
const timeout = setTimeout(() => {
|
|
94
|
+
reject(new Error("GTCRN worker thread initialization timeout"))
|
|
95
|
+
}, 60 * 1000)
|
|
96
|
+
const onMessage = (msg: any) => {
|
|
97
|
+
if (typeof msg === "object" && msg !== null && msg.type === "log")
|
|
98
|
+
this.log(msg.level, msg.message)
|
|
99
|
+
else if (typeof msg === "object" && msg !== null && msg.type === "ready") {
|
|
100
|
+
clearTimeout(timeout)
|
|
101
|
+
this.worker!.off("message", onMessage)
|
|
102
|
+
resolve()
|
|
103
|
+
}
|
|
104
|
+
else if (typeof msg === "object" && msg !== null && msg.type === "failed") {
|
|
105
|
+
clearTimeout(timeout)
|
|
106
|
+
this.worker!.off("message", onMessage)
|
|
107
|
+
reject(new Error(msg.message ?? "GTCRN worker thread initialization failed"))
|
|
108
|
+
}
|
|
109
|
+
}
|
|
110
|
+
this.worker!.on("message", onMessage)
|
|
111
|
+
this.worker!.once("error", (err) => {
|
|
112
|
+
clearTimeout(timeout)
|
|
113
|
+
reject(err)
|
|
114
|
+
})
|
|
115
|
+
})
|
|
116
|
+
|
|
117
|
+
/* receive message from worker */
|
|
118
|
+
const pending = new Map<string, (arr: Float32Array<ArrayBuffer>) => void>()
|
|
119
|
+
this.worker.on("exit", () => {
|
|
120
|
+
pending.clear()
|
|
121
|
+
})
|
|
122
|
+
this.worker.on("message", (msg: any) => {
|
|
123
|
+
if (typeof msg === "object" && msg !== null && msg.type === "process-done") {
|
|
124
|
+
const cb = pending.get(msg.id)
|
|
125
|
+
pending.delete(msg.id)
|
|
126
|
+
if (cb)
|
|
127
|
+
cb(msg.data)
|
|
128
|
+
else
|
|
129
|
+
this.log("warning", `GTCRN worker thread sent back unexpected id: ${msg.id}`)
|
|
130
|
+
}
|
|
131
|
+
else if (typeof msg === "object" && msg !== null && msg.type === "log")
|
|
132
|
+
this.log(msg.level, msg.message)
|
|
133
|
+
else
|
|
134
|
+
this.log("warning", `GTCRN worker thread sent unexpected message: ${JSON.stringify(msg)}`)
|
|
135
|
+
})
|
|
136
|
+
|
|
137
|
+
/* send message to worker */
|
|
138
|
+
let seq = 0
|
|
139
|
+
const workerProcess = async (samples: Float32Array<ArrayBuffer>) => {
|
|
140
|
+
if (this.closing)
|
|
141
|
+
return samples
|
|
142
|
+
const id = `${seq++}`
|
|
143
|
+
return new Promise<Float32Array<ArrayBuffer>>((resolve) => {
|
|
144
|
+
pending.set(id, (result) => { resolve(result) })
|
|
145
|
+
this.worker!.postMessage({ type: "process", id, samples }, [ samples.buffer ])
|
|
146
|
+
})
|
|
147
|
+
}
|
|
148
|
+
|
|
149
|
+
/* establish a transform stream */
|
|
150
|
+
const self = this
|
|
151
|
+
this.stream = new Stream.Transform({
|
|
152
|
+
readableObjectMode: true,
|
|
153
|
+
writableObjectMode: true,
|
|
154
|
+
decodeStrings: false,
|
|
155
|
+
transform (chunk: SpeechFlowChunk & { payload: Buffer }, encoding, callback) {
|
|
156
|
+
if (self.closing) {
|
|
157
|
+
callback(new Error("stream already destroyed"))
|
|
158
|
+
return
|
|
159
|
+
}
|
|
160
|
+
if (!Buffer.isBuffer(chunk.payload))
|
|
161
|
+
callback(new Error("invalid chunk payload type"))
|
|
162
|
+
else {
|
|
163
|
+
/* resample Buffer from 48KHz (SpeechFlow) to 16KHz (GTCRN) */
|
|
164
|
+
const resampledDown = self.resamplerDown!.processChunk(chunk.payload)
|
|
165
|
+
|
|
166
|
+
/* convert Buffer into Float32Array */
|
|
167
|
+
const payload = util.convertBufToF32(resampledDown)
|
|
168
|
+
|
|
169
|
+
/* process with GTCRN */
|
|
170
|
+
workerProcess(payload).then((result: Float32Array<ArrayBuffer>) => {
|
|
171
|
+
/* convert Float32Array into Buffer */
|
|
172
|
+
const buf = util.convertF32ToBuf(result)
|
|
173
|
+
|
|
174
|
+
/* resample Buffer from 16KHz (GTCRN) back to 48KHz (SpeechFlow) */
|
|
175
|
+
const resampledUp = self.resamplerUp!.processChunk(buf)
|
|
176
|
+
|
|
177
|
+
/* update chunk */
|
|
178
|
+
chunk.payload = resampledUp
|
|
179
|
+
|
|
180
|
+
/* forward updated chunk */
|
|
181
|
+
this.push(chunk)
|
|
182
|
+
callback()
|
|
183
|
+
}).catch((err: unknown) => {
|
|
184
|
+
const error = util.ensureError(err)
|
|
185
|
+
self.log("warning", `processing of chunk failed: ${error.message}`)
|
|
186
|
+
callback(error)
|
|
187
|
+
})
|
|
188
|
+
}
|
|
189
|
+
},
|
|
190
|
+
final (callback) {
|
|
191
|
+
callback()
|
|
192
|
+
}
|
|
193
|
+
})
|
|
194
|
+
}
|
|
195
|
+
|
|
196
|
+
/* close node */
|
|
197
|
+
async close () {
|
|
198
|
+
/* indicate closing */
|
|
199
|
+
this.closing = true
|
|
200
|
+
|
|
201
|
+
/* shutdown worker */
|
|
202
|
+
if (this.worker !== null) {
|
|
203
|
+
this.worker.terminate()
|
|
204
|
+
this.worker = null
|
|
205
|
+
}
|
|
206
|
+
|
|
207
|
+
/* shutdown stream */
|
|
208
|
+
if (this.stream !== null) {
|
|
209
|
+
await util.destroyStream(this.stream)
|
|
210
|
+
this.stream = null
|
|
211
|
+
}
|
|
212
|
+
|
|
213
|
+
/* destroy resamplers */
|
|
214
|
+
if (this.resamplerDown !== null)
|
|
215
|
+
this.resamplerDown = null
|
|
216
|
+
if (this.resamplerUp !== null)
|
|
217
|
+
this.resamplerUp = null
|
|
218
|
+
}
|
|
219
|
+
}
|