speechflow 2.4.0 → 2.4.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.ase/service.log +531 -0
- package/.ase/service.yaml +1 -0
- package/.ase/task/TASK-flush.md +44 -0
- package/CHANGELOG.md +33 -2
- package/README.md +10 -9
- package/etc/speechflow.yaml +1 -1
- package/etc/stx.conf +21 -21
- package/package.json +9 -5
- package/speechflow-cli/dst/speechflow-main-api.js +8 -1
- package/speechflow-cli/dst/speechflow-main-api.js.map +1 -1
- package/speechflow-cli/dst/speechflow-main-cli.js +2 -2
- package/speechflow-cli/dst/speechflow-main-cli.js.map +1 -1
- package/speechflow-cli/dst/speechflow-node-a2a-ffmpeg.js +2 -0
- package/speechflow-cli/dst/speechflow-node-a2a-ffmpeg.js.map +1 -1
- package/speechflow-cli/dst/speechflow-node-a2a-gender.js +2 -1
- package/speechflow-cli/dst/speechflow-node-a2a-gender.js.map +1 -1
- package/speechflow-cli/dst/speechflow-node-a2a-meter.js +28 -12
- package/speechflow-cli/dst/speechflow-node-a2a-meter.js.map +1 -1
- package/speechflow-cli/dst/speechflow-node-a2a-mute.js +5 -1
- package/speechflow-cli/dst/speechflow-node-a2a-mute.js.map +1 -1
- package/speechflow-cli/dst/speechflow-node-a2a-pitch.js +9 -15
- package/speechflow-cli/dst/speechflow-node-a2a-pitch.js.map +1 -1
- package/speechflow-cli/dst/speechflow-node-a2t-deepgram.d.ts +3 -0
- package/speechflow-cli/dst/speechflow-node-a2t-deepgram.js +149 -101
- package/speechflow-cli/dst/speechflow-node-a2t-deepgram.js.map +1 -1
- package/speechflow-cli/dst/speechflow-node-t2t-google.js +3 -0
- package/speechflow-cli/dst/speechflow-node-t2t-google.js.map +1 -1
- package/speechflow-cli/dst/speechflow-node-t2t-opus.js.map +1 -1
- package/speechflow-cli/dst/speechflow-node-t2t-proofread.js +17 -9
- package/speechflow-cli/dst/speechflow-node-t2t-proofread.js.map +1 -1
- package/speechflow-cli/dst/speechflow-node-t2t-sentence.js +20 -3
- package/speechflow-cli/dst/speechflow-node-t2t-sentence.js.map +1 -1
- package/speechflow-cli/dst/speechflow-node-xio-exec.js +6 -4
- package/speechflow-cli/dst/speechflow-node-xio-exec.js.map +1 -1
- package/speechflow-cli/dst/speechflow-util-llm.js +9 -2
- package/speechflow-cli/dst/speechflow-util-llm.js.map +1 -1
- package/speechflow-cli/etc/oxlint.jsonc +5 -0
- package/speechflow-cli/etc/stx.conf +6 -6
- package/speechflow-cli/package.json +53 -43
- package/speechflow-cli/src/speechflow-main-api.ts +8 -1
- package/speechflow-cli/src/speechflow-main-cli.ts +1 -1
- package/speechflow-cli/src/speechflow-node-a2a-ffmpeg.ts +3 -1
- package/speechflow-cli/src/speechflow-node-a2a-gender.ts +3 -2
- package/speechflow-cli/src/speechflow-node-a2a-meter.ts +30 -13
- package/speechflow-cli/src/speechflow-node-a2a-mute.ts +6 -1
- package/speechflow-cli/src/speechflow-node-a2a-pitch.ts +12 -20
- package/speechflow-cli/src/speechflow-node-a2t-deepgram.ts +162 -108
- package/speechflow-cli/src/speechflow-node-t2t-google.ts +3 -0
- package/speechflow-cli/src/speechflow-node-t2t-opus.ts +1 -1
- package/speechflow-cli/src/speechflow-node-t2t-proofread.ts +17 -9
- package/speechflow-cli/src/speechflow-node-t2t-sentence.ts +21 -3
- package/speechflow-cli/src/speechflow-node-xio-exec.ts +7 -5
- package/speechflow-cli/src/speechflow-util-llm.ts +11 -3
- package/speechflow-ui-db/dst/app-font-fa-brands-400.woff2 +0 -0
- package/speechflow-ui-db/dst/app-font-fa-regular-400.woff2 +0 -0
- package/speechflow-ui-db/dst/app-font-fa-solid-900.woff2 +0 -0
- package/speechflow-ui-db/dst/app-font-fa-v4compatibility.woff2 +0 -0
- package/speechflow-ui-db/dst/index.css +1 -1
- package/speechflow-ui-db/dst/index.js +579 -57
- package/speechflow-ui-db/etc/stx.conf +5 -4
- package/speechflow-ui-db/etc/vite-client.mts +1 -1
- package/speechflow-ui-db/package.json +27 -22
- package/speechflow-ui-st/dst/app-font-fa-brands-400.woff2 +0 -0
- package/speechflow-ui-st/dst/app-font-fa-regular-400.woff2 +0 -0
- package/speechflow-ui-st/dst/app-font-fa-solid-900.woff2 +0 -0
- package/speechflow-ui-st/dst/app-font-fa-v4compatibility.woff2 +0 -0
- package/speechflow-ui-st/dst/index.css +1 -1
- package/speechflow-ui-st/dst/index.js +593 -71
- package/speechflow-ui-st/etc/stx.conf +5 -4
- package/speechflow-ui-st/etc/vite-client.mts +6 -0
- package/speechflow-ui-st/package.json +27 -22
- package/speechflow-ui-st/src/app.vue +3 -3
- package/speechflow-cli/package.d/sherpa-onnx+1.12.25.patch +0 -12
- /package/speechflow-cli/package.d/{@typescript-eslint+typescript-estree+8.59.3.patch → @typescript-eslint+typescript-estree+8.64.0.patch} +0 -0
- /package/speechflow-ui-db/package.d/{@typescript-eslint+typescript-estree+8.59.3.patch → @typescript-eslint+typescript-estree+8.64.0.patch} +0 -0
- /package/speechflow-ui-st/package.d/{@typescript-eslint+typescript-estree+8.59.3.patch → @typescript-eslint+typescript-estree+8.64.0.patch} +0 -0
|
@@ -93,10 +93,12 @@ export default class SpeechFlowNodeA2AFFMPEG extends SpeechFlowNode {
|
|
|
93
93
|
util.run("starting FFmpeg process", () => this.ffmpeg!.run())
|
|
94
94
|
|
|
95
95
|
/* establish a duplex stream and connect it to FFmpeg */
|
|
96
|
+
/* NOTICE: the object form of Stream.Duplex.from() accepts Node streams at
|
|
97
|
+
run-time, but @types/node types it for Web streams only, so we have to cast */
|
|
96
98
|
const ffmpegStream = Stream.Duplex.from({
|
|
97
99
|
writable: streamInput,
|
|
98
100
|
readable: streamOutput
|
|
99
|
-
})
|
|
101
|
+
} as unknown as Parameters<typeof Stream.Duplex.from>[0])
|
|
100
102
|
|
|
101
103
|
/* wrap streams with conversions for chunk vs plain audio */
|
|
102
104
|
const wrapper1 = util.createTransformStreamForWritableSide("audio", 1)
|
|
@@ -11,7 +11,7 @@ import Stream from "node:stream"
|
|
|
11
11
|
/* external dependencies */
|
|
12
12
|
import * as Transformers from "@huggingface/transformers"
|
|
13
13
|
import { WaveFile } from "wavefile"
|
|
14
|
-
import {
|
|
14
|
+
import { analyze, AudioData } from "audio-inspect"
|
|
15
15
|
import { Duration } from "luxon"
|
|
16
16
|
|
|
17
17
|
/* internal dependencies */
|
|
@@ -145,7 +145,8 @@ export default class SpeechFlowNodeA2AGender extends SpeechFlowNode {
|
|
|
145
145
|
duration: data.length / sampleRateTarget,
|
|
146
146
|
length: data.length
|
|
147
147
|
} satisfies AudioData
|
|
148
|
-
const
|
|
148
|
+
const analysis = await analyze(audioData, { features: { rms: { asDB: true } } })
|
|
149
|
+
const rms = analysis.results.rms ?? -60
|
|
149
150
|
if (rms < this.params.volumeThreshold)
|
|
150
151
|
return genderLast
|
|
151
152
|
|
|
@@ -8,7 +8,7 @@
|
|
|
8
8
|
import Stream from "node:stream"
|
|
9
9
|
|
|
10
10
|
/* external dependencies */
|
|
11
|
-
import {
|
|
11
|
+
import { analyze, AudioData } from "audio-inspect"
|
|
12
12
|
|
|
13
13
|
/* internal dependencies */
|
|
14
14
|
import SpeechFlowNode, { SpeechFlowChunk } from "./speechflow-node"
|
|
@@ -69,8 +69,8 @@ export default class SpeechFlowNodeA2AMeter extends SpeechFlowNode {
|
|
|
69
69
|
const samplesPerChunk = Math.floor(this.config.audioSampleRate * chunkDuration)
|
|
70
70
|
this.chunkBuffer = new Float32Array(0)
|
|
71
71
|
|
|
72
|
-
/*
|
|
73
|
-
|
|
72
|
+
/* calculate the loudness metrics of the accumulated chunk data */
|
|
73
|
+
const calcMeter = async () => {
|
|
74
74
|
/* short-circuit during destruction */
|
|
75
75
|
if (this.closing)
|
|
76
76
|
return
|
|
@@ -99,14 +99,19 @@ export default class SpeechFlowNodeA2AMeter extends SpeechFlowNode {
|
|
|
99
99
|
duration: sampleWindowDuration,
|
|
100
100
|
length: sampleWindow.length
|
|
101
101
|
} satisfies AudioData
|
|
102
|
-
const
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
102
|
+
const analysisLUFS = await analyze(audioDataLUFS, {
|
|
103
|
+
features: {
|
|
104
|
+
lufs: {
|
|
105
|
+
channelMode: this.config.audioChannels === 1 ? "mono" : "stereo",
|
|
106
|
+
calculateShortTerm: false,
|
|
107
|
+
calculateMomentary: true,
|
|
108
|
+
calculateLoudnessRange: false,
|
|
109
|
+
calculateTruePeak: false
|
|
110
|
+
}
|
|
111
|
+
}
|
|
108
112
|
})
|
|
109
|
-
|
|
113
|
+
const lufs = analysisLUFS.results.lufs
|
|
114
|
+
lufsm = lufs?.momentary !== undefined ? Math.max(-60, lufs.momentary) : -60
|
|
110
115
|
|
|
111
116
|
/* calculate the RMS metric */
|
|
112
117
|
const totalSamples = chunkData.length / this.config.audioChannels
|
|
@@ -118,9 +123,14 @@ export default class SpeechFlowNodeA2AMeter extends SpeechFlowNode {
|
|
|
118
123
|
duration,
|
|
119
124
|
length: chunkData.length
|
|
120
125
|
} satisfies AudioData
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
126
|
+
const analysisRMS = await analyze(audioDataRMS, {
|
|
127
|
+
features: {
|
|
128
|
+
rms: {
|
|
129
|
+
asDB: true
|
|
130
|
+
}
|
|
131
|
+
}
|
|
132
|
+
})
|
|
133
|
+
rms = Math.max(-60, analysisRMS.results.rms ?? -60)
|
|
124
134
|
|
|
125
135
|
/* automatically clear measurement (in case no new measurements happen) */
|
|
126
136
|
if (this.silenceTimer !== null)
|
|
@@ -129,6 +139,13 @@ export default class SpeechFlowNodeA2AMeter extends SpeechFlowNode {
|
|
|
129
139
|
lufsm = -60
|
|
130
140
|
rms = -60
|
|
131
141
|
}, 500)
|
|
142
|
+
}
|
|
143
|
+
|
|
144
|
+
/* setup chunking interval */
|
|
145
|
+
this.calcInterval = setInterval(() => {
|
|
146
|
+
calcMeter().catch((err: unknown) => {
|
|
147
|
+
this.log("warning", `meter calculation failed: ${err instanceof Error ? err.message : String(err)}`)
|
|
148
|
+
})
|
|
132
149
|
}, chunkDuration * 1000)
|
|
133
150
|
|
|
134
151
|
/* setup loudness emitting interval */
|
|
@@ -31,7 +31,9 @@ export default class SpeechFlowNodeA2AMute extends SpeechFlowNode {
|
|
|
31
31
|
super(id, cfg, opts, args)
|
|
32
32
|
|
|
33
33
|
/* declare node configuration parameters */
|
|
34
|
-
this.configure({
|
|
34
|
+
this.configure({
|
|
35
|
+
muteMode: { type: "string", val: "none", pos: 0, match: /^(?:none|silenced|unplugged)$/ }
|
|
36
|
+
})
|
|
35
37
|
|
|
36
38
|
/* declare node input/output format */
|
|
37
39
|
this.input = "audio"
|
|
@@ -75,6 +77,9 @@ export default class SpeechFlowNodeA2AMute extends SpeechFlowNode {
|
|
|
75
77
|
/* clear destruction flag */
|
|
76
78
|
this.closing = false
|
|
77
79
|
|
|
80
|
+
/* determine initial mute mode */
|
|
81
|
+
this.muteMode = this.params.muteMode as MuteMode
|
|
82
|
+
|
|
78
83
|
/* establish a transform stream */
|
|
79
84
|
const self = this
|
|
80
85
|
this.stream = new Stream.Transform({
|
|
@@ -18,7 +18,6 @@ import * as util from "./speechflow-util"
|
|
|
18
18
|
/* parameter configuration */
|
|
19
19
|
type AudioPitchShifterConfig = {
|
|
20
20
|
rate?: number
|
|
21
|
-
tempo?: number
|
|
22
21
|
pitch?: number
|
|
23
22
|
semitones?: number
|
|
24
23
|
}
|
|
@@ -38,7 +37,6 @@ class AudioPitchShifter extends util.WebAudio {
|
|
|
38
37
|
super(sampleRate, channels)
|
|
39
38
|
this.config = {
|
|
40
39
|
rate: config.rate ?? 1.0,
|
|
41
|
-
tempo: config.tempo ?? 1.0,
|
|
42
40
|
pitch: config.pitch ?? 1.0,
|
|
43
41
|
semitones: config.semitones ?? 0.0
|
|
44
42
|
}
|
|
@@ -50,10 +48,11 @@ class AudioPitchShifter extends util.WebAudio {
|
|
|
50
48
|
|
|
51
49
|
/* add SoundTouch worklet module */
|
|
52
50
|
const packagePath = path.join(__dirname, "../node_modules/@soundtouchjs/audio-worklet")
|
|
53
|
-
const workletPath = path.join(packagePath, "dist/soundtouch-
|
|
51
|
+
const workletPath = path.join(packagePath, ".dist/soundtouch-processor.js")
|
|
54
52
|
await this.audioContext.audioWorklet.addModule(workletPath)
|
|
55
53
|
|
|
56
|
-
/* create SoundTouch worklet node
|
|
54
|
+
/* create SoundTouch worklet node (the processor always operates on
|
|
55
|
+
stereo internally, but mixes down to the requested channel count) */
|
|
57
56
|
this.pitchNode = new AudioWorkletNode(this.audioContext, "soundtouch-processor", {
|
|
58
57
|
numberOfInputs: 1,
|
|
59
58
|
numberOfOutputs: 1,
|
|
@@ -62,10 +61,9 @@ class AudioPitchShifter extends util.WebAudio {
|
|
|
62
61
|
|
|
63
62
|
/* set initial parameter values */
|
|
64
63
|
const params = this.pitchNode.parameters as Map<string, AudioParam>
|
|
65
|
-
params.get("
|
|
66
|
-
params.get("
|
|
67
|
-
params.get("
|
|
68
|
-
params.get("pitchSemitones")!.value = this.config.semitones
|
|
64
|
+
params.get("playbackRate")!.value = this.config.rate
|
|
65
|
+
params.get("pitch")!.value = this.config.pitch
|
|
66
|
+
params.get("pitchSemitones")!.value = this.config.semitones
|
|
69
67
|
|
|
70
68
|
/* connect nodes: source -> pitch -> capture */
|
|
71
69
|
this.sourceNode!.connect(this.pitchNode)
|
|
@@ -83,14 +81,10 @@ class AudioPitchShifter extends util.WebAudio {
|
|
|
83
81
|
this.config[configField] = value
|
|
84
82
|
}
|
|
85
83
|
|
|
86
|
-
/* update rate value
|
|
84
|
+
/* update rate value (compensates the pitch for an upstream playback
|
|
85
|
+
rate change, i.e. the resulting pitch is divided by this value) */
|
|
87
86
|
public setRate (rate: number): void {
|
|
88
|
-
this.updateParameter("
|
|
89
|
-
}
|
|
90
|
-
|
|
91
|
-
/* update tempo value */
|
|
92
|
-
public setTempo (tempo: number): void {
|
|
93
|
-
this.updateParameter("tempo", tempo, "tempo")
|
|
87
|
+
this.updateParameter("playbackRate", rate, "rate")
|
|
94
88
|
}
|
|
95
89
|
|
|
96
90
|
/* update pitch shift value */
|
|
@@ -131,10 +125,9 @@ export default class SpeechFlowNodeA2APitch extends SpeechFlowNode {
|
|
|
131
125
|
|
|
132
126
|
/* declare node configuration parameters */
|
|
133
127
|
this.configure({
|
|
134
|
-
rate: { type: "number", val: 1.0, match: (n: number) => n >= 0.
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
semitones: { type: "number", val: 0.0, match: (n: number) => n >= -24 && n <= 24 }
|
|
128
|
+
rate: { type: "number", val: 1.0, match: (n: number) => n >= 0.1 && n <= 8.0 },
|
|
129
|
+
pitch: { type: "number", val: 1.0, match: (n: number) => n >= 0.1 && n <= 8.0 },
|
|
130
|
+
semitones: { type: "number", val: 0.0, match: (n: number) => n >= -24 && n <= 24 }
|
|
138
131
|
})
|
|
139
132
|
|
|
140
133
|
/* declare node input/output format */
|
|
@@ -152,7 +145,6 @@ export default class SpeechFlowNodeA2APitch extends SpeechFlowNode {
|
|
|
152
145
|
this.config.audioSampleRate,
|
|
153
146
|
this.config.audioChannels, {
|
|
154
147
|
rate: this.params.rate,
|
|
155
|
-
tempo: this.params.tempo,
|
|
156
148
|
pitch: this.params.pitch,
|
|
157
149
|
semitones: this.params.semitones
|
|
158
150
|
}
|
|
@@ -9,25 +9,33 @@ import Stream from "node:stream"
|
|
|
9
9
|
|
|
10
10
|
/* external dependencies */
|
|
11
11
|
import * as Deepgram from "@deepgram/sdk"
|
|
12
|
+
import { V1Socket } from "@deepgram/sdk/listen/v1"
|
|
12
13
|
import { DateTime, Duration } from "luxon"
|
|
13
14
|
|
|
14
15
|
/* internal dependencies */
|
|
15
16
|
import SpeechFlowNode, { SpeechFlowChunk } from "./speechflow-node"
|
|
16
17
|
import * as util from "./speechflow-util"
|
|
17
18
|
|
|
19
|
+
/* NOTICE: the connection arguments type is not publicly re-exported by the
|
|
20
|
+
Deepgram SDK, so derive it structurally from the connect method instead */
|
|
21
|
+
type DeepgramConnectArgs = Parameters<Deepgram.DeepgramClient["listen"]["v1"]["connect"]>[0]
|
|
22
|
+
|
|
18
23
|
/* SpeechFlow node for Deepgram speech-to-text conversion */
|
|
19
24
|
export default class SpeechFlowNodeA2TDeepgram extends SpeechFlowNode {
|
|
20
25
|
/* declare official node name */
|
|
21
26
|
public static name = "a2t-deepgram"
|
|
22
27
|
|
|
23
28
|
/* internal state */
|
|
24
|
-
private dg:
|
|
29
|
+
private dg: V1Socket | null = null
|
|
25
30
|
private closing = false
|
|
26
31
|
private reconfiguring = false
|
|
27
32
|
private connectionTimeout: ReturnType<typeof setTimeout> | null = null
|
|
28
33
|
private queue: util.AsyncQueue<SpeechFlowChunk | null> | null = null
|
|
29
34
|
private metastore: util.TimeStore<Map<string, any>> | null = null
|
|
30
35
|
private suspended = false
|
|
36
|
+
private opening = false
|
|
37
|
+
private openReject: ((error: Error) => void) | null = null
|
|
38
|
+
private lastChunk: SpeechFlowChunk | null = null
|
|
31
39
|
|
|
32
40
|
/* construct node */
|
|
33
41
|
constructor (id: string, cfg: { [ id: string ]: any }, opts: { [ id: string ]: any }, args: any[]) {
|
|
@@ -85,9 +93,9 @@ export default class SpeechFlowNodeA2TDeepgram extends SpeechFlowNode {
|
|
|
85
93
|
if (suspended === this.suspended)
|
|
86
94
|
return
|
|
87
95
|
this.log("info", `switching to ${suspended ? "SUSPENDED" : "UNSUSPENDED"} operation`)
|
|
88
|
-
this.suspended = suspended
|
|
89
96
|
if (suspended) {
|
|
90
97
|
/* going suspended -- tear down Deepgram API connection */
|
|
98
|
+
this.suspended = true
|
|
91
99
|
this.reconfiguring = true
|
|
92
100
|
try {
|
|
93
101
|
await this.closeConnection()
|
|
@@ -95,10 +103,24 @@ export default class SpeechFlowNodeA2TDeepgram extends SpeechFlowNode {
|
|
|
95
103
|
finally {
|
|
96
104
|
this.reconfiguring = false
|
|
97
105
|
}
|
|
106
|
+
|
|
107
|
+
/* re-emit last intermediate chunk as final one, as its regular
|
|
108
|
+
finalization will never arrive anymore -- with an appended
|
|
109
|
+
ellipsis as truncation marker (and sentence boundary for
|
|
110
|
+
downstream sentence splitting) */
|
|
111
|
+
if (this.lastChunk !== null && this.lastChunk.kind === "intermediate" && this.queue !== null) {
|
|
112
|
+
this.log("info", "sending last non-final chunk as final chunk")
|
|
113
|
+
const chunk = this.lastChunk.clone()
|
|
114
|
+
chunk.kind = "final"
|
|
115
|
+
chunk.payload = `${chunk.payload}…`
|
|
116
|
+
this.queue.write(chunk)
|
|
117
|
+
this.lastChunk = chunk
|
|
118
|
+
}
|
|
98
119
|
}
|
|
99
120
|
else {
|
|
100
121
|
/* going unsuspended -- re-establish Deepgram API connection */
|
|
101
122
|
await this.openConnection()
|
|
123
|
+
this.suspended = false
|
|
102
124
|
}
|
|
103
125
|
}
|
|
104
126
|
|
|
@@ -107,23 +129,32 @@ export default class SpeechFlowNodeA2TDeepgram extends SpeechFlowNode {
|
|
|
107
129
|
/* configure Deepgram connection options */
|
|
108
130
|
const interim = this.params.interim as boolean
|
|
109
131
|
const endpointing = this.params.endpointing as number
|
|
110
|
-
|
|
132
|
+
/* NOTICE: the Deepgram API v5 SDK expects the boolean flags as their
|
|
133
|
+
string representations, as they are passed as URL query parameters.
|
|
134
|
+
The "filler_words" flag is a valid live API parameter, but is not
|
|
135
|
+
covered by the generated SDK types, so pass it via "queryParams". */
|
|
136
|
+
const options: DeepgramConnectArgs = {
|
|
137
|
+
/* NOTICE: the Deepgram API v5 SDK applies the client-level "apiKey" only
|
|
138
|
+
to its REST endpoints -- the WebSocket connect path never consults the
|
|
139
|
+
authentication provider and instead requires the "Authorization" header
|
|
140
|
+
to be passed explicitly, or else the server closes the handshake */
|
|
141
|
+
Authorization: `Token ${this.params.key}`,
|
|
111
142
|
mip_opt_out: true,
|
|
112
143
|
model: this.params.model,
|
|
113
144
|
version: this.params.version,
|
|
114
145
|
channels: this.config.audioChannels,
|
|
115
146
|
sample_rate: this.config.audioSampleRate,
|
|
116
147
|
encoding: "linear16",
|
|
117
|
-
multichannel: false,
|
|
118
|
-
endpointing: endpointing > 0 ? endpointing : false,
|
|
119
|
-
interim_results: interim,
|
|
120
|
-
smart_format: false,
|
|
121
|
-
punctuate: true,
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
148
|
+
multichannel: "false",
|
|
149
|
+
endpointing: endpointing > 0 ? endpointing : "false",
|
|
150
|
+
interim_results: interim ? "true" : "false",
|
|
151
|
+
smart_format: "false",
|
|
152
|
+
punctuate: "true",
|
|
153
|
+
numerals: "false",
|
|
154
|
+
diarize: "false",
|
|
155
|
+
profanity_filter: "false",
|
|
156
|
+
redact: "false",
|
|
157
|
+
queryParams: { filler_words: "true" }
|
|
127
158
|
}
|
|
128
159
|
const model = this.params.model as string
|
|
129
160
|
const language = this.params.language as string
|
|
@@ -149,52 +180,52 @@ export default class SpeechFlowNodeA2TDeepgram extends SpeechFlowNode {
|
|
|
149
180
|
}
|
|
150
181
|
|
|
151
182
|
/* connect to Deepgram API */
|
|
152
|
-
const deepgram = Deepgram.
|
|
153
|
-
this.dg = deepgram.listen.
|
|
183
|
+
const deepgram = new Deepgram.DeepgramClient({ apiKey: this.params.key })
|
|
184
|
+
this.dg = await deepgram.listen.v1.connect(options)
|
|
154
185
|
|
|
155
|
-
/* hook onto Deepgram API
|
|
156
|
-
this.dg.on(
|
|
186
|
+
/* hook onto Deepgram API messages */
|
|
187
|
+
this.dg.on("message", (message: V1Socket.Response) => {
|
|
157
188
|
if (this.closing || this.queue === null || this.metastore === null)
|
|
158
189
|
return
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
if (
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
const
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
const
|
|
181
|
-
const
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
190
|
+
if (message.type === "SpeechStarted")
|
|
191
|
+
this.log("info", "speech started", message)
|
|
192
|
+
else if (message.type === "UtteranceEnd")
|
|
193
|
+
this.log("info", "utterance end received", message)
|
|
194
|
+
else if (message.type === "Metadata")
|
|
195
|
+
this.log("info", "metadata received")
|
|
196
|
+
else if (message.type === "Results") {
|
|
197
|
+
const data = message
|
|
198
|
+
const text = data.channel?.alternatives[0]?.transcript ?? ""
|
|
199
|
+
const words = data.channel?.alternatives[0]?.words ?? []
|
|
200
|
+
const isFinal = data.is_final ?? false
|
|
201
|
+
const speechFinal = data.speech_final ?? false
|
|
202
|
+
const kind = (isFinal || ((this.params.endpointing as number) > 0 && speechFinal)) ? "final" : "intermediate"
|
|
203
|
+
if (text === "")
|
|
204
|
+
this.log("info", `empty/dummy text received (start: ${data.start}s, duration: ${data.duration.toFixed(2)}s)`)
|
|
205
|
+
else {
|
|
206
|
+
this.log("info", `text received (start: ${data.start}s, ` +
|
|
207
|
+
`duration: ${data.duration.toFixed(2)}s, kind: ${kind}): ` +
|
|
208
|
+
`"${text}"`)
|
|
209
|
+
const start = Duration.fromMillis(data.start * 1000).plus(this.timeZeroOffset)
|
|
210
|
+
const end = start.plus({ seconds: data.duration })
|
|
211
|
+
const metas = this.metastore.fetch(start, end)
|
|
212
|
+
const meta = metas.toReversed().reduce((prev: Map<string, any>, curr: Map<string, any>) => {
|
|
213
|
+
curr.forEach((val, key) => { prev.set(key, val) })
|
|
214
|
+
return prev
|
|
215
|
+
}, new Map<string, any>())
|
|
216
|
+
this.metastore.prune(start)
|
|
217
|
+
meta.set("words", words.map((word) => {
|
|
218
|
+
const wordStart = Duration.fromMillis(word.start * 1000).plus(this.timeZeroOffset)
|
|
219
|
+
const wordEnd = Duration.fromMillis(word.end * 1000).plus(this.timeZeroOffset)
|
|
220
|
+
return { word: word.punctuated_word ?? word.word, start: wordStart, end: wordEnd }
|
|
221
|
+
}))
|
|
222
|
+
const chunk = new SpeechFlowChunk(start, end, kind, "text", text, meta)
|
|
223
|
+
this.queue.write(chunk)
|
|
224
|
+
this.lastChunk = chunk
|
|
225
|
+
}
|
|
186
226
|
}
|
|
187
227
|
})
|
|
188
|
-
this.dg.on(
|
|
189
|
-
this.log("info", "speech started", data)
|
|
190
|
-
})
|
|
191
|
-
this.dg.on(Deepgram.LiveTranscriptionEvents.UtteranceEnd, (data) => {
|
|
192
|
-
this.log("info", "utterance end received", data)
|
|
193
|
-
})
|
|
194
|
-
this.dg.on(Deepgram.LiveTranscriptionEvents.Metadata, (data) => {
|
|
195
|
-
this.log("info", "metadata received")
|
|
196
|
-
})
|
|
197
|
-
this.dg.on(Deepgram.LiveTranscriptionEvents.Close, () => {
|
|
228
|
+
this.dg.on("close", () => {
|
|
198
229
|
this.log("info", "connection close")
|
|
199
230
|
/* NOTICE: suppress EOF signalling while reconfiguring (mute toggle),
|
|
200
231
|
since the connection is being torn down deliberately and the
|
|
@@ -203,36 +234,18 @@ export default class SpeechFlowNodeA2TDeepgram extends SpeechFlowNode {
|
|
|
203
234
|
this.queue.write(null)
|
|
204
235
|
})
|
|
205
236
|
|
|
206
|
-
/*
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
this.connectionTimeout = null
|
|
219
|
-
}
|
|
220
|
-
resolve(true)
|
|
221
|
-
})
|
|
222
|
-
this.dg!.once(Deepgram.LiveTranscriptionEvents.Error, (err: Error) => {
|
|
223
|
-
if (this.connectionTimeout !== null) {
|
|
224
|
-
clearTimeout(this.connectionTimeout)
|
|
225
|
-
this.connectionTimeout = null
|
|
226
|
-
}
|
|
227
|
-
reject(err)
|
|
228
|
-
})
|
|
229
|
-
})
|
|
230
|
-
|
|
231
|
-
/* NOTICE: register permanent Error handler only AFTER the open
|
|
232
|
-
handshake -- during open, the transient .once above is the sole
|
|
233
|
-
Error listener so the caller's promise rejects without a parallel
|
|
234
|
-
stream emission tearing down the graph prematurely. */
|
|
235
|
-
this.dg.on(Deepgram.LiveTranscriptionEvents.Error, (error: Error) => {
|
|
237
|
+
/* NOTICE: the Deepgram API socket supports only a single handler per
|
|
238
|
+
event, so the open handshake and the permanent error handling must
|
|
239
|
+
share one error handler which dispatches on the "opening" flag:
|
|
240
|
+
during open, the error is routed solely into the caller's promise
|
|
241
|
+
rejection, without a parallel stream emission tearing down the
|
|
242
|
+
graph prematurely. */
|
|
243
|
+
this.dg.on("error", (error: Error) => {
|
|
244
|
+
if (this.opening) {
|
|
245
|
+
if (this.openReject !== null)
|
|
246
|
+
this.openReject(error)
|
|
247
|
+
return
|
|
248
|
+
}
|
|
236
249
|
this.log("warning", `error: ${error.message}`)
|
|
237
250
|
/* NOTICE: do not write null to the queue here -- a transient error
|
|
238
251
|
must not be misinterpreted as end-of-stream by downstream nodes;
|
|
@@ -244,6 +257,53 @@ export default class SpeechFlowNodeA2TDeepgram extends SpeechFlowNode {
|
|
|
244
257
|
if (!this.closing && this.stream !== null)
|
|
245
258
|
this.stream.emit("error", error)
|
|
246
259
|
})
|
|
260
|
+
|
|
261
|
+
/* NOTICE: the Deepgram API v5 SDK returns a socket which has not yet
|
|
262
|
+
dialed, as it is internally created with the "startClosed" option,
|
|
263
|
+
so the connection has to be established explicitly -- and only once
|
|
264
|
+
all event handlers above are registered, as the connect operation
|
|
265
|
+
re-registers the underlying socket listeners */
|
|
266
|
+
this.dg.connect()
|
|
267
|
+
|
|
268
|
+
/* clear pending connection timer (idempotent) */
|
|
269
|
+
const clearConnectionTimeout = () => {
|
|
270
|
+
if (this.connectionTimeout !== null) {
|
|
271
|
+
clearTimeout(this.connectionTimeout)
|
|
272
|
+
this.connectionTimeout = null
|
|
273
|
+
}
|
|
274
|
+
}
|
|
275
|
+
|
|
276
|
+
/* wait for Deepgram API to be available */
|
|
277
|
+
this.opening = true
|
|
278
|
+
let opened = false
|
|
279
|
+
try {
|
|
280
|
+
await new Promise((resolve, reject) => {
|
|
281
|
+
this.openReject = reject
|
|
282
|
+
this.connectionTimeout = setTimeout(() => {
|
|
283
|
+
if (this.connectionTimeout !== null) {
|
|
284
|
+
this.connectionTimeout = null
|
|
285
|
+
reject(new Error("Deepgram: timeout waiting for connection open"))
|
|
286
|
+
}
|
|
287
|
+
}, 8000)
|
|
288
|
+
this.dg!.waitForOpen().then(() => {
|
|
289
|
+
this.log("info", "connection open")
|
|
290
|
+
clearConnectionTimeout()
|
|
291
|
+
resolve(true)
|
|
292
|
+
}).catch((error: unknown) => {
|
|
293
|
+
clearConnectionTimeout()
|
|
294
|
+
reject(util.ensureError(error, "failed to open Deepgram connection"))
|
|
295
|
+
})
|
|
296
|
+
})
|
|
297
|
+
opened = true
|
|
298
|
+
}
|
|
299
|
+
finally {
|
|
300
|
+
this.opening = false
|
|
301
|
+
this.openReject = null
|
|
302
|
+
|
|
303
|
+
/* discard half-established connection on failure */
|
|
304
|
+
if (!opened)
|
|
305
|
+
await this.closeConnection()
|
|
306
|
+
}
|
|
247
307
|
}
|
|
248
308
|
|
|
249
309
|
/* close Deepgram API connection */
|
|
@@ -257,8 +317,7 @@ export default class SpeechFlowNodeA2TDeepgram extends SpeechFlowNode {
|
|
|
257
317
|
/* close Deepgram connection and remove listeners */
|
|
258
318
|
if (this.dg !== null) {
|
|
259
319
|
try {
|
|
260
|
-
this.dg.
|
|
261
|
-
this.dg.requestClose()
|
|
320
|
+
this.dg.close()
|
|
262
321
|
this.log("info", "connection closed")
|
|
263
322
|
}
|
|
264
323
|
catch (error) {
|
|
@@ -271,18 +330,19 @@ export default class SpeechFlowNodeA2TDeepgram extends SpeechFlowNode {
|
|
|
271
330
|
/* one-time status of node */
|
|
272
331
|
async status () {
|
|
273
332
|
let balance = 0
|
|
333
|
+
if (!this.params.keyAdm) {
|
|
334
|
+
this.log("info", "no Deepgram admin API key configured -- skipping balance fetch")
|
|
335
|
+
return { balance: balance.toFixed(2) }
|
|
336
|
+
}
|
|
274
337
|
try {
|
|
275
|
-
const deepgram = Deepgram.
|
|
276
|
-
const response = await deepgram.manage.
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
}
|
|
338
|
+
const deepgram = new Deepgram.DeepgramClient({ apiKey: this.params.keyAdm })
|
|
339
|
+
const response = await deepgram.manage.v1.projects.list()
|
|
340
|
+
for (const project of response.projects ?? []) {
|
|
341
|
+
if (project.project_id === undefined)
|
|
342
|
+
continue
|
|
343
|
+
const balanceResponse = await deepgram.manage.v1.projects.billing.balances.list(project.project_id)
|
|
344
|
+
balance += balanceResponse.balances?.[0]?.amount ?? 0
|
|
283
345
|
}
|
|
284
|
-
else if (response !== null && response.error !== null)
|
|
285
|
-
this.log("warning", `API error fetching projects: ${response.error}`)
|
|
286
346
|
}
|
|
287
347
|
catch (error) {
|
|
288
348
|
this.log("warning", `failed to fetch balance: ${error}`)
|
|
@@ -305,6 +365,9 @@ export default class SpeechFlowNodeA2TDeepgram extends SpeechFlowNode {
|
|
|
305
365
|
/* create a store for the meta information */
|
|
306
366
|
this.metastore = new util.TimeStore<Map<string, any>>()
|
|
307
367
|
|
|
368
|
+
/* reset last emitted text chunk */
|
|
369
|
+
this.lastChunk = null
|
|
370
|
+
|
|
308
371
|
/* determine initial suspended state from configuration */
|
|
309
372
|
this.suspended = this.params.suspended as boolean
|
|
310
373
|
|
|
@@ -342,10 +405,7 @@ export default class SpeechFlowNodeA2TDeepgram extends SpeechFlowNode {
|
|
|
342
405
|
self.metastore.store(chunk.timestampStart, chunk.timestampEnd, chunk.meta)
|
|
343
406
|
try {
|
|
344
407
|
/* send buffer (and intentionally discard all time information) */
|
|
345
|
-
self.dg.
|
|
346
|
-
chunk.payload.byteOffset,
|
|
347
|
-
chunk.payload.byteOffset + chunk.payload.byteLength
|
|
348
|
-
))
|
|
408
|
+
self.dg.sendMedia(chunk.payload)
|
|
349
409
|
}
|
|
350
410
|
catch (error) {
|
|
351
411
|
callback(util.ensureError(error, "failed to send to Deepgram"))
|
|
@@ -363,14 +423,7 @@ export default class SpeechFlowNodeA2TDeepgram extends SpeechFlowNode {
|
|
|
363
423
|
}
|
|
364
424
|
|
|
365
425
|
/* close Deepgram API */
|
|
366
|
-
|
|
367
|
-
try {
|
|
368
|
-
self.dg.requestClose()
|
|
369
|
-
}
|
|
370
|
-
catch (error) {
|
|
371
|
-
self.log("warning", `error closing Deepgram connection: ${error}`)
|
|
372
|
-
}
|
|
373
|
-
}
|
|
426
|
+
await self.closeConnection()
|
|
374
427
|
|
|
375
428
|
/* await all read operations */
|
|
376
429
|
await reads.awaitAll()
|
|
@@ -421,6 +474,7 @@ export default class SpeechFlowNodeA2TDeepgram extends SpeechFlowNode {
|
|
|
421
474
|
/* signal EOF to any pending read operations */
|
|
422
475
|
if (this.queue !== null) {
|
|
423
476
|
this.queue.write(null)
|
|
477
|
+
this.queue.destroy()
|
|
424
478
|
this.queue = null
|
|
425
479
|
}
|
|
426
480
|
|
|
@@ -86,6 +86,7 @@ export default class SpeechFlowNodeT2TGoogle extends SpeechFlowNode {
|
|
|
86
86
|
})
|
|
87
87
|
|
|
88
88
|
/* establish a transform stream and connect it to Google Translate */
|
|
89
|
+
const self = this
|
|
89
90
|
this.stream = new Stream.Transform({
|
|
90
91
|
readableObjectMode: true,
|
|
91
92
|
writableObjectMode: true,
|
|
@@ -99,7 +100,9 @@ export default class SpeechFlowNodeT2TGoogle extends SpeechFlowNode {
|
|
|
99
100
|
callback()
|
|
100
101
|
}
|
|
101
102
|
else {
|
|
103
|
+
self.log("info", `receive text (${chunk.kind}): "${chunk.payload}"`)
|
|
102
104
|
translate(chunk.payload).then((payload) => {
|
|
105
|
+
self.log("info", `send text (${chunk.kind}): "${payload}"`)
|
|
103
106
|
const chunkNew = chunk.clone()
|
|
104
107
|
chunkNew.payload = payload
|
|
105
108
|
this.push(chunkNew)
|
|
@@ -86,7 +86,7 @@ export default class SpeechFlowNodeT2TOPUS extends SpeechFlowNode {
|
|
|
86
86
|
const translate = async (text: string) => {
|
|
87
87
|
const result = await this.translator!(text)
|
|
88
88
|
const single = Array.isArray(result) ? result[0] : result
|
|
89
|
-
return (single as Transformers.
|
|
89
|
+
return (single as Transformers.TranslationOutput[number]).translation_text
|
|
90
90
|
}
|
|
91
91
|
|
|
92
92
|
/* establish a transform stream and connect it to Transformers */
|