speechflow 2.4.0 → 2.4.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (76) hide show
  1. package/.ase/service.log +531 -0
  2. package/.ase/service.yaml +1 -0
  3. package/.ase/task/TASK-flush.md +44 -0
  4. package/CHANGELOG.md +33 -2
  5. package/README.md +10 -9
  6. package/etc/speechflow.yaml +1 -1
  7. package/etc/stx.conf +21 -21
  8. package/package.json +9 -5
  9. package/speechflow-cli/dst/speechflow-main-api.js +8 -1
  10. package/speechflow-cli/dst/speechflow-main-api.js.map +1 -1
  11. package/speechflow-cli/dst/speechflow-main-cli.js +2 -2
  12. package/speechflow-cli/dst/speechflow-main-cli.js.map +1 -1
  13. package/speechflow-cli/dst/speechflow-node-a2a-ffmpeg.js +2 -0
  14. package/speechflow-cli/dst/speechflow-node-a2a-ffmpeg.js.map +1 -1
  15. package/speechflow-cli/dst/speechflow-node-a2a-gender.js +2 -1
  16. package/speechflow-cli/dst/speechflow-node-a2a-gender.js.map +1 -1
  17. package/speechflow-cli/dst/speechflow-node-a2a-meter.js +28 -12
  18. package/speechflow-cli/dst/speechflow-node-a2a-meter.js.map +1 -1
  19. package/speechflow-cli/dst/speechflow-node-a2a-mute.js +5 -1
  20. package/speechflow-cli/dst/speechflow-node-a2a-mute.js.map +1 -1
  21. package/speechflow-cli/dst/speechflow-node-a2a-pitch.js +9 -15
  22. package/speechflow-cli/dst/speechflow-node-a2a-pitch.js.map +1 -1
  23. package/speechflow-cli/dst/speechflow-node-a2t-deepgram.d.ts +3 -0
  24. package/speechflow-cli/dst/speechflow-node-a2t-deepgram.js +149 -101
  25. package/speechflow-cli/dst/speechflow-node-a2t-deepgram.js.map +1 -1
  26. package/speechflow-cli/dst/speechflow-node-t2t-google.js +3 -0
  27. package/speechflow-cli/dst/speechflow-node-t2t-google.js.map +1 -1
  28. package/speechflow-cli/dst/speechflow-node-t2t-opus.js.map +1 -1
  29. package/speechflow-cli/dst/speechflow-node-t2t-proofread.js +17 -9
  30. package/speechflow-cli/dst/speechflow-node-t2t-proofread.js.map +1 -1
  31. package/speechflow-cli/dst/speechflow-node-t2t-sentence.js +20 -3
  32. package/speechflow-cli/dst/speechflow-node-t2t-sentence.js.map +1 -1
  33. package/speechflow-cli/dst/speechflow-node-xio-exec.js +6 -4
  34. package/speechflow-cli/dst/speechflow-node-xio-exec.js.map +1 -1
  35. package/speechflow-cli/dst/speechflow-util-llm.js +9 -2
  36. package/speechflow-cli/dst/speechflow-util-llm.js.map +1 -1
  37. package/speechflow-cli/etc/oxlint.jsonc +5 -0
  38. package/speechflow-cli/etc/stx.conf +6 -6
  39. package/speechflow-cli/package.json +53 -43
  40. package/speechflow-cli/src/speechflow-main-api.ts +8 -1
  41. package/speechflow-cli/src/speechflow-main-cli.ts +1 -1
  42. package/speechflow-cli/src/speechflow-node-a2a-ffmpeg.ts +3 -1
  43. package/speechflow-cli/src/speechflow-node-a2a-gender.ts +3 -2
  44. package/speechflow-cli/src/speechflow-node-a2a-meter.ts +30 -13
  45. package/speechflow-cli/src/speechflow-node-a2a-mute.ts +6 -1
  46. package/speechflow-cli/src/speechflow-node-a2a-pitch.ts +12 -20
  47. package/speechflow-cli/src/speechflow-node-a2t-deepgram.ts +162 -108
  48. package/speechflow-cli/src/speechflow-node-t2t-google.ts +3 -0
  49. package/speechflow-cli/src/speechflow-node-t2t-opus.ts +1 -1
  50. package/speechflow-cli/src/speechflow-node-t2t-proofread.ts +17 -9
  51. package/speechflow-cli/src/speechflow-node-t2t-sentence.ts +21 -3
  52. package/speechflow-cli/src/speechflow-node-xio-exec.ts +7 -5
  53. package/speechflow-cli/src/speechflow-util-llm.ts +11 -3
  54. package/speechflow-ui-db/dst/app-font-fa-brands-400.woff2 +0 -0
  55. package/speechflow-ui-db/dst/app-font-fa-regular-400.woff2 +0 -0
  56. package/speechflow-ui-db/dst/app-font-fa-solid-900.woff2 +0 -0
  57. package/speechflow-ui-db/dst/app-font-fa-v4compatibility.woff2 +0 -0
  58. package/speechflow-ui-db/dst/index.css +1 -1
  59. package/speechflow-ui-db/dst/index.js +579 -57
  60. package/speechflow-ui-db/etc/stx.conf +5 -4
  61. package/speechflow-ui-db/etc/vite-client.mts +1 -1
  62. package/speechflow-ui-db/package.json +27 -22
  63. package/speechflow-ui-st/dst/app-font-fa-brands-400.woff2 +0 -0
  64. package/speechflow-ui-st/dst/app-font-fa-regular-400.woff2 +0 -0
  65. package/speechflow-ui-st/dst/app-font-fa-solid-900.woff2 +0 -0
  66. package/speechflow-ui-st/dst/app-font-fa-v4compatibility.woff2 +0 -0
  67. package/speechflow-ui-st/dst/index.css +1 -1
  68. package/speechflow-ui-st/dst/index.js +593 -71
  69. package/speechflow-ui-st/etc/stx.conf +5 -4
  70. package/speechflow-ui-st/etc/vite-client.mts +6 -0
  71. package/speechflow-ui-st/package.json +27 -22
  72. package/speechflow-ui-st/src/app.vue +3 -3
  73. package/speechflow-cli/package.d/sherpa-onnx+1.12.25.patch +0 -12
  74. /package/speechflow-cli/package.d/{@typescript-eslint+typescript-estree+8.59.3.patch → @typescript-eslint+typescript-estree+8.64.0.patch} +0 -0
  75. /package/speechflow-ui-db/package.d/{@typescript-eslint+typescript-estree+8.59.3.patch → @typescript-eslint+typescript-estree+8.64.0.patch} +0 -0
  76. /package/speechflow-ui-st/package.d/{@typescript-eslint+typescript-estree+8.59.3.patch → @typescript-eslint+typescript-estree+8.64.0.patch} +0 -0
@@ -93,10 +93,12 @@ export default class SpeechFlowNodeA2AFFMPEG extends SpeechFlowNode {
93
93
  util.run("starting FFmpeg process", () => this.ffmpeg!.run())
94
94
 
95
95
  /* establish a duplex stream and connect it to FFmpeg */
96
+ /* NOTICE: the object form of Stream.Duplex.from() accepts Node streams at
97
+ run-time, but @types/node types it for Web streams only, so we have to cast */
96
98
  const ffmpegStream = Stream.Duplex.from({
97
99
  writable: streamInput,
98
100
  readable: streamOutput
99
- })
101
+ } as unknown as Parameters<typeof Stream.Duplex.from>[0])
100
102
 
101
103
  /* wrap streams with conversions for chunk vs plain audio */
102
104
  const wrapper1 = util.createTransformStreamForWritableSide("audio", 1)
@@ -11,7 +11,7 @@ import Stream from "node:stream"
11
11
  /* external dependencies */
12
12
  import * as Transformers from "@huggingface/transformers"
13
13
  import { WaveFile } from "wavefile"
14
- import { getRMS, AudioData } from "audio-inspect"
14
+ import { analyze, AudioData } from "audio-inspect"
15
15
  import { Duration } from "luxon"
16
16
 
17
17
  /* internal dependencies */
@@ -145,7 +145,8 @@ export default class SpeechFlowNodeA2AGender extends SpeechFlowNode {
145
145
  duration: data.length / sampleRateTarget,
146
146
  length: data.length
147
147
  } satisfies AudioData
148
- const rms = getRMS(audioData, { asDB: true })
148
+ const analysis = await analyze(audioData, { features: { rms: { asDB: true } } })
149
+ const rms = analysis.results.rms ?? -60
149
150
  if (rms < this.params.volumeThreshold)
150
151
  return genderLast
151
152
 
@@ -8,7 +8,7 @@
8
8
  import Stream from "node:stream"
9
9
 
10
10
  /* external dependencies */
11
- import { getLUFS, getRMS, AudioData } from "audio-inspect"
11
+ import { analyze, AudioData } from "audio-inspect"
12
12
 
13
13
  /* internal dependencies */
14
14
  import SpeechFlowNode, { SpeechFlowChunk } from "./speechflow-node"
@@ -69,8 +69,8 @@ export default class SpeechFlowNodeA2AMeter extends SpeechFlowNode {
69
69
  const samplesPerChunk = Math.floor(this.config.audioSampleRate * chunkDuration)
70
70
  this.chunkBuffer = new Float32Array(0)
71
71
 
72
- /* setup chunking interval */
73
- this.calcInterval = setInterval(() => {
72
+ /* calculate the loudness metrics of the accumulated chunk data */
73
+ const calcMeter = async () => {
74
74
  /* short-circuit during destruction */
75
75
  if (this.closing)
76
76
  return
@@ -99,14 +99,19 @@ export default class SpeechFlowNodeA2AMeter extends SpeechFlowNode {
99
99
  duration: sampleWindowDuration,
100
100
  length: sampleWindow.length
101
101
  } satisfies AudioData
102
- const lufs = getLUFS(audioDataLUFS, {
103
- channelMode: this.config.audioChannels === 1 ? "mono" : "stereo",
104
- calculateShortTerm: false,
105
- calculateMomentary: true,
106
- calculateLoudnessRange: false,
107
- calculateTruePeak: false
102
+ const analysisLUFS = await analyze(audioDataLUFS, {
103
+ features: {
104
+ lufs: {
105
+ channelMode: this.config.audioChannels === 1 ? "mono" : "stereo",
106
+ calculateShortTerm: false,
107
+ calculateMomentary: true,
108
+ calculateLoudnessRange: false,
109
+ calculateTruePeak: false
110
+ }
111
+ }
108
112
  })
109
- lufsm = lufs.momentary ? Math.max(-60, lufs.momentary[0]) : -60
113
+ const lufs = analysisLUFS.results.lufs
114
+ lufsm = lufs?.momentary !== undefined ? Math.max(-60, lufs.momentary) : -60
110
115
 
111
116
  /* calculate the RMS metric */
112
117
  const totalSamples = chunkData.length / this.config.audioChannels
@@ -118,9 +123,14 @@ export default class SpeechFlowNodeA2AMeter extends SpeechFlowNode {
118
123
  duration,
119
124
  length: chunkData.length
120
125
  } satisfies AudioData
121
- rms = Math.max(-60, getRMS(audioDataRMS, {
122
- asDB: true
123
- }))
126
+ const analysisRMS = await analyze(audioDataRMS, {
127
+ features: {
128
+ rms: {
129
+ asDB: true
130
+ }
131
+ }
132
+ })
133
+ rms = Math.max(-60, analysisRMS.results.rms ?? -60)
124
134
 
125
135
  /* automatically clear measurement (in case no new measurements happen) */
126
136
  if (this.silenceTimer !== null)
@@ -129,6 +139,13 @@ export default class SpeechFlowNodeA2AMeter extends SpeechFlowNode {
129
139
  lufsm = -60
130
140
  rms = -60
131
141
  }, 500)
142
+ }
143
+
144
+ /* setup chunking interval */
145
+ this.calcInterval = setInterval(() => {
146
+ calcMeter().catch((err: unknown) => {
147
+ this.log("warning", `meter calculation failed: ${err instanceof Error ? err.message : String(err)}`)
148
+ })
132
149
  }, chunkDuration * 1000)
133
150
 
134
151
  /* setup loudness emitting interval */
@@ -31,7 +31,9 @@ export default class SpeechFlowNodeA2AMute extends SpeechFlowNode {
31
31
  super(id, cfg, opts, args)
32
32
 
33
33
  /* declare node configuration parameters */
34
- this.configure({})
34
+ this.configure({
35
+ muteMode: { type: "string", val: "none", pos: 0, match: /^(?:none|silenced|unplugged)$/ }
36
+ })
35
37
 
36
38
  /* declare node input/output format */
37
39
  this.input = "audio"
@@ -75,6 +77,9 @@ export default class SpeechFlowNodeA2AMute extends SpeechFlowNode {
75
77
  /* clear destruction flag */
76
78
  this.closing = false
77
79
 
80
+ /* determine initial mute mode */
81
+ this.muteMode = this.params.muteMode as MuteMode
82
+
78
83
  /* establish a transform stream */
79
84
  const self = this
80
85
  this.stream = new Stream.Transform({
@@ -18,7 +18,6 @@ import * as util from "./speechflow-util"
18
18
  /* parameter configuration */
19
19
  type AudioPitchShifterConfig = {
20
20
  rate?: number
21
- tempo?: number
22
21
  pitch?: number
23
22
  semitones?: number
24
23
  }
@@ -38,7 +37,6 @@ class AudioPitchShifter extends util.WebAudio {
38
37
  super(sampleRate, channels)
39
38
  this.config = {
40
39
  rate: config.rate ?? 1.0,
41
- tempo: config.tempo ?? 1.0,
42
40
  pitch: config.pitch ?? 1.0,
43
41
  semitones: config.semitones ?? 0.0
44
42
  }
@@ -50,10 +48,11 @@ class AudioPitchShifter extends util.WebAudio {
50
48
 
51
49
  /* add SoundTouch worklet module */
52
50
  const packagePath = path.join(__dirname, "../node_modules/@soundtouchjs/audio-worklet")
53
- const workletPath = path.join(packagePath, "dist/soundtouch-worklet.js")
51
+ const workletPath = path.join(packagePath, ".dist/soundtouch-processor.js")
54
52
  await this.audioContext.audioWorklet.addModule(workletPath)
55
53
 
56
- /* create SoundTouch worklet node */
54
+ /* create SoundTouch worklet node (the processor always operates on
55
+ stereo internally, but mixes down to the requested channel count) */
57
56
  this.pitchNode = new AudioWorkletNode(this.audioContext, "soundtouch-processor", {
58
57
  numberOfInputs: 1,
59
58
  numberOfOutputs: 1,
@@ -62,10 +61,9 @@ class AudioPitchShifter extends util.WebAudio {
62
61
 
63
62
  /* set initial parameter values */
64
63
  const params = this.pitchNode.parameters as Map<string, AudioParam>
65
- params.get("rate")!.value = this.config.rate
66
- params.get("tempo")!.value = this.config.tempo
67
- params.get("pitch")!.value = this.config.pitch
68
- params.get("pitchSemitones")!.value = this.config.semitones
64
+ params.get("playbackRate")!.value = this.config.rate
65
+ params.get("pitch")!.value = this.config.pitch
66
+ params.get("pitchSemitones")!.value = this.config.semitones
69
67
 
70
68
  /* connect nodes: source -> pitch -> capture */
71
69
  this.sourceNode!.connect(this.pitchNode)
@@ -83,14 +81,10 @@ class AudioPitchShifter extends util.WebAudio {
83
81
  this.config[configField] = value
84
82
  }
85
83
 
86
- /* update rate value */
84
+ /* update rate value (compensates the pitch for an upstream playback
85
+ rate change, i.e. the resulting pitch is divided by this value) */
87
86
  public setRate (rate: number): void {
88
- this.updateParameter("rate", rate, "rate")
89
- }
90
-
91
- /* update tempo value */
92
- public setTempo (tempo: number): void {
93
- this.updateParameter("tempo", tempo, "tempo")
87
+ this.updateParameter("playbackRate", rate, "rate")
94
88
  }
95
89
 
96
90
  /* update pitch shift value */
@@ -131,10 +125,9 @@ export default class SpeechFlowNodeA2APitch extends SpeechFlowNode {
131
125
 
132
126
  /* declare node configuration parameters */
133
127
  this.configure({
134
- rate: { type: "number", val: 1.0, match: (n: number) => n >= 0.25 && n <= 4.0 },
135
- tempo: { type: "number", val: 1.0, match: (n: number) => n >= 0.25 && n <= 4.0 },
136
- pitch: { type: "number", val: 1.0, match: (n: number) => n >= 0.25 && n <= 4.0 },
137
- semitones: { type: "number", val: 0.0, match: (n: number) => n >= -24 && n <= 24 }
128
+ rate: { type: "number", val: 1.0, match: (n: number) => n >= 0.1 && n <= 8.0 },
129
+ pitch: { type: "number", val: 1.0, match: (n: number) => n >= 0.1 && n <= 8.0 },
130
+ semitones: { type: "number", val: 0.0, match: (n: number) => n >= -24 && n <= 24 }
138
131
  })
139
132
 
140
133
  /* declare node input/output format */
@@ -152,7 +145,6 @@ export default class SpeechFlowNodeA2APitch extends SpeechFlowNode {
152
145
  this.config.audioSampleRate,
153
146
  this.config.audioChannels, {
154
147
  rate: this.params.rate,
155
- tempo: this.params.tempo,
156
148
  pitch: this.params.pitch,
157
149
  semitones: this.params.semitones
158
150
  }
@@ -9,25 +9,33 @@ import Stream from "node:stream"
9
9
 
10
10
  /* external dependencies */
11
11
  import * as Deepgram from "@deepgram/sdk"
12
+ import { V1Socket } from "@deepgram/sdk/listen/v1"
12
13
  import { DateTime, Duration } from "luxon"
13
14
 
14
15
  /* internal dependencies */
15
16
  import SpeechFlowNode, { SpeechFlowChunk } from "./speechflow-node"
16
17
  import * as util from "./speechflow-util"
17
18
 
19
+ /* NOTICE: the connection arguments type is not publicly re-exported by the
20
+ Deepgram SDK, so derive it structurally from the connect method instead */
21
+ type DeepgramConnectArgs = Parameters<Deepgram.DeepgramClient["listen"]["v1"]["connect"]>[0]
22
+
18
23
  /* SpeechFlow node for Deepgram speech-to-text conversion */
19
24
  export default class SpeechFlowNodeA2TDeepgram extends SpeechFlowNode {
20
25
  /* declare official node name */
21
26
  public static name = "a2t-deepgram"
22
27
 
23
28
  /* internal state */
24
- private dg: Deepgram.LiveClient | null = null
29
+ private dg: V1Socket | null = null
25
30
  private closing = false
26
31
  private reconfiguring = false
27
32
  private connectionTimeout: ReturnType<typeof setTimeout> | null = null
28
33
  private queue: util.AsyncQueue<SpeechFlowChunk | null> | null = null
29
34
  private metastore: util.TimeStore<Map<string, any>> | null = null
30
35
  private suspended = false
36
+ private opening = false
37
+ private openReject: ((error: Error) => void) | null = null
38
+ private lastChunk: SpeechFlowChunk | null = null
31
39
 
32
40
  /* construct node */
33
41
  constructor (id: string, cfg: { [ id: string ]: any }, opts: { [ id: string ]: any }, args: any[]) {
@@ -85,9 +93,9 @@ export default class SpeechFlowNodeA2TDeepgram extends SpeechFlowNode {
85
93
  if (suspended === this.suspended)
86
94
  return
87
95
  this.log("info", `switching to ${suspended ? "SUSPENDED" : "UNSUSPENDED"} operation`)
88
- this.suspended = suspended
89
96
  if (suspended) {
90
97
  /* going suspended -- tear down Deepgram API connection */
98
+ this.suspended = true
91
99
  this.reconfiguring = true
92
100
  try {
93
101
  await this.closeConnection()
@@ -95,10 +103,24 @@ export default class SpeechFlowNodeA2TDeepgram extends SpeechFlowNode {
95
103
  finally {
96
104
  this.reconfiguring = false
97
105
  }
106
+
107
+ /* re-emit last intermediate chunk as final one, as its regular
108
+ finalization will never arrive anymore -- with an appended
109
+ ellipsis as truncation marker (and sentence boundary for
110
+ downstream sentence splitting) */
111
+ if (this.lastChunk !== null && this.lastChunk.kind === "intermediate" && this.queue !== null) {
112
+ this.log("info", "sending last non-final chunk as final chunk")
113
+ const chunk = this.lastChunk.clone()
114
+ chunk.kind = "final"
115
+ chunk.payload = `${chunk.payload}…`
116
+ this.queue.write(chunk)
117
+ this.lastChunk = chunk
118
+ }
98
119
  }
99
120
  else {
100
121
  /* going unsuspended -- re-establish Deepgram API connection */
101
122
  await this.openConnection()
123
+ this.suspended = false
102
124
  }
103
125
  }
104
126
 
@@ -107,23 +129,32 @@ export default class SpeechFlowNodeA2TDeepgram extends SpeechFlowNode {
107
129
  /* configure Deepgram connection options */
108
130
  const interim = this.params.interim as boolean
109
131
  const endpointing = this.params.endpointing as number
110
- const options: Deepgram.LiveSchema = {
132
+ /* NOTICE: the Deepgram API v5 SDK expects the boolean flags as their
133
+ string representations, as they are passed as URL query parameters.
134
+ The "filler_words" flag is a valid live API parameter, but is not
135
+ covered by the generated SDK types, so pass it via "queryParams". */
136
+ const options: DeepgramConnectArgs = {
137
+ /* NOTICE: the Deepgram API v5 SDK applies the client-level "apiKey" only
138
+ to its REST endpoints -- the WebSocket connect path never consults the
139
+ authentication provider and instead requires the "Authorization" header
140
+ to be passed explicitly, or else the server closes the handshake */
141
+ Authorization: `Token ${this.params.key}`,
111
142
  mip_opt_out: true,
112
143
  model: this.params.model,
113
144
  version: this.params.version,
114
145
  channels: this.config.audioChannels,
115
146
  sample_rate: this.config.audioSampleRate,
116
147
  encoding: "linear16",
117
- multichannel: false,
118
- endpointing: endpointing > 0 ? endpointing : false,
119
- interim_results: interim,
120
- smart_format: false,
121
- punctuate: true,
122
- filler_words: true,
123
- numerals: false,
124
- diarize: false,
125
- profanity_filter: false,
126
- redact: false
148
+ multichannel: "false",
149
+ endpointing: endpointing > 0 ? endpointing : "false",
150
+ interim_results: interim ? "true" : "false",
151
+ smart_format: "false",
152
+ punctuate: "true",
153
+ numerals: "false",
154
+ diarize: "false",
155
+ profanity_filter: "false",
156
+ redact: "false",
157
+ queryParams: { filler_words: "true" }
127
158
  }
128
159
  const model = this.params.model as string
129
160
  const language = this.params.language as string
@@ -149,52 +180,52 @@ export default class SpeechFlowNodeA2TDeepgram extends SpeechFlowNode {
149
180
  }
150
181
 
151
182
  /* connect to Deepgram API */
152
- const deepgram = Deepgram.createClient(this.params.key)
153
- this.dg = deepgram.listen.live(options)
183
+ const deepgram = new Deepgram.DeepgramClient({ apiKey: this.params.key })
184
+ this.dg = await deepgram.listen.v1.connect(options)
154
185
 
155
- /* hook onto Deepgram API events */
156
- this.dg.on(Deepgram.LiveTranscriptionEvents.Transcript, async (data) => {
186
+ /* hook onto Deepgram API messages */
187
+ this.dg.on("message", (message: V1Socket.Response) => {
157
188
  if (this.closing || this.queue === null || this.metastore === null)
158
189
  return
159
- const text = (data.channel?.alternatives[0]?.transcript ?? "") as string
160
- const words = (data.channel?.alternatives[0]?.words ?? []) as
161
- { word: string, punctuated_word?: string, start: number, end: number }[]
162
- const isFinal = (data.is_final as boolean) ?? false
163
- const speechFinal = (data.speech_final as boolean) ?? false
164
- const kind = (isFinal || (endpointing > 0 && speechFinal)) ? "final" : "intermediate"
165
- if (text === "")
166
- this.log("info", `empty/dummy text received (start: ${data.start}s, duration: ${data.duration.toFixed(2)}s)`)
167
- else {
168
- this.log("info", `text received (start: ${data.start}s, ` +
169
- `duration: ${data.duration.toFixed(2)}s, kind: ${kind}): ` +
170
- `"${text}"`)
171
- const start = Duration.fromMillis(data.start * 1000).plus(this.timeZeroOffset)
172
- const end = start.plus({ seconds: data.duration })
173
- const metas = this.metastore.fetch(start, end)
174
- const meta = metas.toReversed().reduce((prev: Map<string, any>, curr: Map<string, any>) => {
175
- curr.forEach((val, key) => { prev.set(key, val) })
176
- return prev
177
- }, new Map<string, any>())
178
- this.metastore.prune(start)
179
- meta.set("words", words.map((word) => {
180
- const start = Duration.fromMillis(word.start * 1000).plus(this.timeZeroOffset)
181
- const end = Duration.fromMillis(word.end * 1000).plus(this.timeZeroOffset)
182
- return { word: word.punctuated_word ?? word.word, start, end }
183
- }))
184
- const chunk = new SpeechFlowChunk(start, end, kind, "text", text, meta)
185
- this.queue.write(chunk)
190
+ if (message.type === "SpeechStarted")
191
+ this.log("info", "speech started", message)
192
+ else if (message.type === "UtteranceEnd")
193
+ this.log("info", "utterance end received", message)
194
+ else if (message.type === "Metadata")
195
+ this.log("info", "metadata received")
196
+ else if (message.type === "Results") {
197
+ const data = message
198
+ const text = data.channel?.alternatives[0]?.transcript ?? ""
199
+ const words = data.channel?.alternatives[0]?.words ?? []
200
+ const isFinal = data.is_final ?? false
201
+ const speechFinal = data.speech_final ?? false
202
+ const kind = (isFinal || ((this.params.endpointing as number) > 0 && speechFinal)) ? "final" : "intermediate"
203
+ if (text === "")
204
+ this.log("info", `empty/dummy text received (start: ${data.start}s, duration: ${data.duration.toFixed(2)}s)`)
205
+ else {
206
+ this.log("info", `text received (start: ${data.start}s, ` +
207
+ `duration: ${data.duration.toFixed(2)}s, kind: ${kind}): ` +
208
+ `"${text}"`)
209
+ const start = Duration.fromMillis(data.start * 1000).plus(this.timeZeroOffset)
210
+ const end = start.plus({ seconds: data.duration })
211
+ const metas = this.metastore.fetch(start, end)
212
+ const meta = metas.toReversed().reduce((prev: Map<string, any>, curr: Map<string, any>) => {
213
+ curr.forEach((val, key) => { prev.set(key, val) })
214
+ return prev
215
+ }, new Map<string, any>())
216
+ this.metastore.prune(start)
217
+ meta.set("words", words.map((word) => {
218
+ const wordStart = Duration.fromMillis(word.start * 1000).plus(this.timeZeroOffset)
219
+ const wordEnd = Duration.fromMillis(word.end * 1000).plus(this.timeZeroOffset)
220
+ return { word: word.punctuated_word ?? word.word, start: wordStart, end: wordEnd }
221
+ }))
222
+ const chunk = new SpeechFlowChunk(start, end, kind, "text", text, meta)
223
+ this.queue.write(chunk)
224
+ this.lastChunk = chunk
225
+ }
186
226
  }
187
227
  })
188
- this.dg.on(Deepgram.LiveTranscriptionEvents.SpeechStarted, (data) => {
189
- this.log("info", "speech started", data)
190
- })
191
- this.dg.on(Deepgram.LiveTranscriptionEvents.UtteranceEnd, (data) => {
192
- this.log("info", "utterance end received", data)
193
- })
194
- this.dg.on(Deepgram.LiveTranscriptionEvents.Metadata, (data) => {
195
- this.log("info", "metadata received")
196
- })
197
- this.dg.on(Deepgram.LiveTranscriptionEvents.Close, () => {
228
+ this.dg.on("close", () => {
198
229
  this.log("info", "connection close")
199
230
  /* NOTICE: suppress EOF signalling while reconfiguring (mute toggle),
200
231
  since the connection is being torn down deliberately and the
@@ -203,36 +234,18 @@ export default class SpeechFlowNodeA2TDeepgram extends SpeechFlowNode {
203
234
  this.queue.write(null)
204
235
  })
205
236
 
206
- /* wait for Deepgram API to be available */
207
- await new Promise((resolve, reject) => {
208
- this.connectionTimeout = setTimeout(() => {
209
- if (this.connectionTimeout !== null) {
210
- this.connectionTimeout = null
211
- reject(new Error("Deepgram: timeout waiting for connection open"))
212
- }
213
- }, 8000)
214
- this.dg!.once(Deepgram.LiveTranscriptionEvents.Open, () => {
215
- this.log("info", "connection open")
216
- if (this.connectionTimeout !== null) {
217
- clearTimeout(this.connectionTimeout)
218
- this.connectionTimeout = null
219
- }
220
- resolve(true)
221
- })
222
- this.dg!.once(Deepgram.LiveTranscriptionEvents.Error, (err: Error) => {
223
- if (this.connectionTimeout !== null) {
224
- clearTimeout(this.connectionTimeout)
225
- this.connectionTimeout = null
226
- }
227
- reject(err)
228
- })
229
- })
230
-
231
- /* NOTICE: register permanent Error handler only AFTER the open
232
- handshake -- during open, the transient .once above is the sole
233
- Error listener so the caller's promise rejects without a parallel
234
- stream emission tearing down the graph prematurely. */
235
- this.dg.on(Deepgram.LiveTranscriptionEvents.Error, (error: Error) => {
237
+ /* NOTICE: the Deepgram API socket supports only a single handler per
238
+ event, so the open handshake and the permanent error handling must
239
+ share one error handler which dispatches on the "opening" flag:
240
+ during open, the error is routed solely into the caller's promise
241
+ rejection, without a parallel stream emission tearing down the
242
+ graph prematurely. */
243
+ this.dg.on("error", (error: Error) => {
244
+ if (this.opening) {
245
+ if (this.openReject !== null)
246
+ this.openReject(error)
247
+ return
248
+ }
236
249
  this.log("warning", `error: ${error.message}`)
237
250
  /* NOTICE: do not write null to the queue here -- a transient error
238
251
  must not be misinterpreted as end-of-stream by downstream nodes;
@@ -244,6 +257,53 @@ export default class SpeechFlowNodeA2TDeepgram extends SpeechFlowNode {
244
257
  if (!this.closing && this.stream !== null)
245
258
  this.stream.emit("error", error)
246
259
  })
260
+
261
+ /* NOTICE: the Deepgram API v5 SDK returns a socket which has not yet
262
+ dialed, as it is internally created with the "startClosed" option,
263
+ so the connection has to be established explicitly -- and only once
264
+ all event handlers above are registered, as the connect operation
265
+ re-registers the underlying socket listeners */
266
+ this.dg.connect()
267
+
268
+ /* clear pending connection timer (idempotent) */
269
+ const clearConnectionTimeout = () => {
270
+ if (this.connectionTimeout !== null) {
271
+ clearTimeout(this.connectionTimeout)
272
+ this.connectionTimeout = null
273
+ }
274
+ }
275
+
276
+ /* wait for Deepgram API to be available */
277
+ this.opening = true
278
+ let opened = false
279
+ try {
280
+ await new Promise((resolve, reject) => {
281
+ this.openReject = reject
282
+ this.connectionTimeout = setTimeout(() => {
283
+ if (this.connectionTimeout !== null) {
284
+ this.connectionTimeout = null
285
+ reject(new Error("Deepgram: timeout waiting for connection open"))
286
+ }
287
+ }, 8000)
288
+ this.dg!.waitForOpen().then(() => {
289
+ this.log("info", "connection open")
290
+ clearConnectionTimeout()
291
+ resolve(true)
292
+ }).catch((error: unknown) => {
293
+ clearConnectionTimeout()
294
+ reject(util.ensureError(error, "failed to open Deepgram connection"))
295
+ })
296
+ })
297
+ opened = true
298
+ }
299
+ finally {
300
+ this.opening = false
301
+ this.openReject = null
302
+
303
+ /* discard half-established connection on failure */
304
+ if (!opened)
305
+ await this.closeConnection()
306
+ }
247
307
  }
248
308
 
249
309
  /* close Deepgram API connection */
@@ -257,8 +317,7 @@ export default class SpeechFlowNodeA2TDeepgram extends SpeechFlowNode {
257
317
  /* close Deepgram connection and remove listeners */
258
318
  if (this.dg !== null) {
259
319
  try {
260
- this.dg.removeAllListeners()
261
- this.dg.requestClose()
320
+ this.dg.close()
262
321
  this.log("info", "connection closed")
263
322
  }
264
323
  catch (error) {
@@ -271,18 +330,19 @@ export default class SpeechFlowNodeA2TDeepgram extends SpeechFlowNode {
271
330
  /* one-time status of node */
272
331
  async status () {
273
332
  let balance = 0
333
+ if (!this.params.keyAdm) {
334
+ this.log("info", "no Deepgram admin API key configured -- skipping balance fetch")
335
+ return { balance: balance.toFixed(2) }
336
+ }
274
337
  try {
275
- const deepgram = Deepgram.createClient(this.params.keyAdm)
276
- const response = await deepgram.manage.getProjects()
277
- if (response !== null && response.error === null && response.result?.projects) {
278
- for (const project of response.result.projects) {
279
- const balanceResponse = await deepgram.manage.getProjectBalances(project.project_id)
280
- if (balanceResponse !== null && balanceResponse.error === null && balanceResponse.result?.balances)
281
- balance += balanceResponse.result.balances[0]?.amount ?? 0
282
- }
338
+ const deepgram = new Deepgram.DeepgramClient({ apiKey: this.params.keyAdm })
339
+ const response = await deepgram.manage.v1.projects.list()
340
+ for (const project of response.projects ?? []) {
341
+ if (project.project_id === undefined)
342
+ continue
343
+ const balanceResponse = await deepgram.manage.v1.projects.billing.balances.list(project.project_id)
344
+ balance += balanceResponse.balances?.[0]?.amount ?? 0
283
345
  }
284
- else if (response !== null && response.error !== null)
285
- this.log("warning", `API error fetching projects: ${response.error}`)
286
346
  }
287
347
  catch (error) {
288
348
  this.log("warning", `failed to fetch balance: ${error}`)
@@ -305,6 +365,9 @@ export default class SpeechFlowNodeA2TDeepgram extends SpeechFlowNode {
305
365
  /* create a store for the meta information */
306
366
  this.metastore = new util.TimeStore<Map<string, any>>()
307
367
 
368
+ /* reset last emitted text chunk */
369
+ this.lastChunk = null
370
+
308
371
  /* determine initial suspended state from configuration */
309
372
  this.suspended = this.params.suspended as boolean
310
373
 
@@ -342,10 +405,7 @@ export default class SpeechFlowNodeA2TDeepgram extends SpeechFlowNode {
342
405
  self.metastore.store(chunk.timestampStart, chunk.timestampEnd, chunk.meta)
343
406
  try {
344
407
  /* send buffer (and intentionally discard all time information) */
345
- self.dg.send(chunk.payload.buffer.slice(
346
- chunk.payload.byteOffset,
347
- chunk.payload.byteOffset + chunk.payload.byteLength
348
- ))
408
+ self.dg.sendMedia(chunk.payload)
349
409
  }
350
410
  catch (error) {
351
411
  callback(util.ensureError(error, "failed to send to Deepgram"))
@@ -363,14 +423,7 @@ export default class SpeechFlowNodeA2TDeepgram extends SpeechFlowNode {
363
423
  }
364
424
 
365
425
  /* close Deepgram API */
366
- if (self.dg !== null) {
367
- try {
368
- self.dg.requestClose()
369
- }
370
- catch (error) {
371
- self.log("warning", `error closing Deepgram connection: ${error}`)
372
- }
373
- }
426
+ await self.closeConnection()
374
427
 
375
428
  /* await all read operations */
376
429
  await reads.awaitAll()
@@ -421,6 +474,7 @@ export default class SpeechFlowNodeA2TDeepgram extends SpeechFlowNode {
421
474
  /* signal EOF to any pending read operations */
422
475
  if (this.queue !== null) {
423
476
  this.queue.write(null)
477
+ this.queue.destroy()
424
478
  this.queue = null
425
479
  }
426
480
 
@@ -86,6 +86,7 @@ export default class SpeechFlowNodeT2TGoogle extends SpeechFlowNode {
86
86
  })
87
87
 
88
88
  /* establish a transform stream and connect it to Google Translate */
89
+ const self = this
89
90
  this.stream = new Stream.Transform({
90
91
  readableObjectMode: true,
91
92
  writableObjectMode: true,
@@ -99,7 +100,9 @@ export default class SpeechFlowNodeT2TGoogle extends SpeechFlowNode {
99
100
  callback()
100
101
  }
101
102
  else {
103
+ self.log("info", `receive text (${chunk.kind}): "${chunk.payload}"`)
102
104
  translate(chunk.payload).then((payload) => {
105
+ self.log("info", `send text (${chunk.kind}): "${payload}"`)
103
106
  const chunkNew = chunk.clone()
104
107
  chunkNew.payload = payload
105
108
  this.push(chunkNew)
@@ -86,7 +86,7 @@ export default class SpeechFlowNodeT2TOPUS extends SpeechFlowNode {
86
86
  const translate = async (text: string) => {
87
87
  const result = await this.translator!(text)
88
88
  const single = Array.isArray(result) ? result[0] : result
89
- return (single as Transformers.TranslationSingle).translation_text
89
+ return (single as Transformers.TranslationOutput[number]).translation_text
90
90
  }
91
91
 
92
92
  /* establish a transform stream and connect it to Transformers */