speechflow 2.4.1 → 2.4.3

package/speechflow-cli/src/speechflow-node-a2t-deepgram.ts

@@ -9,25 +9,33 @@ import Stream from "node:stream"
 
 /*  external dependencies  */
 import * as Deepgram          from "@deepgram/sdk"
+import { V1Socket }           from "@deepgram/sdk/listen/v1"
 import { DateTime, Duration } from "luxon"
 
 /*  internal dependencies  */
 import SpeechFlowNode, { SpeechFlowChunk } from "./speechflow-node"
 import * as util                           from "./speechflow-util"
 
+/*  NOTICE: the connection arguments type is not publicly re-exported by the
+    Deepgram SDK, so derive it structurally from the connect method instead  */
+type DeepgramConnectArgs = Parameters<Deepgram.DeepgramClient["listen"]["v1"]["connect"]>[0]
+
 /*  SpeechFlow node for Deepgram speech-to-text conversion  */
 export default class SpeechFlowNodeA2TDeepgram extends SpeechFlowNode {
     /*  declare official node name  */
     public static name = "a2t-deepgram"
 
     /*  internal state  */
-    private dg:                Deepgram.LiveClient                      | null = null
+    private dg:                V1Socket                                 | null = null
     private closing                                                            = false
     private reconfiguring                                                      = false
     private connectionTimeout: ReturnType<typeof setTimeout>            | null = null
     private queue:             util.AsyncQueue<SpeechFlowChunk | null>  | null = null
     private metastore:         util.TimeStore<Map<string, any>>         | null = null
     private suspended                                                          = false
+    private opening                                                            = false
+    private openReject:        ((error: Error) => void)                 | null = null
+    private lastChunk:         SpeechFlowChunk                          | null = null
 
     /*  construct node  */
     constructor (id: string, cfg: { [ id: string ]: any }, opts: { [ id: string ]: any }, args: any[]) {
@@ -85,9 +93,9 @@ export default class SpeechFlowNodeA2TDeepgram extends SpeechFlowNode {
         if (suspended === this.suspended)
             return
         this.log("info", `switching to ${suspended ? "SUSPENDED" : "UNSUSPENDED"} operation`)
-        this.suspended = suspended
         if (suspended) {
             /*  going suspended -- tear down Deepgram API connection  */
+            this.suspended = true
             this.reconfiguring = true
             try {
                 await this.closeConnection()
@@ -95,10 +103,24 @@ export default class SpeechFlowNodeA2TDeepgram extends SpeechFlowNode {
             finally {
                 this.reconfiguring = false
             }
+
+            /*  re-emit last intermediate chunk as final one, as its regular
+                finalization will never arrive anymore -- with an appended
+                ellipsis as truncation marker (and sentence boundary for
+                downstream sentence splitting)  */
+            if (this.lastChunk !== null && this.lastChunk.kind === "intermediate" && this.queue !== null) {
+                this.log("info", "sending last non-final chunk as final chunk")
+                const chunk = this.lastChunk.clone()
+                chunk.kind = "final"
+                chunk.payload = `${chunk.payload}…`
+                this.queue.write(chunk)
+                this.lastChunk = chunk
+            }
         }
         else {
             /*  going unsuspended -- re-establish Deepgram API connection  */
             await this.openConnection()
+            this.suspended = false
         }
     }
 
@@ -107,23 +129,32 @@ export default class SpeechFlowNodeA2TDeepgram extends SpeechFlowNode {
         /*  configure Deepgram connection options  */
         const interim     = this.params.interim as boolean
         const endpointing = this.params.endpointing as number
-        const options: Deepgram.LiveSchema = {
+        /*  NOTICE: the Deepgram API v5 SDK expects the boolean flags as their
+            string representations, as they are passed as URL query parameters.
+            The "filler_words" flag is a valid live API parameter, but is not
+            covered by the generated SDK types, so pass it via "queryParams".  */
+        const options: DeepgramConnectArgs = {
+            /*  NOTICE: the Deepgram API v5 SDK applies the client-level "apiKey" only
+                to its REST endpoints -- the WebSocket connect path never consults the
+                authentication provider and instead requires the "Authorization" header
+                to be passed explicitly, or else the server closes the handshake  */
+            Authorization:    `Token ${this.params.key}`,
             mip_opt_out:      true,
             model:            this.params.model,
             version:          this.params.version,
             channels:         this.config.audioChannels,
             sample_rate:      this.config.audioSampleRate,
             encoding:         "linear16",
-            multichannel:     false,
-            endpointing:      endpointing > 0 ? endpointing : false,
-            interim_results:  interim,
-            smart_format:     false,
-            punctuate:        true,
-            filler_words:     true,
-            numerals:         false,
-            diarize:          false,
-            profanity_filter: false,
-            redact:           false
+            multichannel:     "false",
+            endpointing:      endpointing > 0 ? endpointing : "false",
+            interim_results:  interim ? "true" : "false",
+            smart_format:     "false",
+            punctuate:        "true",
+            numerals:         "false",
+            diarize:          "false",
+            profanity_filter: "false",
+            redact:           "false",
+            queryParams:      { filler_words: "true" }
         }
         const model    = this.params.model    as string
         const language = this.params.language as string
@@ -149,52 +180,52 @@ export default class SpeechFlowNodeA2TDeepgram extends SpeechFlowNode {
         }
 
         /*  connect to Deepgram API  */
-        const deepgram = Deepgram.createClient(this.params.key)
-        this.dg = deepgram.listen.live(options)
+        const deepgram = new Deepgram.DeepgramClient({ apiKey: this.params.key })
+        this.dg = await deepgram.listen.v1.connect(options)
 
-        /*  hook onto Deepgram API events  */
-        this.dg.on(Deepgram.LiveTranscriptionEvents.Transcript, async (data) => {
+        /*  hook onto Deepgram API messages  */
+        this.dg.on("message", (message: V1Socket.Response) => {
             if (this.closing || this.queue === null || this.metastore === null)
                 return
-            const text  = (data.channel?.alternatives[0]?.transcript ?? "") as string
-            const words = (data.channel?.alternatives[0]?.words ?? []) as
-                { word: string, punctuated_word?: string, start: number, end: number }[]
-            const isFinal     = (data.is_final     as boolean) ?? false
-            const speechFinal = (data.speech_final as boolean) ?? false
-            const kind = (isFinal || (endpointing > 0 && speechFinal)) ? "final" : "intermediate"
-            if (text === "")
-                this.log("info", `empty/dummy text received (start: ${data.start}s, duration: ${data.duration.toFixed(2)}s)`)
-            else {
-                this.log("info", `text received (start: ${data.start}s, ` +
-                    `duration: ${data.duration.toFixed(2)}s, kind: ${kind}): ` +
-                    `"${text}"`)
-                const start = Duration.fromMillis(data.start * 1000).plus(this.timeZeroOffset)
-                const end   = start.plus({ seconds: data.duration })
-                const metas = this.metastore.fetch(start, end)
-                const meta = metas.toReversed().reduce((prev: Map<string, any>, curr: Map<string, any>) => {
-                    curr.forEach((val, key) => { prev.set(key, val) })
-                    return prev
-                }, new Map<string, any>())
-                this.metastore.prune(start)
-                meta.set("words", words.map((word) => {
-                    const start = Duration.fromMillis(word.start * 1000).plus(this.timeZeroOffset)
-                    const end   = Duration.fromMillis(word.end * 1000).plus(this.timeZeroOffset)
-                    return { word: word.punctuated_word ?? word.word, start, end }
-                }))
-                const chunk = new SpeechFlowChunk(start, end, kind, "text", text, meta)
-                this.queue.write(chunk)
+            if (message.type === "SpeechStarted")
+                this.log("info", "speech started", message)
+            else if (message.type === "UtteranceEnd")
+                this.log("info", "utterance end received", message)
+            else if (message.type === "Metadata")
+                this.log("info", "metadata received")
+            else if (message.type === "Results") {
+                const data  = message
+                const text  = data.channel?.alternatives[0]?.transcript ?? ""
+                const words = data.channel?.alternatives[0]?.words ?? []
+                const isFinal     = data.is_final     ?? false
+                const speechFinal = data.speech_final ?? false
+                const kind = (isFinal || ((this.params.endpointing as number) > 0 && speechFinal)) ? "final" : "intermediate"
+                if (text === "")
+                    this.log("info", `empty/dummy text received (start: ${data.start}s, duration: ${data.duration.toFixed(2)}s)`)
+                else {
+                    this.log("info", `text received (start: ${data.start}s, ` +
+                        `duration: ${data.duration.toFixed(2)}s, kind: ${kind}): ` +
+                        `"${text}"`)
+                    const start = Duration.fromMillis(data.start * 1000).plus(this.timeZeroOffset)
+                    const end   = start.plus({ seconds: data.duration })
+                    const metas = this.metastore.fetch(start, end)
+                    const meta = metas.toReversed().reduce((prev: Map<string, any>, curr: Map<string, any>) => {
+                        curr.forEach((val, key) => { prev.set(key, val) })
+                        return prev
+                    }, new Map<string, any>())
+                    this.metastore.prune(start)
+                    meta.set("words", words.map((word) => {
+                        const wordStart = Duration.fromMillis(word.start * 1000).plus(this.timeZeroOffset)
+                        const wordEnd   = Duration.fromMillis(word.end   * 1000).plus(this.timeZeroOffset)
+                        return { word: word.punctuated_word ?? word.word, start: wordStart, end: wordEnd }
+                    }))
+                    const chunk = new SpeechFlowChunk(start, end, kind, "text", text, meta)
+                    this.queue.write(chunk)
+                    this.lastChunk = chunk
+                }
             }
         })
-        this.dg.on(Deepgram.LiveTranscriptionEvents.SpeechStarted, (data) => {
-            this.log("info", "speech started", data)
-        })
-        this.dg.on(Deepgram.LiveTranscriptionEvents.UtteranceEnd, (data) => {
-            this.log("info", "utterance end received", data)
-        })
-        this.dg.on(Deepgram.LiveTranscriptionEvents.Metadata, (data) => {
-            this.log("info", "metadata received")
-        })
-        this.dg.on(Deepgram.LiveTranscriptionEvents.Close, () => {
+        this.dg.on("close", () => {
             this.log("info", "connection close")
             /*  NOTICE: suppress EOF signalling while reconfiguring (mute toggle),
                 since the connection is being torn down deliberately and the
@@ -203,36 +234,18 @@ export default class SpeechFlowNodeA2TDeepgram extends SpeechFlowNode {
                 this.queue.write(null)
         })
 
-        /*  wait for Deepgram API to be available  */
-        await new Promise((resolve, reject) => {
-            this.connectionTimeout = setTimeout(() => {
-                if (this.connectionTimeout !== null) {
-                    this.connectionTimeout = null
-                    reject(new Error("Deepgram: timeout waiting for connection open"))
-                }
-            }, 8000)
-            this.dg!.once(Deepgram.LiveTranscriptionEvents.Open, () => {
-                this.log("info", "connection open")
-                if (this.connectionTimeout !== null) {
-                    clearTimeout(this.connectionTimeout)
-                    this.connectionTimeout = null
-                }
-                resolve(true)
-            })
-            this.dg!.once(Deepgram.LiveTranscriptionEvents.Error, (err: Error) => {
-                if (this.connectionTimeout !== null) {
-                    clearTimeout(this.connectionTimeout)
-                    this.connectionTimeout = null
-                }
-                reject(err)
-            })
-        })
-
-        /*  NOTICE: register permanent Error handler only AFTER the open
-            handshake -- during open, the transient .once above is the sole
-            Error listener so the caller's promise rejects without a parallel
-            stream emission tearing down the graph prematurely.  */
-        this.dg.on(Deepgram.LiveTranscriptionEvents.Error, (error: Error) => {
+        /*  NOTICE: the Deepgram API socket supports only a single handler per
+            event, so the open handshake and the permanent error handling must
+            share one error handler which dispatches on the "opening" flag:
+            during open, the error is routed solely into the caller's promise
+            rejection, without a parallel stream emission tearing down the
+            graph prematurely.  */
+        this.dg.on("error", (error: Error) => {
+            if (this.opening) {
+                if (this.openReject !== null)
+                    this.openReject(error)
+                return
+            }
             this.log("warning", `error: ${error.message}`)
             /*  NOTICE: do not write null to the queue here -- a transient error
                 must not be misinterpreted as end-of-stream by downstream nodes;
@@ -244,6 +257,53 @@ export default class SpeechFlowNodeA2TDeepgram extends SpeechFlowNode {
             if (!this.closing && this.stream !== null)
                 this.stream.emit("error", error)
         })
+
+        /*  NOTICE: the Deepgram API v5 SDK returns a socket which has not yet
+            dialed, as it is internally created with the "startClosed" option,
+            so the connection has to be established explicitly -- and only once
+            all event handlers above are registered, as the connect operation
+            re-registers the underlying socket listeners  */
+        this.dg.connect()
+
+        /*  clear pending connection timer (idempotent)  */
+        const clearConnectionTimeout = () => {
+            if (this.connectionTimeout !== null) {
+                clearTimeout(this.connectionTimeout)
+                this.connectionTimeout = null
+            }
+        }
+
+        /*  wait for Deepgram API to be available  */
+        this.opening = true
+        let opened = false
+        try {
+            await new Promise((resolve, reject) => {
+                this.openReject = reject
+                this.connectionTimeout = setTimeout(() => {
+                    if (this.connectionTimeout !== null) {
+                        this.connectionTimeout = null
+                        reject(new Error("Deepgram: timeout waiting for connection open"))
+                    }
+                }, 8000)
+                this.dg!.waitForOpen().then(() => {
+                    this.log("info", "connection open")
+                    clearConnectionTimeout()
+                    resolve(true)
+                }).catch((error: unknown) => {
+                    clearConnectionTimeout()
+                    reject(util.ensureError(error, "failed to open Deepgram connection"))
+                })
+            })
+            opened = true
+        }
+        finally {
+            this.opening    = false
+            this.openReject = null
+
+            /*  discard half-established connection on failure  */
+            if (!opened)
+                await this.closeConnection()
+        }
     }
 
     /*  close Deepgram API connection  */
@@ -257,8 +317,7 @@ export default class SpeechFlowNodeA2TDeepgram extends SpeechFlowNode {
         /*  close Deepgram connection and remove listeners  */
         if (this.dg !== null) {
             try {
-                this.dg.removeAllListeners()
-                this.dg.requestClose()
+                this.dg.close()
                 this.log("info", "connection closed")
             }
             catch (error) {
@@ -271,18 +330,19 @@ export default class SpeechFlowNodeA2TDeepgram extends SpeechFlowNode {
     /*  one-time status of node  */
     async status () {
         let balance = 0
+        if (!this.params.keyAdm) {
+            this.log("info", "no Deepgram admin API key configured -- skipping balance fetch")
+            return { balance: balance.toFixed(2) }
+        }
         try {
-            const deepgram = Deepgram.createClient(this.params.keyAdm)
-            const response = await deepgram.manage.getProjects()
-            if (response !== null && response.error === null && response.result?.projects) {
-                for (const project of response.result.projects) {
-                    const balanceResponse = await deepgram.manage.getProjectBalances(project.project_id)
-                    if (balanceResponse !== null && balanceResponse.error === null && balanceResponse.result?.balances)
-                        balance += balanceResponse.result.balances[0]?.amount ?? 0
-                }
+            const deepgram = new Deepgram.DeepgramClient({ apiKey: this.params.keyAdm })
+            const response = await deepgram.manage.v1.projects.list()
+            for (const project of response.projects ?? []) {
+                if (project.project_id === undefined)
+                    continue
+                const balanceResponse = await deepgram.manage.v1.projects.billing.balances.list(project.project_id)
+                balance += balanceResponse.balances?.[0]?.amount ?? 0
             }
-            else if (response !== null && response.error !== null)
-                this.log("warning", `API error fetching projects: ${response.error}`)
         }
         catch (error) {
             this.log("warning", `failed to fetch balance: ${error}`)
@@ -305,6 +365,9 @@ export default class SpeechFlowNodeA2TDeepgram extends SpeechFlowNode {
         /*  create a store for the meta information  */
         this.metastore = new util.TimeStore<Map<string, any>>()
 
+        /*  reset last emitted text chunk  */
+        this.lastChunk = null
+
         /*  determine initial suspended state from configuration  */
         this.suspended = this.params.suspended as boolean
 
@@ -342,10 +405,7 @@ export default class SpeechFlowNodeA2TDeepgram extends SpeechFlowNode {
                             self.metastore.store(chunk.timestampStart, chunk.timestampEnd, chunk.meta)
                         try {
                             /*  send buffer (and intentionally discard all time information)  */
-                            self.dg.send(chunk.payload.buffer.slice(
-                                chunk.payload.byteOffset,
-                                chunk.payload.byteOffset + chunk.payload.byteLength
-                            ))
+                            self.dg.sendMedia(chunk.payload)
                         }
                         catch (error) {
                             callback(util.ensureError(error, "failed to send to Deepgram"))
@@ -363,14 +423,7 @@ export default class SpeechFlowNodeA2TDeepgram extends SpeechFlowNode {
                 }
 
                 /*  close Deepgram API  */
-                if (self.dg !== null) {
-                    try {
-                        self.dg.requestClose()
-                    }
-                    catch (error) {
-                        self.log("warning", `error closing Deepgram connection: ${error}`)
-                    }
-                }
+                await self.closeConnection()
 
                 /*  await all read operations  */
                 await reads.awaitAll()
@@ -421,6 +474,7 @@ export default class SpeechFlowNodeA2TDeepgram extends SpeechFlowNode {
         /*  signal EOF to any pending read operations  */
         if (this.queue !== null) {
             this.queue.write(null)
+            this.queue.destroy()
             this.queue = null
         }
 

package/speechflow-cli/src/speechflow-node-t2t-opus.ts

@@ -86,7 +86,7 @@ export default class SpeechFlowNodeT2TOPUS extends SpeechFlowNode {
         const translate = async (text: string) => {
             const result = await this.translator!(text)
             const single = Array.isArray(result) ? result[0] : result
-            return (single as Transformers.TranslationSingle).translation_text
+            return (single as Transformers.TranslationOutput[number]).translation_text
         }
 
         /*  establish a transform stream and connect it to Transformers  */

package/speechflow-cli/src/speechflow-node-xio-exec.ts

@@ -93,7 +93,7 @@ export default class SpeechFlowNodeXIOExec extends SpeechFlowNode {
         })
 
         /*  handle subprocess errors  */
-        this.subprocess.on("error", (err) => {
+        this.subprocess.nodeChildProcess.on("error", (err: Error) => {
             this.log("error", `subprocess error: ${err.message}`)
             /*  NOTICE: do not emit("error") on the node itself, since nothing
                 listens for it and it would become an uncaughtException tearing
@@ -104,7 +104,7 @@ export default class SpeechFlowNodeXIOExec extends SpeechFlowNode {
         })
 
         /*  handle subprocess exit  */
-        this.subprocess.on("exit", (code, signal) => {
+        this.subprocess.nodeChildProcess.on("exit", (code: number | null, signal: string | null) => {
             if (code !== 0 && code !== null)
                 this.log("warning", `subprocess exited with code ${code}`)
             else if (signal)
@@ -126,10 +126,12 @@ export default class SpeechFlowNodeXIOExec extends SpeechFlowNode {
         /*  dispatch according to mode  */
         if (this.params.mode === "rw") {
             /*  bidirectional mode: both stdin and stdout  */
+            /*  NOTICE: the object form of Stream.Duplex.from() accepts Node streams at
+                run-time, but @types/node types it for Web streams only, so we have to cast  */
             this.stream = Stream.Duplex.from({
                 readable: this.subprocess.stdout,
                 writable: this.subprocess.stdin
-            })
+            } as unknown as Parameters<typeof Stream.Duplex.from>[0])
             const wrapper1 = util.createTransformStreamForWritableSide(this.params.type, highWaterMark)
             const wrapper2 = util.createTransformStreamForReadableSide(
                 this.params.type, () => this.timeZero, highWaterMark,
@@ -176,8 +178,8 @@ export default class SpeechFlowNodeXIOExec extends SpeechFlowNode {
             }
 
             /*  remove event listeners to prevent errors during kill sequence  */
-            this.subprocess.removeAllListeners("error")
-            this.subprocess.removeAllListeners("exit")
+            this.subprocess.nodeChildProcess.removeAllListeners("error")
+            this.subprocess.nodeChildProcess.removeAllListeners("exit")
 
             /*  wait for subprocess to exit gracefully  */
             const ac2 = new AbortController()

package/speechflow-cli/src/speechflow-util-llm.ts

@@ -333,10 +333,18 @@ export class LLM extends EventEmitter {
                 throw new Error(`failed to perform HuggingFace Transformers text generation: ${err}`, { cause: err })
             })
             const single = Array.isArray(result) ? result[0] : result
-            const generatedText = (single as Transformers.TextGenerationSingle).generated_text
-            const content = typeof generatedText === "string" ?
+            const generatedText = (single as Transformers.TextGenerationOutput[number]).generated_text
+            const messageContent = typeof generatedText === "string" ?
                 generatedText :
                 generatedText.at(-1)?.content
+
+            /*  flatten the potentially multi-modal message content into plain text  */
+            const content = typeof messageContent === "string" ?
+                messageContent :
+                messageContent
+                    ?.filter((part) => part.type === "text")
+                    .map((part) => part.text as string)
+                    .join("")
             if (!content)
                 throw new Error("HuggingFace Transformers API returned empty content")
             return content