speechflow 1.3.1 → 1.4.0

package/speechflow-cli/src/speechflow-node-a2t-deepgram.ts

@@ -0,0 +1,313 @@
+/*
+**  SpeechFlow - Speech Processing Flow Graph
+**  Copyright (c) 2024-2025 Dr. Ralf S. Engelschall <rse@engelschall.com>
+**  Licensed under GPL 3.0 <https://spdx.org/licenses/GPL-3.0-only>
+*/
+
+/*  standard dependencies  */
+import Stream from "node:stream"
+
+/*  external dependencies  */
+import * as Deepgram          from "@deepgram/sdk"
+import { DateTime, Duration } from "luxon"
+
+/*  internal dependencies  */
+import SpeechFlowNode, { SpeechFlowChunk } from "./speechflow-node"
+import * as utils                          from "./speechflow-utils"
+
+/*  SpeechFlow node for Deepgram speech-to-text conversion  */
+export default class SpeechFlowNodeDeepgram extends SpeechFlowNode {
+    /*  declare official node name  */
+    public static name = "deepgram"
+
+    /*  internal state  */
+    private dg:                Deepgram.LiveClient | null                       = null
+    private destroyed                                                           = false
+    private initTimeout:       ReturnType<typeof setTimeout> | null             = null
+    private connectionTimeout: ReturnType<typeof setTimeout> | null             = null
+    private queue:             utils.SingleQueue<SpeechFlowChunk | null> | null = null
+
+    /*  construct node  */
+    constructor (id: string, cfg: { [ id: string ]: any }, opts: { [ id: string ]: any }, args: any[]) {
+        super(id, cfg, opts, args)
+
+        /*  declare node configuration parameters  */
+        this.configure({
+            key:      { type: "string",  val: process.env.SPEECHFLOW_DEEPGRAM_KEY },
+            keyAdm:   { type: "string",  val: process.env.SPEECHFLOW_DEEPGRAM_KEY_ADM },
+            model:    { type: "string",  val: "nova-2", pos: 0 },
+            version:  { type: "string",  val: "latest", pos: 1 },
+            language: { type: "string",  val: "multi",  pos: 2 },
+            interim:  { type: "boolean", val: false,    pos: 3 }
+        })
+
+        /*  declare node input/output format  */
+        this.input  = "audio"
+        this.output = "text"
+    }
+
+    /*  one-time status of node  */
+    async status () {
+        let balance = 0
+        try {
+            const deepgram = Deepgram.createClient(this.params.keyAdm)
+            const response = await deepgram.manage.getProjects()
+            if (response !== null && response.error === null) {
+                for (const project of response.result.projects) {
+                    const response = await deepgram.manage.getProjectBalances(project.project_id)
+                    if (response !== null && response.error === null)
+                        balance += response.result.balances[0]?.amount ?? 0
+                }
+            }
+        }
+        catch (error) {
+            this.log("warning", `failed to fetch balance: ${error}`)
+        }
+        return { balance: balance.toFixed(2) }
+    }
+
+    /*  open node  */
+    async open () {
+        /*  sanity check situation  */
+        if (this.config.audioBitDepth !== 16 || !this.config.audioLittleEndian)
+            throw new Error("Deepgram node currently supports PCM-S16LE audio only")
+
+        /*  clear destruction flag  */
+        this.destroyed = false
+
+        /*  create queue for results  */
+        this.queue = new utils.SingleQueue<SpeechFlowChunk | null>()
+
+        /*  create a store for the meta information  */
+        const metastore = new utils.TimeStore<Map<string, any>>()
+
+        /*  connect to Deepgram API  */
+        const deepgram = Deepgram.createClient(this.params.key)
+        let language = "en"
+        if (this.params.model.match(/^nova-2/) && this.params.language !== "en")
+            language = this.params.language
+        else if (this.params.model.match(/^nova-3/) && this.params.language !== "en")
+            language = "multi"
+        this.dg = deepgram.listen.live({
+            mip_opt_out:      true,
+            model:            this.params.model,
+            version:          this.params.version,
+            language,
+            channels:         this.config.audioChannels,
+            sample_rate:      this.config.audioSampleRate,
+            encoding:         "linear16",
+            multichannel:     false,
+            endpointing:      false,
+            interim_results:  this.params.interim,
+            smart_format:     true,
+            punctuate:        true,
+            filler_words:     true,
+            numerals:         true,
+            diarize:          false,
+            profanity_filter: false,
+            redact:           false
+        })
+
+        /*  hook onto Deepgram API events  */
+        this.dg.on(Deepgram.LiveTranscriptionEvents.Transcript, async (data) => {
+            if (this.destroyed || this.queue === null)
+                return
+            const text  = (data.channel?.alternatives[0]?.transcript ?? "") as string
+            const words = (data.channel?.alternatives[0]?.words ?? []) as
+                { word: string, punctuated_word?: string, start: number, end: number }[]
+            const isFinal = (data.is_final ?? false) as boolean
+            if (text === "")
+                this.log("info", `empty/dummy text received (start: ${data.start}s, duration: ${data.duration.toFixed(2)}s)`)
+            else {
+                this.log("info", `text received (start: ${data.start}s, duration: ${data.duration.toFixed(2)}s): "${text}"`)
+                const start = Duration.fromMillis(data.start * 1000).plus(this.timeZeroOffset)
+                const end   = start.plus({ seconds: data.duration })
+                const metas = metastore.fetch(start, end)
+                const meta = metas.reduce((prev: Map<string, any>, curr: Map<string, any>) => {
+                    curr.forEach((val, key) => { prev.set(key, val) })
+                    return prev
+                }, new Map<string, any>())
+                metastore.prune(start)
+                meta.set("words", words.map((word) => {
+                    const start = Duration.fromMillis(word.start * 1000).plus(this.timeZeroOffset)
+                    const end   = Duration.fromMillis(word.end * 1000).plus(this.timeZeroOffset)
+                    return { word: word.punctuated_word ?? word.word, start, end }
+                }))
+                const chunk = new SpeechFlowChunk(start, end,
+                    isFinal ? "final" : "intermediate", "text", text, meta)
+                this.queue.write(chunk)
+            }
+        })
+        this.dg.on(Deepgram.LiveTranscriptionEvents.SpeechStarted, (data) => {
+            this.log("info", "speech started", data)
+        })
+        this.dg.on(Deepgram.LiveTranscriptionEvents.UtteranceEnd, (data) => {
+            this.log("info", "utterance end received", data)
+        })
+        this.dg.on(Deepgram.LiveTranscriptionEvents.Metadata, (data) => {
+            this.log("info", "metadata received")
+        })
+        this.dg.on(Deepgram.LiveTranscriptionEvents.Close, () => {
+            this.log("info", "connection close")
+            if (!this.destroyed && this.queue !== null)
+                this.queue.write(null)
+        })
+        this.dg.on(Deepgram.LiveTranscriptionEvents.Error, (error: Error) => {
+            this.log("error", `error: ${error.message}`)
+            if (!this.destroyed && this.queue !== null)
+                this.queue.write(null)
+            this.emit("error")
+        })
+
+        /*  wait for Deepgram API to be available  */
+        await new Promise((resolve, reject) => {
+            this.connectionTimeout = setTimeout(() => {
+                if (this.connectionTimeout !== null) {
+                    this.connectionTimeout = null
+                    reject(new Error("Deepgram: timeout waiting for connection open"))
+                }
+            }, 8000)
+            this.dg!.once(Deepgram.LiveTranscriptionEvents.Open, () => {
+                this.log("info", "connection open")
+                if (this.connectionTimeout !== null) {
+                    clearTimeout(this.connectionTimeout)
+                    this.connectionTimeout = null
+                }
+                resolve(true)
+            })
+        })
+
+        /*  remember opening time to receive time zero offset  */
+        this.timeOpen = DateTime.now()
+
+        /*  provide Duplex stream and internally attach to Deepgram API  */
+        const self = this
+        this.stream = new Stream.Duplex({
+            writableObjectMode: true,
+            readableObjectMode: true,
+            decodeStrings:      false,
+            highWaterMark:      1,
+            write (chunk: SpeechFlowChunk, encoding, callback) {
+                if (self.destroyed || self.dg === null) {
+                    callback(new Error("stream already destroyed"))
+                    return
+                }
+                if (chunk.type !== "audio")
+                    callback(new Error("expected audio input chunk"))
+                else if (!Buffer.isBuffer(chunk.payload))
+                    callback(new Error("expected Buffer input chunk"))
+                else {
+                    if (chunk.payload.byteLength > 0) {
+                        self.log("debug", `send data (${chunk.payload.byteLength} bytes)`)
+                        if (chunk.meta.size > 0)
+                            metastore.store(chunk.timestampStart, chunk.timestampEnd, chunk.meta)
+                        try {
+                            self.dg.send(chunk.payload.buffer) /* intentionally discard all time information */
+                        }
+                        catch (error) {
+                            callback(error instanceof Error ? error : new Error("failed to send to Deepgram"))
+                            return
+                        }
+                    }
+                    callback()
+                }
+            },
+            read (size) {
+                if (self.destroyed || self.queue === null) {
+                    this.push(null)
+                    return
+                }
+                let readTimeout: ReturnType<typeof setTimeout> | null = setTimeout(() => {
+                    if (readTimeout !== null) {
+                        readTimeout = null
+                        if (!self.destroyed) {
+                            self.log("warning", "read timeout - pushing null to prevent hanging")
+                            this.push(null)
+                        }
+                    }
+                }, 30 * 1000)
+                self.queue.read().then((chunk) => {
+                    if (readTimeout !== null) {
+                        clearTimeout(readTimeout)
+                        readTimeout = null
+                    }
+                    if (self.destroyed) {
+                        this.push(null)
+                        return
+                    }
+                    if (chunk === null) {
+                        self.log("info", "received EOF signal")
+                        this.push(null)
+                    }
+                    else {
+                        self.log("debug", `received data (${chunk.payload.length} bytes)`)
+                        this.push(chunk, self.config.textEncoding)
+                    }
+                }).catch((error) => {
+                    if (readTimeout !== null) {
+                        clearTimeout(readTimeout)
+                        readTimeout = null
+                    }
+                    if (!self.destroyed) {
+                        self.log("error", `queue read error: ${error.message}`)
+                        this.push(null)
+                    }
+                })
+            },
+            final (callback) {
+                if (self.destroyed || self.dg === null) {
+                    callback()
+                    return
+                }
+                try {
+                    self.dg.requestClose()
+                }
+                catch (error) {
+                    self.log("warning", `error closing Deepgram connection: ${error}`)
+                }
+                /*  NOTICE: do not push null here -- let the Deepgram close event handle it  */
+                callback()
+            }
+        })
+    }
+
+    /*  close node  */
+    async close () {
+        /*  indicate destruction first to stop all async operations  */
+        this.destroyed = true
+
+        /*  cleanup all timers  */
+        if (this.initTimeout !== null) {
+            clearTimeout(this.initTimeout)
+            this.initTimeout = null
+        }
+        if (this.connectionTimeout !== null) {
+            clearTimeout(this.connectionTimeout)
+            this.connectionTimeout = null
+        }
+
+        /*  close stream  */
+        if (this.stream !== null) {
+            this.stream.destroy()
+            this.stream = null
+        }
+
+        /*  close Deepgram connection and remove listeners  */
+        if (this.dg !== null) {
+            try {
+                this.dg.removeAllListeners()
+                this.dg.requestClose()
+            }
+            catch (error) {
+                this.log("warning", `error during Deepgram cleanup: ${error}`)
+            }
+            this.dg = null
+        }
+
+        /*  signal EOF to any pending read operations  */
+        if (this.queue !== null) {
+            this.queue.write(null)
+            this.queue = null
+        }
+    }
+}

package/{src → speechflow-cli/src}/speechflow-node-t2a-elevenlabs.ts

@@ -23,6 +23,8 @@ export default class SpeechFlowNodeElevenlabs extends SpeechFlowNode {
     /*  internal state  */
     private elevenlabs: ElevenLabs.ElevenLabsClient | null = null
     private static speexInitialized = false
+    private destroyed = false
+    private resampler: SpeexResampler | null = null
 
     /*  construct node  */
     constructor (id: string, cfg: { [ id: string ]: any }, opts: { [ id: string ]: any }, args: any[]) {
@@ -39,6 +41,10 @@ export default class SpeechFlowNodeElevenlabs extends SpeechFlowNode {
             optimize:   { type: "string", val: "latency", pos: 5, match: /^(?:latency|quality)$/ }
         })
 
+        /*  sanity check parameters  */
+        if (!this.params.key)
+            throw new Error("ElevenLabs API key not configured")
+
         /*  declare node input/output format  */
         this.input  = "text"
         this.output = "audio"
@@ -54,6 +60,9 @@ export default class SpeechFlowNodeElevenlabs extends SpeechFlowNode {
 
     /*  open node  */
     async open () {
+        /*  clear destruction flag  */
+        this.destroyed = false
+
         /*  establish ElevenLabs API connection  */
         this.elevenlabs = new ElevenLabs.ElevenLabsClient({
             apiKey: this.params.key
@@ -120,37 +129,68 @@ export default class SpeechFlowNodeElevenlabs extends SpeechFlowNode {
             await SpeexResampler.initPromise
             SpeechFlowNodeElevenlabs.speexInitialized = true
         }
-        const resampler = new SpeexResampler(1, maxSampleRate, this.config.audioSampleRate, 7)
+        this.resampler = new SpeexResampler(1, maxSampleRate, this.config.audioSampleRate, 7)
 
         /*  create transform stream and connect it to the ElevenLabs API  */
-        const log = (level: string, msg: string) => { this.log(level, msg) }
+        const self = this
         this.stream = new Stream.Transform({
             writableObjectMode: true,
             readableObjectMode: true,
             decodeStrings:      false,
             highWaterMark:      1,
             transform (chunk: SpeechFlowChunk, encoding, callback) {
+                if (self.destroyed) {
+                    callback(new Error("stream already destroyed"))
+                    return
+                }
                 if (Buffer.isBuffer(chunk.payload))
                     callback(new Error("invalid chunk payload type"))
                 else {
-                    speechStream(chunk.payload).then((stream) => {
-                        getStreamAsBuffer(stream).then((buffer) => {
-                            const bufferResampled = resampler.processChunk(buffer)
-                            log("info", `ElevenLabs: received audio (buffer length: ${buffer.byteLength})`)
+                    (async () => {
+                        let processTimeout: ReturnType<typeof setTimeout> | null = setTimeout(() => {
+                            processTimeout = null
+                            callback(new Error("ElevenLabs API timeout"))
+                        }, 60 * 1000)
+                        const clearProcessTimeout = () => {
+                            if (processTimeout !== null) {
+                                clearTimeout(processTimeout)
+                                processTimeout = null
+                            }
+                        }
+                        try {
+                            const stream = await speechStream(chunk.payload as string)
+                            if (self.destroyed) {
+                                clearProcessTimeout()
+                                callback(new Error("stream destroyed during processing"))
+                                return
+                            }
+                            const buffer = await getStreamAsBuffer(stream)
+                            if (self.destroyed) {
+                                clearProcessTimeout()
+                                callback(new Error("stream destroyed during processing"))
+                                return
+                            }
+                            const bufferResampled = self.resampler!.processChunk(buffer)
+                            self.log("info", `ElevenLabs: received audio (buffer length: ${buffer.byteLength})`)
                             const chunkNew = chunk.clone()
                             chunkNew.type = "audio"
                             chunkNew.payload = bufferResampled
+                            clearProcessTimeout()
                             this.push(chunkNew)
                             callback()
-                        }).catch((error) => {
-                            callback(error)
-                        })
-                    }).catch((error) => {
-                        callback(error)
-                    })
+                        }
+                        catch (error) {
+                            clearProcessTimeout()
+                            callback(error instanceof Error ? error : new Error("ElevenLabs processing failed"))
+                        }
+                    })()
                 }
             },
             final (callback) {
+                if (self.destroyed) {
+                    callback()
+                    return
+                }
                 this.push(null)
                 callback()
             }
@@ -159,12 +199,19 @@ export default class SpeechFlowNodeElevenlabs extends SpeechFlowNode {
 
     /*  close node  */
     async close () {
+        /*  indicate destruction  */
+        this.destroyed = true
+
         /*  destroy stream  */
         if (this.stream !== null) {
             this.stream.destroy()
             this.stream = null
         }
 
+        /*  destroy resampler  */
+        if (this.resampler !== null)
+            this.resampler = null
+
         /*  destroy ElevenLabs API  */
         if (this.elevenlabs !== null)
             this.elevenlabs = null

package/{src → speechflow-cli/src}/speechflow-node-t2a-kokoro.ts

@@ -21,6 +21,7 @@ export default class SpeechFlowNodeKokoro extends SpeechFlowNode {
 
     /*  internal state  */
     private kokoro: KokoroTTS | null = null
+    private resampler: SpeexResampler | null = null
     private static speexInitialized = false
 
     /*  construct node  */
@@ -59,9 +60,11 @@ export default class SpeechFlowNodeKokoro extends SpeechFlowNode {
         const interval = setInterval(() => {
             for (const [ artifact, percent ] of progressState) {
                 this.log("info", `downloaded ${percent.toFixed(2)}% of artifact "${artifact}"`)
-                if (percent >= 1.0)
+                if (percent >= 100.0)
                     progressState.delete(artifact)
             }
+            if (progressState.size === 0)
+                clearInterval(interval)
         }, 1000)
         this.kokoro = await KokoroTTS.from_pretrained(model, {
             dtype: "q4f16",
@@ -78,7 +81,7 @@ export default class SpeechFlowNodeKokoro extends SpeechFlowNode {
             await SpeexResampler.initPromise
             SpeechFlowNodeKokoro.speexInitialized = true
         }
-        const resampler = new SpeexResampler(1, 24000, this.config.audioSampleRate, 7)
+        this.resampler = new SpeexResampler(1, 24000, this.config.audioSampleRate, 7)
 
         /*  determine voice for text-to-speech operation  */
         const voices = {
@@ -91,7 +94,7 @@ export default class SpeechFlowNodeKokoro extends SpeechFlowNode {
         if (voice === undefined)
             throw new Error(`invalid Kokoro voice "${this.params.voice}"`)
 
-        /*  perform text-to-speech operation with Elevenlabs API  */
+        /*  perform text-to-speech operation with Kokoro API  */
         const text2speech = async (text: string) => {
             this.log("info", `Kokoro: input: "${text}"`)
             const audio = await this.kokoro!.generate(text, {
@@ -110,7 +113,7 @@ export default class SpeechFlowNodeKokoro extends SpeechFlowNode {
             }
 
             /*  resample audio samples from PCM/I16/24Khz to PCM/I16/48KHz  */
-            const buffer2 = resampler.processChunk(buffer1)
+            const buffer2 = this.resampler!.processChunk(buffer1)
 
             return buffer2
         }
@@ -153,6 +156,10 @@ export default class SpeechFlowNodeKokoro extends SpeechFlowNode {
             this.stream = null
         }
 
+        /*  destroy resampler  */
+        if (this.resampler !== null)
+            this.resampler = null
+
         /*  destroy Kokoro API  */
         if (this.kokoro !== null)
             this.kokoro = null

package/{src → speechflow-cli/src}/speechflow-node-t2t-deepl.ts

@@ -27,12 +27,16 @@ export default class SpeechFlowNodeDeepL extends SpeechFlowNode {
 
         /*  declare node configuration parameters  */
         this.configure({
-            key:      { type: "string", val: process.env.SPEECHFLOW_DEEPL_KEY },
+            key:      { type: "string", val: process.env.SPEECHFLOW_DEEPL_KEY ?? "" },
             src:      { type: "string", pos: 0, val: "de",      match: /^(?:de|en)$/ },
             dst:      { type: "string", pos: 1, val: "en",      match: /^(?:de|en)$/ },
             optimize: { type: "string", pos: 2, val: "latency", match: /^(?:latency|quality)$/ }
         })
 
+        /*  validate API key  */
+        if (this.params.key === "")
+            throw new Error("DeepL API key is required")
+
         /*  sanity check situation  */
         if (this.params.src === this.params.dst)
             throw new Error("source and destination languages cannot be the same")
@@ -44,9 +48,10 @@ export default class SpeechFlowNodeDeepL extends SpeechFlowNode {
 
     /*  one-time status of node  */
     async status () {
-        this.deepl = new DeepL.Translator(this.params.key)
-        const usage = await this.deepl.getUsage()
-        const percent = (usage?.character?.count ?? 0) / (usage?.character?.limit ?? 0) * 100
+        const deepl = new DeepL.Translator(this.params.key)
+        const usage = await deepl.getUsage()
+        const limit = usage?.character?.limit ?? 1
+        const percent = limit > 0 ? ((usage?.character?.count ?? 0) / limit * 100) : 0
         return { usage: `${percent.toFixed(8)}%` }
     }
 

package/{src → speechflow-cli/src}/speechflow-node-t2t-format.ts

@@ -41,7 +41,7 @@ export default class SpeechFlowNodeFormat extends SpeechFlowNode {
             return text
         }
 
-        /*  establish a duplex stream and connect it to DeepL translation  */
+        /*  establish a duplex stream and connect it to text formatting  */
         this.stream = new Stream.Transform({
             readableObjectMode: true,
             writableObjectMode: true,
@@ -74,7 +74,7 @@ export default class SpeechFlowNodeFormat extends SpeechFlowNode {
         })
     }
 
-    /*  open node  */
+    /*  close node  */
     async close () {
         /*  close stream  */
         if (this.stream !== null) {

package/{src → speechflow-cli/src}/speechflow-node-t2t-ollama.ts

@@ -39,7 +39,7 @@ export default class SpeechFlowNodeOllama extends SpeechFlowNode {
                 "Do NOT give any preamble.\n" +
                 "Do NOT give any prolog.\n" +
                 "Do NOT give any epilog.\n" +
-                "Do NOT change the gammar.\n" +
+                "Do NOT change the grammar.\n" +
                 "Do NOT use synonyms for words.\n" +
                 "Keep all words.\n" +
                 "Fill in missing commas.\n" +

package/{src → speechflow-cli/src}/speechflow-node-t2t-openai.ts

@@ -39,7 +39,7 @@ export default class SpeechFlowNodeOpenAI extends SpeechFlowNode {
                 "Do NOT give any preamble.\n" +
                 "Do NOT give any prolog.\n" +
                 "Do NOT give any epilog.\n" +
-                "Do NOT change the gammar.\n" +
+                "Do NOT change the grammar.\n" +
                 "Do NOT use synonyms for words.\n" +
                 "Keep all words.\n" +
                 "Fill in missing commas.\n" +