speechflow 1.3.2 → 1.4.0

package/speechflow-cli/src/speechflow-node-a2a-meter.ts

@@ -0,0 +1,217 @@
+/*
+**  SpeechFlow - Speech Processing Flow Graph
+**  Copyright (c) 2024-2025 Dr. Ralf S. Engelschall <rse@engelschall.com>
+**  Licensed under GPL 3.0 <https://spdx.org/licenses/GPL-3.0-only>
+*/
+
+/*  standard dependencies  */
+import Stream from "node:stream"
+
+/*  external dependencies  */
+import { getLUFS, getRMS, AudioData } from "audio-inspect"
+
+/*  internal dependencies  */
+import SpeechFlowNode, { SpeechFlowChunk } from "./speechflow-node"
+import * as utils                          from "./speechflow-utils"
+
+/*  SpeechFlow node for audio metering  */
+export default class SpeechFlowNodeMeter extends SpeechFlowNode {
+    /*  declare official node name  */
+    public static name = "meter"
+
+    /*  internal state  */
+    private emitInterval: ReturnType<typeof setInterval> | null = null
+    private calcInterval: ReturnType<typeof setInterval> | null = null
+    private pendingCalculations = new Set<ReturnType<typeof setTimeout>>()
+    private chunkBuffer = new Float32Array(0)
+    private destroyed = false
+
+    /*  construct node  */
+    constructor (id: string, cfg: { [ id: string ]: any }, opts: { [ id: string ]: any }, args: any[]) {
+        super(id, cfg, opts, args)
+
+        /*  declare node configuration parameters  */
+        this.configure({
+            interval:  { type: "number", pos: 0, val: 250 },
+            dashboard: { type: "string",         val: "" }
+        })
+
+        /*  declare node input/output format  */
+        this.input  = "audio"
+        this.output = "audio"
+    }
+
+    /*  open node  */
+    async open () {
+        /*  sanity check situation  */
+        if (this.config.audioBitDepth !== 16 || !this.config.audioLittleEndian)
+            throw new Error("meter node currently supports PCM-S16LE audio only")
+
+        /*  clear destruction flag  */
+        this.destroyed = false
+
+        /*  internal state  */
+        const sampleWindowDuration = 3 /* LUFS-S requires 3s */
+        const sampleWindowSize = Math.floor(this.config.audioSampleRate * sampleWindowDuration)
+        let sampleWindow = new Float32Array(sampleWindowSize)
+        sampleWindow.fill(0, 0, sampleWindowSize)
+        let lufss = -60
+        let rms = -60
+
+        /*  chunk processing state  */
+        const chunkDuration = 0.050 /* meter update frequency is about 50ms */
+        const samplesPerChunk = Math.floor(this.config.audioSampleRate * chunkDuration)
+        this.chunkBuffer = new Float32Array(0)
+
+        /*  define chunk processing function  */
+        const processChunk = (chunkData: Float32Array) => {
+            /*  update internal audio sample sliding window  */
+            const newWindow = new Float32Array(sampleWindowSize)
+            const keepSize = sampleWindowSize - chunkData.length
+            newWindow.set(sampleWindow.slice(sampleWindow.length - keepSize), 0)
+            newWindow.set(chunkData, keepSize)
+            sampleWindow = newWindow
+
+            /*  asynchronously calculate the LUFS-S metric  */
+            const calculator = setTimeout(() => {
+                if (this.destroyed)
+                    return
+                try {
+                    this.pendingCalculations.delete(calculator)
+                    const audioData = {
+                        sampleRate:       this.config.audioSampleRate,
+                        numberOfChannels: this.config.audioChannels,
+                        channelData:      [ sampleWindow ],
+                        duration:         sampleWindowDuration,
+                        length:           sampleWindow.length
+                    } satisfies AudioData
+                    const lufs = getLUFS(audioData, {
+                        channelMode: this.config.audioChannels === 1 ? "mono" : "stereo",
+                        calculateShortTerm:     true,
+                        calculateMomentary:     false,
+                        calculateLoudnessRange: false,
+                        calculateTruePeak:      false
+                    })
+                    if (!this.destroyed) {
+                        if (timer !== null) {
+                            clearTimeout(timer)
+                            timer = null
+                        }
+                        lufss = lufs.shortTerm ? lufs.shortTerm[0] : 0
+                        rms = getRMS(audioData, { asDB: true })
+                        timer = setTimeout(() => {
+                            lufss = -60
+                            rms   = -60
+                        }, 500)
+                    }
+                }
+                catch (error) {
+                    if (!this.destroyed)
+                        this.log("warning", `meter calculation error: ${error}`)
+                }
+            }, 0)
+            this.pendingCalculations.add(calculator)
+        }
+
+        /*  setup chunking interval  */
+        this.calcInterval = setInterval(() => {
+            if (this.destroyed)
+                return
+
+            /*  process one single 50ms chunk if available  */
+            if (this.chunkBuffer.length >= samplesPerChunk) {
+                const chunkData = this.chunkBuffer.slice(0, samplesPerChunk)
+                processChunk(chunkData)
+                this.chunkBuffer = this.chunkBuffer.slice(samplesPerChunk)
+            }
+        }, chunkDuration * 1000)
+
+        /*  setup loudness emitting interval  */
+        this.emitInterval = setInterval(() => {
+            if (this.destroyed)
+                return
+            this.log("debug", `LUFS-S: ${lufss.toFixed(1)} dB, RMS: ${rms.toFixed(1)} dB`)
+            this.sendResponse([ "meter", "LUFS-S", lufss ])
+            this.sendResponse([ "meter", "RMS", rms ])
+            if (this.params.dashboard !== "")
+                this.dashboardInfo("audio", this.params.dashboard, "final", lufss)
+        }, this.params.interval)
+
+        /*  provide Duplex stream and internally attach to meter  */
+        const self = this
+        let timer: ReturnType<typeof setTimeout> | null = null
+        this.stream = new Stream.Transform({
+            writableObjectMode: true,
+            readableObjectMode: true,
+            decodeStrings:      false,
+            highWaterMark:      1,
+
+            /*  transform audio chunk  */
+            transform (chunk: SpeechFlowChunk, encoding, callback) {
+                if (self.destroyed) {
+                    callback(new Error("stream already destroyed"))
+                    return
+                }
+                if (!Buffer.isBuffer(chunk.payload))
+                    callback(new Error("expected audio input as Buffer chunks"))
+                else if (chunk.payload.byteLength === 0)
+                    callback()
+                else {
+                    try {
+                        /*  convert audio samples from PCM/I16 to PCM/F32  */
+                        const data = utils.convertBufToF32(chunk.payload, self.config.audioLittleEndian)
+
+                        /*  append new data to buffer  */
+                        const combinedLength = self.chunkBuffer.length + data.length
+                        const newBuffer = new Float32Array(combinedLength)
+                        newBuffer.set(self.chunkBuffer, 0)
+                        newBuffer.set(data, self.chunkBuffer.length)
+                        self.chunkBuffer = newBuffer
+
+                        /*  pass-through original audio chunk  */
+                        this.push(chunk)
+                        callback()
+                    }
+                    catch (error) {
+                        callback(error instanceof Error ? error : new Error("Meter processing failed"))
+                    }
+                }
+            },
+            final (callback) {
+                if (self.destroyed) {
+                    callback()
+                    return
+                }
+                this.push(null)
+                callback()
+            }
+        })
+    }
+
+    /*  close node  */
+    async close () {
+        /*  indicate destruction  */
+        this.destroyed = true
+
+        /*  clear all pending calculations  */
+        for (const timeout of this.pendingCalculations)
+            clearTimeout(timeout)
+        this.pendingCalculations.clear()
+
+        /*  stop intervals  */
+        if (this.emitInterval !== null) {
+            clearInterval(this.emitInterval)
+            this.emitInterval = null
+        }
+        if (this.calcInterval !== null) {
+            clearInterval(this.calcInterval)
+            this.calcInterval = null
+        }
+
+        /*  close stream  */
+        if (this.stream !== null) {
+            this.stream.destroy()
+            this.stream = null
+        }
+    }
+}

package/{src → speechflow-cli/src}/speechflow-node-a2a-vad.ts

@@ -77,6 +77,14 @@ export default class SpeechFlowNodeVAD extends SpeechFlowNode {
         const vadSampleRateTarget = 16000 /* internal target of VAD */
         const vadSamplesPerFrame  = 512   /* required for VAD v5 */
 
+        /*  helper function for timer cleanup  */
+        const clearTailTimer = () => {
+            if (this.tailTimer !== null) {
+                clearTimeout(this.tailTimer)
+                this.tailTimer = null
+            }
+        }
+
         /*  establish Voice Activity Detection (VAD) facility  */
         let tail = false
         try {
@@ -95,10 +103,7 @@ export default class SpeechFlowNodeVAD extends SpeechFlowNode {
                     this.log("info", "VAD: speech start")
                     if (this.params.mode === "unplugged") {
                         tail = false
-                        if (this.tailTimer !== null) {
-                            clearTimeout(this.tailTimer)
-                            this.tailTimer = null
-                        }
+                        clearTailTimer()
                     }
                 },
                 onSpeechEnd: (audio) => {
@@ -108,10 +113,7 @@ export default class SpeechFlowNodeVAD extends SpeechFlowNode {
                     this.log("info", `VAD: speech end (duration: ${duration.toFixed(2)}s)`)
                     if (this.params.mode === "unplugged") {
                         tail = true
-                        if (this.tailTimer !== null) {
-                            clearTimeout(this.tailTimer)
-                            this.tailTimer = null
-                        }
+                        clearTailTimer()
                         this.tailTimer = setTimeout(() => {
                             if (this.destroyed || this.tailTimer === null)
                                 return
@@ -121,14 +123,12 @@ export default class SpeechFlowNodeVAD extends SpeechFlowNode {
                     }
                 },
                 onVADMisfire: () => {
-                    if (this.destroyed) return
+                    if (this.destroyed)
+                        return
                     this.log("info", "VAD: speech end (segment too short)")
                     if (this.params.mode === "unplugged") {
                         tail = true
-                        if (this.tailTimer !== null) {
-                            clearTimeout(this.tailTimer)
-                            this.tailTimer = null
-                        }
+                        clearTailTimer()
                         this.tailTimer = setTimeout(() => {
                             if (this.destroyed || this.tailTimer === null)
                                 return
@@ -152,14 +152,7 @@ export default class SpeechFlowNodeVAD extends SpeechFlowNode {
 
                         /*  annotate the entire audio chunk  */
                         if (element.segmentIdx >= element.segmentData.length) {
-                            let isSpeech = false
-                            for (const segment of element.segmentData) {
-                                if (segment.isSpeech) {
-                                    isSpeech = true
-                                    break
-                                }
-                            }
-                            element.isSpeech = isSpeech
+                            element.isSpeech = element.segmentData.some(segment => segment.isSpeech)
                             this.queueVAD.touch()
                             this.queueVAD.walk(+1)
                         }

package/{src → speechflow-cli/src}/speechflow-node-a2t-deepgram.ts

@@ -33,11 +33,12 @@ export default class SpeechFlowNodeDeepgram extends SpeechFlowNode {
 
         /*  declare node configuration parameters  */
         this.configure({
-            key:      { type: "string", val: process.env.SPEECHFLOW_DEEPGRAM_KEY },
-            keyAdm:   { type: "string", val: process.env.SPEECHFLOW_DEEPGRAM_KEY_ADM },
-            model:    { type: "string", val: "nova-3", pos: 0 },
-            version:  { type: "string", val: "latest", pos: 1 },
-            language: { type: "string", val: "multi",  pos: 2 }
+            key:      { type: "string",  val: process.env.SPEECHFLOW_DEEPGRAM_KEY },
+            keyAdm:   { type: "string",  val: process.env.SPEECHFLOW_DEEPGRAM_KEY_ADM },
+            model:    { type: "string",  val: "nova-2", pos: 0 },
+            version:  { type: "string",  val: "latest", pos: 1 },
+            language: { type: "string",  val: "multi",  pos: 2 },
+            interim:  { type: "boolean", val: false,    pos: 3 }
         })
 
         /*  declare node input/output format  */
@@ -96,14 +97,15 @@ export default class SpeechFlowNodeDeepgram extends SpeechFlowNode {
             sample_rate:      this.config.audioSampleRate,
             encoding:         "linear16",
             multichannel:     false,
-            endpointing:      10,
-            interim_results:  false,
+            endpointing:      false,
+            interim_results:  this.params.interim,
             smart_format:     true,
             punctuate:        true,
             filler_words:     true,
-            diarize:          false,
             numerals:         true,
-            profanity_filter: false
+            diarize:          false,
+            profanity_filter: false,
+            redact:           false
         })
 
         /*  hook onto Deepgram API events  */
@@ -113,6 +115,7 @@ export default class SpeechFlowNodeDeepgram extends SpeechFlowNode {
             const text  = (data.channel?.alternatives[0]?.transcript ?? "") as string
             const words = (data.channel?.alternatives[0]?.words ?? []) as
                 { word: string, punctuated_word?: string, start: number, end: number }[]
+            const isFinal = (data.is_final ?? false) as boolean
             if (text === "")
                 this.log("info", `empty/dummy text received (start: ${data.start}s, duration: ${data.duration.toFixed(2)}s)`)
             else {
@@ -130,10 +133,17 @@ export default class SpeechFlowNodeDeepgram extends SpeechFlowNode {
                     const end   = Duration.fromMillis(word.end * 1000).plus(this.timeZeroOffset)
                     return { word: word.punctuated_word ?? word.word, start, end }
                 }))
-                const chunk = new SpeechFlowChunk(start, end, "final", "text", text, meta)
+                const chunk = new SpeechFlowChunk(start, end,
+                    isFinal ? "final" : "intermediate", "text", text, meta)
                 this.queue.write(chunk)
             }
         })
+        this.dg.on(Deepgram.LiveTranscriptionEvents.SpeechStarted, (data) => {
+            this.log("info", "speech started", data)
+        })
+        this.dg.on(Deepgram.LiveTranscriptionEvents.UtteranceEnd, (data) => {
+            this.log("info", "utterance end received", data)
+        })
         this.dg.on(Deepgram.LiveTranscriptionEvents.Metadata, (data) => {
             this.log("info", "metadata received")
         })
@@ -170,31 +180,6 @@ export default class SpeechFlowNodeDeepgram extends SpeechFlowNode {
         /*  remember opening time to receive time zero offset  */
         this.timeOpen = DateTime.now()
 
-        /*  workaround Deepgram initialization problems  */
-        let initDone = false
-        const initTimeoutStart = () => {
-            if (initDone || this.destroyed)
-                return
-            this.initTimeout = setTimeout(async () => {
-                if (this.initTimeout === null || this.destroyed)
-                    return
-                this.initTimeout = null
-                this.log("warning", "initialization timeout -- restarting service usage")
-                await this.close()
-                if (!this.destroyed)
-                    await this.open()
-            }, 3 * 1000)
-        }
-        const initTimeoutStop = () => {
-            if (initDone)
-                return
-            initDone = true
-            if (this.initTimeout !== null) {
-                clearTimeout(this.initTimeout)
-                this.initTimeout = null
-            }
-        }
-
         /*  provide Duplex stream and internally attach to Deepgram API  */
         const self = this
         this.stream = new Stream.Duplex({
@@ -214,7 +199,6 @@ export default class SpeechFlowNodeDeepgram extends SpeechFlowNode {
                 else {
                     if (chunk.payload.byteLength > 0) {
                         self.log("debug", `send data (${chunk.payload.byteLength} bytes)`)
-                        initTimeoutStart()
                         if (chunk.meta.size > 0)
                             metastore.store(chunk.timestampStart, chunk.timestampEnd, chunk.meta)
                         try {
@@ -256,8 +240,7 @@ export default class SpeechFlowNodeDeepgram extends SpeechFlowNode {
                         this.push(null)
                     }
                     else {
-                        self.log("info", `received data (${chunk.payload.length} bytes)`)
-                        initTimeoutStop()
+                        self.log("debug", `received data (${chunk.payload.length} bytes)`)
                         this.push(chunk, self.config.textEncoding)
                     }
                 }).catch((error) => {

package/{src → speechflow-cli/src}/speechflow-node-t2a-elevenlabs.ts

@@ -151,22 +151,22 @@ export default class SpeechFlowNodeElevenlabs extends SpeechFlowNode {
                             processTimeout = null
                             callback(new Error("ElevenLabs API timeout"))
                         }, 60 * 1000)
+                        const clearProcessTimeout = () => {
+                            if (processTimeout !== null) {
+                                clearTimeout(processTimeout)
+                                processTimeout = null
+                            }
+                        }
                         try {
                             const stream = await speechStream(chunk.payload as string)
                             if (self.destroyed) {
-                                if (processTimeout !== null) {
-                                    clearTimeout(processTimeout)
-                                    processTimeout = null
-                                }
+                                clearProcessTimeout()
                                 callback(new Error("stream destroyed during processing"))
                                 return
                             }
                             const buffer = await getStreamAsBuffer(stream)
                             if (self.destroyed) {
-                                if (processTimeout !== null) {
-                                    clearTimeout(processTimeout)
-                                    processTimeout = null
-                                }
+                                clearProcessTimeout()
                                 callback(new Error("stream destroyed during processing"))
                                 return
                             }
@@ -175,18 +175,12 @@ export default class SpeechFlowNodeElevenlabs extends SpeechFlowNode {
                             const chunkNew = chunk.clone()
                             chunkNew.type = "audio"
                             chunkNew.payload = bufferResampled
-                            if (processTimeout !== null) {
-                                clearTimeout(processTimeout)
-                                processTimeout = null
-                            }
+                            clearProcessTimeout()
                             this.push(chunkNew)
                             callback()
                         }
                         catch (error) {
-                            if (processTimeout !== null) {
-                                clearTimeout(processTimeout)
-                                processTimeout = null
-                            }
+                            clearProcessTimeout()
                             callback(error instanceof Error ? error : new Error("ElevenLabs processing failed"))
                         }
                     })()