speechflow 0.9.9 → 1.0.0

package/src/speechflow-node-a2a-mute.ts

@@ -0,0 +1,101 @@
+/*
+**  SpeechFlow - Speech Processing Flow Graph
+**  Copyright (c) 2024-2025 Dr. Ralf S. Engelschall <rse@engelschall.com>
+**  Licensed under GPL 3.0 <https://spdx.org/licenses/GPL-3.0-only>
+*/
+
+/*  standard dependencies  */
+import Stream from "node:stream"
+
+/*  internal dependencies  */
+import SpeechFlowNode, { SpeechFlowChunk } from "./speechflow-node"
+
+/*  the type of muting  */
+type MuteMode =
+    "none"      |  /*  not muted  */
+    "silenced"  |  /*  muted by changing audio samples to silence  */
+    "unplugged"    /*  muted by unplugging the audio sample flow   */
+
+/*  SpeechFlow node for muting in audio-to-audio passing  */
+export default class SpeechFlowNodeMute extends SpeechFlowNode {
+    /*  declare official node name  */
+    public static name = "mute"
+
+    /*  internal state  */
+    private muteMode: MuteMode = "none"
+
+    /*  construct node  */
+    constructor (id: string, cfg: { [ id: string ]: any }, opts: { [ id: string ]: any }, args: any[]) {
+        super(id, cfg, opts, args)
+
+        /*  declare node configuration parameters  */
+        this.configure({})
+
+        /*  declare node input/output format  */
+        this.input  = "audio"
+        this.output = "audio"
+    }
+
+    /*  receive external request  */
+    async receiveRequest (params: any[]) {
+        if (params.length === 2 && params[0] === "mode") {
+            if (!params[1].match(/^(?:none|silenced|unplugged)$/))
+                throw new Error("mute: invalid mode argument in external request")
+            const muteMode: MuteMode = params[1] as MuteMode
+            this.setMuteMode(muteMode)
+            this.sendResponse([ "mute", "mode", muteMode ])
+        }
+        else
+            throw new Error("mute: invalid arguments in external request")
+    }
+
+    /*  change mute mode  */
+    setMuteMode (mode: MuteMode) {
+        this.log("info", `setting mute mode to "${mode}"`)
+        this.muteMode = mode
+    }
+
+    /*  open node  */
+    async open () {
+        /*  establish a transform stream  */
+        const self = this
+        this.stream = new Stream.Transform({
+            readableObjectMode: true,
+            writableObjectMode: true,
+            decodeStrings:      false,
+            transform (chunk: SpeechFlowChunk, encoding, callback) {
+                if (!Buffer.isBuffer(chunk.payload))
+                    callback(new Error("invalid chunk payload type"))
+                else if (self.muteMode === "unplugged")
+                    /*  pass-through nothing  */
+                    callback()
+                else if (self.muteMode === "silenced") {
+                    /*  pass-through a silenced chunk  */
+                    chunk = chunk.clone()
+                    const buffer = chunk.payload as Buffer
+                    buffer.fill(0)
+                    callback()
+                }
+                else {
+                    /*  pass-through original chunk  */
+                    this.push(chunk)
+                    callback()
+                }
+            },
+            final (callback) {
+                this.push(null)
+                callback()
+            }
+        })
+    }
+
+    /*  close node  */
+    async close () {
+        /*  close stream  */
+        if (this.stream !== null) {
+            this.stream.destroy()
+            this.stream = null
+        }
+    }
+}
+

package/src/speechflow-node-a2a-vad.ts

@@ -0,0 +1,266 @@
+/*
+**  SpeechFlow - Speech Processing Flow Graph
+**  Copyright (c) 2024-2025 Dr. Ralf S. Engelschall <rse@engelschall.com>
+**  Licensed under GPL 3.0 <https://spdx.org/licenses/GPL-3.0-only>
+*/
+
+/*  standard dependencies  */
+import Stream             from "node:stream"
+
+/*  external dependencies  */
+import { RealTimeVAD }    from "@ericedouard/vad-node-realtime"
+import { Duration }       from "luxon"
+
+/*  internal dependencies  */
+import SpeechFlowNode, { SpeechFlowChunk } from "./speechflow-node"
+import * as utils                          from "./speechflow-utils"
+
+/*  audio stream queue element */
+type AudioQueueElement = {
+    type:      "audio-frame",
+    chunk:     SpeechFlowChunk,
+    isSpeech?: boolean
+} | {
+    type:      "audio-eof"
+}
+
+/*  SpeechFlow node for VAD speech-to-speech processing  */
+export default class SpeechFlowNodeVAD extends SpeechFlowNode {
+    /*  declare official node name  */
+    public static name = "vad"
+
+    /*  internal state  */
+    private vad: RealTimeVAD | null = null
+    private queue     = new utils.Queue<AudioQueueElement>()
+    private queueRecv = this.queue.pointerUse("recv")
+    private queueVAD  = this.queue.pointerUse("vad")
+    private queueSend = this.queue.pointerUse("send")
+
+    /*  construct node  */
+    constructor (id: string, cfg: { [ id: string ]: any }, opts: { [ id: string ]: any }, args: any[]) {
+        super(id, cfg, opts, args)
+
+        /*  declare node configuration parameters  */
+        this.configure({
+            mode:               { type: "string", val: "unplugged", match: /^(?:silenced|unplugged)$/ },
+            posSpeechThreshold: { type: "number", val: 0.50 },
+            negSpeechThreshold: { type: "number", val: 0.35 },
+            minSpeechFrames:    { type: "number", val: 2    },
+            redemptionFrames:   { type: "number", val: 12   },
+            preSpeechPadFrames: { type: "number", val: 1    }
+        })
+
+        /*  declare node input/output format  */
+        this.input  = "audio"
+        this.output = "audio"
+    }
+
+    /*  open node  */
+    async open () {
+        /*  sanity check situation  */
+        if (this.config.audioBitDepth !== 16 || !this.config.audioLittleEndian)
+            throw new Error("VAD node currently supports PCM-S16LE audio only")
+
+        /*  pass-through logging  */
+        const log = (level: string, msg: string) => { this.log(level, msg) }
+
+        /*  internal processing constants  */
+        const vadSampleRateTarget = 16000 /* internal target of VAD */
+        const vadSamplesPerFrame  = 512   /* required for VAD v5 */
+
+        /*  establish Voice Activity Detection (VAD) facility  */
+        this.vad = await RealTimeVAD.new({
+            model:                   "v5",
+            sampleRate:              this.config.audioSampleRate, /* before resampling to 16KHz */
+            frameSamples:            vadSamplesPerFrame,          /* after  resampling to 16KHz */
+            positiveSpeechThreshold: this.params.posSpeechThreshold,
+            negativeSpeechThreshold: this.params.negSpeechThreshold,
+            minSpeechFrames:         this.params.minSpeechFrames,
+            redemptionFrames:        this.params.redemptionFrames,
+            preSpeechPadFrames:      this.params.preSpeechPadFrames,
+            onSpeechStart: () => {
+                log("info", "VAD: speech start")
+            },
+            onSpeechEnd: (audio) => {
+                const duration = utils.audioArrayDuration(audio, vadSampleRateTarget)
+                log("info", `VAD: speech end (duration: ${duration.toFixed(2)}s)`)
+            },
+            onVADMisfire: () => {
+                log("info", "VAD: speech end (segment too short)")
+            },
+            onFrameProcessed: (audio) => {
+                /*  annotate the current audio frame  */
+                const element = this.queueVAD.peek()
+                if (element !== undefined && element.type === "audio-frame") {
+                    const isSpeech = audio.isSpeech > audio.notSpeech
+                    element.isSpeech = isSpeech
+                    this.queueVAD.touch()
+                    this.queueVAD.walk(+1)
+                }
+            }
+        })
+        this.vad.start()
+
+        /*  provide Duplex stream and internally attach to VAD  */
+        const vad       = this.vad
+        const cfg       = this.config
+        const queue     = this.queue
+        const queueRecv = this.queueRecv
+        const queueSend = this.queueSend
+        const mode      = this.params.mode
+        let carrySamples = new Float32Array()
+        let carryStart   = Duration.fromDurationLike(0)
+        this.stream = new Stream.Duplex({
+            writableObjectMode: true,
+            readableObjectMode: true,
+            decodeStrings:      false,
+
+            /*  receive audio chunk (writable side of stream)  */
+            write (chunk: SpeechFlowChunk, encoding, callback) {
+                if (!Buffer.isBuffer(chunk.payload))
+                    callback(new Error("expected audio input as Buffer chunks"))
+                else if (chunk.payload.byteLength === 0)
+                    callback()
+                else {
+                    /*  convert audio samples from PCM/I16 to PCM/F32  */
+                    let data = utils.convertBufToF32(chunk.payload, cfg.audioLittleEndian)
+                    let start = chunk.timestampStart
+
+                    /*  merge previous carry samples  */
+                    if (carrySamples.length > 0) {
+                        start = carryStart
+                        const merged = new Float32Array(carrySamples.length + data.length)
+                        merged.set(carrySamples)
+                        merged.set(data, carrySamples.length)
+                        data = merged
+                        carrySamples = new Float32Array()
+                    }
+
+                    /*  queue audio samples as individual VAD-sized frames
+                        and in parallel send it into the Voice Activity Detection (VAD)  */
+                    const chunkSize = (vadSamplesPerFrame * (cfg.audioSampleRate / vadSampleRateTarget))
+                    const chunks = Math.trunc(data.length / chunkSize)
+                    for (let i = 0; i < chunks; i++) {
+                        const frame = data.slice(i * chunkSize, (i + 1) * chunkSize)
+                        const buf = utils.convertF32ToBuf(frame)
+                        const duration = utils.audioBufferDuration(buf)
+                        const end = start.plus(duration)
+                        const chunk = new SpeechFlowChunk(start, end, "final", "audio", buf)
+                        queueRecv.append({ type: "audio-frame", chunk })
+                        vad.processAudio(frame)
+                        start = end
+                    }
+
+                    /*  remember new carry samples  */
+                    const bulkLen = chunks * chunkSize
+                    carrySamples = data.slice(bulkLen)
+                    carryStart = start
+
+                    callback()
+                }
+            },
+
+            /*  receive no more audio chunks (writable side of stream)  */
+            final (callback) {
+                /*  flush pending audio chunks  */
+                if (carrySamples.length > 0) {
+                    const chunkSize = (vadSamplesPerFrame * (cfg.audioSampleRate / vadSampleRateTarget))
+                    if (carrySamples.length < chunkSize) {
+                        const merged = new Float32Array(chunkSize)
+                        merged.set(carrySamples)
+                        merged.fill(0.0, carrySamples.length, chunkSize)
+                        carrySamples = merged
+                    }
+                    const buf = utils.convertF32ToBuf(carrySamples)
+                    const duration = utils.audioBufferDuration(buf)
+                    const end = carryStart.plus(duration)
+                    const chunk = new SpeechFlowChunk(carryStart, end, "final", "audio", buf)
+                    queueRecv.append({ type: "audio-frame", chunk })
+                    vad.processAudio(carrySamples)
+                }
+
+                /*  signal end of file  */
+                queueRecv.append({ type: "audio-eof" })
+                callback()
+            },
+
+            /*  send audio chunk(s) (readable side of stream)  */
+            read (_size) {
+                /*  try to perform read operation from scratch  */
+                const tryToRead = () => {
+                    /*  flush pending audio chunks  */
+                    const flushPendingChunks = () => {
+                        let pushed = 0
+                        while (true) {
+                            const element = queueSend.peek()
+                            if (element === undefined)
+                                break
+                            else if (element.type === "audio-eof") {
+                                this.push(null)
+                                break
+                            }
+                            else if (element.type === "audio-frame"
+                                && element.isSpeech === undefined)
+                                break
+                            queueSend.walk(+1)
+                            if (element.isSpeech) {
+                                this.push(element.chunk)
+                                pushed++
+                            }
+                            else if (mode === "silenced") {
+                                const chunk = element.chunk.clone()
+                                const buffer = chunk.payload as Buffer
+                                buffer.fill(0)
+                                this.push(chunk)
+                                pushed++
+                            }
+                            else if (mode === "unplugged" && pushed === 0)
+                                /*  we have to await chunks now, as in unplugged
+                                    mode we else would be never called again until
+                                    we at least once push a new chunk as the result  */
+                                tryToRead()
+                        }
+                    }
+
+                    /*  await forthcoming audio chunks  */
+                    const awaitForthcomingChunks = () => {
+                        const element = queueSend.peek()
+                        if (element !== undefined
+                            && element.type === "audio-frame"
+                            && element.isSpeech !== undefined)
+                            flushPendingChunks()
+                        else
+                            queue.once("write", awaitForthcomingChunks)
+                    }
+
+                    const element = queueSend.peek()
+                    if (element !== undefined && element.type === "audio-eof")
+                        this.push(null)
+                    else if (element !== undefined
+                        && element.type === "audio-frame"
+                        && element.isSpeech !== undefined)
+                        flushPendingChunks()
+                    else
+                        queue.once("write", awaitForthcomingChunks)
+                }
+                tryToRead()
+            }
+        })
+    }
+
+    /*  close node  */
+    async close () {
+        /*  close stream  */
+        if (this.stream !== null) {
+            this.stream.destroy()
+            this.stream = null
+        }
+
+        /*  close VAD  */
+        if (this.vad !== null) {
+            await this.vad.flush()
+            this.vad.destroy()
+            this.vad = null
+        }
+    }
+}

package/src/speechflow-node-a2t-deepgram.ts

@@ -164,7 +164,7 @@ export default class SpeechFlowNodeDeepgram extends SpeechFlowNode {
                     if (chunk.payload.byteLength > 0) {
                         log("info", `Deepgram: send data (${chunk.payload.byteLength} bytes)`)
                         initTimeoutStart()
-                        dg.send(chunk.payload) /* intentionally discard all time information  */
+                        dg.send(chunk.payload.buffer) /* intentionally discard all time information  */
                     }
                     callback()
                 }

package/src/speechflow-node-t2t-transformers.ts

@@ -104,10 +104,11 @@ export default class SpeechFlowNodeTransformers extends SpeechFlowNode {
 
     /*  open node  */
     async open () {
-        /*  instantiate Transformers engine and model  */
         let model: string = ""
+
+        /*  track download progress when instantiating Transformers engine and model  */
         const progressState = new Map<string, number>()
-        const progressCallback = (progress: any) => {
+        const progressCallback: Transformers.ProgressCallback = (progress: any) => {
             let artifact = model
             if (typeof progress.file === "string")
                 artifact += `:${progress.file}`
@@ -126,26 +127,30 @@ export default class SpeechFlowNodeTransformers extends SpeechFlowNode {
                     progressState.delete(artifact)
             }
         }, 1000)
+
+        /*  instantiate Transformers engine and model  */
         if (this.params.model === "OPUS") {
             model = `onnx-community/opus-mt-${this.params.src}-${this.params.dst}`
-            this.translator = await Transformers.pipeline("translation", model, {
-                cache_dir: path.join(this.config.cacheDir, "opus"),
+            const pipeline = Transformers.pipeline("translation", model, {
+                cache_dir: path.join(this.config.cacheDir, "transformers"),
                 dtype:     "q4",
-                device:    "gpu",
+                device:    "auto",
                 progress_callback: progressCallback
             })
+            this.translator = await pipeline
             clearInterval(interval)
             if (this.translator === null)
                 throw new Error("failed to instantiate translator pipeline")
         }
         else if (this.params.model === "SmolLM3") {
             model = "HuggingFaceTB/SmolLM3-3B-ONNX"
-            this.generator = await Transformers.pipeline("text-generation", model, {
+            const pipeline = Transformers.pipeline("text-generation", model, {
                 cache_dir: path.join(this.config.cacheDir, "transformers"),
                 dtype:     "q4",
-                device:    "gpu",
+                device:    "auto",
                 progress_callback: progressCallback
             })
+            this.generator = await pipeline
             clearInterval(interval)
             if (this.generator === null)
                 throw new Error("failed to instantiate generator pipeline")

package/src/speechflow-node-xio-websocket.ts

@@ -9,7 +9,7 @@ import Stream                          from "node:stream"
 
 /*  external dependencies  */
 import ws                              from "ws"
-import ReconnWebsocket, { ErrorEvent } from "@opensumi/reconnecting-websocket"
+import ReconnWebSocket, { ErrorEvent } from "@opensumi/reconnecting-websocket"
 
 /*  internal dependencies  */
 import SpeechFlowNode, { SpeechFlowChunk } from "./speechflow-node"
@@ -22,7 +22,7 @@ export default class SpeechFlowNodeWebsocket extends SpeechFlowNode {
 
     /*  internal state  */
     private server: ws.WebSocketServer | null = null
-    private client: WebSocket          | null = null
+    private client: ReconnWebSocket    | null = null
 
     /*  construct node  */
     constructor (id: string, cfg: { [ id: string ]: any }, opts: { [ id: string ]: any }, args: any[]) {
@@ -153,7 +153,7 @@ export default class SpeechFlowNodeWebsocket extends SpeechFlowNode {
         }
         else if (this.params.connect !== "") {
             /*  connect remotely to a Websocket port  */
-            this.client = new ReconnWebsocket(this.params.connect, [], {
+            this.client = new ReconnWebSocket(this.params.connect, [], {
                 WebSocket:                   ws,
                 WebSocketOptions:            {},
                 reconnectionDelayGrowFactor: 1.3,
@@ -162,10 +162,10 @@ export default class SpeechFlowNodeWebsocket extends SpeechFlowNode {
                 connectionTimeout:           4000,
                 minUptime:                   5000
             })
-            this.client.addEventListener("open", (ev: Event) => {
+            this.client.addEventListener("open", (ev) => {
                 this.log("info", `connection opened to URL ${this.params.connect}`)
             })
-            this.client.addEventListener("close", (ev: Event) => {
+            this.client.addEventListener("close", (ev) => {
                 this.log("info", `connection closed to URL ${this.params.connect}`)
             })
             this.client.addEventListener("error", (ev: ErrorEvent) => {

package/src/speechflow-node.ts

@@ -36,6 +36,8 @@ export class SpeechFlowChunk {
 
 /*  the base class for all SpeechFlow nodes  */
 export default class SpeechFlowNode extends Events.EventEmitter {
+    public static name: string | undefined
+
     /*  general constant configuration (for reference)  */
     config = {
         audioChannels:     1,                            /* audio mono channel        */
@@ -80,6 +82,16 @@ export default class SpeechFlowNode extends Events.EventEmitter {
         this.timeZeroOffset = this.timeZero.diff(this.timeOpen)
     }
 
+    /*  receive external request  */
+    async receiveRequest (args: any[]) {
+        /*  no-op  */
+    }
+
+    /*  send external response  */
+    sendResponse (args: any[]) {
+        this.emit("send-response", args)
+    }
+
     /*  INTERNAL: utility function: create "params" attribute from constructor of sub-classes  */
     configure (spec: { [ id: string ]: { type: string, pos?: number, val?: any, match?: RegExp | ((x: any) => boolean) } }) {
         for (const name of Object.keys(spec)) {