speechflow 0.9.8 → 1.0.0

package/package.json

@@ -1,8 +1,8 @@
 {
     "name":                                 "speechflow",
-    "version":                              "0.9.8",
-    "x-stdver":                             "0.9.8-EA",
-    "x-release":                            "2025-07-12",
+    "version":                              "1.0.0",
+    "x-stdver":                             "1.0.0-GA",
+    "x-release":                            "2025-07-16",
     "homepage":                             "https://github.com/rse/speechflow",
     "description":                          "Speech Processing Flow Graph",
     "license":                              "GPL-3.0-only",
@@ -26,15 +26,20 @@
         "@elevenlabs/elevenlabs-js":        "2.6.0",
         "stream-transform":                 "3.4.0",
         "get-stream":                       "9.0.1",
-        "@dotenvx/dotenvx":                 "1.47.5",
+        "@dotenvx/dotenvx":                 "1.47.6",
         "speex-resampler":                  "3.0.1",
         "pcm-convert":                      "1.6.5",
         "object-path":                      "0.11.8",
         "ws":                               "8.18.3",
         "bufferutil":                       "4.0.9",
         "utf-8-validate":                   "6.0.5",
+        "@hapi/hapi":                       "21.4.0",
+        "@hapi/boom":                       "10.0.1",
+        "hapi-plugin-header":               "1.1.8",
+        "hapi-plugin-websocket":            "2.4.11",
         "@opensumi/reconnecting-websocket": "4.4.0",
         "ollama":                           "0.5.16",
+        "openai":                           "5.10.0",
         "@rse/ffmpeg":                      "1.4.2",
         "ffmpeg-stream":                    "1.0.1",
         "installed-packages":               "1.0.13",
@@ -42,13 +47,15 @@
         "wav":                              "1.0.2",
         "mqtt":                             "5.13.2",
         "cbor2":                            "2.0.1",
+        "arktype":                          "2.1.20",
         "pure-uuid":                        "1.8.1",
         "wavefile":                         "11.0.0",
+        "audio-inspect":                    "0.0.2",
         "@huggingface/transformers":        "3.6.3",
+        "kokoro-js":                        "1.2.1",
         "@ericedouard/vad-node-realtime":   "0.2.0",
         "luxon":                            "3.7.1",
-        "wrap-text":                        "1.0.10",
-        "smart-whisper":                    "0.8.1"
+        "wrap-text":                        "1.0.10"
     },
     "devDependencies": {
         "eslint":                           "9.31.0",
@@ -57,14 +64,14 @@
         "eslint-plugin-promise":            "7.2.1",
         "eslint-plugin-import":             "2.32.0",
         "eslint-plugin-node":               "11.1.0",
-        "@typescript-eslint/eslint-plugin": "8.36.0",
-        "@typescript-eslint/parser":        "8.36.0",
+        "@typescript-eslint/eslint-plugin": "8.37.0",
+        "@typescript-eslint/parser":        "8.37.0",
         "oxlint":                           "1.6.0",
         "eslint-plugin-oxlint":             "1.6.0",
         "@biomejs/biome":                   "2.0.6",
         "eslint-config-biome":              "1.9.4",
 
-        "@types/node":                      "24.0.13",
+        "@types/node":                      "24.0.14",
         "@types/yargs":                     "17.0.33",
         "@types/js-yaml":                   "4.0.9",
         "@types/object-path":               "0.11.4",
@@ -84,7 +91,7 @@
         "cross-env":                        "7.0.3"
     },
     "overrides": {
-        "onnxruntime-node":                 "1.22.0-dev.20250418-c19a49615b"
+        "@huggingface/transformers":        { "onnxruntime-node": "1.23.0-dev.20250703-7fc6235861" }
     },
     "upd":                                  [ "!@biomejs/biome" ],
     "engines": {

package/src/speechflow-node-a2a-meter.ts

@@ -0,0 +1,125 @@
+/*
+**  SpeechFlow - Speech Processing Flow Graph
+**  Copyright (c) 2024-2025 Dr. Ralf S. Engelschall <rse@engelschall.com>
+**  Licensed under GPL 3.0 <https://spdx.org/licenses/GPL-3.0-only>
+*/
+
+/*  standard dependencies  */
+import Stream             from "node:stream"
+
+/*  external dependencies  */
+import { getLUFS, getRMS, AudioData } from "audio-inspect"
+
+/*  internal dependencies  */
+import SpeechFlowNode, { SpeechFlowChunk } from "./speechflow-node"
+import * as utils                          from "./speechflow-utils"
+
+/*  SpeechFlow node for audio metering  */
+export default class SpeechFlowNodeMeter extends SpeechFlowNode {
+    /*  declare official node name  */
+    public static name = "meter"
+
+    /*  internal state  */
+    interval: ReturnType<typeof setInterval> | null = null
+
+    /*  construct node  */
+    constructor (id: string, cfg: { [ id: string ]: any }, opts: { [ id: string ]: any }, args: any[]) {
+        super(id, cfg, opts, args)
+
+        /*  declare node configuration parameters  */
+        this.configure({
+            interval: { type: "number", val: 250 }
+        })
+
+        /*  declare node input/output format  */
+        this.input  = "audio"
+        this.output = "audio"
+    }
+
+    /*  open node  */
+    async open () {
+        /*  sanity check situation  */
+        if (this.config.audioBitDepth !== 16 || !this.config.audioLittleEndian)
+            throw new Error("VAD node currently supports PCM-S16LE audio only")
+
+        /*  internal state  */
+        const sampleWindowDuration = 3 /* LUFS-S requires 3s */
+        const sampleWindowSize = this.config.audioSampleRate * sampleWindowDuration
+        let sampleWindow = new Float32Array(sampleWindowSize)
+        sampleWindow.fill(0, 0, sampleWindowSize)
+        let lufss = 0
+        let rms = 0
+
+        /*  setup loundess emitting interval  */
+        this.interval = setInterval(() => {
+            this.log("info", `LUFS-S: ${lufss.toFixed(1)} dB, RMS: ${rms.toFixed(1)} dB`)
+            this.sendResponse([ "meter", "LUFS-S", lufss ])
+            this.sendResponse([ "meter", "RMS", rms ])
+        }, this.params.interval)
+
+        /*  provide Duplex stream and internally attach to VAD  */
+        const self = this
+        this.stream = new Stream.Transform({
+            writableObjectMode: true,
+            readableObjectMode: true,
+            decodeStrings:      false,
+
+            /*  transform audio chunk  */
+            transform (chunk: SpeechFlowChunk, encoding, callback) {
+                if (!Buffer.isBuffer(chunk.payload))
+                    callback(new Error("expected audio input as Buffer chunks"))
+                else if (chunk.payload.byteLength === 0)
+                    callback()
+                else {
+                    /*  convert audio samples from PCM/I16 to PCM/F32  */
+                    const data = utils.convertBufToF32(chunk.payload, self.config.audioLittleEndian)
+
+                    /*  update internal audio sample sliding window  */
+                    const fusion = new Float32Array(sampleWindow.length + data.length)
+                    fusion.set(sampleWindow, 0)
+                    fusion.set(data, sampleWindow.length)
+                    sampleWindow = fusion.slice(fusion.length - sampleWindowSize)
+
+                    /*  asynchronously calculate the LUFS-S metric  */
+                    setTimeout(() => {
+                        const audioData = {
+                            sampleRate:       self.config.audioSampleRate,
+                            numberOfChannels: self.config.audioChannels,
+                            channelData:      [ sampleWindow ],
+                            duration:         sampleWindowDuration,
+                            length:           sampleWindow.length
+                        } satisfies AudioData
+                        const lufs = getLUFS(audioData, {
+                            channelMode: self.config.audioChannels === 1 ? "mono" : "stereo",
+                            calculateShortTerm:     true,
+                            calculateMomentary:     false,
+                            calculateLoudnessRange: false,
+                            calculateTruePeak:      false
+                        })
+                        lufss = lufs.shortTerm ? lufs.shortTerm[0] : 0
+                        rms = getRMS(audioData, { asDB: true })
+                    }, 0)
+
+                    /*  pass-through original audio chunk  */
+                    this.push(chunk)
+                    callback()
+                }
+            }
+        })
+    }
+
+    /*  close node  */
+    async close () {
+        /*  close stream  */
+        if (this.stream !== null) {
+            this.stream.destroy()
+            this.stream = null
+        }
+
+        /*  stop interval  */
+        if (this.interval !== null) {
+            clearInterval(this.interval)
+            this.interval = null
+        }
+    }
+}

package/src/speechflow-node-a2a-mute.ts

@@ -0,0 +1,101 @@
+/*
+**  SpeechFlow - Speech Processing Flow Graph
+**  Copyright (c) 2024-2025 Dr. Ralf S. Engelschall <rse@engelschall.com>
+**  Licensed under GPL 3.0 <https://spdx.org/licenses/GPL-3.0-only>
+*/
+
+/*  standard dependencies  */
+import Stream from "node:stream"
+
+/*  internal dependencies  */
+import SpeechFlowNode, { SpeechFlowChunk } from "./speechflow-node"
+
+/*  the type of muting  */
+type MuteMode =
+    "none"      |  /*  not muted  */
+    "silenced"  |  /*  muted by changing audio samples to silence  */
+    "unplugged"    /*  muted by unplugging the audio sample flow   */
+
+/*  SpeechFlow node for muting in audio-to-audio passing  */
+export default class SpeechFlowNodeMute extends SpeechFlowNode {
+    /*  declare official node name  */
+    public static name = "mute"
+
+    /*  internal state  */
+    private muteMode: MuteMode = "none"
+
+    /*  construct node  */
+    constructor (id: string, cfg: { [ id: string ]: any }, opts: { [ id: string ]: any }, args: any[]) {
+        super(id, cfg, opts, args)
+
+        /*  declare node configuration parameters  */
+        this.configure({})
+
+        /*  declare node input/output format  */
+        this.input  = "audio"
+        this.output = "audio"
+    }
+
+    /*  receive external request  */
+    async receiveRequest (params: any[]) {
+        if (params.length === 2 && params[0] === "mode") {
+            if (!params[1].match(/^(?:none|silenced|unplugged)$/))
+                throw new Error("mute: invalid mode argument in external request")
+            const muteMode: MuteMode = params[1] as MuteMode
+            this.setMuteMode(muteMode)
+            this.sendResponse([ "mute", "mode", muteMode ])
+        }
+        else
+            throw new Error("mute: invalid arguments in external request")
+    }
+
+    /*  change mute mode  */
+    setMuteMode (mode: MuteMode) {
+        this.log("info", `setting mute mode to "${mode}"`)
+        this.muteMode = mode
+    }
+
+    /*  open node  */
+    async open () {
+        /*  establish a transform stream  */
+        const self = this
+        this.stream = new Stream.Transform({
+            readableObjectMode: true,
+            writableObjectMode: true,
+            decodeStrings:      false,
+            transform (chunk: SpeechFlowChunk, encoding, callback) {
+                if (!Buffer.isBuffer(chunk.payload))
+                    callback(new Error("invalid chunk payload type"))
+                else if (self.muteMode === "unplugged")
+                    /*  pass-through nothing  */
+                    callback()
+                else if (self.muteMode === "silenced") {
+                    /*  pass-through a silenced chunk  */
+                    chunk = chunk.clone()
+                    const buffer = chunk.payload as Buffer
+                    buffer.fill(0)
+                    callback()
+                }
+                else {
+                    /*  pass-through original chunk  */
+                    this.push(chunk)
+                    callback()
+                }
+            },
+            final (callback) {
+                this.push(null)
+                callback()
+            }
+        })
+    }
+
+    /*  close node  */
+    async close () {
+        /*  close stream  */
+        if (this.stream !== null) {
+            this.stream.destroy()
+            this.stream = null
+        }
+    }
+}
+

package/src/speechflow-node-a2a-vad.ts

@@ -0,0 +1,266 @@
+/*
+**  SpeechFlow - Speech Processing Flow Graph
+**  Copyright (c) 2024-2025 Dr. Ralf S. Engelschall <rse@engelschall.com>
+**  Licensed under GPL 3.0 <https://spdx.org/licenses/GPL-3.0-only>
+*/
+
+/*  standard dependencies  */
+import Stream             from "node:stream"
+
+/*  external dependencies  */
+import { RealTimeVAD }    from "@ericedouard/vad-node-realtime"
+import { Duration }       from "luxon"
+
+/*  internal dependencies  */
+import SpeechFlowNode, { SpeechFlowChunk } from "./speechflow-node"
+import * as utils                          from "./speechflow-utils"
+
+/*  audio stream queue element */
+type AudioQueueElement = {
+    type:      "audio-frame",
+    chunk:     SpeechFlowChunk,
+    isSpeech?: boolean
+} | {
+    type:      "audio-eof"
+}
+
+/*  SpeechFlow node for VAD speech-to-speech processing  */
+export default class SpeechFlowNodeVAD extends SpeechFlowNode {
+    /*  declare official node name  */
+    public static name = "vad"
+
+    /*  internal state  */
+    private vad: RealTimeVAD | null = null
+    private queue     = new utils.Queue<AudioQueueElement>()
+    private queueRecv = this.queue.pointerUse("recv")
+    private queueVAD  = this.queue.pointerUse("vad")
+    private queueSend = this.queue.pointerUse("send")
+
+    /*  construct node  */
+    constructor (id: string, cfg: { [ id: string ]: any }, opts: { [ id: string ]: any }, args: any[]) {
+        super(id, cfg, opts, args)
+
+        /*  declare node configuration parameters  */
+        this.configure({
+            mode:               { type: "string", val: "unplugged", match: /^(?:silenced|unplugged)$/ },
+            posSpeechThreshold: { type: "number", val: 0.50 },
+            negSpeechThreshold: { type: "number", val: 0.35 },
+            minSpeechFrames:    { type: "number", val: 2    },
+            redemptionFrames:   { type: "number", val: 12   },
+            preSpeechPadFrames: { type: "number", val: 1    }
+        })
+
+        /*  declare node input/output format  */
+        this.input  = "audio"
+        this.output = "audio"
+    }
+
+    /*  open node  */
+    async open () {
+        /*  sanity check situation  */
+        if (this.config.audioBitDepth !== 16 || !this.config.audioLittleEndian)
+            throw new Error("VAD node currently supports PCM-S16LE audio only")
+
+        /*  pass-through logging  */
+        const log = (level: string, msg: string) => { this.log(level, msg) }
+
+        /*  internal processing constants  */
+        const vadSampleRateTarget = 16000 /* internal target of VAD */
+        const vadSamplesPerFrame  = 512   /* required for VAD v5 */
+
+        /*  establish Voice Activity Detection (VAD) facility  */
+        this.vad = await RealTimeVAD.new({
+            model:                   "v5",
+            sampleRate:              this.config.audioSampleRate, /* before resampling to 16KHz */
+            frameSamples:            vadSamplesPerFrame,          /* after  resampling to 16KHz */
+            positiveSpeechThreshold: this.params.posSpeechThreshold,
+            negativeSpeechThreshold: this.params.negSpeechThreshold,
+            minSpeechFrames:         this.params.minSpeechFrames,
+            redemptionFrames:        this.params.redemptionFrames,
+            preSpeechPadFrames:      this.params.preSpeechPadFrames,
+            onSpeechStart: () => {
+                log("info", "VAD: speech start")
+            },
+            onSpeechEnd: (audio) => {
+                const duration = utils.audioArrayDuration(audio, vadSampleRateTarget)
+                log("info", `VAD: speech end (duration: ${duration.toFixed(2)}s)`)
+            },
+            onVADMisfire: () => {
+                log("info", "VAD: speech end (segment too short)")
+            },
+            onFrameProcessed: (audio) => {
+                /*  annotate the current audio frame  */
+                const element = this.queueVAD.peek()
+                if (element !== undefined && element.type === "audio-frame") {
+                    const isSpeech = audio.isSpeech > audio.notSpeech
+                    element.isSpeech = isSpeech
+                    this.queueVAD.touch()
+                    this.queueVAD.walk(+1)
+                }
+            }
+        })
+        this.vad.start()
+
+        /*  provide Duplex stream and internally attach to VAD  */
+        const vad       = this.vad
+        const cfg       = this.config
+        const queue     = this.queue
+        const queueRecv = this.queueRecv
+        const queueSend = this.queueSend
+        const mode      = this.params.mode
+        let carrySamples = new Float32Array()
+        let carryStart   = Duration.fromDurationLike(0)
+        this.stream = new Stream.Duplex({
+            writableObjectMode: true,
+            readableObjectMode: true,
+            decodeStrings:      false,
+
+            /*  receive audio chunk (writable side of stream)  */
+            write (chunk: SpeechFlowChunk, encoding, callback) {
+                if (!Buffer.isBuffer(chunk.payload))
+                    callback(new Error("expected audio input as Buffer chunks"))
+                else if (chunk.payload.byteLength === 0)
+                    callback()
+                else {
+                    /*  convert audio samples from PCM/I16 to PCM/F32  */
+                    let data = utils.convertBufToF32(chunk.payload, cfg.audioLittleEndian)
+                    let start = chunk.timestampStart
+
+                    /*  merge previous carry samples  */
+                    if (carrySamples.length > 0) {
+                        start = carryStart
+                        const merged = new Float32Array(carrySamples.length + data.length)
+                        merged.set(carrySamples)
+                        merged.set(data, carrySamples.length)
+                        data = merged
+                        carrySamples = new Float32Array()
+                    }
+
+                    /*  queue audio samples as individual VAD-sized frames
+                        and in parallel send it into the Voice Activity Detection (VAD)  */
+                    const chunkSize = (vadSamplesPerFrame * (cfg.audioSampleRate / vadSampleRateTarget))
+                    const chunks = Math.trunc(data.length / chunkSize)
+                    for (let i = 0; i < chunks; i++) {
+                        const frame = data.slice(i * chunkSize, (i + 1) * chunkSize)
+                        const buf = utils.convertF32ToBuf(frame)
+                        const duration = utils.audioBufferDuration(buf)
+                        const end = start.plus(duration)
+                        const chunk = new SpeechFlowChunk(start, end, "final", "audio", buf)
+                        queueRecv.append({ type: "audio-frame", chunk })
+                        vad.processAudio(frame)
+                        start = end
+                    }
+
+                    /*  remember new carry samples  */
+                    const bulkLen = chunks * chunkSize
+                    carrySamples = data.slice(bulkLen)
+                    carryStart = start
+
+                    callback()
+                }
+            },
+
+            /*  receive no more audio chunks (writable side of stream)  */
+            final (callback) {
+                /*  flush pending audio chunks  */
+                if (carrySamples.length > 0) {
+                    const chunkSize = (vadSamplesPerFrame * (cfg.audioSampleRate / vadSampleRateTarget))
+                    if (carrySamples.length < chunkSize) {
+                        const merged = new Float32Array(chunkSize)
+                        merged.set(carrySamples)
+                        merged.fill(0.0, carrySamples.length, chunkSize)
+                        carrySamples = merged
+                    }
+                    const buf = utils.convertF32ToBuf(carrySamples)
+                    const duration = utils.audioBufferDuration(buf)
+                    const end = carryStart.plus(duration)
+                    const chunk = new SpeechFlowChunk(carryStart, end, "final", "audio", buf)
+                    queueRecv.append({ type: "audio-frame", chunk })
+                    vad.processAudio(carrySamples)
+                }
+
+                /*  signal end of file  */
+                queueRecv.append({ type: "audio-eof" })
+                callback()
+            },
+
+            /*  send audio chunk(s) (readable side of stream)  */
+            read (_size) {
+                /*  try to perform read operation from scratch  */
+                const tryToRead = () => {
+                    /*  flush pending audio chunks  */
+                    const flushPendingChunks = () => {
+                        let pushed = 0
+                        while (true) {
+                            const element = queueSend.peek()
+                            if (element === undefined)
+                                break
+                            else if (element.type === "audio-eof") {
+                                this.push(null)
+                                break
+                            }
+                            else if (element.type === "audio-frame"
+                                && element.isSpeech === undefined)
+                                break
+                            queueSend.walk(+1)
+                            if (element.isSpeech) {
+                                this.push(element.chunk)
+                                pushed++
+                            }
+                            else if (mode === "silenced") {
+                                const chunk = element.chunk.clone()
+                                const buffer = chunk.payload as Buffer
+                                buffer.fill(0)
+                                this.push(chunk)
+                                pushed++
+                            }
+                            else if (mode === "unplugged" && pushed === 0)
+                                /*  we have to await chunks now, as in unplugged
+                                    mode we else would be never called again until
+                                    we at least once push a new chunk as the result  */
+                                tryToRead()
+                        }
+                    }
+
+                    /*  await forthcoming audio chunks  */
+                    const awaitForthcomingChunks = () => {
+                        const element = queueSend.peek()
+                        if (element !== undefined
+                            && element.type === "audio-frame"
+                            && element.isSpeech !== undefined)
+                            flushPendingChunks()
+                        else
+                            queue.once("write", awaitForthcomingChunks)
+                    }
+
+                    const element = queueSend.peek()
+                    if (element !== undefined && element.type === "audio-eof")
+                        this.push(null)
+                    else if (element !== undefined
+                        && element.type === "audio-frame"
+                        && element.isSpeech !== undefined)
+                        flushPendingChunks()
+                    else
+                        queue.once("write", awaitForthcomingChunks)
+                }
+                tryToRead()
+            }
+        })
+    }
+
+    /*  close node  */
+    async close () {
+        /*  close stream  */
+        if (this.stream !== null) {
+            this.stream.destroy()
+            this.stream = null
+        }
+
+        /*  close VAD  */
+        if (this.vad !== null) {
+            await this.vad.flush()
+            this.vad.destroy()
+            this.vad = null
+        }
+    }
+}

package/src/speechflow-node-a2t-deepgram.ts

@@ -164,7 +164,7 @@ export default class SpeechFlowNodeDeepgram extends SpeechFlowNode {
                     if (chunk.payload.byteLength > 0) {
                         log("info", `Deepgram: send data (${chunk.payload.byteLength} bytes)`)
                         initTimeoutStart()
-                        dg.send(chunk.payload) /* intentionally discard all time information  */
+                        dg.send(chunk.payload.buffer) /* intentionally discard all time information  */
                     }
                     callback()
                 }