speechflow 1.1.0 → 1.2.1

package/src/speechflow-node-a2a-gender.ts

@@ -0,0 +1,272 @@
+/*
+**  SpeechFlow - Speech Processing Flow Graph
+**  Copyright (c) 2024-2025 Dr. Ralf S. Engelschall <rse@engelschall.com>
+**  Licensed under GPL 3.0 <https://spdx.org/licenses/GPL-3.0-only>
+*/
+
+/*  standard dependencies  */
+import path               from "node:path"
+import Stream             from "node:stream"
+
+/*  external dependencies  */
+import * as Transformers  from "@huggingface/transformers"
+import { WaveFile }       from "wavefile"
+
+/*  internal dependencies  */
+import SpeechFlowNode, { SpeechFlowChunk } from "./speechflow-node"
+import * as utils                          from "./speechflow-utils"
+
+/*  audio stream queue element */
+type AudioQueueElement = {
+    type:         "audio-frame",
+    chunk:        SpeechFlowChunk,
+    data:         Float32Array,
+    gender?:      "male" | "female"
+} | {
+    type:         "audio-eof"
+}
+
+/*  SpeechFlow node for Gender recognition  */
+export default class SpeechFlowNodeGender extends SpeechFlowNode {
+    /*  declare official node name  */
+    public static name = "gender"
+
+    /*  internal state  */
+    private static speexInitialized = false
+    private classifier: Transformers.AudioClassificationPipeline | null = null
+    private queue     = new utils.Queue<AudioQueueElement>()
+    private queueRecv = this.queue.pointerUse("recv")
+    private queueAC   = this.queue.pointerUse("ac")
+    private queueSend = this.queue.pointerUse("send")
+
+    /*  construct node  */
+    constructor (id: string, cfg: { [ id: string ]: any }, opts: { [ id: string ]: any }, args: any[]) {
+        super(id, cfg, opts, args)
+
+        /*  declare node configuration parameters  */
+        this.configure({
+            window: { type: "number", pos: 0, val: 500 }
+        })
+
+        /*  declare node input/output format  */
+        this.input  = "audio"
+        this.output = "audio"
+    }
+
+    /*  open node  */
+    async open () {
+        /*  sanity check situation  */
+        if (this.config.audioBitDepth !== 16 || !this.config.audioLittleEndian)
+            throw new Error("Gender node currently supports PCM-S16LE audio only")
+
+        /*  pass-through logging  */
+        const log = (level: string, msg: string) => { this.log(level, msg) }
+
+        /*  the used model  */
+        const model = "Xenova/wav2vec2-large-xlsr-53-gender-recognition-librispeech"
+
+        /*  track download progress when instantiating Transformers engine and model  */
+        const progressState = new Map<string, number>()
+        const progressCallback: Transformers.ProgressCallback = (progress: any) => {
+            let artifact = model
+            if (typeof progress.file === "string")
+                artifact += `:${progress.file}`
+            let percent = 0
+            if (typeof progress.loaded === "number" && typeof progress.total === "number")
+                percent = (progress.loaded as number / progress.total as number) * 100
+            else if (typeof progress.progress === "number")
+                percent = progress.progress
+            if (percent > 0)
+                progressState.set(artifact, percent)
+        }
+        const interval = setInterval(() => {
+            for (const [ artifact, percent ] of progressState) {
+                this.log("info", `downloaded ${percent.toFixed(2)}% of artifact "${artifact}"`)
+                if (percent >= 1.0)
+                    progressState.delete(artifact)
+            }
+        }, 1000)
+
+        /*  instantiate Transformers engine and model  */
+        const pipeline = Transformers.pipeline("audio-classification", model, {
+            cache_dir: path.join(this.config.cacheDir, "gender"),
+            dtype:     "q4",
+            device:    "auto",
+            progress_callback: progressCallback
+        })
+        this.classifier = await pipeline
+        clearInterval(interval)
+        if (this.classifier === null)
+            throw new Error("failed to instantiate classifier pipeline")
+
+        /*  classify a single large-enough concatenated audio frame  */
+        const classify = async (data: Float32Array) => {
+            const result = await this.classifier!(data)
+            const classified: Transformers.AudioClassificationOutput =
+                Array.isArray(result) ? result as Transformers.AudioClassificationOutput : [ result ]
+            const c1 = classified.find((c: any) => c.label === "male")
+            const c2 = classified.find((c: any) => c.label === "female")
+            const male   = c1 ? c1.score : 0.0
+            const female = c2 ? c2.score : 0.0
+            return (male > female ? "male" : "female")
+        }
+
+        /*  work off queued audio frames  */
+        const frameWindowDuration = 0.5
+        const frameWindowSamples  = frameWindowDuration * this.config.audioSampleRate
+        let lastGender = ""
+        let workingOffTimer: ReturnType<typeof setTimeout> | null = null
+        let workingOff = false
+        const workOffQueue = async () => {
+            /*  control working off round  */
+            if (workingOff)
+                return
+            workingOff = true
+            if (workingOffTimer !== null) {
+                clearTimeout(workingOffTimer)
+                workingOffTimer = null
+            }
+
+            let pos0 = this.queueAC.position()
+            const posL = this.queueAC.maxPosition()
+            const data = new Float32Array(frameWindowSamples)
+            data.fill(0)
+            let samples = 0
+            let pos = pos0
+            while (pos < posL && samples < frameWindowSamples) {
+                const element = this.queueAC.peek(pos)
+                if (element === undefined || element.type !== "audio-frame")
+                    break
+                if ((samples + element.data.length) < frameWindowSamples) {
+                    data.set(element.data, samples)
+                    samples += element.data.length
+                }
+                pos++
+            }
+            if (pos0 < pos && samples > frameWindowSamples * 0.75) {
+                const gender = await classify(data)
+                const posM = pos0 + Math.trunc((pos - pos0) * 0.25)
+                while (pos0 < posM && pos0 < posL) {
+                    const element = this.queueAC.peek(pos0)
+                    if (element === undefined || element.type !== "audio-frame")
+                        break
+                    element.gender = gender
+                    this.queueAC.touch()
+                    this.queueAC.walk(+1)
+                    pos0++
+                }
+                if (lastGender !== gender) {
+                    log("info", `gender now recognized as <${gender}>`)
+                    lastGender = gender
+                }
+            }
+
+            /*  re-initiate working off round  */
+            workingOff = false
+            workingOffTimer = setTimeout(() => { workOffQueue() }, 100)
+            this.queue.once("write", () => { workOffQueue() })
+        }
+        this.queue.once("write", () => { workOffQueue() })
+
+        /*  define sample rate required by model  */
+        const sampleRateTarget = 16000
+
+        /*  provide Duplex stream and internally attach to classifier  */
+        const self = this
+        this.stream = new Stream.Duplex({
+            writableObjectMode: true,
+            readableObjectMode: true,
+            decodeStrings:      false,
+
+            /*  receive audio chunk (writable side of stream)  */
+            write (chunk: SpeechFlowChunk, encoding, callback) {
+                if (!Buffer.isBuffer(chunk.payload))
+                    callback(new Error("expected audio input as Buffer chunks"))
+                else if (chunk.payload.byteLength === 0)
+                    callback()
+                else {
+                    /*  convert audio samples from PCM/I16/48KHz to PCM/F32/16KHz  */
+                    let data = utils.convertBufToF32(chunk.payload, self.config.audioLittleEndian)
+                    const wav = new WaveFile()
+                    wav.fromScratch(self.config.audioChannels, self.config.audioSampleRate, "32f", data)
+                    wav.toSampleRate(sampleRateTarget, { method: "cubic" })
+                    data = wav.getSamples(false, Float32Array<ArrayBuffer>) as
+                        any as Float32Array<ArrayBuffer>
+
+                    /*  queue chunk and converted data  */
+                    self.queueRecv.append({ type: "audio-frame", chunk, data })
+
+                    callback()
+                }
+            },
+
+            /*  receive no more audio chunks (writable side of stream)  */
+            final (callback) {
+                /*  signal end of file  */
+                self.queueRecv.append({ type: "audio-eof" })
+                callback()
+            },
+
+            /*  send audio chunk(s) (readable side of stream)  */
+            read (_size) {
+                /*  flush pending audio chunks  */
+                const flushPendingChunks = () => {
+                    while (true) {
+                        const element = self.queueSend.peek()
+                        if (element === undefined)
+                            break
+                        else if (element.type === "audio-eof") {
+                            this.push(null)
+                            break
+                        }
+                        else if (element.type === "audio-frame"
+                            && element.gender === undefined)
+                            break
+                        const duration = utils.audioArrayDuration(element.data)
+                        log("info", `send chunk (${duration.toFixed(3)}s) with gender <${element.gender}>`)
+                        element.chunk.meta.set("gender", element.gender)
+                        this.push(element.chunk)
+                        self.queueSend.walk(+1)
+                        self.queue.trim()
+                    }
+                }
+
+                /*  await forthcoming audio chunks  */
+                const awaitForthcomingChunks = () => {
+                    const element = self.queueSend.peek()
+                    if (element !== undefined
+                        && element.type === "audio-frame"
+                        && element.gender !== undefined)
+                        flushPendingChunks()
+                    else
+                        self.queue.once("write", awaitForthcomingChunks)
+                }
+
+                const element = self.queueSend.peek()
+                if (element !== undefined && element.type === "audio-eof")
+                    this.push(null)
+                else if (element !== undefined
+                    && element.type === "audio-frame"
+                    && element.gender !== undefined)
+                    flushPendingChunks()
+                else
+                    self.queue.once("write", awaitForthcomingChunks)
+            }
+        })
+    }
+
+    /*  close node  */
+    async close () {
+        /*  close stream  */
+        if (this.stream !== null) {
+            this.stream.destroy()
+            this.stream = null
+        }
+
+        /*  close classifier  */
+        if (this.classifier !== null) {
+            this.classifier.dispose()
+            this.classifier = null
+        }
+    }
+}

package/src/speechflow-node-a2a-meter.ts

@@ -5,7 +5,7 @@
 */
 
 /*  standard dependencies  */
-import Stream             from "node:stream"
+import Stream from "node:stream"
 
 /*  external dependencies  */
 import { getLUFS, getRMS, AudioData } from "audio-inspect"
@@ -40,7 +40,7 @@ export default class SpeechFlowNodeMeter extends SpeechFlowNode {
     async open () {
         /*  sanity check situation  */
         if (this.config.audioBitDepth !== 16 || !this.config.audioLittleEndian)
-            throw new Error("VAD node currently supports PCM-S16LE audio only")
+            throw new Error("meter node currently supports PCM-S16LE audio only")
 
         /*  internal state  */
         const sampleWindowDuration = 3 /* LUFS-S requires 3s */
@@ -50,7 +50,7 @@ export default class SpeechFlowNodeMeter extends SpeechFlowNode {
         let lufss = 0
         let rms = 0
 
-        /*  setup loundess emitting interval  */
+        /*  setup loudness emitting interval  */
         this.interval = setInterval(() => {
             this.log("info", `LUFS-S: ${lufss.toFixed(1)} dB, RMS: ${rms.toFixed(1)} dB`)
             this.sendResponse([ "meter", "LUFS-S", lufss ])

package/src/speechflow-node-a2a-mute.ts

@@ -72,6 +72,7 @@ export default class SpeechFlowNodeMute extends SpeechFlowNode {
                 else if (self.muteMode === "silenced") {
                     /*  pass-through a silenced chunk  */
                     chunk = chunk.clone()
+                    chunk.meta.set("muted", true)
                     const buffer = chunk.payload as Buffer
                     buffer.fill(0)
                     callback()

package/src/speechflow-node-a2a-vad.ts

@@ -9,19 +9,24 @@ import Stream             from "node:stream"
 
 /*  external dependencies  */
 import { RealTimeVAD }    from "@ericedouard/vad-node-realtime"
-import { Duration }       from "luxon"
 
 /*  internal dependencies  */
 import SpeechFlowNode, { SpeechFlowChunk } from "./speechflow-node"
 import * as utils                          from "./speechflow-utils"
 
 /*  audio stream queue element */
+type AudioQueueElementSegment = {
+    data:        Float32Array,
+    isSpeech?:   boolean
+}
 type AudioQueueElement = {
-    type:      "audio-frame",
-    chunk:     SpeechFlowChunk,
-    isSpeech?: boolean
+    type:       "audio-frame",
+    chunk:       SpeechFlowChunk,
+    segmentIdx:  number,
+    segmentData: AudioQueueElementSegment[],
+    isSpeech?:   boolean
 } | {
-    type:      "audio-eof"
+    type:        "audio-eof"
 }
 
 /*  SpeechFlow node for VAD speech-to-speech processing  */
@@ -89,10 +94,22 @@ export default class SpeechFlowNodeVAD extends SpeechFlowNode {
                 log("info", "VAD: speech end (segment too short)")
             },
             onFrameProcessed: (audio) => {
-                /*  annotate the current audio frame  */
+                /*  annotate the current audio segment  */
                 const element = this.queueVAD.peek()
-                if (element !== undefined && element.type === "audio-frame") {
-                    const isSpeech = audio.isSpeech > audio.notSpeech
+                if (element === undefined || element.type !== "audio-frame")
+                    throw new Error("internal error which cannot happen: no more queued element")
+                const segment = element.segmentData[element.segmentIdx++]
+                segment.isSpeech = (audio.isSpeech > audio.notSpeech)
+
+                /*  annotate the entire audio chunk  */
+                if (element.segmentIdx >= element.segmentData.length) {
+                    let isSpeech = false
+                    for (const segment of element.segmentData) {
+                        if (segment.isSpeech) {
+                            isSpeech = true
+                            break
+                        }
+                    }
                     element.isSpeech = isSpeech
                     this.queueVAD.touch()
                     this.queueVAD.walk(+1)
@@ -102,14 +119,7 @@ export default class SpeechFlowNodeVAD extends SpeechFlowNode {
         this.vad.start()
 
         /*  provide Duplex stream and internally attach to VAD  */
-        const vad       = this.vad
-        const cfg       = this.config
-        const queue     = this.queue
-        const queueRecv = this.queueRecv
-        const queueSend = this.queueSend
-        const mode      = this.params.mode
-        let carrySamples = new Float32Array()
-        let carryStart   = Duration.fromDurationLike(0)
+        const self = this
         this.stream = new Stream.Duplex({
             writableObjectMode: true,
             readableObjectMode: true,
@@ -123,38 +133,34 @@ export default class SpeechFlowNodeVAD extends SpeechFlowNode {
                     callback()
                 else {
                     /*  convert audio samples from PCM/I16 to PCM/F32  */
-                    let data = utils.convertBufToF32(chunk.payload, cfg.audioLittleEndian)
-                    let start = chunk.timestampStart
-
-                    /*  merge previous carry samples  */
-                    if (carrySamples.length > 0) {
-                        start = carryStart
-                        const merged = new Float32Array(carrySamples.length + data.length)
-                        merged.set(carrySamples)
-                        merged.set(data, carrySamples.length)
-                        data = merged
-                        carrySamples = new Float32Array()
-                    }
+                    const data = utils.convertBufToF32(chunk.payload, self.config.audioLittleEndian)
 
-                    /*  queue audio samples as individual VAD-sized frames
-                        and in parallel send it into the Voice Activity Detection (VAD)  */
-                    const chunkSize = (vadSamplesPerFrame * (cfg.audioSampleRate / vadSampleRateTarget))
+                    /*  segment audio samples as individual VAD-sized frames  */
+                    const segmentData: AudioQueueElementSegment[] = []
+                    const chunkSize = vadSamplesPerFrame * (self.config.audioSampleRate / vadSampleRateTarget)
                     const chunks = Math.trunc(data.length / chunkSize)
                     for (let i = 0; i < chunks; i++) {
                         const frame = data.slice(i * chunkSize, (i + 1) * chunkSize)
-                        const buf = utils.convertF32ToBuf(frame)
-                        const duration = utils.audioBufferDuration(buf)
-                        const end = start.plus(duration)
-                        const chunk = new SpeechFlowChunk(start, end, "final", "audio", buf)
-                        queueRecv.append({ type: "audio-frame", chunk })
-                        vad.processAudio(frame)
-                        start = end
+                        const segment: AudioQueueElementSegment = { data: frame }
+                        segmentData.push(segment)
                     }
+                    if ((chunks * chunkSize) < data.length) {
+                        const frame = new Float32Array(chunkSize)
+                        frame.fill(0)
+                        frame.set(data.slice(chunks * chunkSize, data.length))
+                        const segment: AudioQueueElementSegment = { data: frame }
+                        segmentData.push(segment)
+                    }
+
+                    /*  queue the results  */
+                    self.queueRecv.append({
+                        type: "audio-frame", chunk,
+                        segmentIdx: 0, segmentData
+                    })
 
-                    /*  remember new carry samples  */
-                    const bulkLen = chunks * chunkSize
-                    carrySamples = data.slice(bulkLen)
-                    carryStart = start
+                    /*  push segments through Voice Activity Detection (VAD)  */
+                    for (const segment of segmentData)
+                        self.vad!.processAudio(segment.data)
 
                     callback()
                 }
@@ -162,25 +168,8 @@ export default class SpeechFlowNodeVAD extends SpeechFlowNode {
 
             /*  receive no more audio chunks (writable side of stream)  */
             final (callback) {
-                /*  flush pending audio chunks  */
-                if (carrySamples.length > 0) {
-                    const chunkSize = (vadSamplesPerFrame * (cfg.audioSampleRate / vadSampleRateTarget))
-                    if (carrySamples.length < chunkSize) {
-                        const merged = new Float32Array(chunkSize)
-                        merged.set(carrySamples)
-                        merged.fill(0.0, carrySamples.length, chunkSize)
-                        carrySamples = merged
-                    }
-                    const buf = utils.convertF32ToBuf(carrySamples)
-                    const duration = utils.audioBufferDuration(buf)
-                    const end = carryStart.plus(duration)
-                    const chunk = new SpeechFlowChunk(carryStart, end, "final", "audio", buf)
-                    queueRecv.append({ type: "audio-frame", chunk })
-                    vad.processAudio(carrySamples)
-                }
-
                 /*  signal end of file  */
-                queueRecv.append({ type: "audio-eof" })
+                self.queueRecv.append({ type: "audio-eof" })
                 callback()
             },
 
@@ -192,7 +181,7 @@ export default class SpeechFlowNodeVAD extends SpeechFlowNode {
                     const flushPendingChunks = () => {
                         let pushed = 0
                         while (true) {
-                            const element = queueSend.peek()
+                            const element = self.queueSend.peek()
                             if (element === undefined)
                                 break
                             else if (element.type === "audio-eof") {
@@ -202,19 +191,20 @@ export default class SpeechFlowNodeVAD extends SpeechFlowNode {
                             else if (element.type === "audio-frame"
                                 && element.isSpeech === undefined)
                                 break
-                            queueSend.walk(+1)
+                            self.queueSend.walk(+1)
+                            self.queue.trim()
                             if (element.isSpeech) {
                                 this.push(element.chunk)
                                 pushed++
                             }
-                            else if (mode === "silenced") {
+                            else if (self.params.mode === "silenced") {
                                 const chunk = element.chunk.clone()
                                 const buffer = chunk.payload as Buffer
                                 buffer.fill(0)
                                 this.push(chunk)
                                 pushed++
                             }
-                            else if (mode === "unplugged" && pushed === 0)
+                            else if (self.params.mode === "unplugged" && pushed === 0)
                                 /*  we have to await chunks now, as in unplugged
                                     mode we else would be never called again until
                                     we at least once push a new chunk as the result  */
@@ -224,16 +214,16 @@ export default class SpeechFlowNodeVAD extends SpeechFlowNode {
 
                     /*  await forthcoming audio chunks  */
                     const awaitForthcomingChunks = () => {
-                        const element = queueSend.peek()
+                        const element = self.queueSend.peek()
                         if (element !== undefined
                             && element.type === "audio-frame"
                             && element.isSpeech !== undefined)
                             flushPendingChunks()
                         else
-                            queue.once("write", awaitForthcomingChunks)
+                            self.queue.once("write", awaitForthcomingChunks)
                     }
 
-                    const element = queueSend.peek()
+                    const element = self.queueSend.peek()
                     if (element !== undefined && element.type === "audio-eof")
                         this.push(null)
                     else if (element !== undefined
@@ -241,7 +231,7 @@ export default class SpeechFlowNodeVAD extends SpeechFlowNode {
                         && element.isSpeech !== undefined)
                         flushPendingChunks()
                     else
-                        queue.once("write", awaitForthcomingChunks)
+                        self.queue.once("write", awaitForthcomingChunks)
                 }
                 tryToRead()
             }