speechflow 1.4.5 → 1.5.1

package/speechflow-cli/src/speechflow-node-xio-mqtt.ts

@@ -23,6 +23,7 @@ export default class SpeechFlowNodeMQTT extends SpeechFlowNode {
     /*  internal state  */
     private broker: MQTT.MqttClient | null = null
     private clientId: string = (new UUID(1)).format()
+    private chunkQueue: utils.SingleQueue<SpeechFlowChunk> | null = null
 
     /*  construct node  */
     constructor (id: string, cfg: { [ id: string ]: any }, opts: { [ id: string ]: any }, args: any[]) {
@@ -63,6 +64,10 @@ export default class SpeechFlowNodeMQTT extends SpeechFlowNode {
             throw new Error("writing to MQTT requires a topicWrite parameter")
         if ((this.params.mode === "r" || this.params.mode === "rw") && this.params.topicRead === "")
             throw new Error("reading from MQTT requires a topicRead parameter")
+        if (this.params.username !== "" && this.params.password === "")
+            throw new Error("username provided but password is missing")
+        if (this.params.username === "" && this.params.password !== "")
+            throw new Error("password provided but username is missing")
 
         /*  connect remotely to a MQTT broker  */
         this.broker = MQTT.connect(this.params.url, {
@@ -85,7 +90,7 @@ export default class SpeechFlowNodeMQTT extends SpeechFlowNode {
             if (this.params.mode !== "w" && !packet.sessionPresent)
                 this.broker!.subscribe([ this.params.topicRead ], (err) => {
                     if (err)
-                        this.log("error", `failed to subscribe to MQTT topic "${this.params.topicRead}": ${err.message}`)
+                        this.log("warning", `failed to subscribe to MQTT topic "${this.params.topicRead}": ${err.message}`)
                 })
         })
         this.broker.on("reconnect", () => {
@@ -94,49 +99,48 @@ export default class SpeechFlowNodeMQTT extends SpeechFlowNode {
         this.broker.on("disconnect", (packet: MQTT.IDisconnectPacket) => {
             this.log("info", `connection closed to MQTT ${this.params.url}`)
         })
-        const chunkQueue = new utils.SingleQueue<SpeechFlowChunk>()
+        this.chunkQueue = new utils.SingleQueue<SpeechFlowChunk>()
         this.broker.on("message", (topic: string, payload: Buffer, packet: MQTT.IPublishPacket) => {
-            if (topic !== this.params.topicRead)
+            if (topic !== this.params.topicRead || this.params.mode === "w")
                 return
             try {
                 const chunk = utils.streamChunkDecode(payload)
-                chunkQueue.write(chunk)
+                this.chunkQueue!.write(chunk)
             }
             catch (_err: any) {
                 this.log("warning", `received invalid CBOR chunk from MQTT ${this.params.url}`)
             }
         })
-        const broker     = this.broker
-        const topicWrite = this.params.topicWrite
-        const type       = this.params.type
-        const mode       = this.params.mode
+        const self = this
         this.stream = new Stream.Duplex({
             writableObjectMode: true,
             readableObjectMode: true,
             decodeStrings:      false,
             highWaterMark:      1,
             write (chunk: SpeechFlowChunk, encoding, callback) {
-                if (mode === "r")
+                if (self.params.mode === "r")
                     callback(new Error("write operation on read-only node"))
-                else if (chunk.type !== type)
-                    callback(new Error(`written chunk is not of ${type} type`))
-                else if (!broker.connected)
+                else if (chunk.type !== self.params.type)
+                    callback(new Error(`written chunk is not of ${self.params.type} type`))
+                else if (!self.broker!.connected)
                     callback(new Error("still no MQTT connection available"))
                 else {
                     const data = Buffer.from(utils.streamChunkEncode(chunk))
-                    broker.publish(topicWrite, data, { qos: 2, retain: false }, (err) => {
+                    self.broker!.publish(self.params.topicWrite, data, { qos: 2, retain: false }, (err) => {
                         if (err)
-                            callback(new Error(`failed to publish to MQTT topic "${topicWrite}": ${err}`))
+                            callback(new Error(`failed to publish to MQTT topic "${self.params.topicWrite}": ${err}`))
                         else
                             callback()
                     })
                 }
             },
             read (size: number) {
-                if (mode === "w")
+                if (self.params.mode === "w")
                     throw new Error("read operation on write-only node")
-                chunkQueue.read().then((chunk) => {
+                self.chunkQueue!.read().then((chunk) => {
                     this.push(chunk, "binary")
+                }).catch((err: Error) => {
+                    self.log("warning", `read on chunk queue operation failed: ${err}`)
                 })
             }
         })
@@ -144,6 +148,9 @@ export default class SpeechFlowNodeMQTT extends SpeechFlowNode {
 
     /*  close node  */
     async close () {
+        /*  clear chunk queue reference  */
+        this.chunkQueue = null
+
         /*  close MQTT broker  */
         if (this.broker !== null) {
             if (this.broker.connected)

package/speechflow-cli/src/speechflow-node-xio-websocket.ts

@@ -66,7 +66,7 @@ export default class SpeechFlowNodeWebsocket extends SpeechFlowNode {
             const chunkQueue = new utils.SingleQueue<SpeechFlowChunk>()
             this.server = new ws.WebSocketServer({
                 host: url.hostname,
-                port: Number.parseInt(url.port),
+                port: Number.parseInt(url.port, 10),
                 path: url.pathname
             })
             this.server.on("listening", () => {
@@ -108,18 +108,17 @@ export default class SpeechFlowNodeWebsocket extends SpeechFlowNode {
             this.server.on("error", (error) => {
                 this.log("error", `error of some connection on URL ${this.params.listen}: ${error.message}`)
             })
-            const type = this.params.type
-            const mode = this.params.mode
+            const self = this
             this.stream = new Stream.Duplex({
                 writableObjectMode: true,
                 readableObjectMode: true,
                 decodeStrings:      false,
                 highWaterMark:      1,
                 write (chunk: SpeechFlowChunk, encoding, callback) {
-                    if (mode === "r")
+                    if (self.params.mode === "r")
                         callback(new Error("write operation on read-only node"))
-                    else if (chunk.type !== type)
-                        callback(new Error(`written chunk is not of ${type} type`))
+                    else if (chunk.type !== self.params.type)
+                        callback(new Error(`written chunk is not of ${self.params.type} type`))
                     else if (websockets.size === 0)
                         callback(new Error("still no Websocket connections available"))
                     else {
@@ -137,17 +136,18 @@ export default class SpeechFlowNodeWebsocket extends SpeechFlowNode {
                         }
                         Promise.all(results).then(() => {
                             callback()
-                        }).catch((errors: Error[]) => {
-                            const error = new Error(errors.map((e) => e.message).join("; "))
+                        }).catch((error: Error) => {
                             callback(error)
                         })
                     }
                 },
                 read (size: number) {
-                    if (mode === "w")
+                    if (self.params.mode === "w")
                         throw new Error("read operation on write-only node")
                     chunkQueue.read().then((chunk) => {
                         this.push(chunk, "binary")
+                    }).catch((err: Error) => {
+                        self.log("warning", `read on chunk queue operation failed: ${err}`)
                     })
                 }
             })
@@ -188,33 +188,33 @@ export default class SpeechFlowNodeWebsocket extends SpeechFlowNode {
                 const chunk = utils.streamChunkDecode(buffer)
                 chunkQueue.write(chunk)
             })
-            const client = this.client
-            client.binaryType = "arraybuffer"
-            const type = this.params.type
-            const mode = this.params.mode
+            this.client.binaryType = "arraybuffer"
+            const self = this
             this.stream = new Stream.Duplex({
                 writableObjectMode: true,
                 readableObjectMode: true,
                 decodeStrings:      false,
                 highWaterMark:      1,
                 write (chunk: SpeechFlowChunk, encoding, callback) {
-                    if (mode === "r")
+                    if (self.params.mode === "r")
                         callback(new Error("write operation on read-only node"))
-                    else if (chunk.type !== type)
-                        callback(new Error(`written chunk is not of ${type} type`))
-                    else if (!client.OPEN)
+                    else if (chunk.type !== self.params.type)
+                        callback(new Error(`written chunk is not of ${self.params.type} type`))
+                    else if (!self.client!.OPEN)
                         callback(new Error("still no Websocket connection available"))
                     else {
                         const data = utils.streamChunkEncode(chunk)
-                        client.send(data)
+                        self.client!.send(data)
                         callback()
                     }
                 },
                 read (size: number) {
-                    if (mode === "w")
+                    if (self.params.mode === "w")
                         throw new Error("read operation on write-only node")
                     chunkQueue.read().then((chunk) => {
                         this.push(chunk, "binary")
+                    }).catch((err: Error) => {
+                        self.log("warning", `read on chunk queue operation failed: ${err}`)
                     })
                 }
             })

package/speechflow-cli/src/speechflow-node.ts

@@ -5,7 +5,7 @@
 */
 
 /*  standard dependencies  */
-import Events from "node:events"
+import Events, { EventEmitter } from "node:events"
 import Stream from "node:stream"
 
 /*  external dependencies  */
@@ -62,6 +62,7 @@ export default class SpeechFlowNode extends Events.EventEmitter {
     timeOpen:       DateTime<boolean> | undefined
     timeZero:       DateTime<boolean> = DateTime.fromMillis(0)
     timeZeroOffset: Duration<boolean> = Duration.fromMillis(0)
+    _accessBus: ((name: string) => EventEmitter) | null = null
 
     /*  the default constructor  */
     constructor (
@@ -87,20 +88,32 @@ export default class SpeechFlowNode extends Events.EventEmitter {
     }
 
     /*  receive external request  */
-    async receiveRequest (args: any[]) {
+    async receiveRequest (args: any[]): Promise<void> {
         /*  no-op  */
     }
 
     /*  send external response  */
-    sendResponse (args: any[]) {
+    sendResponse (args: any[]): void {
         this.emit("send-response", args)
     }
 
-    /*  emit dashboard information  */
-    dashboardInfo (type: "audio", id: string, kind: "final" | "intermediate", value: number): void
-    dashboardInfo (type: "text", id: string, kind: "final" | "intermediate", value: string): void
-    dashboardInfo (type: "audio" | "text", id: string, kind: "final" | "intermediate", value: number | string): void {
-        this.emit("dashboard-info", { type, id, kind, value })
+    /*  receive dashboard information  */
+    async receiveDashboard (type: "audio" | "text", id: string, kind: "final" | "intermediate", value: number | string): Promise<void> {
+        /*  no-op  */
+    }
+
+    /*  send dashboard information  */
+    sendDashboard (type: "audio", id: string, kind: "final" | "intermediate", value: number): void
+    sendDashboard (type: "text", id: string, kind: "final" | "intermediate", value: string): void
+    sendDashboard (type: "audio" | "text", id: string, kind: "final" | "intermediate", value: number | string): void {
+        this.emit("send-dashboard", { type, id, kind, value })
+    }
+
+    /*  access communication bus  */
+    accessBus (name: string): EventEmitter {
+        if (this._accessBus === null)
+            throw new Error("access to communication bus still not possible")
+        return this._accessBus(name)
     }
 
     /*  INTERNAL: utility function: create "params" attribute from constructor of sub-classes  */

package/speechflow-cli/src/speechflow-utils-audio-wt.ts

@@ -0,0 +1,172 @@
+/*
+**  SpeechFlow - Speech Processing Flow Graph
+**  Copyright (c) 2024-2025 Dr. Ralf S. Engelschall <rse@engelschall.com>
+**  Licensed under GPL 3.0 <https://spdx.org/licenses/GPL-3.0-only>
+*/
+
+/*  internal types  */
+interface InputChunkMessage {
+    type: "input-chunk"
+    chunkId: string
+    data: { pcmData: Float32Array, channels: number }
+}
+interface StartCaptureMessage {
+    type: "start-capture"
+    chunkId: string
+    expectedSamples: number
+}
+type WorkletMessage = InputChunkMessage | StartCaptureMessage
+interface ChunkData {
+    data: Float32Array
+    chunkId: string
+}
+interface ChunkStartedMessage {
+    type: "chunk-started"
+    chunkId: string
+}
+interface CaptureCompleteMessage {
+    type: "capture-complete"
+    chunkId: string
+    data: number[]
+}
+
+/*  audio source node  */
+class AudioSourceProcessor extends AudioWorkletProcessor {
+    /*  internal state  */
+    private pendingData:  ChunkData[] = []
+    private currentChunk: ChunkData | null = null
+    private currentOffset = 0
+
+    /*  node construction  */
+    constructor() {
+        super()
+
+        /*  receive input chunks  */
+        this.port.addEventListener("message", (event: MessageEvent<WorkletMessage>) => {
+            const { type, chunkId } = event.data
+            if (type === "input-chunk")
+                this.pendingData.push({ data: event.data.data.pcmData, chunkId })
+        })
+    }
+
+    /*  process audio frame  */
+    process(
+        inputs:     Float32Array[][],            /* unused */
+        outputs:    Float32Array[][],
+        parameters: Record<string, Float32Array> /* unused */
+    ): boolean {
+        /*  determine output  */
+        const output = outputs[0]
+        if (!output || output.length === 0)
+            return true
+        const frameCount = output[0].length
+        const channelCount = output.length
+
+        /*  get current chunk if we don't have one  */
+        if (this.currentChunk === null && this.pendingData.length > 0) {
+            this.currentChunk = this.pendingData.shift()!
+            this.currentOffset = 0
+
+            /*  signal chunk start  */
+            const message: ChunkStartedMessage = {
+                type: "chunk-started",
+                chunkId: this.currentChunk.chunkId
+            }
+            this.port.postMessage(message)
+        }
+
+        /*  process input  */
+        if (this.currentChunk) {
+            /*  output current chunk  */
+            const samplesPerChannel = this.currentChunk.data.length / channelCount
+            const remainingFrames   = samplesPerChannel - this.currentOffset
+            const framesToProcess   = Math.min(frameCount, remainingFrames)
+
+            /*  copy data from current chunk (interleaved to planar)  */
+            for (let frame = 0; frame < framesToProcess; frame++) {
+                for (let ch = 0; ch < channelCount; ch++) {
+                    const interleavedIndex = (this.currentOffset + frame) * channelCount + ch
+                    output[ch][frame] = this.currentChunk.data[interleavedIndex] ?? 0
+                }
+            }
+
+            /*  zero-pad remaining output if needed  */
+            for (let frame = framesToProcess; frame < frameCount; frame++)
+                for (let ch = 0; ch < channelCount; ch++)
+                    output[ch][frame] = 0
+
+            /*  check if current chunk is finished  */
+            this.currentOffset += framesToProcess
+            if (this.currentOffset >= samplesPerChannel) {
+                this.currentChunk  = null
+                this.currentOffset = 0
+            }
+        }
+        else {
+            /*  output silence when no input  */
+            for (let ch = 0; ch < channelCount; ch++)
+                output[ch].fill(0)
+        }
+        return true
+    }
+}
+
+/*  audio capture node  */
+class AudioCaptureProcessor extends AudioWorkletProcessor {
+    /*  internal state  */
+    private activeCaptures = new Map<string, { data: number[], expectedSamples: number }>()
+
+    /*  node construction  */
+    constructor() {
+        super()
+
+        /*  receive start of capturing command  */
+        this.port.addEventListener("message", (event: MessageEvent<WorkletMessage>) => {
+            const { type, chunkId } = event.data
+            if (type === "start-capture") {
+                this.activeCaptures.set(chunkId, {
+                    data: [],
+                    expectedSamples: event.data.expectedSamples
+                })
+            }
+        })
+    }
+
+    /*  process audio frame  */
+    process(
+        inputs:     Float32Array[][],
+        outputs:    Float32Array[][],             /* unused */
+        parameters: Record<string, Float32Array>  /* unused */
+    ): boolean {
+        /*  determine input  */
+        const input = inputs[0]
+        if (!input || input.length === 0 || this.activeCaptures.size === 0)
+            return true
+        const frameCount = input[0].length
+        const channelCount = input.length
+
+        /*  iterate over all active captures  */
+        for (const [ chunkId, capture ] of this.activeCaptures) {
+            /*  convert planar to interleaved  */
+            for (let frame = 0; frame < frameCount; frame++)
+                for (let ch = 0; ch < channelCount; ch++)
+                    capture.data.push(input[ch][frame])
+
+            /*  send back captured data  */
+            if (capture.data.length >= capture.expectedSamples) {
+                const message: CaptureCompleteMessage = {
+                    type: "capture-complete",
+                    chunkId,
+                    data: capture.data.slice(0, capture.expectedSamples)
+                }
+                this.port.postMessage(message)
+                this.activeCaptures.delete(chunkId)
+            }
+        }
+        return true
+    }
+}
+
+/*  register the new audio nodes  */
+registerProcessor("source",  AudioSourceProcessor)
+registerProcessor("capture", AudioCaptureProcessor)

package/speechflow-cli/src/speechflow-utils-audio.ts

@@ -0,0 +1,147 @@
+/*
+**  SpeechFlow - Speech Processing Flow Graph
+**  Copyright (c) 2024-2025 Dr. Ralf S. Engelschall <rse@engelschall.com>
+**  Licensed under GPL 3.0 <https://spdx.org/licenses/GPL-3.0-only>
+*/
+
+/*  standard dependencies  */
+import path from "node:path"
+
+/*  external dependencies  */
+import { AudioContext, AudioWorkletNode } from "node-web-audio-api"
+
+export class WebAudio {
+    /*  internal state  */
+    public audioContext: AudioContext
+    public sourceNode:   AudioWorkletNode | null = null
+    public captureNode:  AudioWorkletNode | null = null
+    private pendingPromises = new Map<string, {
+        resolve: (value: Int16Array) => void
+        reject: (error: Error) => void
+        timeout: ReturnType<typeof setTimeout>
+    }>()
+
+    /*  construct object  */
+    constructor(
+        public sampleRate: number,
+        public channels: number
+    ) {
+        /*  create new audio context  */
+        this.audioContext = new AudioContext({
+            sampleRate,
+            latencyHint: "interactive"
+        })
+    }
+
+    /*  setup object  */
+    public async setup (): Promise<void> {
+        /*  ensure audio context is not suspended  */
+        if (this.audioContext.state === "suspended")
+            await this.audioContext.resume()
+
+        /*  add audio worklet module  */
+        const url = path.resolve(__dirname, "speechflow-utils-audio-wt.js")
+        await this.audioContext.audioWorklet.addModule(url)
+
+        /*  create source node  */
+        this.sourceNode = new AudioWorkletNode(this.audioContext, "source", {
+            numberOfInputs:  0,
+            numberOfOutputs: 1,
+            outputChannelCount: [ this.channels ]
+        })
+
+        /*  create capture node  */
+        this.captureNode = new AudioWorkletNode(this.audioContext, "capture", {
+            numberOfInputs:  1,
+            numberOfOutputs: 0
+        })
+        this.captureNode!.port.addEventListener("message", (event) => {
+            const { type, chunkId, data } = event.data ?? {}
+            if (type === "capture-complete") {
+                const promise = this.pendingPromises.get(chunkId)
+                if (promise) {
+                    clearTimeout(promise.timeout)
+                    this.pendingPromises.delete(chunkId)
+                    const int16Data = new Int16Array(data.length)
+                    for (let i = 0; i < data.length; i++)
+                        int16Data[i] = Math.max(-32768, Math.min(32767, Math.round(data[i] * 32767)))
+                    promise.resolve(int16Data)
+                }
+            }
+        })
+
+        /*  start ports  */
+        this.sourceNode.port.start()
+        this.captureNode!.port.start()
+    }
+
+    /*  process single audio chunk  */
+    public async process (int16Array: Int16Array): Promise<Int16Array> {
+        const chunkId = `chunk_${Date.now()}_${Math.random().toString(36).substring(2, 11)}`
+        return new Promise<Int16Array>((resolve, reject) => {
+            const timeout = setTimeout(() => {
+                this.pendingPromises.delete(chunkId)
+                reject(new Error("processing timeout"))
+            }, (int16Array.length / this.audioContext.sampleRate) * 1000 + 250)
+            if (this.captureNode !== null)
+                this.pendingPromises.set(chunkId, { resolve, reject, timeout })
+            try {
+                const float32Data = new Float32Array(int16Array.length)
+                for (let i = 0; i < int16Array.length; i++)
+                    float32Data[i] = int16Array[i] / 32768.0
+
+                /*  start capture first  */
+                if (this.captureNode !== null) {
+                    this.captureNode?.port.postMessage({
+                        type: "start-capture",
+                        chunkId,
+                        expectedSamples: int16Array.length
+                    })
+                }
+
+                /*  small delay to ensure capture is ready before sending data  */
+                setTimeout(() => {
+                    /*  send input to source node  */
+                    this.sourceNode?.port.postMessage({
+                        type: "input-chunk",
+                        chunkId,
+                        data: { pcmData: float32Data, channels: this.channels }
+                    }, [ float32Data.buffer ])
+                }, 5)
+            }
+            catch (error) {
+                clearTimeout(timeout)
+                if (this.captureNode !== null)
+                    this.pendingPromises.delete(chunkId)
+                reject(new Error(`failed to process chunk: ${error}`))
+            }
+        })
+    }
+
+    public async destroy (): Promise<void> {
+        /*  reject all pending promises  */
+        try {
+            this.pendingPromises.forEach(({ reject, timeout }) => {
+                clearTimeout(timeout)
+                reject(new Error("WebAudio destroyed"))
+            })
+            this.pendingPromises.clear()
+        }
+        catch (_err) {
+            /* ignored - cleanup during shutdown */
+        }
+
+        /*  disconnect nodes  */
+        if (this.sourceNode !== null) {
+            this.sourceNode.disconnect()
+            this.sourceNode = null
+        }
+        if (this.captureNode !== null) {
+            this.captureNode.disconnect()
+            this.captureNode = null
+        }
+
+        /*  stop context  */
+        await this.audioContext.close()
+    }
+}