speechflow 1.3.1 → 1.4.0

package/{src → speechflow-cli/src}/speechflow-node-t2t-sentence.ts

@@ -29,12 +29,12 @@ export default class SpeechFlowNodeSentence extends SpeechFlowNode {
     public static name = "sentence"
 
     /*  internal state  */
-    private static speexInitialized = false
     private queue      = new utils.Queue<TextQueueElement>()
     private queueRecv  = this.queue.pointerUse("recv")
     private queueSplit = this.queue.pointerUse("split")
     private queueSend  = this.queue.pointerUse("send")
     private destroyed  = false
+    private workingOffTimer: ReturnType<typeof setTimeout> | null = null
 
     /*  construct node  */
     constructor (id: string, cfg: { [ id: string ]: any }, opts: { [ id: string ]: any }, args: any[]) {
@@ -53,11 +53,7 @@ export default class SpeechFlowNodeSentence extends SpeechFlowNode {
         /*  clear destruction flag  */
         this.destroyed = false
 
-        /*  pass-through logging  */
-        const log = (level: string, msg: string) => { this.log(level, msg) }
-
         /*  work off queued audio frames  */
-        let workingOffTimer: ReturnType<typeof setTimeout> | null = null
         let workingOff = false
         const workOffQueue = async () => {
             if (this.destroyed)
@@ -67,14 +63,14 @@ export default class SpeechFlowNodeSentence extends SpeechFlowNode {
             if (workingOff)
                 return
             workingOff = true
-            if (workingOffTimer !== null) {
-                clearTimeout(workingOffTimer)
-                workingOffTimer = null
+            if (this.workingOffTimer !== null) {
+                clearTimeout(this.workingOffTimer)
+                this.workingOffTimer = null
             }
             this.queue.off("write", workOffQueue)
 
             /*  try to work off one or more chunks  */
-            while (true) {
+            while (!this.destroyed) {
                 const element = this.queueSplit.peek()
                 if (element === undefined)
                     break
@@ -136,10 +132,12 @@ export default class SpeechFlowNodeSentence extends SpeechFlowNode {
                 }
             }
 
-            /*  re-initiate working off round  */
+            /*  re-initiate working off round (if still not destroyed)  */
             workingOff = false
-            workingOffTimer = setTimeout(workOffQueue, 100)
-            this.queue.once("write", workOffQueue)
+            if (!this.destroyed) {
+                this.workingOffTimer = setTimeout(workOffQueue, 100)
+                this.queue.once("write", workOffQueue)
+            }
         }
         this.queue.once("write", workOffQueue)
 
@@ -153,12 +151,14 @@ export default class SpeechFlowNodeSentence extends SpeechFlowNode {
 
             /*  receive text chunk (writable side of stream)  */
             write (chunk: SpeechFlowChunk, encoding, callback) {
-                if (Buffer.isBuffer(chunk.payload))
+                if (self.destroyed)
+                    callback(new Error("stream already destroyed"))
+                else if (Buffer.isBuffer(chunk.payload))
                     callback(new Error("expected text input as string chunks"))
                 else if (chunk.payload.length === 0)
                     callback()
                 else {
-                    log("info", `received text: ${JSON.stringify(chunk.payload)}`)
+                    self.log("info", `received text: ${JSON.stringify(chunk.payload)}`)
                     self.queueRecv.append({ type: "text-frame", chunk })
                     callback()
                 }
@@ -166,6 +166,10 @@ export default class SpeechFlowNodeSentence extends SpeechFlowNode {
 
             /*  receive no more text chunks (writable side of stream)  */
             final (callback) {
+                if (self.destroyed) {
+                    callback()
+                    return
+                }
                 /*  signal end of file  */
                 self.queueRecv.append({ type: "text-eof" })
                 callback()
@@ -173,8 +177,12 @@ export default class SpeechFlowNodeSentence extends SpeechFlowNode {
 
             /*  send text chunk(s) (readable side of stream)  */
             read (_size) {
-                /*  flush pending audio chunks  */
+                /*  flush pending text chunks  */
                 const flushPendingChunks = () => {
+                    if (self.destroyed) {
+                        this.push(null)
+                        return
+                    }
                     const element = self.queueSend.peek()
                     if (element !== undefined
                         && element.type === "text-eof") {
@@ -196,13 +204,13 @@ export default class SpeechFlowNodeSentence extends SpeechFlowNode {
                             else if (element.type === "text-frame"
                                 && element.complete !== true)
                                 break
-                            log("info", `send text: ${JSON.stringify(element.chunk.payload)}`)
+                            self.log("info", `send text: ${JSON.stringify(element.chunk.payload)}`)
                             this.push(element.chunk)
                             self.queueSend.walk(+1)
                             self.queue.trim()
                         }
                     }
-                    else
+                    else if (!self.destroyed)
                         self.queue.once("write", flushPendingChunks)
                 }
                 flushPendingChunks()
@@ -212,13 +220,22 @@ export default class SpeechFlowNodeSentence extends SpeechFlowNode {
 
     /*  close node  */
     async close () {
+        /*  indicate destruction  */
+        this.destroyed = true
+
+        /*  clean up timer  */
+        if (this.workingOffTimer !== null) {
+            clearTimeout(this.workingOffTimer)
+            this.workingOffTimer = null
+        }
+
+        /*  remove any pending event listeners  */
+        this.queue.removeAllListeners("write")
+
         /*  close stream  */
         if (this.stream !== null) {
             this.stream.destroy()
             this.stream = null
         }
-
-        /*  indicate destruction  */
-        this.destroyed = true
     }
 }

package/speechflow-cli/src/speechflow-node-t2t-subtitle.ts

@@ -0,0 +1,276 @@
+/*
+**  SpeechFlow - Speech Processing Flow Graph
+**  Copyright (c) 2024-2025 Dr. Ralf S. Engelschall <rse@engelschall.com>
+**  Licensed under GPL 3.0 <https://spdx.org/licenses/GPL-3.0-only>
+*/
+
+/*  standard dependencies  */
+import path          from "node:path"
+import http          from "node:http"
+import Stream        from "node:stream"
+
+/*  external dependencies  */
+import { Duration }  from "luxon"
+import * as HAPI     from "@hapi/hapi"
+import Inert         from "@hapi/inert"
+import WebSocket     from "ws"
+import HAPIWebSocket from "hapi-plugin-websocket"
+
+/*  internal dependencies  */
+import SpeechFlowNode, { SpeechFlowChunk } from "./speechflow-node"
+
+type wsPeerCtx = {
+    peer: string
+}
+type wsPeerInfo = {
+    ctx:        wsPeerCtx
+    ws:         WebSocket
+    req:        http.IncomingMessage
+}
+
+/*  SpeechFlow node for subtitle (text-to-text) "translations"  */
+export default class SpeechFlowNodeSubtitle extends SpeechFlowNode {
+    /*  declare official node name  */
+    public static name = "subtitle"
+
+    /*  internal state  */
+    private sequenceNo = 1
+    private hapi: HAPI.Server | null = null
+
+    /*  construct node  */
+    constructor (id: string, cfg: { [ id: string ]: any }, opts: { [ id: string ]: any }, args: any[]) {
+        super(id, cfg, opts, args)
+
+        /*  declare node configuration parameters  */
+        this.configure({
+            format: { type: "string", pos: 0, val: "srt", match: /^(?:srt|vtt)$/ },
+            words:  { type: "boolean", val: false },
+            mode:   { type: "string",  val: "export", match: /^(?:export|render)$/ },
+            addr:   { type: "string",  val: "127.0.0.1" },
+            port:   { type: "number",  val: 8585 }
+        })
+
+        /*  declare node input/output format  */
+        this.input  = "text"
+        this.output = this.params.mode === "export" ? "text" : "none"
+    }
+
+    /*  open node  */
+    async open () {
+        if (this.params.mode === "export") {
+            this.sequenceNo = 1
+
+            /*  provide text-to-subtitle conversion  */
+            const convert = async (chunk: SpeechFlowChunk) => {
+                if (typeof chunk.payload !== "string")
+                    throw new Error("chunk payload type must be string")
+                const convertSingle = (
+                    start:      Duration,
+                    end:        Duration,
+                    text:       string,
+                    word?:      string,
+                    occurence?: number
+                ) => {
+                    if (word) {
+                        occurence ??= 1
+                        let match = 1
+                        word = word.replace(/[.*+?^${}()|[\]\\]/g, "\\$&")
+                        text = text.replaceAll(new RegExp(`\\b${word}\\b`, "g"), (m) => {
+                            if (match++ === occurence)
+                                return `<b>${m}</b>`
+                            else
+                                return m
+                        })
+                    }
+                    if (this.params.format === "srt") {
+                        const startFmt = start.toFormat("hh:mm:ss,SSS")
+                        const endFmt   = end.toFormat("hh:mm:ss,SSS")
+                        text = `${this.sequenceNo++}\n` +
+                            `${startFmt} --> ${endFmt}\n` +
+                            `${text}\n\n`
+                    }
+                    else if (this.params.format === "vtt") {
+                        const startFmt = start.toFormat("hh:mm:ss.SSS")
+                        const endFmt   = end.toFormat("hh:mm:ss.SSS")
+                        text = `${startFmt} --> ${endFmt}\n` +
+                            `${text}\n\n`
+                    }
+                    return text
+                }
+                let output = ""
+                if (this.params.words) {
+                    output += convertSingle(chunk.timestampStart, chunk.timestampEnd, chunk.payload)
+                    const words = (chunk.meta.get("words") ?? []) as
+                        { word: string, start: Duration, end: Duration }[]
+                    const occurences = new Map<string, number>()
+                    for (const word of words) {
+                        let occurence = occurences.get(word.word) ?? 0
+                        occurence++
+                        occurences.set(word.word, occurence)
+                        output += convertSingle(word.start, word.end, chunk.payload, word.word, occurence)
+                    }
+                }
+                else
+                    output += convertSingle(chunk.timestampStart, chunk.timestampEnd, chunk.payload)
+                return output
+            }
+
+            /*  establish a duplex stream  */
+            const self = this
+            let firstChunk = true
+            this.stream = new Stream.Transform({
+                readableObjectMode: true,
+                writableObjectMode: true,
+                decodeStrings:      false,
+                highWaterMark:      1,
+                transform (chunk: SpeechFlowChunk, encoding, callback) {
+                    if (firstChunk && self.params.format === "vtt") {
+                        this.push(new SpeechFlowChunk(
+                            Duration.fromMillis(0), Duration.fromMillis(0),
+                            "final", "text",
+                            "WEBVTT\n\n"
+                        ))
+                        firstChunk = false
+                    }
+                    if (Buffer.isBuffer(chunk.payload))
+                        callback(new Error("invalid chunk payload type"))
+                    else {
+                        if (chunk.payload === "") {
+                            this.push(chunk)
+                            callback()
+                        }
+                        else {
+                            convert(chunk).then((payload) => {
+                                const chunkNew = chunk.clone()
+                                chunkNew.payload = payload
+                                this.push(chunkNew)
+                                callback()
+                            }).catch((err) => {
+                                callback(err)
+                            })
+                        }
+                    }
+                },
+                final (callback) {
+                    this.push(null)
+                    callback()
+                }
+            })
+        }
+        else if (this.params.mode === "render") {
+            /*  establish REST/WebSocket API  */
+            const wsPeers = new Map<string, wsPeerInfo>()
+            this.hapi = new HAPI.Server({
+                address: this.params.addr,
+                port:    this.params.port
+            })
+            await this.hapi.register({ plugin: Inert })
+            await this.hapi.register({ plugin: HAPIWebSocket })
+            this.hapi.events.on({ name: "request", channels: [ "error" ] }, (request: HAPI.Request, event: HAPI.RequestEvent, tags: { [key: string]: true }) => {
+                if (event.error instanceof Error)
+                    this.log("error", `HAPI: request-error: ${event.error.message}`)
+                else
+                    this.log("error", `HAPI: request-error: ${event.error}`)
+            })
+            this.hapi.events.on("log", (event: HAPI.LogEvent, tags: { [key: string]: true }) => {
+                if (tags.error) {
+                    const err = event.error
+                    if (err instanceof Error)
+                        this.log("error", `HAPI: log: ${err.message}`)
+                    else
+                        this.log("error", `HAPI: log: ${err}`)
+                }
+            })
+            this.hapi.route({
+                method: "GET",
+                path: "/{param*}",
+                handler: {
+                    directory: {
+                        path: path.join(__dirname, "../../speechflow-ui-st/dst"),
+                        redirectToSlash: true,
+                        index: true
+                    }
+                }
+            })
+            this.hapi.route({
+                method: "POST",
+                path:   "/api",
+                options: {
+                    payload: {
+                        output: "data",
+                        parse:  true,
+                        allow:  "application/json"
+                    },
+                    plugins: {
+                        websocket: {
+                            autoping: 30 * 1000,
+                            connect: (args: any) => {
+                                const ctx: wsPeerCtx            = args.ctx
+                                const ws:  WebSocket            = args.ws
+                                const req: http.IncomingMessage = args.req
+                                const peer = `${req.socket.remoteAddress}:${req.socket.remotePort}`
+                                ctx.peer = peer
+                                wsPeers.set(peer, { ctx, ws, req })
+                                this.log("info", `HAPI: WebSocket: connect: peer ${peer}`)
+                            },
+                            disconnect: (args: any) => {
+                                const ctx: wsPeerCtx = args.ctx
+                                const peer = ctx.peer
+                                wsPeers.delete(peer)
+                                this.log("info", `HAPI: WebSocket: disconnect: peer ${peer}`)
+                            }
+                        }
+                    }
+                },
+                handler: (request: HAPI.Request, h: HAPI.ResponseToolkit) => {
+                    return h.response({}).code(204)
+                }
+            })
+
+            await this.hapi.start()
+            this.log("info", `HAPI: started REST/WebSocket network service: http://${this.params.addr}:${this.params.port}`)
+
+            const emit = (chunk: SpeechFlowChunk) => {
+                const data = JSON.stringify(chunk)
+                for (const info of wsPeers.values())
+                    info.ws.send(data)
+            }
+
+            this.stream = new Stream.Writable({
+                objectMode:     true,
+                decodeStrings:  false,
+                highWaterMark:  1,
+                write (chunk: SpeechFlowChunk, encoding, callback) {
+                    if (Buffer.isBuffer(chunk.payload))
+                        callback(new Error("invalid chunk payload type"))
+                    else {
+                        if (chunk.payload === "")
+                            callback()
+                        else {
+                            emit(chunk)
+                            callback()
+                        }
+                    }
+                },
+                final (callback) {
+                    callback()
+                }
+            })
+        }
+    }
+
+    /*  open node  */
+    async close () {
+        /*  close stream  */
+        if (this.stream !== null) {
+            this.stream.destroy()
+            this.stream = null
+        }
+
+        /*  shutdown HAPI  */
+        if (this.hapi !== null) {
+            await this.hapi.stop()
+            this.hapi = null
+        }
+    }
+}

package/{src → speechflow-cli/src}/speechflow-node-t2t-transformers.ts

@@ -104,7 +104,7 @@ export default class SpeechFlowNodeTransformers extends SpeechFlowNode {
 
     /*  open node  */
     async open () {
-        let model: string = ""
+        let model = ""
 
         /*  track download progress when instantiating Transformers engine and model  */
         const progressState = new Map<string, number>()
@@ -138,7 +138,6 @@ export default class SpeechFlowNodeTransformers extends SpeechFlowNode {
                 progress_callback: progressCallback
             })
             this.translator = await pipeline
-            clearInterval(interval)
             if (this.translator === null)
                 throw new Error("failed to instantiate translator pipeline")
         }
@@ -151,13 +150,15 @@ export default class SpeechFlowNodeTransformers extends SpeechFlowNode {
                 progress_callback: progressCallback
             })
             this.generator = await pipeline
-            clearInterval(interval)
             if (this.generator === null)
                 throw new Error("failed to instantiate generator pipeline")
         }
         else
             throw new Error("invalid model")
 
+        /*  clear progress interval again  */
+        clearInterval(interval)
+
         /*  provide text-to-text translation  */
         const translate = async (text: string) => {
             if (this.params.model === "OPUS") {

package/{src → speechflow-cli/src}/speechflow-node-x2x-filter.ts

@@ -23,7 +23,7 @@ export default class SpeechFlowNodeFilter extends SpeechFlowNode {
         this.configure({
             type: { type: "string", pos: 0, val: "audio",  match: /^(?:audio|text)$/ },
             name: { type: "string", pos: 1, val: "filter", match: /^.+?$/ },
-            var:  { type: "string", pos: 2, val: "",       match: /^(?:meta:.+|payload:(?:length|text)|time:(?:start|end))$/ },
+            var:  { type: "string", pos: 2, val: "",       match: /^(?:meta:.+|payload:(?:length|text)|time:(?:start|end)|kind|type)$/ },
             op:   { type: "string", pos: 3, val: "==",     match: /^(?:<|<=|==|!=|~~|!~|>=|>)$/ },
             val:  { type: "string", pos: 4, val: "",       match: /^.*$/ }
         })
@@ -94,13 +94,17 @@ export default class SpeechFlowNodeFilter extends SpeechFlowNode {
                 const m = self.params.var.match(/^meta:(.+)$/)
                 if (m !== null)
                     val1 = chunk.meta.get(m[1]) ?? ""
-                else if (self.params.key === "payload:length")
+                else if (self.params.var === "kind")
+                    val1 = chunk.kind
+                else if (self.params.var === "type")
+                    val1 = chunk.type
+                else if (self.params.var === "payload:length")
                     val1 = chunk.payload.length
-                else if (self.params.key === "payload:text")
+                else if (self.params.var === "payload:text")
                     val1 = (self.params.type === "text" ? chunk.payload as string : "")
-                else if (self.params.key === "time:start")
+                else if (self.params.var === "time:start")
                     val1 = chunk.timestampStart.toMillis()
-                else if (self.params.key === "time:end")
+                else if (self.params.var === "time:end")
                     val1 = chunk.timestampEnd.toMillis()
                 if (comparison(val1, self.params.op, val2)) {
                     self.log("info", `[${self.params.name}]: passing through ${chunk.type} chunk`)

package/{src → speechflow-cli/src}/speechflow-node-x2x-trace.ts

@@ -24,10 +24,15 @@ export default class SpeechFlowNodeTrace extends SpeechFlowNode {
 
         /*  declare node configuration parameters  */
         this.configure({
-            type: { type: "string", pos: 0, val: "audio", match: /^(?:audio|text)$/ },
-            name: { type: "string", pos: 1, val: "trace" }
+            type:      { type: "string", pos: 0, val: "audio", match: /^(?:audio|text)$/ },
+            name:      { type: "string", pos: 1, val: "trace" },
+            dashboard: { type: "string",         val: "" }
         })
 
+        /*  sanity check parameters  */
+        if (this.params.dashboard !== "" && this.params.type === "audio")
+            throw new Error("only trace nodes of type \"text\" can export to dashboard")
+
         /*  declare node input/output format  */
         this.input  = this.params.type
         this.output = this.params.type
@@ -44,7 +49,7 @@ export default class SpeechFlowNodeTrace extends SpeechFlowNode {
         }
 
         /*  provide Transform stream  */
-        const type = this.params.type
+        const self = this
         this.stream = new Stream.Transform({
             writableObjectMode: true,
             readableObjectMode: true,
@@ -63,7 +68,7 @@ export default class SpeechFlowNodeTrace extends SpeechFlowNode {
                         } }`
                 }
                 if (Buffer.isBuffer(chunk.payload)) {
-                    if (type === "audio")
+                    if (self.params.type === "audio")
                         log("debug", `chunk: type=${chunk.type} ` +
                             `kind=${chunk.kind} ` +
                             `start=${fmtTime(chunk.timestampStart)} ` +
@@ -71,19 +76,22 @@ export default class SpeechFlowNodeTrace extends SpeechFlowNode {
                             `payload-type=Buffer payload-length=${chunk.payload.byteLength} ` +
                             `meta=${fmtMeta(chunk.meta)}`)
                     else
-                        error = new Error(`${type} chunk: seen Buffer instead of String chunk type`)
+                        error = new Error(`${self.params.type} chunk: seen Buffer instead of String chunk type`)
                 }
                 else {
-                    if (type === "text")
-                        log("debug", `${type} chunk: type=${chunk.type} ` +
+                    if (self.params.type === "text") {
+                        log("debug", `chunk: type=${chunk.type} ` +
                             `kind=${chunk.kind} ` +
                             `start=${fmtTime(chunk.timestampStart)} ` +
                             `end=${fmtTime(chunk.timestampEnd)} ` +
                             `payload-type=String payload-length=${chunk.payload.length} ` +
                             `payload-content="${chunk.payload.toString()}" ` +
                             `meta=${fmtMeta(chunk.meta)}`)
+                        if (self.params.dashboard !== "")
+                            self.dashboardInfo("text", self.params.dashboard, chunk.kind, chunk.payload.toString())
+                    }
                     else
-                        error = new Error(`${type} chunk: seen String instead of Buffer chunk type`)
+                        error = new Error(`${self.params.type} chunk: seen String instead of Buffer chunk type`)
                 }
                 if (error !== undefined)
                     callback(error)

package/{src → speechflow-cli/src}/speechflow-node-xio-device.ts

@@ -159,8 +159,7 @@ export default class SpeechFlowNodeDevice extends SpeechFlowNode {
 
             /*  convert regular stream into object-mode stream  */
             const wrapper = utils.createTransformStreamForReadableSide("audio", () => this.timeZero)
-            this.stream.pipe(wrapper)
-            this.stream = wrapper
+            this.stream = Stream.compose(this.stream, wrapper)
         }
         else if (this.params.mode === "w") {
             /*  output device  */
@@ -180,8 +179,7 @@ export default class SpeechFlowNodeDevice extends SpeechFlowNode {
 
             /*  convert regular stream into object-mode stream  */
             const wrapper = utils.createTransformStreamForWritableSide()
-            wrapper.pipe(this.stream)
-            this.stream = wrapper
+            this.stream = Stream.compose(wrapper, this.stream)
         }
         else
             throw new Error(`device "${device.id}" does not have any input or output channels`)
@@ -200,13 +198,19 @@ export default class SpeechFlowNodeDevice extends SpeechFlowNode {
         /*  shutdown PortAudio  */
         if (this.io !== null) {
             await new Promise<void>((resolve, reject) => {
-                this.io!.abort(() => {
-                    resolve()
+                this.io!.abort((err?: Error) => {
+                    if (err)
+                        reject(err)
+                    else
+                        resolve()
                 })
             })
             await new Promise<void>((resolve, reject) => {
-                this.io!.quit(() => {
-                    resolve()
+                this.io!.quit((err?: Error) => {
+                    if (err)
+                        reject(err)
+                    else
+                        resolve()
                 })
             })
             this.io = null

package/{src → speechflow-cli/src}/speechflow-node-xio-file.ts

@@ -178,9 +178,15 @@ export default class SpeechFlowNodeFile extends SpeechFlowNode {
     async close () {
         /*  shutdown stream  */
         if (this.stream !== null) {
-            await new Promise<void>((resolve) => {
-                if (this.stream instanceof Stream.Writable || this.stream instanceof Stream.Duplex)
-                    this.stream.end(() => { resolve() })
+            await new Promise<void>((resolve, reject) => {
+                if (this.stream instanceof Stream.Writable || this.stream instanceof Stream.Duplex) {
+                    this.stream.end((err?: Error) => {
+                        if (err)
+                            reject(err)
+                        else
+                            resolve()
+                    })
+                }
                 else
                     resolve()
             })

package/{src → speechflow-cli/src}/speechflow-node-xio-mqtt.ts

@@ -83,7 +83,10 @@ export default class SpeechFlowNodeMQTT extends SpeechFlowNode {
         this.broker.on("connect", (packet: MQTT.IConnackPacket) => {
             this.log("info", `connection opened to MQTT ${this.params.url}`)
             if (this.params.mode !== "w" && !packet.sessionPresent)
-                this.broker!.subscribe([ this.params.topicRead ], () => {})
+                this.broker!.subscribe([ this.params.topicRead ], (err) => {
+                    if (err)
+                        this.log("error", `failed to subscribe to MQTT topic "${this.params.topicRead}": ${err.message}`)
+                })
         })
         this.broker.on("reconnect", () => {
             this.log("info", `connection re-opened to MQTT ${this.params.url}`)
@@ -141,7 +144,7 @@ export default class SpeechFlowNodeMQTT extends SpeechFlowNode {
 
     /*  close node  */
     async close () {
-        /*  close Websocket server  */
+        /*  close MQTT broker  */
         if (this.broker !== null) {
             if (this.broker.connected)
                 this.broker.end()