speechflow 1.3.0 → 1.3.2

package/src/speechflow-node-t2t-sentence.ts

@@ -29,12 +29,12 @@ export default class SpeechFlowNodeSentence extends SpeechFlowNode {
     public static name = "sentence"
 
     /*  internal state  */
-    private static speexInitialized = false
     private queue      = new utils.Queue<TextQueueElement>()
     private queueRecv  = this.queue.pointerUse("recv")
     private queueSplit = this.queue.pointerUse("split")
     private queueSend  = this.queue.pointerUse("send")
     private destroyed  = false
+    private workingOffTimer: ReturnType<typeof setTimeout> | null = null
 
     /*  construct node  */
     constructor (id: string, cfg: { [ id: string ]: any }, opts: { [ id: string ]: any }, args: any[]) {
@@ -53,11 +53,7 @@ export default class SpeechFlowNodeSentence extends SpeechFlowNode {
         /*  clear destruction flag  */
         this.destroyed = false
 
-        /*  pass-through logging  */
-        const log = (level: string, msg: string) => { this.log(level, msg) }
-
         /*  work off queued audio frames  */
-        let workingOffTimer: ReturnType<typeof setTimeout> | null = null
         let workingOff = false
         const workOffQueue = async () => {
             if (this.destroyed)
@@ -67,17 +63,14 @@ export default class SpeechFlowNodeSentence extends SpeechFlowNode {
             if (workingOff)
                 return
             workingOff = true
-            if (workingOffTimer !== null) {
-                clearTimeout(workingOffTimer)
-                workingOffTimer = null
+            if (this.workingOffTimer !== null) {
+                clearTimeout(this.workingOffTimer)
+                this.workingOffTimer = null
             }
             this.queue.off("write", workOffQueue)
 
-            const position = this.queueSplit.position()
-            const maxPosition = this.queueSplit.maxPosition()
-            log("info", `SPLIT: ${maxPosition - position} elements in queue`)
-
-            while (true) {
+            /*  try to work off one or more chunks  */
+            while (!this.destroyed) {
                 const element = this.queueSplit.peek()
                 if (element === undefined)
                     break
@@ -139,10 +132,12 @@ export default class SpeechFlowNodeSentence extends SpeechFlowNode {
                 }
             }
 
-            /*  re-initiate working off round  */
+            /*  re-initiate working off round (if still not destroyed)  */
             workingOff = false
-            workingOffTimer = setTimeout(workOffQueue, 100)
-            this.queue.once("write", workOffQueue)
+            if (!this.destroyed) {
+                this.workingOffTimer = setTimeout(workOffQueue, 100)
+                this.queue.once("write", workOffQueue)
+            }
         }
         this.queue.once("write", workOffQueue)
 
@@ -156,12 +151,14 @@ export default class SpeechFlowNodeSentence extends SpeechFlowNode {
 
             /*  receive text chunk (writable side of stream)  */
             write (chunk: SpeechFlowChunk, encoding, callback) {
-                if (Buffer.isBuffer(chunk.payload))
+                if (self.destroyed)
+                    callback(new Error("stream already destroyed"))
+                else if (Buffer.isBuffer(chunk.payload))
                     callback(new Error("expected text input as string chunks"))
                 else if (chunk.payload.length === 0)
                     callback()
                 else {
-                    log("info", `received text: ${JSON.stringify(chunk.payload)}`)
+                    self.log("info", `received text: ${JSON.stringify(chunk.payload)}`)
                     self.queueRecv.append({ type: "text-frame", chunk })
                     callback()
                 }
@@ -169,6 +166,10 @@ export default class SpeechFlowNodeSentence extends SpeechFlowNode {
 
             /*  receive no more text chunks (writable side of stream)  */
             final (callback) {
+                if (self.destroyed) {
+                    callback()
+                    return
+                }
                 /*  signal end of file  */
                 self.queueRecv.append({ type: "text-eof" })
                 callback()
@@ -176,11 +177,12 @@ export default class SpeechFlowNodeSentence extends SpeechFlowNode {
 
             /*  send text chunk(s) (readable side of stream)  */
             read (_size) {
-                /*  flush pending audio chunks  */
-                const position = self.queueSend.position()
-                const maxPosition = self.queueSend.maxPosition()
-                log("info", `SEND: ${maxPosition - position} elements in queue`)
+                /*  flush pending text chunks  */
                 const flushPendingChunks = () => {
+                    if (self.destroyed) {
+                        this.push(null)
+                        return
+                    }
                     const element = self.queueSend.peek()
                     if (element !== undefined
                         && element.type === "text-eof") {
@@ -202,13 +204,13 @@ export default class SpeechFlowNodeSentence extends SpeechFlowNode {
                             else if (element.type === "text-frame"
                                 && element.complete !== true)
                                 break
-                            log("info", `send text: ${JSON.stringify(element.chunk.payload)}`)
+                            self.log("info", `send text: ${JSON.stringify(element.chunk.payload)}`)
                             this.push(element.chunk)
                             self.queueSend.walk(+1)
                             self.queue.trim()
                         }
                     }
-                    else
+                    else if (!self.destroyed)
                         self.queue.once("write", flushPendingChunks)
                 }
                 flushPendingChunks()
@@ -218,13 +220,22 @@ export default class SpeechFlowNodeSentence extends SpeechFlowNode {
 
     /*  close node  */
     async close () {
+        /*  indicate destruction  */
+        this.destroyed = true
+
+        /*  clean up timer  */
+        if (this.workingOffTimer !== null) {
+            clearTimeout(this.workingOffTimer)
+            this.workingOffTimer = null
+        }
+
+        /*  remove any pending event listeners  */
+        this.queue.removeAllListeners("write")
+
         /*  close stream  */
         if (this.stream !== null) {
             this.stream.destroy()
             this.stream = null
         }
-
-        /*  indicate destruction  */
-        this.destroyed = true
     }
 }

package/src/speechflow-node-t2t-subtitle.ts

@@ -7,6 +7,9 @@
 /*  standard dependencies  */
 import Stream from "node:stream"
 
+/*  external dependencies  */
+import { Duration } from "luxon"
+
 /*  internal dependencies  */
 import SpeechFlowNode, { SpeechFlowChunk } from "./speechflow-node"
 
@@ -24,7 +27,8 @@ export default class SpeechFlowNodeSubtitle extends SpeechFlowNode {
 
         /*  declare node configuration parameters  */
         this.configure({
-            format:   { type: "string", pos: 0, val: "srt", match: /^(?:srt|vtt)$/ }
+            format: { type: "string", pos: 0, val: "srt", match: /^(?:srt|vtt)$/ },
+            words:  { type: "boolean", val: false }
         })
 
         /*  declare node input/output format  */
@@ -40,31 +44,74 @@ export default class SpeechFlowNodeSubtitle extends SpeechFlowNode {
         const convert = async (chunk: SpeechFlowChunk) => {
             if (typeof chunk.payload !== "string")
                 throw new Error("chunk payload type must be string")
-            let text = chunk.payload
-            if (this.params.format === "srt") {
-                const start = chunk.timestampStart.toFormat("hh:mm:ss,SSS")
-                const end   = chunk.timestampEnd.toFormat("hh:mm:ss,SSS")
-                text = `${this.sequenceNo++}\n` +
-                    `${start} --> ${end}\n` +
-                    `${text}\n\n`
+            const convertSingle = (
+                start:      Duration,
+                end:        Duration,
+                text:       string,
+                word?:      string,
+                occurence?: number
+            ) => {
+                if (word) {
+                    occurence ??= 1
+                    let match = 1
+                    word = word.replace(/[.*+?^${}()|[\]\\]/g, "\\$&")
+                    text = text.replaceAll(new RegExp(`\\b${word}\\b`, "g"), (m) => {
+                        if (match++ === occurence)
+                            return `<b>${m}</b>`
+                        else
+                            return m
+                    })
+                }
+                if (this.params.format === "srt") {
+                    const startFmt = start.toFormat("hh:mm:ss,SSS")
+                    const endFmt   = end.toFormat("hh:mm:ss,SSS")
+                    text = `${this.sequenceNo++}\n` +
+                        `${startFmt} --> ${endFmt}\n` +
+                        `${text}\n\n`
+                }
+                else if (this.params.format === "vtt") {
+                    const startFmt = start.toFormat("hh:mm:ss.SSS")
+                    const endFmt   = end.toFormat("hh:mm:ss.SSS")
+                    text = `${startFmt} --> ${endFmt}\n` +
+                        `${text}\n\n`
+                }
+                return text
             }
-            else if (this.params.format === "vtt") {
-                const start = chunk.timestampStart.toFormat("hh:mm:ss.SSS")
-                const end   = chunk.timestampEnd.toFormat("hh:mm:ss.SSS")
-                text = `${this.sequenceNo++}\n` +
-                    `${start} --> ${end}\n` +
-                    `${text}\n\n`
+            let output = ""
+            if (this.params.words) {
+                output += convertSingle(chunk.timestampStart, chunk.timestampEnd, chunk.payload)
+                const words = (chunk.meta.get("words") ?? []) as
+                    { word: string, start: Duration, end: Duration }[]
+                const occurences = new Map<string, number>()
+                for (const word of words) {
+                    let occurence = occurences.get(word.word) ?? 0
+                    occurence++
+                    occurences.set(word.word, occurence)
+                    output += convertSingle(word.start, word.end, chunk.payload, word.word, occurence)
+                }
             }
-            return text
+            else
+                output += convertSingle(chunk.timestampStart, chunk.timestampEnd, chunk.payload)
+            return output
         }
 
         /*  establish a duplex stream  */
+        const self = this
+        let firstChunk = true
         this.stream = new Stream.Transform({
             readableObjectMode: true,
             writableObjectMode: true,
             decodeStrings:      false,
             highWaterMark:      1,
             transform (chunk: SpeechFlowChunk, encoding, callback) {
+                if (firstChunk && self.params.format === "vtt") {
+                    this.push(new SpeechFlowChunk(
+                        Duration.fromMillis(0), Duration.fromMillis(0),
+                        "final", "text",
+                        "WEBVTT\n\n"
+                    ))
+                    firstChunk = false
+                }
                 if (Buffer.isBuffer(chunk.payload))
                     callback(new Error("invalid chunk payload type"))
                 else {

package/src/speechflow-node-t2t-transformers.ts

@@ -104,7 +104,7 @@ export default class SpeechFlowNodeTransformers extends SpeechFlowNode {
 
     /*  open node  */
     async open () {
-        let model: string = ""
+        let model = ""
 
         /*  track download progress when instantiating Transformers engine and model  */
         const progressState = new Map<string, number>()
@@ -138,7 +138,6 @@ export default class SpeechFlowNodeTransformers extends SpeechFlowNode {
                 progress_callback: progressCallback
             })
             this.translator = await pipeline
-            clearInterval(interval)
             if (this.translator === null)
                 throw new Error("failed to instantiate translator pipeline")
         }
@@ -151,13 +150,15 @@ export default class SpeechFlowNodeTransformers extends SpeechFlowNode {
                 progress_callback: progressCallback
             })
             this.generator = await pipeline
-            clearInterval(interval)
             if (this.generator === null)
                 throw new Error("failed to instantiate generator pipeline")
         }
         else
             throw new Error("invalid model")
 
+        /*  clear progress interval again  */
+        clearInterval(interval)
+
         /*  provide text-to-text translation  */
         const translate = async (text: string) => {
             if (this.params.model === "OPUS") {

package/src/speechflow-node-x2x-filter.ts

@@ -94,13 +94,13 @@ export default class SpeechFlowNodeFilter extends SpeechFlowNode {
                 const m = self.params.var.match(/^meta:(.+)$/)
                 if (m !== null)
                     val1 = chunk.meta.get(m[1]) ?? ""
-                else if (self.params.key === "payload:length")
+                else if (self.params.var === "payload:length")
                     val1 = chunk.payload.length
-                else if (self.params.key === "payload:text")
+                else if (self.params.var === "payload:text")
                     val1 = (self.params.type === "text" ? chunk.payload as string : "")
-                else if (self.params.key === "time:start")
+                else if (self.params.var === "time:start")
                     val1 = chunk.timestampStart.toMillis()
-                else if (self.params.key === "time:end")
+                else if (self.params.var === "time:end")
                     val1 = chunk.timestampEnd.toMillis()
                 if (comparison(val1, self.params.op, val2)) {
                     self.log("info", `[${self.params.name}]: passing through ${chunk.type} chunk`)

package/src/speechflow-node-x2x-trace.ts

@@ -75,7 +75,7 @@ export default class SpeechFlowNodeTrace extends SpeechFlowNode {
                 }
                 else {
                     if (type === "text")
-                        log("debug", `${type} chunk: type=${chunk.type} ` +
+                        log("debug", `chunk: type=${chunk.type} ` +
                             `kind=${chunk.kind} ` +
                             `start=${fmtTime(chunk.timestampStart)} ` +
                             `end=${fmtTime(chunk.timestampEnd)} ` +

package/src/speechflow-node-xio-device.ts

@@ -159,8 +159,7 @@ export default class SpeechFlowNodeDevice extends SpeechFlowNode {
 
             /*  convert regular stream into object-mode stream  */
             const wrapper = utils.createTransformStreamForReadableSide("audio", () => this.timeZero)
-            this.stream.pipe(wrapper)
-            this.stream = wrapper
+            this.stream = Stream.compose(this.stream, wrapper)
         }
         else if (this.params.mode === "w") {
             /*  output device  */
@@ -180,8 +179,7 @@ export default class SpeechFlowNodeDevice extends SpeechFlowNode {
 
             /*  convert regular stream into object-mode stream  */
             const wrapper = utils.createTransformStreamForWritableSide()
-            wrapper.pipe(this.stream)
-            this.stream = wrapper
+            this.stream = Stream.compose(wrapper, this.stream)
         }
         else
             throw new Error(`device "${device.id}" does not have any input or output channels`)
@@ -200,13 +198,19 @@ export default class SpeechFlowNodeDevice extends SpeechFlowNode {
         /*  shutdown PortAudio  */
         if (this.io !== null) {
             await new Promise<void>((resolve, reject) => {
-                this.io!.abort(() => {
-                    resolve()
+                this.io!.abort((err?: Error) => {
+                    if (err)
+                        reject(err)
+                    else
+                        resolve()
                 })
             })
             await new Promise<void>((resolve, reject) => {
-                this.io!.quit(() => {
-                    resolve()
+                this.io!.quit((err?: Error) => {
+                    if (err)
+                        reject(err)
+                    else
+                        resolve()
                 })
             })
             this.io = null

package/src/speechflow-node-xio-file.ts

@@ -178,9 +178,15 @@ export default class SpeechFlowNodeFile extends SpeechFlowNode {
     async close () {
         /*  shutdown stream  */
         if (this.stream !== null) {
-            await new Promise<void>((resolve) => {
-                if (this.stream instanceof Stream.Writable || this.stream instanceof Stream.Duplex)
-                    this.stream.end(() => { resolve() })
+            await new Promise<void>((resolve, reject) => {
+                if (this.stream instanceof Stream.Writable || this.stream instanceof Stream.Duplex) {
+                    this.stream.end((err?: Error) => {
+                        if (err)
+                            reject(err)
+                        else
+                            resolve()
+                    })
+                }
                 else
                     resolve()
             })

package/src/speechflow-node-xio-mqtt.ts

@@ -83,7 +83,10 @@ export default class SpeechFlowNodeMQTT extends SpeechFlowNode {
         this.broker.on("connect", (packet: MQTT.IConnackPacket) => {
             this.log("info", `connection opened to MQTT ${this.params.url}`)
             if (this.params.mode !== "w" && !packet.sessionPresent)
-                this.broker!.subscribe([ this.params.topicRead ], () => {})
+                this.broker!.subscribe([ this.params.topicRead ], (err) => {
+                    if (err)
+                        this.log("error", `failed to subscribe to MQTT topic "${this.params.topicRead}": ${err.message}`)
+                })
         })
         this.broker.on("reconnect", () => {
             this.log("info", `connection re-opened to MQTT ${this.params.url}`)
@@ -141,7 +144,7 @@ export default class SpeechFlowNodeMQTT extends SpeechFlowNode {
 
     /*  close node  */
     async close () {
-        /*  close Websocket server  */
+        /*  close MQTT broker  */
         if (this.broker !== null) {
             if (this.broker.connected)
                 this.broker.end()

package/src/speechflow-node-xio-websocket.ts

@@ -64,15 +64,15 @@ export default class SpeechFlowNodeWebsocket extends SpeechFlowNode {
             const url = new URL(this.params.listen)
             const websockets = new Set<ws.WebSocket>()
             const chunkQueue = new utils.SingleQueue<SpeechFlowChunk>()
-            const server = new ws.WebSocketServer({
+            this.server = new ws.WebSocketServer({
                 host: url.hostname,
                 port: Number.parseInt(url.port),
                 path: url.pathname
             })
-            server.on("listening", () => {
+            this.server.on("listening", () => {
                 this.log("info", `listening on URL ${this.params.listen}`)
             })
-            server.on("connection", (ws, request) => {
+            this.server.on("connection", (ws, request) => {
                 const peer = `${request.socket.remoteAddress}:${request.socket.remotePort}`
                 this.log("info", `connection opened on URL ${this.params.listen} by peer ${peer}`)
                 websockets.add(ws)
@@ -105,7 +105,7 @@ export default class SpeechFlowNodeWebsocket extends SpeechFlowNode {
                     chunkQueue.write(chunk)
                 })
             })
-            server.on("error", (error) => {
+            this.server.on("error", (error) => {
                 this.log("error", `error of some connection on URL ${this.params.listen}: ${error.message}`)
             })
             const type = this.params.type
@@ -124,7 +124,7 @@ export default class SpeechFlowNodeWebsocket extends SpeechFlowNode {
                         callback(new Error("still no Websocket connections available"))
                     else {
                         const data = utils.streamChunkEncode(chunk)
-                        const results = []
+                        const results: Promise<void>[] = []
                         for (const websocket of websockets.values()) {
                             results.push(new Promise<void>((resolve, reject) => {
                                 websocket.send(data, (error) => {
@@ -175,12 +175,12 @@ export default class SpeechFlowNodeWebsocket extends SpeechFlowNode {
             const chunkQueue = new utils.SingleQueue<SpeechFlowChunk>()
             this.client.addEventListener("message", (ev: MessageEvent) => {
                 if (this.params.mode === "w") {
-                    this.log("warning", `connection to URL ${this.params.listen}: ` +
+                    this.log("warning", `connection to URL ${this.params.connect}: ` +
                         "received remote data on write-only node")
                     return
                 }
                 if (!(ev.data instanceof ArrayBuffer)) {
-                    this.log("warning", `connection to URL ${this.params.listen}: ` +
+                    this.log("warning", `connection to URL ${this.params.connect}: ` +
                         "received non-binary message")
                     return
                 }
@@ -204,15 +204,15 @@ export default class SpeechFlowNodeWebsocket extends SpeechFlowNode {
                         callback(new Error(`written chunk is not of ${type} type`))
                     else if (!client.OPEN)
                         callback(new Error("still no Websocket connection available"))
-                    const data = utils.streamChunkEncode(chunk)
-                    client.send(data)
-                    callback()
+                    else {
+                        const data = utils.streamChunkEncode(chunk)
+                        client.send(data)
+                        callback()
+                    }
                 },
                 read (size: number) {
                     if (mode === "w")
                         throw new Error("read operation on write-only node")
-                    if (!client.OPEN)
-                        throw new Error("still no Websocket connection available")
                     chunkQueue.read().then((chunk) => {
                         this.push(chunk, "binary")
                     })