speechflow 1.2.8 → 1.3.1

package/src/speechflow-node-t2t-sentence.ts

@@ -0,0 +1,224 @@
+/*
+**  SpeechFlow - Speech Processing Flow Graph
+**  Copyright (c) 2024-2025 Dr. Ralf S. Engelschall <rse@engelschall.com>
+**  Licensed under GPL 3.0 <https://spdx.org/licenses/GPL-3.0-only>
+*/
+
+/*  standard dependencies  */
+import Stream             from "node:stream"
+
+/*  external dependencies  */
+import { Duration }       from "luxon"
+
+/*  internal dependencies  */
+import SpeechFlowNode, { SpeechFlowChunk } from "./speechflow-node"
+import * as utils                          from "./speechflow-utils"
+
+/*  text stream queue element */
+type TextQueueElement = {
+    type:         "text-frame",
+    chunk:        SpeechFlowChunk,
+    complete?:    boolean
+} | {
+    type:         "text-eof"
+}
+
+/*  SpeechFlow node for sentence splitting  */
+export default class SpeechFlowNodeSentence extends SpeechFlowNode {
+    /*  declare official node name  */
+    public static name = "sentence"
+
+    /*  internal state  */
+    private static speexInitialized = false
+    private queue      = new utils.Queue<TextQueueElement>()
+    private queueRecv  = this.queue.pointerUse("recv")
+    private queueSplit = this.queue.pointerUse("split")
+    private queueSend  = this.queue.pointerUse("send")
+    private destroyed  = false
+
+    /*  construct node  */
+    constructor (id: string, cfg: { [ id: string ]: any }, opts: { [ id: string ]: any }, args: any[]) {
+        super(id, cfg, opts, args)
+
+        /*  declare node configuration parameters  */
+        this.configure({})
+
+        /*  declare node input/output format  */
+        this.input  = "text"
+        this.output = "text"
+    }
+
+    /*  open node  */
+    async open () {
+        /*  clear destruction flag  */
+        this.destroyed = false
+
+        /*  pass-through logging  */
+        const log = (level: string, msg: string) => { this.log(level, msg) }
+
+        /*  work off queued audio frames  */
+        let workingOffTimer: ReturnType<typeof setTimeout> | null = null
+        let workingOff = false
+        const workOffQueue = async () => {
+            if (this.destroyed)
+                return
+
+            /*  control working off round  */
+            if (workingOff)
+                return
+            workingOff = true
+            if (workingOffTimer !== null) {
+                clearTimeout(workingOffTimer)
+                workingOffTimer = null
+            }
+            this.queue.off("write", workOffQueue)
+
+            /*  try to work off one or more chunks  */
+            while (true) {
+                const element = this.queueSplit.peek()
+                if (element === undefined)
+                    break
+                if (element.type === "text-eof") {
+                    this.queueSplit.walk(+1)
+                    break
+                }
+                const chunk = element.chunk
+                const payload = chunk.payload as string
+                const m = payload.match(/^((?:.|\r?\n)+?[.;?!])\s*((?:.|\r?\n)*)$/)
+                if (m !== null) {
+                    /*  contains a sentence  */
+                    const [ , sentence, rest ] = m
+                    if (rest !== "") {
+                        /*  contains more than a sentence  */
+                        const chunk2 = chunk.clone()
+                        const duration = Duration.fromMillis(
+                            chunk.timestampEnd.minus(chunk.timestampStart).toMillis() *
+                            (sentence.length / payload.length))
+                        chunk2.timestampStart = chunk.timestampStart.plus(duration)
+                        chunk.timestampEnd    = chunk2.timestampStart
+                        chunk.payload  = sentence
+                        chunk2.payload = rest
+                        element.complete = true
+                        this.queueSplit.touch()
+                        this.queueSplit.walk(+1)
+                        this.queueSplit.insert({ type: "text-frame", chunk: chunk2 })
+                    }
+                    else {
+                        /*  contains just the sentence  */
+                        element.complete = true
+                        this.queueSplit.touch()
+                        this.queueSplit.walk(+1)
+                    }
+                }
+                else {
+                    /*  contains less than a sentence  */
+                    const position = this.queueSplit.position()
+                    if (position < this.queueSplit.maxPosition() - 1) {
+                        /*  merge into following chunk  */
+                        const element2 = this.queueSplit.peek(position + 1)
+                        if (element2 === undefined)
+                            break
+                        if (element2.type === "text-eof") {
+                            element.complete = true
+                            this.queueSplit.touch()
+                            this.queueSplit.walk(+1)
+                            break
+                        }
+                        element2.chunk.timestampStart = element.chunk.timestampStart
+                        element2.chunk.payload =
+                            element.chunk.payload  as string + " " +
+                            element2.chunk.payload as string
+                        this.queueSplit.delete()
+                        this.queueSplit.touch()
+                    }
+                    else
+                        break
+                }
+            }
+
+            /*  re-initiate working off round  */
+            workingOff = false
+            workingOffTimer = setTimeout(workOffQueue, 100)
+            this.queue.once("write", workOffQueue)
+        }
+        this.queue.once("write", workOffQueue)
+
+        /*  provide Duplex stream and internally attach to classifier  */
+        const self = this
+        this.stream = new Stream.Duplex({
+            writableObjectMode: true,
+            readableObjectMode: true,
+            decodeStrings:      false,
+            highWaterMark:      1,
+
+            /*  receive text chunk (writable side of stream)  */
+            write (chunk: SpeechFlowChunk, encoding, callback) {
+                if (Buffer.isBuffer(chunk.payload))
+                    callback(new Error("expected text input as string chunks"))
+                else if (chunk.payload.length === 0)
+                    callback()
+                else {
+                    log("info", `received text: ${JSON.stringify(chunk.payload)}`)
+                    self.queueRecv.append({ type: "text-frame", chunk })
+                    callback()
+                }
+            },
+
+            /*  receive no more text chunks (writable side of stream)  */
+            final (callback) {
+                /*  signal end of file  */
+                self.queueRecv.append({ type: "text-eof" })
+                callback()
+            },
+
+            /*  send text chunk(s) (readable side of stream)  */
+            read (_size) {
+                /*  flush pending audio chunks  */
+                const flushPendingChunks = () => {
+                    const element = self.queueSend.peek()
+                    if (element !== undefined
+                        && element.type === "text-eof") {
+                        this.push(null)
+                        self.queueSend.walk(+1)
+                    }
+                    else if (element !== undefined
+                        && element.type === "text-frame"
+                        && element.complete === true) {
+                        while (true) {
+                            const element = self.queueSend.peek()
+                            if (element === undefined)
+                                break
+                            else if (element.type === "text-eof") {
+                                this.push(null)
+                                self.queueSend.walk(+1)
+                                break
+                            }
+                            else if (element.type === "text-frame"
+                                && element.complete !== true)
+                                break
+                            log("info", `send text: ${JSON.stringify(element.chunk.payload)}`)
+                            this.push(element.chunk)
+                            self.queueSend.walk(+1)
+                            self.queue.trim()
+                        }
+                    }
+                    else
+                        self.queue.once("write", flushPendingChunks)
+                }
+                flushPendingChunks()
+            }
+        })
+    }
+
+    /*  close node  */
+    async close () {
+        /*  close stream  */
+        if (this.stream !== null) {
+            this.stream.destroy()
+            this.stream = null
+        }
+
+        /*  indicate destruction  */
+        this.destroyed = true
+    }
+}

package/src/speechflow-node-t2t-subtitle.ts

@@ -7,6 +7,9 @@
 /*  standard dependencies  */
 import Stream from "node:stream"
 
+/*  external dependencies  */
+import { Duration } from "luxon"
+
 /*  internal dependencies  */
 import SpeechFlowNode, { SpeechFlowChunk } from "./speechflow-node"
 
@@ -24,7 +27,8 @@ export default class SpeechFlowNodeSubtitle extends SpeechFlowNode {
 
         /*  declare node configuration parameters  */
         this.configure({
-            format:   { type: "string", pos: 0, val: "srt", match: /^(?:srt|vtt)$/ }
+            format: { type: "string", pos: 0, val: "srt", match: /^(?:srt|vtt)$/ },
+            words:  { type: "boolean", val: false }
         })
 
         /*  declare node input/output format  */
@@ -40,31 +44,74 @@ export default class SpeechFlowNodeSubtitle extends SpeechFlowNode {
         const convert = async (chunk: SpeechFlowChunk) => {
             if (typeof chunk.payload !== "string")
                 throw new Error("chunk payload type must be string")
-            let text = chunk.payload
-            if (this.params.format === "srt") {
-                const start = chunk.timestampStart.toFormat("hh:mm:ss,SSS")
-                const end   = chunk.timestampEnd.toFormat("hh:mm:ss,SSS")
-                text = `${this.sequenceNo++}\n` +
-                    `${start} --> ${end}\n` +
-                    `${text}\n\n`
+            const convertSingle = (
+                start:      Duration,
+                end:        Duration,
+                text:       string,
+                word?:      string,
+                occurence?: number
+            ) => {
+                if (word) {
+                    occurence ??= 1
+                    let match = 1
+                    word = word.replace(/[.*+?^${}()|[\]\\]/g, "\\$&")
+                    text = text.replaceAll(new RegExp(`\\b${word}\\b`, "g"), (m) => {
+                        if (match++ === occurence)
+                            return `<b>${m}</b>`
+                        else
+                            return m
+                    })
+                }
+                if (this.params.format === "srt") {
+                    const startFmt = start.toFormat("hh:mm:ss,SSS")
+                    const endFmt   = end.toFormat("hh:mm:ss,SSS")
+                    text = `${this.sequenceNo++}\n` +
+                        `${startFmt} --> ${endFmt}\n` +
+                        `${text}\n\n`
+                }
+                else if (this.params.format === "vtt") {
+                    const startFmt = start.toFormat("hh:mm:ss.SSS")
+                    const endFmt   = end.toFormat("hh:mm:ss.SSS")
+                    text = `${startFmt} --> ${endFmt}\n` +
+                        `${text}\n\n`
+                }
+                return text
             }
-            else if (this.params.format === "vtt") {
-                const start = chunk.timestampStart.toFormat("hh:mm:ss.SSS")
-                const end   = chunk.timestampEnd.toFormat("hh:mm:ss.SSS")
-                text = `${this.sequenceNo++}\n` +
-                    `${start} --> ${end}\n` +
-                    `${text}\n\n`
+            let output = ""
+            if (this.params.words) {
+                output += convertSingle(chunk.timestampStart, chunk.timestampEnd, chunk.payload)
+                const words = (chunk.meta.get("words") ?? []) as
+                    { word: string, start: Duration, end: Duration }[]
+                const occurences = new Map<string, number>()
+                for (const word of words) {
+                    let occurence = occurences.get(word.word) ?? 0
+                    occurence++
+                    occurences.set(word.word, occurence)
+                    output += convertSingle(word.start, word.end, chunk.payload, word.word, occurence)
+                }
             }
-            return text
+            else
+                output += convertSingle(chunk.timestampStart, chunk.timestampEnd, chunk.payload)
+            return output
         }
 
         /*  establish a duplex stream  */
+        const self = this
+        let firstChunk = true
         this.stream = new Stream.Transform({
             readableObjectMode: true,
             writableObjectMode: true,
             decodeStrings:      false,
             highWaterMark:      1,
             transform (chunk: SpeechFlowChunk, encoding, callback) {
+                if (firstChunk && self.params.format === "vtt") {
+                    this.push(new SpeechFlowChunk(
+                        Duration.fromMillis(0), Duration.fromMillis(0),
+                        "final", "text",
+                        "WEBVTT\n\n"
+                    ))
+                    firstChunk = false
+                }
                 if (Buffer.isBuffer(chunk.payload))
                     callback(new Error("invalid chunk payload type"))
                 else {

package/src/speechflow-node-xio-device.ts

@@ -199,8 +199,16 @@ export default class SpeechFlowNodeDevice extends SpeechFlowNode {
     async close () {
         /*  shutdown PortAudio  */
         if (this.io !== null) {
-            this.io.abort()
-            this.io.quit()
+            await new Promise<void>((resolve, reject) => {
+                this.io!.abort(() => {
+                    resolve()
+                })
+            })
+            await new Promise<void>((resolve, reject) => {
+                this.io!.quit(() => {
+                    resolve()
+                })
+            })
             this.io = null
         }
     }

package/src/speechflow-utils.ts

@@ -378,7 +378,7 @@ export class QueuePointer<T extends QueueElement> extends EventEmitter {
         this.queue.emit("write", { start: this.index - 1, end: this.index - 1 })
     }
     insert (element: T) {
-        this.queue.elements.splice(this.index++, 0, element)
+        this.queue.elements.splice(this.index, 0, element)
         this.queue.emit("write", { start: this.index - 1, end: this.index })
     }
     delete () {

package/src/speechflow.ts

@@ -244,6 +244,7 @@ type wsPeerInfo = {
         "./speechflow-node-t2t-format.js",
         "./speechflow-node-t2t-ollama.js",
         "./speechflow-node-t2t-openai.js",
+        "./speechflow-node-t2t-sentence.js",
         "./speechflow-node-t2t-subtitle.js",
         "./speechflow-node-t2t-transformers.js",
         "./speechflow-node-x2x-filter.js",
@@ -506,8 +507,10 @@ type wsPeerInfo = {
         const name = req.node as string
         const args = req.args as any[]
         const foundNode = Array.from(graphNodes).find((node) => node.id === name)
-        if (foundNode === undefined)
+        if (foundNode === undefined) {
             cli!.log("warning", `external request failed: no such node <${name}>`)
+            throw new Error(`external request failed: no such node <${name}>`)
+        }
         else {
             await foundNode.receiveRequest(args).catch((err: Error) => {
                 cli!.log("warning", `external request to node <${name}> failed: ${err}`)
@@ -644,12 +647,14 @@ type wsPeerInfo = {
         shuttingDown = true
         if (signal === "finished")
             cli!.log("info", "**** streams of all nodes finished -- shutting down service ****")
+        else if (signal === "exception")
+            cli!.log("warning", "**** exception occurred -- shutting down service ****")
         else
             cli!.log("warning", `**** received signal ${signal} -- shutting down service ****`)
 
         /*  shutdown HAPI service  */
         cli!.log("info", `HAPI: stopping REST/WebSocket network service: http://${args.address}:${args.port}`)
-        await hapi.stop()
+        await hapi.stop({ timeout: 2000 })
 
         /*  graph processing: PASS 1: disconnect node streams  */
         for (const node of graphNodes) {
@@ -715,6 +720,14 @@ type wsPeerInfo = {
     process.on("SIGUSR1",       () => { shutdown("SIGUSR1")  })
     process.on("SIGUSR2",       () => { shutdown("SIGUSR2")  })
     process.on("SIGTERM",       () => { shutdown("SIGTERM")  })
+    process.on("uncaughtException", (err) => {
+        cli!.log("error", `uncaught exception: ${err}`)
+        shutdown("exception")
+    })
+    process.on("unhandledRejection", (reason) => {
+        cli!.log("error", `unhandled rejection: ${reason}`)
+        shutdown("exception")
+    })
 })().catch((err: Error) => {
     if (cli !== null)
         cli.log("error", err.message)