speechflow 1.3.2 → 1.4.0

package/speechflow-cli/src/speechflow-node-t2t-subtitle.ts

@@ -0,0 +1,276 @@
+/*
+**  SpeechFlow - Speech Processing Flow Graph
+**  Copyright (c) 2024-2025 Dr. Ralf S. Engelschall <rse@engelschall.com>
+**  Licensed under GPL 3.0 <https://spdx.org/licenses/GPL-3.0-only>
+*/
+
+/*  standard dependencies  */
+import path          from "node:path"
+import http          from "node:http"
+import Stream        from "node:stream"
+
+/*  external dependencies  */
+import { Duration }  from "luxon"
+import * as HAPI     from "@hapi/hapi"
+import Inert         from "@hapi/inert"
+import WebSocket     from "ws"
+import HAPIWebSocket from "hapi-plugin-websocket"
+
+/*  internal dependencies  */
+import SpeechFlowNode, { SpeechFlowChunk } from "./speechflow-node"
+
+type wsPeerCtx = {
+    peer: string
+}
+type wsPeerInfo = {
+    ctx:        wsPeerCtx
+    ws:         WebSocket
+    req:        http.IncomingMessage
+}
+
+/*  SpeechFlow node for subtitle (text-to-text) "translations"  */
+export default class SpeechFlowNodeSubtitle extends SpeechFlowNode {
+    /*  declare official node name  */
+    public static name = "subtitle"
+
+    /*  internal state  */
+    private sequenceNo = 1
+    private hapi: HAPI.Server | null = null
+
+    /*  construct node  */
+    constructor (id: string, cfg: { [ id: string ]: any }, opts: { [ id: string ]: any }, args: any[]) {
+        super(id, cfg, opts, args)
+
+        /*  declare node configuration parameters  */
+        this.configure({
+            format: { type: "string", pos: 0, val: "srt", match: /^(?:srt|vtt)$/ },
+            words:  { type: "boolean", val: false },
+            mode:   { type: "string",  val: "export", match: /^(?:export|render)$/ },
+            addr:   { type: "string",  val: "127.0.0.1" },
+            port:   { type: "number",  val: 8585 }
+        })
+
+        /*  declare node input/output format  */
+        this.input  = "text"
+        this.output = this.params.mode === "export" ? "text" : "none"
+    }
+
+    /*  open node  */
+    async open () {
+        if (this.params.mode === "export") {
+            this.sequenceNo = 1
+
+            /*  provide text-to-subtitle conversion  */
+            const convert = async (chunk: SpeechFlowChunk) => {
+                if (typeof chunk.payload !== "string")
+                    throw new Error("chunk payload type must be string")
+                const convertSingle = (
+                    start:      Duration,
+                    end:        Duration,
+                    text:       string,
+                    word?:      string,
+                    occurence?: number
+                ) => {
+                    if (word) {
+                        occurence ??= 1
+                        let match = 1
+                        word = word.replace(/[.*+?^${}()|[\]\\]/g, "\\$&")
+                        text = text.replaceAll(new RegExp(`\\b${word}\\b`, "g"), (m) => {
+                            if (match++ === occurence)
+                                return `<b>${m}</b>`
+                            else
+                                return m
+                        })
+                    }
+                    if (this.params.format === "srt") {
+                        const startFmt = start.toFormat("hh:mm:ss,SSS")
+                        const endFmt   = end.toFormat("hh:mm:ss,SSS")
+                        text = `${this.sequenceNo++}\n` +
+                            `${startFmt} --> ${endFmt}\n` +
+                            `${text}\n\n`
+                    }
+                    else if (this.params.format === "vtt") {
+                        const startFmt = start.toFormat("hh:mm:ss.SSS")
+                        const endFmt   = end.toFormat("hh:mm:ss.SSS")
+                        text = `${startFmt} --> ${endFmt}\n` +
+                            `${text}\n\n`
+                    }
+                    return text
+                }
+                let output = ""
+                if (this.params.words) {
+                    output += convertSingle(chunk.timestampStart, chunk.timestampEnd, chunk.payload)
+                    const words = (chunk.meta.get("words") ?? []) as
+                        { word: string, start: Duration, end: Duration }[]
+                    const occurences = new Map<string, number>()
+                    for (const word of words) {
+                        let occurence = occurences.get(word.word) ?? 0
+                        occurence++
+                        occurences.set(word.word, occurence)
+                        output += convertSingle(word.start, word.end, chunk.payload, word.word, occurence)
+                    }
+                }
+                else
+                    output += convertSingle(chunk.timestampStart, chunk.timestampEnd, chunk.payload)
+                return output
+            }
+
+            /*  establish a duplex stream  */
+            const self = this
+            let firstChunk = true
+            this.stream = new Stream.Transform({
+                readableObjectMode: true,
+                writableObjectMode: true,
+                decodeStrings:      false,
+                highWaterMark:      1,
+                transform (chunk: SpeechFlowChunk, encoding, callback) {
+                    if (firstChunk && self.params.format === "vtt") {
+                        this.push(new SpeechFlowChunk(
+                            Duration.fromMillis(0), Duration.fromMillis(0),
+                            "final", "text",
+                            "WEBVTT\n\n"
+                        ))
+                        firstChunk = false
+                    }
+                    if (Buffer.isBuffer(chunk.payload))
+                        callback(new Error("invalid chunk payload type"))
+                    else {
+                        if (chunk.payload === "") {
+                            this.push(chunk)
+                            callback()
+                        }
+                        else {
+                            convert(chunk).then((payload) => {
+                                const chunkNew = chunk.clone()
+                                chunkNew.payload = payload
+                                this.push(chunkNew)
+                                callback()
+                            }).catch((err) => {
+                                callback(err)
+                            })
+                        }
+                    }
+                },
+                final (callback) {
+                    this.push(null)
+                    callback()
+                }
+            })
+        }
+        else if (this.params.mode === "render") {
+            /*  establish REST/WebSocket API  */
+            const wsPeers = new Map<string, wsPeerInfo>()
+            this.hapi = new HAPI.Server({
+                address: this.params.addr,
+                port:    this.params.port
+            })
+            await this.hapi.register({ plugin: Inert })
+            await this.hapi.register({ plugin: HAPIWebSocket })
+            this.hapi.events.on({ name: "request", channels: [ "error" ] }, (request: HAPI.Request, event: HAPI.RequestEvent, tags: { [key: string]: true }) => {
+                if (event.error instanceof Error)
+                    this.log("error", `HAPI: request-error: ${event.error.message}`)
+                else
+                    this.log("error", `HAPI: request-error: ${event.error}`)
+            })
+            this.hapi.events.on("log", (event: HAPI.LogEvent, tags: { [key: string]: true }) => {
+                if (tags.error) {
+                    const err = event.error
+                    if (err instanceof Error)
+                        this.log("error", `HAPI: log: ${err.message}`)
+                    else
+                        this.log("error", `HAPI: log: ${err}`)
+                }
+            })
+            this.hapi.route({
+                method: "GET",
+                path: "/{param*}",
+                handler: {
+                    directory: {
+                        path: path.join(__dirname, "../../speechflow-ui-st/dst"),
+                        redirectToSlash: true,
+                        index: true
+                    }
+                }
+            })
+            this.hapi.route({
+                method: "POST",
+                path:   "/api",
+                options: {
+                    payload: {
+                        output: "data",
+                        parse:  true,
+                        allow:  "application/json"
+                    },
+                    plugins: {
+                        websocket: {
+                            autoping: 30 * 1000,
+                            connect: (args: any) => {
+                                const ctx: wsPeerCtx            = args.ctx
+                                const ws:  WebSocket            = args.ws
+                                const req: http.IncomingMessage = args.req
+                                const peer = `${req.socket.remoteAddress}:${req.socket.remotePort}`
+                                ctx.peer = peer
+                                wsPeers.set(peer, { ctx, ws, req })
+                                this.log("info", `HAPI: WebSocket: connect: peer ${peer}`)
+                            },
+                            disconnect: (args: any) => {
+                                const ctx: wsPeerCtx = args.ctx
+                                const peer = ctx.peer
+                                wsPeers.delete(peer)
+                                this.log("info", `HAPI: WebSocket: disconnect: peer ${peer}`)
+                            }
+                        }
+                    }
+                },
+                handler: (request: HAPI.Request, h: HAPI.ResponseToolkit) => {
+                    return h.response({}).code(204)
+                }
+            })
+
+            await this.hapi.start()
+            this.log("info", `HAPI: started REST/WebSocket network service: http://${this.params.addr}:${this.params.port}`)
+
+            const emit = (chunk: SpeechFlowChunk) => {
+                const data = JSON.stringify(chunk)
+                for (const info of wsPeers.values())
+                    info.ws.send(data)
+            }
+
+            this.stream = new Stream.Writable({
+                objectMode:     true,
+                decodeStrings:  false,
+                highWaterMark:  1,
+                write (chunk: SpeechFlowChunk, encoding, callback) {
+                    if (Buffer.isBuffer(chunk.payload))
+                        callback(new Error("invalid chunk payload type"))
+                    else {
+                        if (chunk.payload === "")
+                            callback()
+                        else {
+                            emit(chunk)
+                            callback()
+                        }
+                    }
+                },
+                final (callback) {
+                    callback()
+                }
+            })
+        }
+    }
+
+    /*  open node  */
+    async close () {
+        /*  close stream  */
+        if (this.stream !== null) {
+            this.stream.destroy()
+            this.stream = null
+        }
+
+        /*  shutdown HAPI  */
+        if (this.hapi !== null) {
+            await this.hapi.stop()
+            this.hapi = null
+        }
+    }
+}

package/{src → speechflow-cli/src}/speechflow-node-x2x-filter.ts

@@ -23,7 +23,7 @@ export default class SpeechFlowNodeFilter extends SpeechFlowNode {
         this.configure({
             type: { type: "string", pos: 0, val: "audio",  match: /^(?:audio|text)$/ },
             name: { type: "string", pos: 1, val: "filter", match: /^.+?$/ },
-            var:  { type: "string", pos: 2, val: "",       match: /^(?:meta:.+|payload:(?:length|text)|time:(?:start|end))$/ },
+            var:  { type: "string", pos: 2, val: "",       match: /^(?:meta:.+|payload:(?:length|text)|time:(?:start|end)|kind|type)$/ },
             op:   { type: "string", pos: 3, val: "==",     match: /^(?:<|<=|==|!=|~~|!~|>=|>)$/ },
             val:  { type: "string", pos: 4, val: "",       match: /^.*$/ }
         })
@@ -94,6 +94,10 @@ export default class SpeechFlowNodeFilter extends SpeechFlowNode {
                 const m = self.params.var.match(/^meta:(.+)$/)
                 if (m !== null)
                     val1 = chunk.meta.get(m[1]) ?? ""
+                else if (self.params.var === "kind")
+                    val1 = chunk.kind
+                else if (self.params.var === "type")
+                    val1 = chunk.type
                 else if (self.params.var === "payload:length")
                     val1 = chunk.payload.length
                 else if (self.params.var === "payload:text")

package/{src → speechflow-cli/src}/speechflow-node-x2x-trace.ts

@@ -24,10 +24,15 @@ export default class SpeechFlowNodeTrace extends SpeechFlowNode {
 
         /*  declare node configuration parameters  */
         this.configure({
-            type: { type: "string", pos: 0, val: "audio", match: /^(?:audio|text)$/ },
-            name: { type: "string", pos: 1, val: "trace" }
+            type:      { type: "string", pos: 0, val: "audio", match: /^(?:audio|text)$/ },
+            name:      { type: "string", pos: 1, val: "trace" },
+            dashboard: { type: "string",         val: "" }
         })
 
+        /*  sanity check parameters  */
+        if (this.params.dashboard !== "" && this.params.type === "audio")
+            throw new Error("only trace nodes of type \"text\" can export to dashboard")
+
         /*  declare node input/output format  */
         this.input  = this.params.type
         this.output = this.params.type
@@ -44,7 +49,7 @@ export default class SpeechFlowNodeTrace extends SpeechFlowNode {
         }
 
         /*  provide Transform stream  */
-        const type = this.params.type
+        const self = this
         this.stream = new Stream.Transform({
             writableObjectMode: true,
             readableObjectMode: true,
@@ -63,7 +68,7 @@ export default class SpeechFlowNodeTrace extends SpeechFlowNode {
                         } }`
                 }
                 if (Buffer.isBuffer(chunk.payload)) {
-                    if (type === "audio")
+                    if (self.params.type === "audio")
                         log("debug", `chunk: type=${chunk.type} ` +
                             `kind=${chunk.kind} ` +
                             `start=${fmtTime(chunk.timestampStart)} ` +
@@ -71,10 +76,10 @@ export default class SpeechFlowNodeTrace extends SpeechFlowNode {
                             `payload-type=Buffer payload-length=${chunk.payload.byteLength} ` +
                             `meta=${fmtMeta(chunk.meta)}`)
                     else
-                        error = new Error(`${type} chunk: seen Buffer instead of String chunk type`)
+                        error = new Error(`${self.params.type} chunk: seen Buffer instead of String chunk type`)
                 }
                 else {
-                    if (type === "text")
+                    if (self.params.type === "text") {
                         log("debug", `chunk: type=${chunk.type} ` +
                             `kind=${chunk.kind} ` +
                             `start=${fmtTime(chunk.timestampStart)} ` +
@@ -82,8 +87,11 @@ export default class SpeechFlowNodeTrace extends SpeechFlowNode {
                             `payload-type=String payload-length=${chunk.payload.length} ` +
                             `payload-content="${chunk.payload.toString()}" ` +
                             `meta=${fmtMeta(chunk.meta)}`)
+                        if (self.params.dashboard !== "")
+                            self.dashboardInfo("text", self.params.dashboard, chunk.kind, chunk.payload.toString())
+                    }
                     else
-                        error = new Error(`${type} chunk: seen String instead of Buffer chunk type`)
+                        error = new Error(`${self.params.type} chunk: seen String instead of Buffer chunk type`)
                 }
                 if (error !== undefined)
                     callback(error)

package/{src → speechflow-cli/src}/speechflow-node.ts

@@ -96,6 +96,13 @@ export default class SpeechFlowNode extends Events.EventEmitter {
         this.emit("send-response", args)
     }
 
+    /*  emit dashboard information  */
+    dashboardInfo (type: "audio", id: string, kind: "final" | "intermediate", value: number): void
+    dashboardInfo (type: "text", id: string, kind: "final" | "intermediate", value: string): void
+    dashboardInfo (type: "audio" | "text", id: string, kind: "final" | "intermediate", value: number | string): void {
+        this.emit("dashboard-info", { type, id, kind, value })
+    }
+
     /*  INTERNAL: utility function: create "params" attribute from constructor of sub-classes  */
     configure (spec: { [ id: string ]: { type: string, pos?: number, val?: any, match?: RegExp | ((x: any) => boolean) } }) {
         for (const name of Object.keys(spec)) {

package/{src → speechflow-cli/src}/speechflow.ts

@@ -11,6 +11,7 @@ import Stream            from "node:stream"
 import { EventEmitter }  from "node:events"
 import http              from "node:http"
 import * as HAPI         from "@hapi/hapi"
+import Inert             from "@hapi/inert"
 import WebSocket         from "ws"
 import HAPIWebSocket     from "hapi-plugin-websocket"
 import HAPIHeader        from "hapi-plugin-header"
@@ -32,7 +33,7 @@ import chalk             from "chalk"
 
 /*  internal dependencies  */
 import SpeechFlowNode    from "./speechflow-node"
-import pkg               from "../package.json"
+import pkg               from "../../package.json"
 
 /*  central CLI context  */
 let cli: CLIio | null = null
@@ -67,6 +68,7 @@ type wsPeerInfo = {
             "[-a|--address <ip-address>] " +
             "[-p|--port <tcp-port>] " +
             "[-C|--cache <directory>] " +
+            "[-d|--dashboard <type>:<id>:<name>[,...]] " +
             "[-e|--expression <expression>] " +
             "[-f|--file <file>] " +
             "[-c|--config <id>@<yaml-config-file>] " +
@@ -125,6 +127,15 @@ type wsPeerInfo = {
             default:  path.join(dataDir, "cache"),
             describe: "directory for cached files (primarily AI model files)"
         })
+        .option("d", {
+            alias:    "dashboard",
+            type:     "string",
+            array:    false,
+            coerce,
+            nargs:    1,
+            default:  "",
+            describe: "list of dashboard block types and names"
+        })
         .option("e", {
             alias:    "expression",
             type:     "string",
@@ -319,7 +330,7 @@ type wsPeerInfo = {
             const node = new nodes[name](name, cfg, {}, [])
             const status = await Promise.race<{ [ key: string ]: string | number }>([
                 node.status(),
-                new Promise((resolve, reject) => setTimeout(() =>
+                new Promise<never>((resolve, reject) => setTimeout(() =>
                     reject(new Error("timeout")), 10 * 1000))
             ]).catch((err: Error) => {
                 cli!.log("warning", `[${node.id}]: failed to gather status of node <${node.id}>: ${err.message}`)
@@ -352,12 +363,9 @@ type wsPeerInfo = {
         ast = flowlink.compile(config)
     }
     catch (err) {
-        if (err instanceof Error && err.name === "FlowLinkError")
-            cli!.log("error", `failed to parse SpeechFlow configuration: ${err.toString()}`)
-        else if (err instanceof Error)
-            cli!.log("error", `failed to parse SpeechFlow configuration: ${err.message}`)
-        else
-            cli!.log("error", "failed to parse SpeechFlow configuration: internal error")
+        const errorMsg = err instanceof Error && err.name === "FlowLinkError"
+            ? err.toString() : (err instanceof Error ? err.message : "internal error")
+        cli!.log("error", `failed to parse SpeechFlow configuration: ${errorMsg}`)
         process.exit(1)
     }
     try {
@@ -374,10 +382,11 @@ type wsPeerInfo = {
                     throw new Error(`unknown node <${id}>`)
                 let node: SpeechFlowNode
                 try {
-                    let num = nodeNums.get(nodes[id]) ?? 0
-                    nodeNums.set(nodes[id], ++num)
+                    const NodeClass = nodes[id]
+                    let num = nodeNums.get(NodeClass) ?? 0
+                    nodeNums.set(NodeClass, ++num)
                     const name = num === 1 ? id : `${id}:${num}`
-                    node = new nodes[id](name, cfg, opts, args)
+                    node = new NodeClass(name, cfg, opts, args)
                 }
                 catch (err) {
                     /*  fatal error  */
@@ -400,12 +409,9 @@ type wsPeerInfo = {
         })
     }
     catch (err) {
-        if (err instanceof Error && err.name === "FlowLinkError")
-            cli!.log("error", `failed to materialize SpeechFlow configuration: ${err.toString()}`)
-        else if (err instanceof Error)
-            cli!.log("error", `failed to materialize SpeechFlow configuration: ${err.message}`)
-        else
-            cli!.log("error", "failed to materialize SpeechFlow configuration: internal error")
+        const errorMsg = err instanceof Error && err.name === "FlowLinkError"
+            ? err.toString() : (err instanceof Error ? err.message : "internal error")
+        cli!.log("error", `failed to materialize SpeechFlow configuration: ${errorMsg}`)
         process.exit(1)
     }
 
@@ -456,7 +462,7 @@ type wsPeerInfo = {
         node.setTimeZero(timeZero)
         await Promise.race<void>([
             node.open(),
-            new Promise((resolve, reject) => setTimeout(() =>
+            new Promise<never>((resolve, reject) => setTimeout(() =>
                 reject(new Error("timeout")), 10 * 1000))
         ]).catch((err: Error) => {
             cli!.log("error", `[${node.id}]: failed to open node <${node.id}>: ${err.message}`)
@@ -535,7 +541,7 @@ type wsPeerInfo = {
         else {
             await Promise.race<void>([
                 foundNode.receiveRequest(args),
-                new Promise((resolve, reject) => setTimeout(() =>
+                new Promise<never>((resolve, reject) => setTimeout(() =>
                     reject(new Error("timeout")), 10 * 1000))
             ]).catch((err: Error) => {
                 cli!.log("warning", `external request to node <${name}> failed: ${err.message}`)
@@ -549,6 +555,7 @@ type wsPeerInfo = {
         address: args.a,
         port:    args.p
     })
+    await hapi.register({ plugin: Inert })
     await hapi.register({ plugin: HAPIHeader, options: { Server: `${pkg.name}/${pkg.version}` } })
     await hapi.register({ plugin: HAPIWebSocket })
     hapi.events.on("response", (request: HAPI.Request) => {
@@ -582,6 +589,29 @@ type wsPeerInfo = {
                 cli!.log("error", `HAPI: log: ${err}`)
         }
     })
+    hapi.route({
+        method: "GET",
+        path: "/{param*}",
+        handler: {
+            directory: {
+                path: path.join(__dirname, "../../speechflow-ui-db/dst"),
+                redirectToSlash: true,
+                index: true
+            }
+        }
+    })
+    hapi.route({
+        method: "GET",
+        path:   "/api/dashboard",
+        handler: (request: HAPI.Request, h: HAPI.ResponseToolkit) => {
+            const config = []
+            for (const block of args.d.split(",")) {
+                const [ type, id, name ] = block.split(":")
+                config.push({ type, id, name })
+            }
+            return h.response(config).code(200)
+        }
+    })
     hapi.route({
         method: "GET",
         path:   "/api/{req}/{node}/{params*}",
@@ -589,10 +619,13 @@ type wsPeerInfo = {
         },
         handler: (request: HAPI.Request, h: HAPI.ResponseToolkit) => {
             const peer = request.info.remoteAddress
+            const params = request.params.params as string ?? ""
+            if (params.length > 1000)
+                return h.response({ response: "ERROR", data: "parameters too long" }).code(400)
             const req = {
                 request: request.params.req,
                 node:    request.params.node,
-                args:    (request.params.params as string ?? "").split("/").filter((seg) => seg !== "")
+                args:    params.split("/").filter((seg) => seg !== "")
             }
             cli!.log("info", `HAPI: peer ${peer}: GET: ${JSON.stringify(req)}`)
             return consumeExternalRequest(req).then(() => {
@@ -607,9 +640,10 @@ type wsPeerInfo = {
         path:   "/api",
         options: {
             payload: {
-                output: "data",
-                parse:  true,
-                allow:  "application/json"
+                output:   "data",
+                parse:    true,
+                allow:    "application/json",
+                maxBytes: 1 * 1024 * 1024
             },
             plugins: {
                 websocket: {
@@ -625,8 +659,10 @@ type wsPeerInfo = {
                     },
                     disconnect: (args: any) => {
                         const ctx: wsPeerCtx = args.ctx
+                        const ws: WebSocket = args.ws
                         const peer = ctx.peer
                         wsPeers.delete(peer)
+                        ws.removeAllListeners()
                         cli!.log("info", `HAPI: WebSocket: disconnect: peer ${peer}`)
                     }
                 }
@@ -654,13 +690,33 @@ type wsPeerInfo = {
         node.on("send-response", (args: any[]) => {
             const data = JSON.stringify({ response: "NOTIFY", node: node.id, args })
             for (const [ peer, info ] of wsPeers.entries()) {
-                cli!.log("info", `HAPI: peer ${peer}: ${data}`)
+                cli!.log("debug", `HAPI: remote peer ${peer}: sending ${data}`)
                 if (info.ws.readyState === WebSocket.OPEN)
                     info.ws.send(data)
             }
         })
     }
 
+    /*  hook for dashboardInfo method of nodes  */
+    for (const node of graphNodes) {
+        node.on("dashboard-info", (info: {
+            type: string,
+            id:   string,
+            kind: "final" | "intermediate",
+            value: string | number
+        }) => {
+            const data = JSON.stringify({
+                response: "DASHBOARD",
+                node:     "",
+                args:     [ info.type, info.id, info.kind, info.value ]
+            })
+            for (const [ peer, info ] of wsPeers.entries()) {
+                cli!.log("debug", `HAPI: dashboard peer ${peer}: send ${data}`)
+                info.ws.send(data)
+            }
+        })
+    }
+
     /*  start of internal stream processing  */
     cli!.log("info", "**** everything established -- stream processing in SpeechFlow graph starts ****")
 
@@ -742,7 +798,7 @@ type wsPeerInfo = {
             cli!.log("info", `close node <${node.id}>`)
             await Promise.race<void>([
                 node.close(),
-                new Promise((resolve, reject) => setTimeout(() =>
+                new Promise<never>((resolve, reject) => setTimeout(() =>
                     reject(new Error("timeout")), 10 * 1000))
             ]).catch((err: Error) => {
                 cli!.log("warning", `node <${node.id}> failed to close: ${err.message}`)