speechflow 1.7.0 → 2.0.0

package/speechflow-cli/src/speechflow-node-xio-exec.ts

@@ -0,0 +1,210 @@
+/*
+**  SpeechFlow - Speech Processing Flow Graph
+**  Copyright (c) 2024-2025 Dr. Ralf S. Engelschall <rse@engelschall.com>
+**  Licensed under GPL 3.0 <https://spdx.org/licenses/GPL-3.0-only>
+*/
+
+/*  standard dependencies  */
+import Stream                     from "node:stream"
+
+/*  external dependencies  */
+import { execa, type Subprocess, type Options } from "execa"
+import shellParser                from "shell-parser"
+
+/*  internal dependencies  */
+import SpeechFlowNode             from "./speechflow-node"
+import * as util                  from "./speechflow-util"
+
+/*  SpeechFlow node for external command execution  */
+export default class SpeechFlowNodeXIOExec extends SpeechFlowNode {
+    /*  declare official node name  */
+    public static name = "xio-exec"
+
+    /*  internal state  */
+    private subprocess: Subprocess | null = null
+
+    /*  construct node  */
+    constructor (id: string, cfg: { [ id: string ]: any }, opts: { [ id: string ]: any }, args: any[]) {
+        super(id, cfg, opts, args)
+
+        /*  declare node configuration parameters  */
+        this.configure({
+            command:    { type: "string", pos: 0, val: "" },
+            mode:       { type: "string", pos: 1, val: "r",     match: /^(?:r|w|rw)$/ },
+            type:       { type: "string", pos: 2, val: "audio", match: /^(?:audio|text)$/ },
+            chunkAudio: { type: "number",         val: 200,     match: (n: number) => n >= 10 && n <= 1000 },
+            chunkText:  { type: "number",         val: 65536,   match: (n: number) => n >= 1024 && n <= 131072 }
+        })
+
+        /*  sanity check parameters  */
+        if (this.params.command === "")
+            throw new Error("required parameter \"command\" has to be given")
+
+        /*  declare node input/output format  */
+        if (this.params.mode === "rw") {
+            this.input  = this.params.type
+            this.output = this.params.type
+        }
+        else if (this.params.mode === "r") {
+            this.input  = "none"
+            this.output = this.params.type
+        }
+        else if (this.params.mode === "w") {
+            this.input  = this.params.type
+            this.output = "none"
+        }
+    }
+
+    /*  open node  */
+    async open () {
+        /*  determine how many bytes we need per chunk when
+            the chunk should be of the required duration/size  */
+        const highWaterMarkAudio = (
+            this.config.audioSampleRate *
+            (this.config.audioBitDepth / 8)
+        ) / (1000 / this.params.chunkAudio)
+        const highWaterMarkText = this.params.chunkText
+
+        /*  parse command into executable and arguments
+            (SECURITY: caller must ensure command parameter is properly validated
+            and does not contain untrusted user input to prevent command injection)  */
+        const cmdParts = shellParser(this.params.command)
+        if (cmdParts.length === 0)
+            throw new Error("failed to parse command: no executable found")
+
+        /*  warn about potentially dangerous shell metacharacters  */
+        if (/[;&|`$()<>]/.test(this.params.command))
+            this.log("warning", "command contains shell metacharacters -- ensure input is trusted")
+        const executable = cmdParts[0]
+        const args = cmdParts.slice(1)
+
+        /*  determine subprocess options  */
+        const encoding = (this.params.type === "text" ?
+            this.config.textEncoding : "buffer") as Options["encoding"]
+
+        /*  spawn subprocess  */
+        this.log("info", `executing command: ${this.params.command}`)
+        this.subprocess = execa(executable, args, {
+            buffer: false,
+            encoding,
+            ...(this.params.mode === "rw" ? { stdin: "pipe",   stdout: "pipe"   } : {}),
+            ...(this.params.mode === "r"  ? { stdin: "ignore", stdout: "pipe"   } : {}),
+            ...(this.params.mode === "w"  ? { stdin: "pipe",   stdout: "ignore" } : {})
+        })
+
+        /*  handle subprocess errors  */
+        this.subprocess.on("error", (err) => {
+            this.log("error", `subprocess error: ${err.message}`)
+            this.emit("error", err)
+            if (this.stream !== null)
+                this.stream.emit("error", err)
+        })
+
+        /*  handle subprocess exit  */
+        this.subprocess.on("exit", (code, signal) => {
+            if (code !== 0 && code !== null)
+                this.log("warning", `subprocess exited with code ${code}`)
+            else if (signal)
+                this.log("warning", `subprocess terminated by signal ${signal}`)
+            else
+                this.log("info", "subprocess terminated gracefully")
+        })
+
+        /*  determine high water mark based on type  */
+        const highWaterMark = this.params.type === "audio" ? highWaterMarkAudio : highWaterMarkText
+
+        /*  configure stream encoding  */
+        if (this.subprocess.stdout && this.params.type === "text")
+            this.subprocess.stdout.setEncoding(this.config.textEncoding)
+        if (this.subprocess.stdin)
+            this.subprocess.stdin.setDefaultEncoding(this.params.type === "text" ?
+                this.config.textEncoding : "binary")
+
+        /*  dispatch according to mode  */
+        if (this.params.mode === "rw") {
+            /*  bidirectional mode: both stdin and stdout  */
+            this.stream = Stream.Duplex.from({
+                readable: this.subprocess.stdout,
+                writable: this.subprocess.stdin
+            })
+            const wrapper1 = util.createTransformStreamForWritableSide(this.params.type, highWaterMark)
+            const wrapper2 = util.createTransformStreamForReadableSide(
+                this.params.type, () => this.timeZero, highWaterMark)
+            this.stream = Stream.compose(wrapper1, this.stream, wrapper2)
+        }
+        else if (this.params.mode === "r") {
+            /*  read-only mode: stdout only  */
+            const wrapper = util.createTransformStreamForReadableSide(
+                this.params.type, () => this.timeZero, highWaterMark)
+            this.stream = Stream.compose(this.subprocess.stdout!, wrapper)
+        }
+        else if (this.params.mode === "w") {
+            /*  write-only mode: stdin only  */
+            const wrapper = util.createTransformStreamForWritableSide(
+                this.params.type, highWaterMark)
+            this.stream = Stream.compose(wrapper, this.subprocess.stdin!)
+        }
+    }
+
+    /*  close node  */
+    async close () {
+        /*  terminate subprocess  */
+        if (this.subprocess !== null) {
+            /*  gracefully end stdin if in write or read/write mode  */
+            if ((this.params.mode === "w" || this.params.mode === "rw") && this.subprocess.stdin &&
+                !this.subprocess.stdin.destroyed && !this.subprocess.stdin.writableEnded) {
+                await Promise.race([
+                    new Promise<void>((resolve, reject) => {
+                        this.subprocess!.stdin!.end((err?: Error) => {
+                            if (err) reject(err)
+                            else     resolve()
+                        })
+                    }),
+                    util.timeout(2000)
+                ]).catch((err: unknown) => {
+                    const error = util.ensureError(err)
+                    this.log("warning", `failed to gracefully close stdin: ${error.message}`)
+                })
+            }
+
+            /*  wait for subprocess to exit gracefully  */
+            await Promise.race([
+                this.subprocess,
+                util.timeout(5000, "subprocess exit timeout")
+            ]).catch(async (err: unknown) => {
+                /*  force kill with SIGTERM  */
+                const error = util.ensureError(err)
+                if (error.message.includes("timeout")) {
+                    this.log("warning", "subprocess did not exit gracefully, forcing termination")
+                    this.subprocess!.kill("SIGTERM")
+                    return Promise.race([
+                        this.subprocess,
+                        util.timeout(2000)
+                    ])
+                }
+            }).catch(async () => {
+                /*  force kill with SIGKILL  */
+                this.log("warning", "subprocess did not respond to SIGTERM, forcing SIGKILL")
+                this.subprocess!.kill("SIGKILL")
+                return Promise.race([
+                    this.subprocess,
+                    util.timeout(1000)
+                ])
+            }).catch(() => {
+                this.log("error", "subprocess did not terminate even after SIGKILL")
+            })
+
+            /*  remove event listeners to prevent memory leaks  */
+            this.subprocess.removeAllListeners("error")
+            this.subprocess.removeAllListeners("exit")
+
+            this.subprocess = null
+        }
+
+        /*  shutdown stream  */
+        if (this.stream !== null) {
+            await util.destroyStream(this.stream)
+            this.stream = null
+        }
+    }
+}

package/speechflow-cli/src/speechflow-node-xio-file.ts

@@ -9,37 +9,39 @@ import fs               from "node:fs"
 import Stream           from "node:stream"
 
 /*  internal dependencies  */
-import SpeechFlowNode   from "./speechflow-node"
-import * as util        from "./speechflow-util"
+import SpeechFlowNode, { SpeechFlowChunk } from "./speechflow-node"
+import * as util                           from "./speechflow-util"
 
 /*  SpeechFlow node for file access  */
 export default class SpeechFlowNodeXIOFile extends SpeechFlowNode {
     /*  declare official node name  */
     public static name = "xio-file"
 
+    /*  file descriptor for seekable write mode  */
+    private fd: number | null = null
+
     /*  construct node  */
     constructor (id: string, cfg: { [ id: string ]: any }, opts: { [ id: string ]: any }, args: any[]) {
         super(id, cfg, opts, args)
 
         /*  declare node configuration parameters  */
         this.configure({
-            path:   { type: "string", pos: 0, val: "" },
-            mode:   { type: "string", pos: 1, val: "r",     match: /^(?:r|w|rw)$/ },
-            type:   { type: "string", pos: 2, val: "audio", match: /^(?:audio|text)$/ },
-            chunka: { type: "number",         val: 200,     match: (n: number) => n >= 10 && n <= 1000 },
-            chunkt: { type: "number",         val: 65536,   match: (n: number) => n >= 1024 && n <= 131072 }
+            path:       { type: "string",  pos: 0, val: "" },
+            mode:       { type: "string",  pos: 1, val: "r",     match: /^(?:r|w)$/ },
+            type:       { type: "string",  pos: 2, val: "audio", match: /^(?:audio|text)$/ },
+            seekable:   { type: "boolean",         val: false },
+            chunkAudio: { type: "number",          val: 200,     match: (n: number) => n >= 10 && n <= 1000 },
+            chunkText:  { type: "number",          val: 65536,   match: (n: number) => n >= 1024 && n <= 131072 }
         })
 
         /*  sanity check parameters  */
         if (this.params.path === "")
             throw new Error("required parameter \"path\" has to be given")
+        if (this.params.seekable && this.params.path === "-")
+            throw new Error("parameter \"seekable\" cannot be used with standard I/O")
 
         /*  declare node input/output format  */
-        if (this.params.mode === "rw") {
-            this.input  = this.params.type
-            this.output = this.params.type
-        }
-        else if (this.params.mode === "r") {
+        if (this.params.mode === "r") {
             this.input  = "none"
             this.output = this.params.type
         }
@@ -56,8 +58,8 @@ export default class SpeechFlowNodeXIOFile extends SpeechFlowNode {
         const highWaterMarkAudio = (
             this.config.audioSampleRate *
             (this.config.audioBitDepth / 8)
-        ) / (1000 / this.params.chunka)
-        const highWaterMarkText = this.params.chunkt
+        ) / (1000 / this.params.chunkAudio)
+        const highWaterMarkText = this.params.chunkText
 
         /*  utility function: create a writable stream as chunker that
             writes to process.stdout but properly handles finish events.
@@ -81,59 +83,7 @@ export default class SpeechFlowNodeXIOFile extends SpeechFlowNode {
             })
 
         /*  dispatch according to mode and path  */
-        if (this.params.mode === "rw") {
-            if (this.params.path === "-") {
-                /*  standard I/O  */
-                if (this.params.type === "audio") {
-                    process.stdin.setEncoding()
-                    process.stdout.setEncoding()
-                    const streamR = new Stream.PassThrough({ highWaterMark: highWaterMarkAudio })
-                    process.stdin.pipe(streamR)
-                    const streamW = new Stream.PassThrough({ highWaterMark: highWaterMarkAudio })
-                    streamW.pipe(process.stdout)
-                    this.stream = Stream.Duplex.from({ readable: streamR, writable: streamW })
-                }
-                else {
-                    process.stdin.setEncoding(this.config.textEncoding)
-                    process.stdout.setEncoding(this.config.textEncoding)
-                    const streamR = new Stream.PassThrough({ highWaterMark: highWaterMarkText })
-                    process.stdin.pipe(streamR)
-                    const streamW = new Stream.PassThrough({ highWaterMark: highWaterMarkText })
-                    streamW.pipe(process.stdout)
-                    this.stream = Stream.Duplex.from({ readable: streamR, writable: streamW })
-                }
-            }
-            else {
-                /*  file I/O  */
-                if (this.params.type === "audio") {
-                    this.stream = Stream.Duplex.from({
-                        readable: fs.createReadStream(this.params.path,
-                            { highWaterMark: highWaterMarkAudio }),
-                        writable: fs.createWriteStream(this.params.path,
-                            { highWaterMark: highWaterMarkAudio })
-                    })
-                }
-                else {
-                    this.stream = Stream.Duplex.from({
-                        readable: fs.createReadStream(this.params.path, {
-                            highWaterMark: highWaterMarkText,
-                            encoding: this.config.textEncoding
-                        }),
-                        writable: fs.createWriteStream(this.params.path, {
-                            highWaterMark: highWaterMarkText,
-                            encoding: this.config.textEncoding
-                        })
-                    })
-                }
-            }
-
-            /*  convert regular stream into object-mode stream  */
-            const wrapper1 = util.createTransformStreamForWritableSide(this.params.type, 1)
-            const wrapper2 = util.createTransformStreamForReadableSide(
-                this.params.type, () => this.timeZero)
-            this.stream = Stream.compose(wrapper1, this.stream, wrapper2)
-        }
-        else if (this.params.mode === "r") {
+        if (this.params.mode === "r") {
             if (this.params.path === "-") {
                 /*  standard I/O  */
                 let chunker: Stream.PassThrough
@@ -176,15 +126,63 @@ export default class SpeechFlowNodeXIOFile extends SpeechFlowNode {
             }
             else {
                 /*  file I/O  */
-                let writable: Stream.Writable
-                if (this.params.type === "audio")
-                    writable = fs.createWriteStream(this.params.path,
-                        { highWaterMark: highWaterMarkAudio })
-                else
-                    writable = fs.createWriteStream(this.params.path,
-                        { highWaterMark: highWaterMarkText, encoding: this.config.textEncoding })
-                const wrapper = util.createTransformStreamForWritableSide(this.params.type, 1)
-                this.stream = Stream.compose(wrapper, writable)
+                if (this.params.seekable) {
+                    /*  seekable file I/O with file descriptor  */
+                    this.fd = fs.openSync(this.params.path, "w")
+                    let writePosition = 0
+                    const self = this
+                    const writable = new Stream.Writable({
+                        objectMode:    true,
+                        decodeStrings: false,
+                        highWaterMark: 1,
+                        write (chunk: SpeechFlowChunk, encoding, callback) {
+                            const payload = Buffer.isBuffer(chunk.payload) ?
+                                chunk.payload : Buffer.from(chunk.payload)
+                            const seekPosition = chunk.meta.get("chunk:seek") as number | undefined
+                            if (seekPosition !== undefined) {
+                                /*  seek to specified position and write (overload)  */
+                                fs.write(self.fd!, payload, 0, payload.byteLength, seekPosition, callback)
+                            }
+                            else {
+                                /*  append at current position  */
+                                fs.write(self.fd!, payload, 0, payload.byteLength, writePosition, (err) => {
+                                    if (err)
+                                        callback(err)
+                                    else {
+                                        writePosition += payload.byteLength
+                                        callback()
+                                    }
+                                })
+                            }
+                        },
+                        final (callback) {
+                            callback()
+                        },
+                        destroy (err, callback) {
+                            if (self.fd !== null) {
+                                fs.close(self.fd, () => {
+                                    self.fd = null
+                                    callback(err)
+                                })
+                            }
+                            else
+                                callback(err)
+                        }
+                    })
+                    this.stream = writable
+                }
+                else {
+                    /*  non-seekable file I/O with stream  */
+                    let writable: Stream.Writable
+                    if (this.params.type === "audio")
+                        writable = fs.createWriteStream(this.params.path,
+                            { highWaterMark: highWaterMarkAudio })
+                    else
+                        writable = fs.createWriteStream(this.params.path,
+                            { highWaterMark: highWaterMarkText, encoding: this.config.textEncoding })
+                    const wrapper = util.createTransformStreamForWritableSide(this.params.type, 1)
+                    this.stream = Stream.compose(wrapper, writable)
+                }
             }
         }
         else
@@ -202,20 +200,35 @@ export default class SpeechFlowNodeXIOFile extends SpeechFlowNode {
                 /*  for stdio streams, just end without destroying  */
                 const stream = this.stream
                 if ((stream instanceof Stream.Writable || stream instanceof Stream.Duplex) &&
-                    (!stream.writableEnded && !stream.destroyed)                             ) {
+                    (!stream.writableEnded && !stream.destroyed)) {
                     await Promise.race([
                         new Promise<void>((resolve, reject) => {
                             stream.end((err?: Error) => {
-                                if (err) reject(err)
-                                else     resolve()
+                                if (err)
+                                    reject(err)
+                                else
+                                    resolve()
                             })
                         }),
-                        util.timeoutPromise(5000)
+                        util.timeout(5000)
                     ])
                 }
             }
             this.stream = null
         }
+
+        /*  ensure file descriptor is closed  */
+        if (this.fd !== null) {
+            await new Promise<void>((resolve, reject) => {
+                fs.close(this.fd!, (err) => {
+                    this.fd = null
+                    if (err)
+                        reject(err)
+                    else
+                        resolve()
+                })
+            })
+        }
     }
 }
 

package/speechflow-cli/src/speechflow-node-xio-mqtt.ts

@@ -97,7 +97,8 @@ export default class SpeechFlowNodeXIOMQTT extends SpeechFlowNode {
             this.log("info", `connection re-opened to MQTT ${this.params.url}`)
         })
         this.broker.on("disconnect", (packet: MQTT.IDisconnectPacket) => {
-            this.log("info", `connection closed to MQTT ${this.params.url}`)
+            const reasonCode = packet.reasonCode ?? 0
+            this.log("info", `connection closed to MQTT ${this.params.url} (reason code: ${reasonCode})`)
         })
         this.chunkQueue = new util.SingleQueue<SpeechFlowChunk>()
         this.broker.on("message", (topic: string, payload: Buffer, packet: MQTT.IPublishPacket) => {
@@ -107,7 +108,7 @@ export default class SpeechFlowNodeXIOMQTT extends SpeechFlowNode {
                 const chunk = util.streamChunkDecode(payload)
                 this.chunkQueue!.write(chunk)
             }
-            catch (_err: any) {
+            catch (_err: unknown) {
                 this.log("warning", `received invalid CBOR chunk from MQTT ${this.params.url}`)
             }
         })