speechflow 0.9.4 → 0.9.7

package/package.json

@@ -1,8 +1,8 @@
 {
     "name":                                 "speechflow",
-    "version":                              "0.9.4",
-    "x-stdver":                             "0.9.4-EA",
-    "x-release":                            "2025-04-27",
+    "version":                              "0.9.7",
+    "x-stdver":                             "0.9.7-EA",
+    "x-release":                            "2025-07-12",
     "homepage":                             "https://github.com/rse/speechflow",
     "description":                          "Speech Processing Flow Graph",
     "license":                              "GPL-3.0-only",
@@ -17,58 +17,76 @@
     },
     "dependencies": {
         "cli-io":                           "0.9.13",
-        "yargs":                            "17.7.2",
-        "flowlink":                         "0.9.9",
+        "yargs":                            "18.0.0",
+        "flowlink":                         "0.9.11",
         "js-yaml":                          "4.1.0",
-        "@gpeng/naudiodon":                 "2.4.0",
-        "@deepgram/sdk":                    "3.12.1",
-        "deepl-node":                       "1.17.3",
-        "elevenlabs":                       "1.57.0",
-        "stream-transform":                 "3.3.3",
+        "@gpeng/naudiodon":                 "2.4.1",
+        "@deepgram/sdk":                    "4.9.1",
+        "deepl-node":                       "1.19.0",
+        "@elevenlabs/elevenlabs-js":        "2.6.0",
+        "stream-transform":                 "3.4.0",
         "get-stream":                       "9.0.1",
-        "@dotenvx/dotenvx":                 "1.41.0",
+        "@dotenvx/dotenvx":                 "1.47.5",
         "speex-resampler":                  "3.0.1",
         "pcm-convert":                      "1.6.5",
         "object-path":                      "0.11.8",
-        "ws":                               "8.18.1",
+        "ws":                               "8.18.3",
         "bufferutil":                       "4.0.9",
         "utf-8-validate":                   "6.0.5",
         "@opensumi/reconnecting-websocket": "4.4.0",
-        "ollama":                           "0.5.15",
+        "ollama":                           "0.5.16",
         "@rse/ffmpeg":                      "1.4.2",
-        "ffmpeg-stream":                    "1.0.0",
-        "installed-packages":               "1.0.13"
+        "ffmpeg-stream":                    "1.0.1",
+        "installed-packages":               "1.0.13",
+        "syspath":                          "1.0.8",
+        "wav":                              "1.0.2",
+        "mqtt":                             "5.13.2",
+        "cbor2":                            "2.0.1",
+        "pure-uuid":                        "1.8.1",
+        "wavefile":                         "11.0.0",
+        "@huggingface/transformers":        "3.6.3",
+        "@ericedouard/vad-node-realtime":   "0.2.0",
+        "luxon":                            "3.7.1",
+        "wrap-text":                        "1.0.10",
+        "smart-whisper":                    "0.8.1"
     },
     "devDependencies": {
-        "eslint":                           "9.25.1",
-        "@eslint/js":                       "9.25.1",
-        "neostandard":                      "0.12.1",
+        "eslint":                           "9.31.0",
+        "@eslint/js":                       "9.31.0",
+        "neostandard":                      "0.12.2",
         "eslint-plugin-promise":            "7.2.1",
-        "eslint-plugin-import":             "2.31.0",
+        "eslint-plugin-import":             "2.32.0",
         "eslint-plugin-node":               "11.1.0",
-        "@typescript-eslint/eslint-plugin": "8.31.0",
-        "@typescript-eslint/parser":        "8.31.0",
-        "oxlint":                           "0.16.8",
-        "eslint-plugin-oxlint":             "0.16.8",
-        "@biomejs/biome":                   "1.9.4",
+        "@typescript-eslint/eslint-plugin": "8.36.0",
+        "@typescript-eslint/parser":        "8.36.0",
+        "oxlint":                           "1.6.0",
+        "eslint-plugin-oxlint":             "1.6.0",
+        "@biomejs/biome":                   "2.0.6",
         "eslint-config-biome":              "1.9.4",
 
-        "@types/node":                      "22.15.2",
+        "@types/node":                      "24.0.13",
         "@types/yargs":                     "17.0.33",
         "@types/js-yaml":                   "4.0.9",
         "@types/object-path":               "0.11.4",
         "@types/ws":                        "8.18.1",
         "@types/resolve":                   "1.20.6",
+        "@types/wav":                       "1.0.4",
+        "@types/luxon":                     "3.6.2",
+        "@types/wrap-text":                 "1.0.2",
 
-        "stmux":                            "1.8.10",
+        "patch-package":                    "8.0.0",
+        "stmux":                            "1.8.11",
         "nodemon":                          "3.1.10",
-        "rimraf":                           "6.0.1",
+        "shx":                              "0.4.0",
         "typescript":                       "5.8.3",
         "delay-cli":                        "2.0.0",
-        "nps":                              "5.10.0",
+        "@rse/stx":                         "1.0.2",
         "cross-env":                        "7.0.3"
     },
-    "upd":                                  [],
+    "overrides": {
+        "onnxruntime-node":                 "1.22.0-dev.20250418-c19a49615b"
+    },
+    "upd":                                  [ "!@biomejs/biome" ],
     "engines": {
         "node":                             ">=22.0.0"
     },
@@ -83,7 +101,7 @@
         }
     },
     "scripts": {
-        "start":                            "nps -c etc/nps.yaml",
-        "speechflow":                       "node dst/speechflow.js"
+        "postinstall":                      "npm start patch-apply",
+        "start":                            "stx -v4 -c etc/stx.conf"
     }
 }

package/sample.yaml

@@ -2,32 +2,70 @@
 ##  sample.yaml -- Speechflow Sample Audio Processing Graphs
 ##
 
-#  capture audio from microphone to file
-capture-microphone: |
-     device(device: "wasapi:VoiceMeeter Output", mode: "r") |
-     file(path: "capture.pcm", mode: "w", type: "audio")
+#   Capture audio from microphone device into WAV audio file
+capturing: |
+    device(device: "wasapi:VoiceMeeter Out B1", mode: "r") |
+        wav(mode: "encode") |
+            file(path: "capture.wav", mode: "w", type: "audio")
 
-#  generate audio file with narration of text file
-generate-narration: |
-     file(path: argv.0, mode: "r", type: "audio") |
-     deepgram(key: env.SPEECHFLOW_KEY_DEEPGRAM)   |
-     file(path: argv.1, mode: "w", type: "text")
+#   Pass-through audio from microphone device to speaker
+#   device and in parallel record it to WAV audio file
+pass-through: |
+    device(device: "wasapi:VoiceMeeter Out B1", mode: "r") | {
+        wav(mode: "encode") |
+            file(path: "capture.wav", mode: "w", type: "audio"),
+        device(device: "wasapi:VoiceMeeter VAIO3 Input", mode: "w")
+    }
 
-#  pass-through audio from microphone to speaker and in parallel record it to file
-microphone-to-speaker: |
-     device(device: "wasapi:VoiceMeeter Output", mode: "r") | {
-         file(path: "capture.pcm", mode: "w", type: "audio"),
-         device(device: "wasapi:VoiceMeeter VAIO3 Input", mode: "w")
-     }
+#   Generate text file with German narration of MP3 audio file
+narration: |
+    file(path: argv.0, mode: "r", type: "audio") |
+        ffmpeg(src: "mp3", dst: "pcm") |
+            deepgram(language: "de", key: env.SPEECHFLOW_KEY_DEEPGRAM) |
+                format(width: 80) |
+                    file(path: argv.1, mode: "w", type: "text")
 
-#  translate stdin to stdout
+#  Generate text file with German subtitles of MP3 audio file
+subtitling: |
+    file(path: argv.0, mode: "r", type: "audio") |
+        ffmpeg(src: "mp3", dst: "pcm") |
+            deepgram(language: "de", key: env.SPEECHFLOW_KEY_DEEPGRAM) |
+                subtitle(format: "vtt") |
+                    file(path: argv.1, mode: "w", type: "text")
+
+#  Ad-Hoc text translation from German to English
 translation: |
-     file(path: "-", mode: "r", type: "text") |
-     deepl(key: env.SPEECHFLOW_KEY_DEEPL, src: "de", dst: "en-US") |
-     file(path: "-", mode: "w", type: "text")
+    file(path: "-", mode: "r", type: "text") |
+        deepl(src: "de", dst: "en") |
+            file(path: "-", mode: "w", type: "text")
 
-#  sample for development
-sample: |
-     device(device: "coreaudio:Elgato Wave:3", mode: "r") |
-     file(path: "capture.pcm", mode: "w", type: "audio")
+#  Real-time studio translation from German to English,
+#  including the capturing of all involved inputs and outputs:
+studio: |
+    device(device: "coreaudio:Elgato Wave:3", mode: "r") | {
+        wav(mode: "encode") |
+            file(path: "program-de.wav", mode: "w", type: "audio"),
+        deepgram(key: env.SPEECHFLOW_KEY_DEEPGRAM, language: "de") | {
+            format(width: 80) |
+                file(path: "program-de.txt", mode: "w", type: "text"),
+            deepl(key: env.SPEECHFLOW_KEY_DEEPL, src: "de", dst: "en") | {
+                format(width: 80) |
+                    file(path: "program-en.txt", mode: "w", type: "text"),
+                subtitle(format: "vtt") | {
+                    file(path: "program-en.vtt", mode: "w", type: "text"),
+                    mqtt(url: "mqtt://10.1.0.10:1883",
+                        username: env.SPEECHFLOW_MQTT_USER,
+                        password: env.SPEECHFLOW_MQTT_PASS,
+                        topicWrite: "stream/studio/sender")
+                },
+                subtitle(format: "srt") |
+                    file(path: "program-en.srt", mode: "w", type: "text"),
+                elevenlabs(voice: "Mark", speed: 1.05, language: "en") | {
+                    wav(mode: "encode") |
+                        file(path: "program-en.wav", mode: "w", type: "audio"),
+                    device(device: "coreaudio:USBAudio2.0", mode: "w")
+                }
+            }
+        }
+    }
 

package/src/lib.d.ts

@@ -15,6 +15,11 @@ declare module "pcm-convert" {
         data:        Buffer,
         srcFormat:   Format,
         dstFormat:   Format
-    ): Buffer
+    ): any
+}
+
+declare module "node:stream" {
+    import { Stream, Duplex } from "node:stream"
+    export function compose (...streams: Stream[]): Duplex
 }
 

package/src/{speechflow-node-ffmpeg.ts → speechflow-node-a2a-ffmpeg.ts}

@@ -13,6 +13,7 @@ import { Converter as FFmpegStream } from "ffmpeg-stream"
 
 /*  internal dependencies  */
 import SpeechFlowNode                from "./speechflow-node"
+import * as utils                    from "./speechflow-utils"
 
 /*  SpeechFlow node for FFmpeg  */
 export default class SpeechFlowNodeFFmpeg extends SpeechFlowNode {
@@ -24,8 +25,8 @@ export default class SpeechFlowNodeFFmpeg extends SpeechFlowNode {
     private ffmpeg: FFmpegStream | null = null
 
     /*  construct node  */
-    constructor (id: string, opts: { [ id: string ]: any }, args: any[]) {
-        super(id, opts, args)
+    constructor (id: string, cfg: { [ id: string ]: any }, opts: { [ id: string ]: any }, args: any[]) {
+        super(id, cfg, opts, args)
 
         /*  declare node configuration parameters  */
         this.configure({
@@ -93,9 +94,14 @@ export default class SpeechFlowNodeFFmpeg extends SpeechFlowNode {
 
         /*  establish a duplex stream and connect it to FFmpeg  */
         this.stream = Stream.Duplex.from({
-            readable: streamOutput,
-            writable: streamInput
+            writable: streamInput,
+            readable: streamOutput
         })
+
+        /*  wrap streams with conversions for chunk vs plain audio  */
+        const wrapper1 = utils.createTransformStreamForWritableSide()
+        const wrapper2 = utils.createTransformStreamForReadableSide("audio", () => this.timeZero)
+        this.stream = Stream.compose(wrapper1, this.stream, wrapper2)
     }
 
     /*  close node  */

package/src/speechflow-node-a2a-wav.ts

@@ -0,0 +1,143 @@
+/*
+**  SpeechFlow - Speech Processing Flow Graph
+**  Copyright (c) 2024-2025 Dr. Ralf S. Engelschall <rse@engelschall.com>
+**  Licensed under GPL 3.0 <https://spdx.org/licenses/GPL-3.0-only>
+*/
+
+/*  standard dependencies  */
+import Stream           from "node:stream"
+
+/*  external dependencies  */
+import wav              from "wav"
+
+/*  internal dependencies  */
+import SpeechFlowNode   from "./speechflow-node"
+import * as utils       from "./speechflow-utils"
+
+/*  utility class for wrapping a custom stream into a regular Transform stream  */
+class StreamWrapper extends Stream.Transform {
+    private foreignStream: any
+    constructor (foreignStream: any, options: Stream.TransformOptions = {}) {
+        options.readableObjectMode = true
+        options.writableObjectMode = true
+        super(options)
+        this.foreignStream = foreignStream
+        this.foreignStream.on("data", (chunk: any) => {
+            this.push(chunk)
+        })
+        this.foreignStream.on("error", (err: Error) => {
+            this.emit("error", err)
+        })
+        this.foreignStream.on("end", () => {
+            this.push(null)
+        })
+    }
+    _transform (chunk: any, encoding: BufferEncoding, callback: Stream.TransformCallback): void {
+        try {
+            const canContinue = this.foreignStream.write(chunk)
+            if (canContinue)
+                callback()
+            else
+                this.foreignStream.once("drain", callback)
+        }
+        catch (err) {
+            callback(err as Error)
+        }
+    }
+    _flush (callback: Stream.TransformCallback): void {
+        try {
+            if (typeof this.foreignStream.end === "function")
+                this.foreignStream.end()
+            callback()
+        }
+        catch (err) {
+            callback(err as Error)
+        }
+    }
+}
+
+/*  SpeechFlow node for WAV format conversion  */
+export default class SpeechFlowNodeWAV extends SpeechFlowNode {
+    /*  declare official node name  */
+    public static name = "wav"
+
+    /*  construct node  */
+    constructor (id: string, cfg: { [ id: string ]: any }, opts: { [ id: string ]: any }, args: any[]) {
+        super(id, cfg, opts, args)
+
+        /*  declare node configuration parameters  */
+        this.configure({
+            mode: { type: "string", pos: 1, val: "encode", match: /^(?:encode|decode)$/ }
+        })
+
+        /*  declare node input/output format  */
+        this.input  = "audio"
+        this.output = "audio"
+    }
+
+    /*  open node  */
+    async open () {
+        if (this.params.mode === "encode") {
+            /*  convert raw/PCM to WAV/PCM  */
+            /*  NOTICE: as this is a continuous stream, the resulting WAV header is not 100%
+                conforming to the WAV standard, as it has to use a zero duration information.
+                This cannot be changed in a stream-based processing.  */
+            const writer = new wav.Writer({
+                format:     0x0001 /* PCM */,
+                channels:   this.config.audioChannels,
+                sampleRate: this.config.audioSampleRate,
+                bitDepth:   this.config.audioBitDepth
+            })
+            this.stream = new StreamWrapper(writer)
+        }
+        else if (this.params.mode === "decode") {
+            /*  convert WAV/PCM to raw/PCM  */
+            const reader = new wav.Reader()
+            reader.on("format", (format: any) => {
+                this.log("info", `WAV audio stream: format=${format.audioFormat === 0x0001 ? "PCM" :
+                    "0x" + (format.audioFormat as number).toString(16).padStart(4, "0")} ` +
+                    `bitDepth=${format.bitDepth} ` +
+                    `signed=${format.signed ? "yes" : "no"} ` +
+                    `endian=${format.endianness} ` +
+                    `sampleRate=${format.sampleRate} ` +
+                    `channels=${format.channels}`)
+                if (format.audioFormat !== 0x0001 /* PCM */)
+                    throw new Error("WAV not based on PCM format")
+                if (format.bitDepth !== 16)
+                    throw new Error("WAV not based on 16 bit samples")
+                if (!format.signed)
+                    throw new Error("WAV not based on signed integers")
+                if (format.endianness !== "LE")
+                    throw new Error("WAV not based on little endianness")
+                if (format.sampleRate !== 48000)
+                    throw new Error("WAV not based on 48Khz sample rate")
+                if (format.channels !== 1)
+                    throw new Error("WAV not based on mono channel")
+            })
+            this.stream = new StreamWrapper(reader)
+        }
+        else
+            throw new Error(`invalid operation mode "${this.params.mode}"`)
+
+        /*  convert regular stream into object-mode stream  */
+        const wrapper1 = utils.createTransformStreamForWritableSide()
+        const wrapper2 = utils.createTransformStreamForReadableSide("audio", () => this.timeZero)
+        this.stream = Stream.compose(wrapper1, this.stream, wrapper2)
+    }
+
+    /*  close node  */
+    async close () {
+        /*  shutdown stream  */
+        if (this.stream !== null) {
+            await new Promise<void>((resolve) => {
+                if (this.stream instanceof Stream.Duplex)
+                    this.stream.end(() => { resolve() })
+                else
+                    resolve()
+            })
+            this.stream.destroy()
+            this.stream = null
+        }
+    }
+}
+

package/src/speechflow-node-a2t-deepgram.ts

@@ -0,0 +1,199 @@
+/*
+**  SpeechFlow - Speech Processing Flow Graph
+**  Copyright (c) 2024-2025 Dr. Ralf S. Engelschall <rse@engelschall.com>
+**  Licensed under GPL 3.0 <https://spdx.org/licenses/GPL-3.0-only>
+*/
+
+/*  standard dependencies  */
+import Stream           from "node:stream"
+
+/*  external dependencies  */
+import * as Deepgram          from "@deepgram/sdk"
+import { DateTime, Duration } from "luxon"
+
+/*  internal dependencies  */
+import SpeechFlowNode, { SpeechFlowChunk } from "./speechflow-node"
+import * as utils                          from "./speechflow-utils"
+
+/*  SpeechFlow node for Deepgram speech-to-text conversion  */
+export default class SpeechFlowNodeDeepgram extends SpeechFlowNode {
+    /*  declare official node name  */
+    public static name = "deepgram"
+
+    /*  internal state  */
+    private dg: Deepgram.LiveClient | null = null
+
+    /*  construct node  */
+    constructor (id: string, cfg: { [ id: string ]: any }, opts: { [ id: string ]: any }, args: any[]) {
+        super(id, cfg, opts, args)
+
+        /*  declare node configuration parameters  */
+        this.configure({
+            key:      { type: "string", val: process.env.SPEECHFLOW_KEY_DEEPGRAM },
+            model:    { type: "string", val: "nova-3", pos: 0 },
+            version:  { type: "string", val: "latest", pos: 1 },
+            language: { type: "string", val: "multi",  pos: 2 }
+        })
+
+        /*  declare node input/output format  */
+        this.input  = "audio"
+        this.output = "text"
+    }
+
+    /*  open node  */
+    async open () {
+        /*  sanity check situation  */
+        if (this.config.audioBitDepth !== 16 || !this.config.audioLittleEndian)
+            throw new Error("Deepgram node currently supports PCM-S16LE audio only")
+
+        /*  create queue for results  */
+        const queue = new utils.SingleQueue<SpeechFlowChunk>()
+
+        /*  connect to Deepgram API  */
+        const deepgram = Deepgram.createClient(this.params.key)
+        let language = "en"
+        if (this.params.model.match(/^nova-2/) && this.params.language !== "en")
+            language = this.params.language
+        else if (this.params.model.match(/^nova-3/) && this.params.language !== "en")
+            language = "multi"
+        this.dg = deepgram.listen.live({
+            mip_opt_out:      true,
+            model:            this.params.model,
+            version:          this.params.version,
+            language,
+            channels:         this.config.audioChannels,
+            sample_rate:      this.config.audioSampleRate,
+            encoding:         "linear16",
+            multichannel:     false,
+            endpointing:      10,
+            interim_results:  false,
+            smart_format:     true,
+            punctuate:        true,
+            filler_words:     true,
+            diarize:          true, /* still not used by us */
+            numerals:         true,
+            profanity_filter: false
+        })
+
+        /*  hook onto Deepgram API events  */
+        this.dg.on(Deepgram.LiveTranscriptionEvents.Transcript, async (data) => {
+            const text = (data.channel?.alternatives[0].transcript as string) ?? ""
+            if (text === "")
+                this.log("info", `Deepgram: empty/dummy text received (start: ${data.start}s, duration: ${data.duration}s)`)
+            else {
+                this.log("info", `Deepgram: text received (start: ${data.start}s, duration: ${data.duration}s): "${text}"`)
+                const start = Duration.fromMillis(data.start * 1000).plus(this.timeZeroOffset)
+                const end   = start.plus({ seconds: data.duration })
+                const chunk = new SpeechFlowChunk(start, end, "final", "text", text)
+                queue.write(chunk)
+            }
+        })
+        this.dg.on(Deepgram.LiveTranscriptionEvents.Metadata, (data) => {
+            this.log("info", "Deepgram: metadata received")
+        })
+        this.dg.on(Deepgram.LiveTranscriptionEvents.Close, () => {
+            this.log("info", "Deepgram: connection close")
+        })
+        this.dg.on(Deepgram.LiveTranscriptionEvents.Error, (error: Error) => {
+            this.log("error", `Deepgram: ${error.message}`)
+            this.emit("error")
+        })
+
+        /*  wait for Deepgram API to be available  */
+        await new Promise((resolve, reject) => {
+            let timer: ReturnType<typeof setTimeout> | null = setTimeout(() => {
+                if (timer !== null) {
+                    timer = null
+                    reject(new Error("Deepgram: timeout waiting for connection open"))
+                }
+            }, 3000)
+            this.dg!.once(Deepgram.LiveTranscriptionEvents.Open, () => {
+                this.log("info", "Deepgram: connection open")
+                if (timer !== null) {
+                    clearTimeout(timer)
+                    timer = null
+                }
+                resolve(true)
+            })
+        })
+
+        /*  remember opening time to receive time zero offset  */
+        this.timeOpen = DateTime.now()
+
+        /*  workaround Deepgram initialization problems  */
+        let initDone = false
+        let initTimeout: ReturnType<typeof setTimeout> | null = null
+        const initTimeoutStart = () => {
+            if (initDone)
+                return
+            setTimeout(async () => {
+                if (initTimeout === null)
+                    return
+                initTimeout = null
+                this.log("warning", "Deepgram: initialization timeout -- restarting service usage")
+                await this.close()
+                this.open()
+            }, 3000)
+        }
+        const initTimeoutStop = () => {
+            if (initDone)
+                return
+            initDone = true
+            if (initTimeout !== null) {
+                clearTimeout(initTimeout)
+                initTimeout = null
+            }
+        }
+
+        /*  provide Duplex stream and internally attach to Deepgram API  */
+        const dg = this.dg
+        const log = (level: string, msg: string) => {
+            this.log(level, msg)
+        }
+        const encoding = this.config.textEncoding
+        this.stream = new Stream.Duplex({
+            writableObjectMode: true,
+            readableObjectMode: true,
+            decodeStrings:      false,
+            write (chunk: SpeechFlowChunk, encoding, callback) {
+                if (chunk.type !== "audio")
+                    callback(new Error("expected audio input chunk"))
+                else if (!Buffer.isBuffer(chunk.payload))
+                    callback(new Error("expected Buffer input chunk"))
+                else {
+                    if (chunk.payload.byteLength > 0) {
+                        log("info", `Deepgram: send data (${chunk.payload.byteLength} bytes)`)
+                        initTimeoutStart()
+                        dg.send(chunk.payload) /* intentionally discard all time information  */
+                    }
+                    callback()
+                }
+            },
+            read (size) {
+                queue.read().then((chunk) => {
+                    log("info", `Deepgram: receive data (${chunk.payload.length} bytes)`)
+                    initTimeoutStop()
+                    this.push(chunk, encoding)
+                })
+            },
+            final (callback) {
+                dg.requestClose()
+                this.push(null)
+                callback()
+            }
+        })
+    }
+
+    /*  close node  */
+    async close () {
+        /*  close stream  */
+        if (this.stream !== null) {
+            this.stream.destroy()
+            this.stream = null
+        }
+
+        /*  shutdown Deepgram API  */
+        if (this.dg !== null)
+            this.dg.requestClose()
+    }
+}