speechflow 0.9.8 → 1.0.0

package/src/speechflow-node-t2t-transformers.ts

@@ -0,0 +1,249 @@
+/*
+**  SpeechFlow - Speech Processing Flow Graph
+**  Copyright (c) 2024-2025 Dr. Ralf S. Engelschall <rse@engelschall.com>
+**  Licensed under GPL 3.0 <https://spdx.org/licenses/GPL-3.0-only>
+*/
+
+/*  standard dependencies  */
+import path             from "node:path"
+import Stream           from "node:stream"
+
+/*  external dependencies  */
+import * as Transformers from "@huggingface/transformers"
+
+/*  internal dependencies  */
+import SpeechFlowNode, { SpeechFlowChunk } from "./speechflow-node"
+
+/*  internal utility types  */
+type ConfigEntry = { systemPrompt: string, chat: Array<{ role: string, content: string }> }
+type Config      = { [ key: string ]: ConfigEntry }
+
+/*  SpeechFlow node for Transformers text-to-text translation  */
+export default class SpeechFlowNodeTransformers extends SpeechFlowNode {
+    /*  declare official node name  */
+    public static name = "transformers"
+
+    /*  internal state  */
+    private translator: Transformers.TranslationPipeline    | null = null
+    private generator:  Transformers.TextGenerationPipeline | null = null
+
+    /*  internal LLM setup  */
+    private setup: Config = {
+        /*  SmolLM3: English (EN) to German (DE) translation  */
+        "SmolLM3:en-de": {
+            systemPrompt:
+                "/no_think\n" +
+                "You are a translator.\n" +
+                "Output only the requested text.\n" +
+                "Do not use markdown.\n" +
+                "Do not chat.\n" +
+                "Do not show any explanations.\n" +
+                "Do not show any introduction.\n" +
+                "Do not show any preamble.\n" +
+                "Do not show any prolog.\n" +
+                "Do not show any epilog.\n" +
+                "Get to the point.\n" +
+                "Preserve the original meaning, tone, and nuance.\n" +
+                "Directly translate text from English (EN) to fluent and natural German (DE) language.\n",
+            chat: [
+                { role: "user",   content: "I love my wife." },
+                { role: "assistant", content: "Ich liebe meine Frau." },
+                { role: "user",   content: "The weather is wonderful." },
+                { role: "assistant", content: "Das Wetter ist wunderschön." },
+                { role: "user",   content: "The live is awesome." },
+                { role: "assistant", content: "Das Leben ist einfach großartig." }
+            ]
+        },
+
+        /*  SmolLM3: German (DE) to English (EN) translation  */
+        "SmolLM3:de-en": {
+            systemPrompt:
+                "/no_think\n" +
+                "You are a translator.\n" +
+                "Output only the requested text.\n" +
+                "Do not use markdown.\n" +
+                "Do not chat.\n" +
+                "Do not show any explanations.\n" +
+                "Do not show any introduction.\n" +
+                "Do not show any preamble. \n" +
+                "Do not show any prolog. \n" +
+                "Do not show any epilog. \n" +
+                "Get to the point.\n" +
+                "Preserve the original meaning, tone, and nuance.\n" +
+                "Directly translate text from German (DE) to fluent and natural English (EN) language.\n",
+            chat: [
+                { role: "user",   content: "Ich liebe meine Frau." },
+                { role: "assistant", content: "I love my wife." },
+                { role: "user",   content: "Das Wetter ist wunderschön." },
+                { role: "assistant", content: "The weather is wonderful." },
+                { role: "user",   content: "Das Leben ist einfach großartig." },
+                { role: "assistant", content: "The live is awesome." }
+            ]
+        }
+    }
+
+    /*  construct node  */
+    constructor (id: string, cfg: { [ id: string ]: any }, opts: { [ id: string ]: any }, args: any[]) {
+        super(id, cfg, opts, args)
+
+        /*  declare node configuration parameters  */
+        this.configure({
+            src:   { type: "string", pos: 0, val: "de", match: /^(?:de|en)$/ },
+            dst:   { type: "string", pos: 1, val: "en", match: /^(?:de|en)$/ },
+            model: { type: "string", val: "OPUS", match: /^(?:OPUS|SmolLM3)$/ }
+        })
+
+        /*  sanity check parameters  */
+        if (this.params.src === this.params.dst)
+            throw new Error("source and destination languages cannot be the same")
+
+        /*  declare node input/output format  */
+        this.input  = "text"
+        this.output = "text"
+    }
+
+    /*  open node  */
+    async open () {
+        let model: string = ""
+
+        /*  track download progress when instantiating Transformers engine and model  */
+        const progressState = new Map<string, number>()
+        const progressCallback: Transformers.ProgressCallback = (progress: any) => {
+            let artifact = model
+            if (typeof progress.file === "string")
+                artifact += `:${progress.file}`
+            let percent = 0
+            if (typeof progress.loaded === "number" && typeof progress.total === "number")
+                percent = (progress.loaded as number / progress.total as number) * 100
+            else if (typeof progress.progress === "number")
+                percent = progress.progress
+            if (percent > 0)
+                progressState.set(artifact, percent)
+        }
+        const interval = setInterval(() => {
+            for (const [ artifact, percent ] of progressState) {
+                this.log("info", `downloaded ${percent.toFixed(2)}% of artifact "${artifact}"`)
+                if (percent >= 1.0)
+                    progressState.delete(artifact)
+            }
+        }, 1000)
+
+        /*  instantiate Transformers engine and model  */
+        if (this.params.model === "OPUS") {
+            model = `onnx-community/opus-mt-${this.params.src}-${this.params.dst}`
+            const pipeline = Transformers.pipeline("translation", model, {
+                cache_dir: path.join(this.config.cacheDir, "transformers"),
+                dtype:     "q4",
+                device:    "auto",
+                progress_callback: progressCallback
+            })
+            this.translator = await pipeline
+            clearInterval(interval)
+            if (this.translator === null)
+                throw new Error("failed to instantiate translator pipeline")
+        }
+        else if (this.params.model === "SmolLM3") {
+            model = "HuggingFaceTB/SmolLM3-3B-ONNX"
+            const pipeline = Transformers.pipeline("text-generation", model, {
+                cache_dir: path.join(this.config.cacheDir, "transformers"),
+                dtype:     "q4",
+                device:    "auto",
+                progress_callback: progressCallback
+            })
+            this.generator = await pipeline
+            clearInterval(interval)
+            if (this.generator === null)
+                throw new Error("failed to instantiate generator pipeline")
+        }
+        else
+            throw new Error("invalid model")
+
+        /*  provide text-to-text translation  */
+        const translate = async (text: string) => {
+            if (this.params.model === "OPUS") {
+                const result = await this.translator!(text)
+                return Array.isArray(result) ?
+                    (result[0] as Transformers.TranslationSingle).translation_text :
+                    (result as Transformers.TranslationSingle).translation_text
+            }
+            else if (this.params.model === "SmolLM3") {
+                const key = `SmolLM3:${this.params.src}-${this.params.dst}`
+                const cfg = this.setup[key]
+                const messages = [
+                    { role: "system", content: cfg.systemPrompt },
+                    ...cfg.chat,
+                    { role: "user", content: text }
+                ]
+                const result = await this.generator!(messages, {
+                    max_new_tokens: 100,
+                    temperature:    0.6,
+                    top_p:          0.95,
+                    streamer: new Transformers.TextStreamer(this.generator!.tokenizer, {
+                        skip_prompt:         true,
+                        skip_special_tokens: true
+                    })
+                })
+                const generatedText = Array.isArray(result) ?
+                    (result[0] as Transformers.TextGenerationSingle).generated_text :
+                    (result as Transformers.TextGenerationSingle).generated_text
+                const response = typeof generatedText === "string" ?
+                    generatedText :
+                    generatedText.at(-1)!.content
+                return response
+            }
+            else
+                throw new Error("invalid model")
+        }
+
+        /*  establish a duplex stream and connect it to Transformers  */
+        this.stream = new Stream.Transform({
+            readableObjectMode: true,
+            writableObjectMode: true,
+            decodeStrings:      false,
+            transform (chunk: SpeechFlowChunk, encoding, callback) {
+                if (Buffer.isBuffer(chunk.payload))
+                    callback(new Error("invalid chunk payload type"))
+                else {
+                    if (chunk.payload === "") {
+                        this.push(chunk)
+                        callback()
+                    }
+                    else {
+                        translate(chunk.payload).then((payload) => {
+                            chunk = chunk.clone()
+                            chunk.payload = payload
+                            this.push(chunk)
+                            callback()
+                        }).catch((err) => {
+                            callback(err)
+                        })
+                    }
+                }
+            },
+            final (callback) {
+                this.push(null)
+                callback()
+            }
+        })
+    }
+
+    /*  close node  */
+    async close () {
+        /*  close stream  */
+        if (this.stream !== null) {
+            this.stream.destroy()
+            this.stream = null
+        }
+
+        /*  shutdown Transformers  */
+        if (this.translator !== null) {
+            this.translator.dispose()
+            this.translator = null
+        }
+        if (this.generator !== null) {
+            this.generator.dispose()
+            this.generator = null
+        }
+    }
+}
+

package/src/speechflow-node-x2x-trace.ts

@@ -52,7 +52,7 @@ export default class SpeechFlowNodeTrace extends SpeechFlowNode {
                 const fmt = (t: Duration) => t.toFormat("hh:mm:ss.SSS")
                 if (Buffer.isBuffer(chunk.payload)) {
                     if (type === "audio")
-                        log("info", `writing ${type} chunk: start=${fmt(chunk.timestampStart)} ` +
+                        log("debug", `writing ${type} chunk: start=${fmt(chunk.timestampStart)} ` +
                             `end=${fmt(chunk.timestampEnd)} kind=${chunk.kind} type=${chunk.type} ` +
                             `payload-type=Buffer payload-bytes=${chunk.payload.byteLength}`)
                     else
@@ -60,7 +60,7 @@ export default class SpeechFlowNodeTrace extends SpeechFlowNode {
                 }
                 else {
                     if (type === "text")
-                        log("info", `writing ${type} chunk: start=${fmt(chunk.timestampStart)} ` +
+                        log("debug", `writing ${type} chunk: start=${fmt(chunk.timestampStart)} ` +
                             `end=${fmt(chunk.timestampEnd)} kind=${chunk.kind} type=${chunk.type}` +
                             `payload-type=String payload-length=${chunk.payload.length} ` +
                             `payload-encoding=${encoding} payload-content="${chunk.payload.toString()}"`)

package/src/speechflow-node-xio-websocket.ts

@@ -9,7 +9,7 @@ import Stream                          from "node:stream"
 
 /*  external dependencies  */
 import ws                              from "ws"
-import ReconnWebsocket, { ErrorEvent } from "@opensumi/reconnecting-websocket"
+import ReconnWebSocket, { ErrorEvent } from "@opensumi/reconnecting-websocket"
 
 /*  internal dependencies  */
 import SpeechFlowNode, { SpeechFlowChunk } from "./speechflow-node"
@@ -22,7 +22,7 @@ export default class SpeechFlowNodeWebsocket extends SpeechFlowNode {
 
     /*  internal state  */
     private server: ws.WebSocketServer | null = null
-    private client: WebSocket          | null = null
+    private client: ReconnWebSocket    | null = null
 
     /*  construct node  */
     constructor (id: string, cfg: { [ id: string ]: any }, opts: { [ id: string ]: any }, args: any[]) {
@@ -153,7 +153,7 @@ export default class SpeechFlowNodeWebsocket extends SpeechFlowNode {
         }
         else if (this.params.connect !== "") {
             /*  connect remotely to a Websocket port  */
-            this.client = new ReconnWebsocket(this.params.connect, [], {
+            this.client = new ReconnWebSocket(this.params.connect, [], {
                 WebSocket:                   ws,
                 WebSocketOptions:            {},
                 reconnectionDelayGrowFactor: 1.3,
@@ -162,10 +162,10 @@ export default class SpeechFlowNodeWebsocket extends SpeechFlowNode {
                 connectionTimeout:           4000,
                 minUptime:                   5000
             })
-            this.client.addEventListener("open", (ev: Event) => {
+            this.client.addEventListener("open", (ev) => {
                 this.log("info", `connection opened to URL ${this.params.connect}`)
             })
-            this.client.addEventListener("close", (ev: Event) => {
+            this.client.addEventListener("close", (ev) => {
                 this.log("info", `connection closed to URL ${this.params.connect}`)
             })
             this.client.addEventListener("error", (ev: ErrorEvent) => {

package/src/speechflow-node.ts

@@ -36,6 +36,8 @@ export class SpeechFlowChunk {
 
 /*  the base class for all SpeechFlow nodes  */
 export default class SpeechFlowNode extends Events.EventEmitter {
+    public static name: string | undefined
+
     /*  general constant configuration (for reference)  */
     config = {
         audioChannels:     1,                            /* audio mono channel        */
@@ -80,6 +82,16 @@ export default class SpeechFlowNode extends Events.EventEmitter {
         this.timeZeroOffset = this.timeZero.diff(this.timeOpen)
     }
 
+    /*  receive external request  */
+    async receiveRequest (args: any[]) {
+        /*  no-op  */
+    }
+
+    /*  send external response  */
+    sendResponse (args: any[]) {
+        this.emit("send-response", args)
+    }
+
     /*  INTERNAL: utility function: create "params" attribute from constructor of sub-classes  */
     configure (spec: { [ id: string ]: { type: string, pos?: number, val?: any, match?: RegExp | ((x: any) => boolean) } }) {
         for (const name of Object.keys(spec)) {

package/src/speechflow-utils.ts

@@ -31,6 +31,33 @@ export function audioBufferDuration (
     return totalSamples / sampleRate
 }
 
+/*  calculate duration of an audio array  */
+export function audioArrayDuration (
+    arr: Float32Array,
+    sampleRate   = 48000,
+    channels     = 1
+) {
+    const totalSamples = arr.length / channels
+    return totalSamples / sampleRate
+}
+
+/*  helper function: convert Buffer in PCM/I16 to Float32Array in PCM/F32 format  */
+export function convertBufToF32 (buf: Buffer, littleEndian = true) {
+    const dataView = new DataView(buf.buffer)
+    const arr = new Float32Array(buf.length / 2)
+    for (let i = 0; i < arr.length; i++)
+        arr[i] = dataView.getInt16(i * 2, littleEndian) / 32768
+    return arr
+}
+
+/*  helper function: convert Float32Array in PCM/F32 to Buffer in PCM/I16 format  */
+export function convertF32ToBuf (arr: Float32Array) {
+    const int16Array = new Int16Array(arr.length)
+    for (let i = 0; i < arr.length; i++)
+        int16Array[i] = Math.max(-32768, Math.min(32767, Math.round(arr[i] * 32768)))
+    return Buffer.from(int16Array.buffer)
+}
+
 /*  create a Duplex/Transform stream which has
     object-mode on Writable side and buffer/string-mode on Readable side  */
 export function createTransformStreamForWritableSide () {
@@ -210,3 +237,171 @@ export class DoubleQueue<T0, T1> extends EventEmitter {
         })
     }
 }
+
+/*  queue element */
+export type QueueElement = { type: string }
+
+/*  queue pointer  */
+export class QueuePointer<T extends QueueElement> extends EventEmitter {
+    /*  internal state  */
+    private index = 0
+
+    /*  construction  */
+    constructor (
+        private name: string,
+        private queue: Queue<T>
+    ) {
+        super()
+    }
+
+    /*  positioning operations  */
+    maxPosition () {
+        return this.queue.elements.length
+    }
+    position (index?: number): number {
+        if (index !== undefined) {
+            this.index = index
+            if (this.index < 0)
+                this.index = 0
+            else if (this.index >= this.queue.elements.length)
+                this.index = this.queue.elements.length
+            this.emit("position", this.index)
+        }
+        return this.index
+    }
+    walk (num: number) {
+        if (num > 0) {
+            for (let i = 0; i < num && this.index < this.queue.elements.length; i++)
+                this.index++
+            this.emit("position", { start: this.index })
+        }
+        else if (num < 0) {
+            for (let i = 0; i < Math.abs(num) && this.index > 0; i++)
+                this.index--
+            this.emit("position", { start: this.index })
+        }
+    }
+    walkForwardUntil (type: T["type"]) {
+        while (this.index < this.queue.elements.length
+            && this.queue.elements[this.index].type !== type)
+            this.index++
+        this.emit("position", { start: this.index })
+    }
+    walkBackwardUntil (type: T["type"]) {
+        while (this.index > 0
+            && this.queue.elements[this.index].type !== type)
+            this.index--
+        this.emit("position", { start: this.index })
+    }
+
+    /*  search operations  */
+    searchForward (type: T["type"]) {
+        let position = this.index
+        while (position < this.queue.elements.length
+            && this.queue.elements[position].type !== type)
+            position++
+        this.emit("search", { start: this.index, end: position })
+        return position
+    }
+    searchBackward (type: T["type"]) {
+        let position = this.index
+        while (position > 0
+            && this.queue.elements[position].type !== type)
+            position--
+        this.emit("search", { start: position, end: this.index })
+        return position
+    }
+
+    /*  reading operations  */
+    peek (position?: number) {
+        if (position === undefined)
+            position = this.index
+        else {
+            if (position < 0)
+                position = 0
+            else if (position > this.queue.elements.length)
+                position = this.queue.elements.length
+        }
+        const element = this.queue.elements[position]
+        this.queue.emit("read", { start: position, end: position })
+        return element
+    }
+    read () {
+        const element = this.queue.elements[this.index]
+        if (this.index < this.queue.elements.length)
+            this.index++
+        this.queue.emit("read", { start: this.index - 1, end: this.index - 1 })
+        return element
+    }
+    slice (size?: number) {
+        let slice: T[]
+        const start = this.index
+        if (size !== undefined) {
+            if (size < 0)
+                size = 0
+            else if (size > this.queue.elements.length - this.index)
+                size = this.queue.elements.length - this.index
+            slice = this.queue.elements.slice(this.index, size)
+            this.index += size
+        }
+        else {
+            slice = this.queue.elements.slice(this.index)
+            this.index = this.queue.elements.length
+        }
+        this.queue.emit("read", { start, end: this.index })
+        return slice
+    }
+
+    /*  writing operations  */
+    touch () {
+        if (this.index >= this.queue.elements.length)
+            throw new Error("cannot touch after last element")
+        this.queue.emit("write", { start: this.index, end: this.index + 1 })
+    }
+    append (element: T) {
+        this.queue.elements.push(element)
+        this.index = this.queue.elements.length
+        this.queue.emit("write", { start: this.index - 1, end: this.index - 1 })
+    }
+    insert (element: T) {
+        this.queue.elements.splice(this.index++, 0, element)
+        this.queue.emit("write", { start: this.index - 1, end: this.index })
+    }
+    delete () {
+        if (this.index >= this.queue.elements.length)
+            throw new Error("cannot delete after last element")
+        this.queue.elements.splice(this.index, 1)
+        this.queue.emit("write", { start: this.index, end: this.index })
+    }
+}
+
+/*  queue  */
+export class Queue<T extends QueueElement> extends EventEmitter {
+    public elements: T[] = []
+    private pointers = new Map<string, QueuePointer<T>>()
+    pointerUse (name: string): QueuePointer<T> {
+        if (!this.pointers.has(name))
+            this.pointers.set(name, new QueuePointer<T>(name, this))
+        return this.pointers.get(name)!
+    }
+    pointerDelete (name: string): void {
+        if (!this.pointers.has(name))
+            throw new Error("pointer not exists")
+        this.pointers.delete(name)
+    }
+    trim (): void {
+        /*  determine minimum pointer position  */
+        let min = this.elements.length
+        for (const pointer of this.pointers.values())
+            if (min > pointer.position())
+                min = pointer.position()
+
+        /*  trim the maximum amount of first elements  */
+        this.elements.splice(0, min)
+
+        /*  shift all pointers  */
+        for (const pointer of this.pointers.values())
+            pointer.position(pointer.position() - min)
+    }
+}
+