speechflow 0.9.4 → 0.9.7

package/src/speechflow-node-xio-websocket.ts

@@ -0,0 +1,248 @@
+/*
+**  SpeechFlow - Speech Processing Flow Graph
+**  Copyright (c) 2024-2025 Dr. Ralf S. Engelschall <rse@engelschall.com>
+**  Licensed under GPL 3.0 <https://spdx.org/licenses/GPL-3.0-only>
+*/
+
+/*  standard dependencies  */
+import Stream                          from "node:stream"
+
+/*  external dependencies  */
+import ws                              from "ws"
+import ReconnWebsocket, { ErrorEvent } from "@opensumi/reconnecting-websocket"
+
+/*  internal dependencies  */
+import SpeechFlowNode, { SpeechFlowChunk } from "./speechflow-node"
+import * as utils                          from "./speechflow-utils"
+
+/*  SpeechFlow node for Websocket networking  */
+export default class SpeechFlowNodeWebsocket extends SpeechFlowNode {
+    /*  declare official node name  */
+    public static name = "websocket"
+
+    /*  internal state  */
+    private server: ws.WebSocketServer | null = null
+    private client: WebSocket          | null = null
+
+    /*  construct node  */
+    constructor (id: string, cfg: { [ id: string ]: any }, opts: { [ id: string ]: any }, args: any[]) {
+        super(id, cfg, opts, args)
+
+        /*  declare node configuration parameters  */
+        this.configure({
+            listen:  { type: "string", val: "",     match: /^(?:|ws:\/\/(.+?):(\d+))$/ },
+            connect: { type: "string", val: "",     match: /^(?:|ws:\/\/(.+?):(\d+)(?:\/.*)?)$/ },
+            mode:    { type: "string", val: "r",    match: /^(?:r|w|rw)$/ },
+            type:    { type: "string", val: "text", match: /^(?:audio|text)$/ }
+        })
+
+        /*  sanity check usage  */
+        if (this.params.listen !== "" && this.params.connect !== "")
+            throw new Error("Websocket node cannot listen and connect at the same time")
+        else if (this.params.listen === "" && this.params.connect === "")
+            throw new Error("Websocket node requires either listen or connect mode")
+
+        /*  declare node input/output format  */
+        if (this.params.mode === "rw") {
+            this.input  = this.params.type
+            this.output = this.params.type
+        }
+        else if (this.params.mode === "r") {
+            this.input  = "none"
+            this.output = this.params.type
+        }
+        else if (this.params.mode === "w") {
+            this.input  = this.params.type
+            this.output = "none"
+        }
+    }
+
+    /*  open node  */
+    async open () {
+        if (this.params.listen !== "") {
+            /*  listen locally on a Websocket port  */
+            const url = new URL(this.params.listen)
+            const websockets = new Set<ws.WebSocket>()
+            const chunkQueue = new utils.SingleQueue<SpeechFlowChunk>()
+            const server = new ws.WebSocketServer({
+                host: url.hostname,
+                port: Number.parseInt(url.port),
+                path: url.pathname
+            })
+            server.on("listening", () => {
+                this.log("info", `listening on URL ${this.params.listen}`)
+            })
+            server.on("connection", (ws, request) => {
+                const peer = `${request.socket.remoteAddress}:${request.socket.remotePort}`
+                this.log("info", `connection opened on URL ${this.params.listen} by peer ${peer}`)
+                websockets.add(ws)
+                ws.on("close", () => {
+                    this.log("info", `connection closed on URL ${this.params.listen} by peer ${peer}`)
+                    websockets.delete(ws)
+                })
+                ws.on("error", (error) => {
+                    this.log("error", `error of connection on URL ${this.params.listen} for peer ${peer}: ${error.message}`)
+                })
+                ws.on("message", (data, isBinary) => {
+                    if (this.params.mode === "w") {
+                        this.log("warning", `connection on URL ${this.params.listen} by peer ${peer}: ` +
+                            "received remote data on write-only node")
+                        return
+                    }
+                    if (!isBinary) {
+                        this.log("warning", `connection on URL ${this.params.listen} by peer ${peer}: ` +
+                            "received non-binary message")
+                        return
+                    }
+                    let buffer: Buffer
+                    if (Buffer.isBuffer(data))
+                        buffer = data
+                    else if (data instanceof ArrayBuffer)
+                        buffer = Buffer.from(data)
+                    else
+                        buffer = Buffer.concat(data)
+                    const chunk = utils.streamChunkDecode(buffer)
+                    chunkQueue.write(chunk)
+                })
+            })
+            server.on("error", (error) => {
+                this.log("error", `error of some connection on URL ${this.params.listen}: ${error.message}`)
+            })
+            const type = this.params.type
+            const mode = this.params.mode
+            this.stream = new Stream.Duplex({
+                writableObjectMode: true,
+                readableObjectMode: true,
+                decodeStrings:      false,
+                write (chunk: SpeechFlowChunk, encoding, callback) {
+                    if (mode === "r")
+                        callback(new Error("write operation on read-only node"))
+                    else if (chunk.type !== type)
+                        callback(new Error(`written chunk is not of ${type} type`))
+                    else if (websockets.size === 0)
+                        callback(new Error("still no Websocket connections available"))
+                    else {
+                        const data = utils.streamChunkEncode(chunk)
+                        const results = []
+                        for (const websocket of websockets.values()) {
+                            results.push(new Promise<void>((resolve, reject) => {
+                                websocket.send(data, (error) => {
+                                    if (error)
+                                        reject(error)
+                                    else
+                                        resolve()
+                                })
+                            }))
+                        }
+                        Promise.all(results).then(() => {
+                            callback()
+                        }).catch((errors: Error[]) => {
+                            const error = new Error(errors.map((e) => e.message).join("; "))
+                            callback(error)
+                        })
+                    }
+                },
+                read (size: number) {
+                    if (mode === "w")
+                        throw new Error("read operation on write-only node")
+                    chunkQueue.read().then((chunk) => {
+                        this.push(chunk, "binary")
+                    })
+                }
+            })
+        }
+        else if (this.params.connect !== "") {
+            /*  connect remotely to a Websocket port  */
+            this.client = new ReconnWebsocket(this.params.connect, [], {
+                WebSocket:                   ws,
+                WebSocketOptions:            {},
+                reconnectionDelayGrowFactor: 1.3,
+                maxReconnectionDelay:        4000,
+                minReconnectionDelay:        1000,
+                connectionTimeout:           4000,
+                minUptime:                   5000
+            })
+            this.client.addEventListener("open", (ev: Event) => {
+                this.log("info", `connection opened to URL ${this.params.connect}`)
+            })
+            this.client.addEventListener("close", (ev: Event) => {
+                this.log("info", `connection closed to URL ${this.params.connect}`)
+            })
+            this.client.addEventListener("error", (ev: ErrorEvent) => {
+                this.log("error", `error of connection on URL ${this.params.connect}: ${ev.error.message}`)
+            })
+            const chunkQueue = new utils.SingleQueue<SpeechFlowChunk>()
+            this.client.addEventListener("message", (ev: MessageEvent) => {
+                if (this.params.mode === "w") {
+                    this.log("warning", `connection to URL ${this.params.listen}: ` +
+                        "received remote data on write-only node")
+                    return
+                }
+                if (!(ev.data instanceof ArrayBuffer)) {
+                    this.log("warning", `connection to URL ${this.params.listen}: ` +
+                        "received non-binary message")
+                    return
+                }
+                const buffer = Buffer.from(ev.data)
+                const chunk = utils.streamChunkDecode(buffer)
+                chunkQueue.write(chunk)
+            })
+            const client = this.client
+            client.binaryType = "arraybuffer"
+            const type = this.params.type
+            const mode = this.params.mode
+            this.stream = new Stream.Duplex({
+                writableObjectMode: true,
+                readableObjectMode: true,
+                decodeStrings:      false,
+                write (chunk: SpeechFlowChunk, encoding, callback) {
+                    if (mode === "r")
+                        callback(new Error("write operation on read-only node"))
+                    else if (chunk.type !== type)
+                        callback(new Error(`written chunk is not of ${type} type`))
+                    else if (!client.OPEN)
+                        callback(new Error("still no Websocket connection available"))
+                    const data = utils.streamChunkEncode(chunk)
+                    client.send(data)
+                    callback()
+                },
+                read (size: number) {
+                    if (mode === "w")
+                        throw new Error("read operation on write-only node")
+                    if (!client.OPEN)
+                        throw new Error("still no Websocket connection available")
+                    chunkQueue.read().then((chunk) => {
+                        this.push(chunk, "binary")
+                    })
+                }
+            })
+        }
+    }
+
+    /*  close node  */
+    async close () {
+        /*  close Websocket server  */
+        if (this.server !== null) {
+            await new Promise<void>((resolve, reject) => {
+                this.server!.close((error) => {
+                    if (error) reject(error)
+                    else       resolve()
+                })
+            })
+            this.server = null
+        }
+
+        /*  close Websocket client  */
+        if (this.client !== null) {
+            this.client!.close()
+            this.client = null
+        }
+
+        /*  close stream  */
+        if (this.stream !== null) {
+            this.stream.destroy()
+            this.stream = null
+        }
+    }
+}
+

package/src/speechflow-node.ts

@@ -7,17 +7,44 @@
 /*  standard dependencies  */
 import Events from "node:events"
 import Stream from "node:stream"
+import { DateTime, Duration } from "luxon"
+
+/*  the definition of a single payload chunk passed through the SpeechFlow nodes  */
+export class SpeechFlowChunk {
+    constructor (
+        public timestampStart: Duration,
+        public timestampEnd:   Duration,
+        public kind:           "intermediate" | "final",
+        public type:           "audio" | "text",
+        public payload:        Buffer | string
+    ) {}
+    clone () {
+        let payload: Buffer | string
+        if (Buffer.isBuffer(this.payload))
+            payload = Buffer.from(this.payload)
+        else
+            payload = String(this.payload)
+        return new SpeechFlowChunk(
+            Duration.fromMillis(this.timestampStart.toMillis()),
+            Duration.fromMillis(this.timestampEnd.toMillis()),
+            this.kind,
+            this.type,
+            payload
+        )
+    }
+}
 
 /*  the base class for all SpeechFlow nodes  */
 export default class SpeechFlowNode extends Events.EventEmitter {
     /*  general constant configuration (for reference)  */
     config = {
-        audioChannels:     1,      /* audio mono channel       */
-        audioBitDepth:     16,     /* audio PCM 16-bit integer */
-        audioLittleEndian: true,   /* audio PCM little-endian  */
-        audioSampleRate:   48000,  /* audio 48kHz sample rate  */
-        textEncoding:      "utf8"  /* UTF-8 text encoding      */
-    } as const
+        audioChannels:     1,                            /* audio mono channel        */
+        audioBitDepth:     16 as (1 | 8 | 16 | 24 | 32), /* audio PCM 16-bit integer  */
+        audioLittleEndian: true,                         /* audio PCM little-endian   */
+        audioSampleRate:   48000,                        /* audio 48kHz sample rate   */
+        textEncoding:      "utf8" as BufferEncoding,     /* UTF-8 text encoding       */
+        cacheDir:          ""                            /* directory for cache files */
+    }
 
     /*  announced information  */
     input  = "none"
@@ -26,18 +53,35 @@ export default class SpeechFlowNode extends Events.EventEmitter {
     stream: Stream.Writable | Stream.Readable | Stream.Duplex | null = null
     connectionsIn  = new Set<SpeechFlowNode>()
     connectionsOut = new Set<SpeechFlowNode>()
+    timeOpen:       DateTime<boolean> | undefined
+    timeZero:       DateTime<boolean> = DateTime.fromMillis(0)
+    timeZeroOffset: Duration<boolean> = Duration.fromMillis(0)
 
     /*  the default constructor  */
     constructor (
         public  id:   string,
+        private cfg:  { [ id: string ]: any },
         private opts: { [ id: string ]: any },
         private args: any[]
     ) {
         super()
+        for (const key of Object.keys(cfg)) {
+            const idx = key as keyof typeof this.config
+            if (this.config[idx] !== undefined)
+                (this.config[idx] as any) = cfg[key]
+        }
+    }
+
+    /*  set base/zero time for relative timestamp calculations  */
+    setTimeZero (time: DateTime) {
+        this.timeZero = time
+        if (this.timeOpen === undefined)
+            this.timeOpen = this.timeZero
+        this.timeZeroOffset = this.timeZero.diff(this.timeOpen)
     }
 
     /*  INTERNAL: utility function: create "params" attribute from constructor of sub-classes  */
-    configure (spec: { [ id: string ]: { type: string, pos?: number, val?: any, match?: RegExp } }) {
+    configure (spec: { [ id: string ]: { type: string, pos?: number, val?: any, match?: RegExp | ((x: any) => boolean) } }) {
         for (const name of Object.keys(spec)) {
             if (this.opts[name] !== undefined) {
                 /*  named parameter  */
@@ -45,9 +89,11 @@ export default class SpeechFlowNode extends Events.EventEmitter {
                     throw new Error(`invalid type of named parameter "${name}" ` +
                         `(has to be ${spec[name].type})`)
                 if ("match" in spec[name]
-                    && this.opts[name].match(spec[name].match) === null)
-                    throw new Error(`invalid value of named parameter "${name}" ` +
-                        `(has to match ${spec[name].match})`)
+                    && (   (   spec[name].match instanceof RegExp
+                            && this.opts[name].match(spec[name].match) === null)
+                        || (   typeof spec[name].match === "function"
+                            && !spec[name].match(this.opts[name])    )          ))
+                    throw new Error(`invalid value "${this.opts[name]}" of named parameter "${name}"`)
                 this.params[name] = this.opts[name]
             }
             else if (this.opts[name] === undefined
@@ -55,14 +101,20 @@ export default class SpeechFlowNode extends Events.EventEmitter {
                 && typeof spec[name].pos === "number"
                 && spec[name].pos < this.args.length) {
                 /*  positional argument  */
-                if (typeof this.args[spec[name].pos!] !== spec[name].type)
+                if (typeof this.args[spec[name].pos] !== spec[name].type)
                     throw new Error(`invalid type of positional parameter "${name}" ` +
                         `(has to be ${spec[name].type})`)
                 if ("match" in spec[name]
-                    && this.args[spec[name].pos!].match(spec[name].match) === null)
+                    && this.args[spec[name].pos].match(spec[name].match) === null)
                     throw new Error(`invalid value of positional parameter "${name}" ` +
                         `(has to match ${spec[name].match})`)
-                this.params[name] = this.args[spec[name].pos!]
+                if ("match" in spec[name]
+                    && (   (   spec[name].match instanceof RegExp
+                            && this.args[spec[name].pos].match(spec[name].match) === null)
+                        || (   typeof spec[name].match === "function"
+                            && !spec[name].match(this.args[spec[name].pos])    )          ))
+                    throw new Error(`invalid value "${this.opts[name]}" of positional parameter "${name}"`)
+                this.params[name] = this.args[spec[name].pos]
             }
             else if ("val" in spec[name] && spec[name].val !== undefined)
                 /*  default argument  */
@@ -70,6 +122,19 @@ export default class SpeechFlowNode extends Events.EventEmitter {
             else
                 throw new Error(`required parameter "${name}" not given`)
         }
+        for (const name of Object.keys(this.opts)) {
+            if (spec[name] === undefined)
+                throw new Error(`named parameter "${name}" not known`)
+        }
+        for (let i = 0; i < this.args.length; i++) {
+            let found = false
+            for (const name of Object.keys(spec))
+                if (spec[name].pos === i)
+                    found = true
+            if (!found)
+                throw new Error(`positional parameter #${i} ("${this.args[i]}") ` +
+                    "not mappable to any known argument")
+        }
     }
 
     /*  connect node to another one  */

package/src/speechflow-utils.ts

@@ -0,0 +1,212 @@
+/*
+**  SpeechFlow - Speech Processing Flow Graph
+**  Copyright (c) 2024-2025 Dr. Ralf S. Engelschall <rse@engelschall.com>
+**  Licensed under GPL 3.0 <https://spdx.org/licenses/GPL-3.0-only>
+*/
+
+/*  external dependencies  */
+import Stream                 from "node:stream"
+import { EventEmitter }       from "node:events"
+import { DateTime, Duration } from "luxon"
+import CBOR                   from "cbor2"
+
+/*  internal dependencies  */
+import { SpeechFlowChunk } from "./speechflow-node"
+
+/*  calculate duration of an audio buffer  */
+export function audioBufferDuration (
+    buffer: Buffer,
+    sampleRate   = 48000,
+    bitDepth     = 16,
+    channels     = 1,
+    littleEndian = true
+) {
+    if (!Buffer.isBuffer(buffer))
+        throw new Error("invalid input (Buffer expected)")
+    if (littleEndian !== true)
+        throw new Error("only Little Endian supported")
+
+    const bytesPerSample = bitDepth / 8
+    const totalSamples = buffer.length / (bytesPerSample * channels)
+    return totalSamples / sampleRate
+}
+
+/*  create a Duplex/Transform stream which has
+    object-mode on Writable side and buffer/string-mode on Readable side  */
+export function createTransformStreamForWritableSide () {
+    return new Stream.Transform({
+        readableObjectMode: true,
+        writableObjectMode: true,
+        decodeStrings: false,
+        transform (chunk: SpeechFlowChunk, encoding, callback) {
+            this.push(chunk.payload)
+            callback()
+        }
+    })
+}
+
+/*  create a Duplex/Transform stream which has
+    object-mode on Readable side and buffer/string-mode on Writable side  */
+export function createTransformStreamForReadableSide (type: "text" | "audio", getTimeZero: () => DateTime) {
+    return new Stream.Transform({
+        readableObjectMode: true,
+        writableObjectMode: true,
+        decodeStrings: false,
+        transform (chunk: Buffer | string, encoding, callback) {
+            const timeZero = getTimeZero()
+            const start = DateTime.now().diff(timeZero)
+            let end = start
+            if (type === "audio") {
+                const duration = audioBufferDuration(chunk as Buffer)
+                end = start.plus(duration * 1000)
+            }
+            const obj = new SpeechFlowChunk(start, end, "final", type, chunk)
+            this.push(obj)
+            callback()
+        }
+    })
+}
+
+/*  ensure a chunk is of a certain type and format  */
+export function ensureStreamChunk (type: "audio" | "text", chunk: SpeechFlowChunk | Buffer | string) {
+    if (chunk instanceof SpeechFlowChunk) {
+        if (chunk.type !== type)
+            throw new Error(`invalid payload chunk (expected ${type} type, received ${chunk.type} type)`)
+    }
+    else {
+        if (type === "text" && Buffer.isBuffer(chunk))
+            chunk = chunk.toString("utf8")
+        else if (type === "audio" && !Buffer.isBuffer(chunk))
+            chunk = Buffer.from(chunk)
+    }
+    return chunk
+}
+
+/*  type of a serialized SpeechFlow chunk  */
+type SpeechFlowChunkSerialized = {
+    timestampStart: number,
+    timestampEnd:   number,
+    kind:           string,
+    type:           string,
+    payload:        Uint8Array
+}
+
+/*  encode/serialize chunk of data  */
+export function streamChunkEncode (chunk: SpeechFlowChunk) {
+    let payload: Uint8Array
+    if (Buffer.isBuffer(chunk.payload))
+        payload = new Uint8Array(chunk.payload)
+    else {
+        const encoder = new TextEncoder()
+        payload = encoder.encode(chunk.payload)
+    }
+    const data = {
+        timestampStart: chunk.timestampStart.toMillis(),
+        timestampEnd:   chunk.timestampEnd.toMillis(),
+        kind:           chunk.kind,
+        type:           chunk.type,
+        payload
+    } satisfies SpeechFlowChunkSerialized
+    const _data = CBOR.encode(data)
+    return _data
+}
+
+/*  decode/unserialize chunk of data  */
+export function streamChunkDecode (_data: Uint8Array) {
+    let data: SpeechFlowChunkSerialized
+    try {
+        data = CBOR.decode<SpeechFlowChunkSerialized>(_data)
+    }
+    catch (err: any) {
+        throw new Error(`CBOR decoding failed: ${err}`)
+    }
+    let payload: Buffer | string
+    if (data.type === "audio")
+        payload = Buffer.from(data.payload)
+    else
+        payload = (new TextDecoder()).decode(data.payload)
+    const chunk = new SpeechFlowChunk(
+        Duration.fromMillis(data.timestampStart),
+        Duration.fromMillis(data.timestampEnd),
+        data.kind as "intermediate" | "final",
+        data.type as "audio" | "text",
+        payload
+    )
+    return chunk
+}
+
+/*  helper class for single item queue  */
+export class SingleQueue<T> extends EventEmitter {
+    private queue = new Array<T>()
+    write (item: T) {
+        this.queue.unshift(item)
+        this.emit("dequeue")
+    }
+    read () {
+        return new Promise<T>((resolve, reject) => {
+            const consume = () => {
+                if (this.queue.length > 0)
+                    return this.queue.pop()!
+                else
+                    return null
+            }
+            let item = consume()
+            if (item !== null)
+                resolve(item)
+            else {
+                const tryToConsume = () => {
+                    item = consume()
+                    if (item !== null)
+                        resolve(item)
+                    else
+                        this.once("dequeue", tryToConsume)
+                }
+                this.once("dequeue", tryToConsume)
+            }
+        })
+    }
+}
+
+/*  helper class for double-item queue  */
+export class DoubleQueue<T0, T1> extends EventEmitter {
+    private queue0 = new Array<T0>()
+    private queue1 = new Array<T1>()
+    private notify () {
+        if (this.queue0.length > 0 && this.queue1.length > 0)
+            this.emit("dequeue")
+    }
+    write0 (item: T0) {
+        this.queue0.unshift(item)
+        this.notify()
+    }
+    write1 (item: T1) {
+        this.queue1.unshift(item)
+        this.notify()
+    }
+    read () {
+        return new Promise<[ T0, T1 ]>((resolve, reject) => {
+            const consume = (): [ T0, T1 ] | null => {
+                if (this.queue0.length > 0 && this.queue1.length > 0) {
+                    const item0 = this.queue0.pop() as T0
+                    const item1 = this.queue1.pop() as T1
+                    return [ item0, item1 ]
+                }
+                else
+                    return null
+            }
+            let items = consume()
+            if (items !== null)
+                resolve(items)
+            else {
+                const tryToConsume = () => {
+                    items = consume()
+                    if (items !== null)
+                        resolve(items)
+                    else
+                        this.once("dequeue", tryToConsume)
+                }
+                this.once("dequeue", tryToConsume)
+            }
+        })
+    }
+}