speechflow 1.0.0 → 1.2.0

package/src/speechflow-node-x2x-trace.ts

@@ -5,7 +5,9 @@
 */
 
 /*  standard dependencies  */
-import Stream       from "node:stream"
+import Stream from "node:stream"
+
+/*  external dependencies  */
 import { Duration } from "luxon"
 
 /*  internal dependencies  */
@@ -23,7 +25,7 @@ export default class SpeechFlowNodeTrace extends SpeechFlowNode {
         /*  declare node configuration parameters  */
         this.configure({
             type: { type: "string", pos: 0, val: "audio", match: /^(?:audio|text)$/ },
-            name: { type: "string", pos: 1 }
+            name: { type: "string", pos: 1, val: "trace" }
         })
 
         /*  declare node input/output format  */
@@ -41,7 +43,7 @@ export default class SpeechFlowNodeTrace extends SpeechFlowNode {
                 this.log(level, msg)
         }
 
-        /*  provide Duplex stream and internally attach to Deepgram API  */
+        /*  provide Transform stream  */
         const type = this.params.type
         this.stream = new Stream.Transform({
             writableObjectMode: true,
@@ -49,23 +51,38 @@ export default class SpeechFlowNodeTrace extends SpeechFlowNode {
             decodeStrings:      false,
             transform (chunk: SpeechFlowChunk, encoding, callback) {
                 let error: Error | undefined
-                const fmt = (t: Duration) => t.toFormat("hh:mm:ss.SSS")
+                const fmtTime = (t: Duration) => t.toFormat("hh:mm:ss.SSS")
+                const fmtMeta = (meta: Map<string, any>) => {
+                    if (meta.size === 0)
+                        return "none"
+                    else
+                        return `{ ${Array.from(meta.entries())
+                            .map(([ k, v ]) => `${k}: ${JSON.stringify(v)}`)
+                            .join(", ")
+                        } }`
+                }
                 if (Buffer.isBuffer(chunk.payload)) {
                     if (type === "audio")
-                        log("debug", `writing ${type} chunk: start=${fmt(chunk.timestampStart)} ` +
-                            `end=${fmt(chunk.timestampEnd)} kind=${chunk.kind} type=${chunk.type} ` +
-                            `payload-type=Buffer payload-bytes=${chunk.payload.byteLength}`)
+                        log("debug", `chunk: type=${chunk.type} ` +
+                            `kind=${chunk.kind} ` +
+                            `start=${fmtTime(chunk.timestampStart)} ` +
+                            `end=${fmtTime(chunk.timestampEnd)} ` +
+                            `payload-type=Buffer payload-length=${chunk.payload.byteLength} ` +
+                            `meta=${fmtMeta(chunk.meta)}`)
                     else
-                        error = new Error(`writing ${type} chunk: seen Buffer instead of String chunk type`)
+                        error = new Error(`${type} chunk: seen Buffer instead of String chunk type`)
                 }
                 else {
                     if (type === "text")
-                        log("debug", `writing ${type} chunk: start=${fmt(chunk.timestampStart)} ` +
-                            `end=${fmt(chunk.timestampEnd)} kind=${chunk.kind} type=${chunk.type}` +
+                        log("debug", `${type} chunk: type=${chunk.type}` +
+                            `kind=${chunk.kind} ` +
+                            `start=${fmtTime(chunk.timestampStart)} ` +
+                            `end=${fmtTime(chunk.timestampEnd)} ` +
                             `payload-type=String payload-length=${chunk.payload.length} ` +
-                            `payload-encoding=${encoding} payload-content="${chunk.payload.toString()}"`)
+                            `payload-encoding=${encoding} payload-content="${chunk.payload.toString()}" ` +
+                            `meta=${fmtMeta(chunk.meta)}`)
                     else
-                        error = new Error(`writing ${type} chunk: seen String instead of Buffer chunk type`)
+                        error = new Error(`${type} chunk: seen String instead of Buffer chunk type`)
                 }
                 if (error !== undefined)
                     callback(error)

package/src/speechflow-node-xio-device.ts

@@ -31,8 +31,9 @@ export default class SpeechFlowNodeDevice extends SpeechFlowNode {
 
         /*  declare node configuration parameters  */
         this.configure({
-            device: { type: "string", pos: 0,            match: /^(.+?):(.+)$/ },
-            mode:   { type: "string", pos: 1, val: "rw", match: /^(?:r|w|rw)$/ }
+            device: { type: "string", pos: 0, val: "",   match: /^(.+?):(.+)$/ },
+            mode:   { type: "string", pos: 1, val: "rw", match: /^(?:r|w|rw)$/ },
+            chunk:  { type: "number", pos: 2, val: 200,  match: (n: number) => n >= 10 && n <= 1000 }
         })
 
         /*  declare node input/output format  */
@@ -86,6 +87,9 @@ export default class SpeechFlowNodeDevice extends SpeechFlowNode {
 
     /*  open node  */
     async open () {
+        if (this.params.device === "")
+            throw new Error("required parameter \"device\" has to be given")
+
         /*  determine device  */
         const device = this.audioDeviceFromURL(this.params.mode, this.params.device)
 
@@ -95,6 +99,13 @@ export default class SpeechFlowNodeDevice extends SpeechFlowNode {
             throw new Error(`audio device sample rate ${device.defaultSampleRate} is ` +
                 `incompatible with required sample rate ${this.config.audioSampleRate}`)
 
+        /*  determine how many bytes we need per chunk when
+            the chunk should be the requested duration  */
+        const highwaterMark = (
+            this.config.audioSampleRate *
+            (this.config.audioBitDepth / 8)
+        ) / (1000 / this.params.chunk)
+
         /*  establish device connection
             Notice: "naudion" actually implements Stream.{Readable,Writable,Duplex}, but
             declares just its sub-interface NodeJS.{Readable,Writable,Duplex}Stream,
@@ -112,13 +123,15 @@ export default class SpeechFlowNodeDevice extends SpeechFlowNode {
                     deviceId:     device.id,
                     channelCount: this.config.audioChannels,
                     sampleRate:   this.config.audioSampleRate,
-                    sampleFormat: this.config.audioBitDepth
+                    sampleFormat: this.config.audioBitDepth,
+                    highwaterMark
                 },
                 outOptions: {
                     deviceId:     device.id,
                     channelCount: this.config.audioChannels,
                     sampleRate:   this.config.audioSampleRate,
-                    sampleFormat: this.config.audioBitDepth
+                    sampleFormat: this.config.audioBitDepth,
+                    highwaterMark
                 }
             })
             this.stream = this.io as unknown as Stream.Duplex
@@ -135,10 +148,11 @@ export default class SpeechFlowNodeDevice extends SpeechFlowNode {
             this.log("info", `resolved "${this.params.device}" to input device "${device.id}"`)
             this.io = PortAudio.AudioIO({
                 inOptions: {
-                    deviceId:     device.id,
-                    channelCount: this.config.audioChannels,
-                    sampleRate:   this.config.audioSampleRate,
-                    sampleFormat: this.config.audioBitDepth
+                    deviceId:      device.id,
+                    channelCount:  this.config.audioChannels,
+                    sampleRate:    this.config.audioSampleRate,
+                    sampleFormat:  this.config.audioBitDepth,
+                    highwaterMark
                 }
             })
             this.stream = this.io as unknown as Stream.Readable
@@ -158,7 +172,8 @@ export default class SpeechFlowNodeDevice extends SpeechFlowNode {
                     deviceId:     device.id,
                     channelCount: this.config.audioChannels,
                     sampleRate:   this.config.audioSampleRate,
-                    sampleFormat: this.config.audioBitDepth
+                    sampleFormat: this.config.audioBitDepth,
+                    highwaterMark
                 }
             })
             this.stream = this.io as unknown as Stream.Writable

package/src/speechflow-node-xio-file.ts

@@ -23,9 +23,11 @@ export default class SpeechFlowNodeFile extends SpeechFlowNode {
 
         /*  declare node configuration parameters  */
         this.configure({
-            path: { type: "string", pos: 0 },
-            mode: { type: "string", pos: 1, val: "r",     match: /^(?:r|w|rw)$/ },
-            type: { type: "string", pos: 2, val: "audio", match: /^(?:audio|text)$/ }
+            path:   { type: "string", pos: 0, val: "" },
+            mode:   { type: "string", pos: 1, val: "r",     match: /^(?:r|w|rw)$/ },
+            type:   { type: "string", pos: 2, val: "audio", match: /^(?:audio|text)$/ },
+            chunka: { type: "number",         val: 200,     match: (n: number) => n >= 10 && n <= 1000 },
+            chunkt: { type: "number",         val: 65536,   match: (n: number) => n >= 1024 && n <= 131072 }
         })
 
         /*  declare node input/output format  */
@@ -45,90 +47,128 @@ export default class SpeechFlowNodeFile extends SpeechFlowNode {
 
     /*  open node  */
     async open () {
+        /*  determine how many bytes we need per chunk when
+            the chunk should be of the required duration/size */
+        const highWaterMarkAudio = (
+            this.config.audioSampleRate *
+            (this.config.audioBitDepth / 8)
+        ) / (1000 / this.params.chunka)
+        const highWaterMarkText = this.params.chunkt
+
+        /*  sanity check  */
+        if (this.params.path === "")
+            throw new Error("required parameter \"path\" has to be given")
+
+        /*  dispatch according to mode and path  */
         if (this.params.mode === "rw") {
             if (this.params.path === "-") {
                 /*  standard I/O  */
                 if (this.params.type === "audio") {
                     process.stdin.setEncoding()
                     process.stdout.setEncoding()
+                    const streamR = new Stream.PassThrough({ highWaterMark: highWaterMarkAudio })
+                    process.stdin.pipe(streamR)
+                    const streamW = new Stream.PassThrough({ highWaterMark: highWaterMarkAudio })
+                    streamW.pipe(process.stdout)
+                    this.stream = Stream.Duplex.from({ readable: streamR, writable: streamW })
                 }
                 else {
                     process.stdin.setEncoding(this.config.textEncoding)
                     process.stdout.setEncoding(this.config.textEncoding)
+                    const streamR = new Stream.PassThrough({ highWaterMark: highWaterMarkText })
+                    process.stdin.pipe(streamR)
+                    const streamW = new Stream.PassThrough({ highWaterMark: highWaterMarkText })
+                    streamW.pipe(process.stdout)
+                    this.stream = Stream.Duplex.from({ readable: streamR, writable: streamW })
                 }
-                this.stream = Stream.Duplex.from({
-                    readable: process.stdin,
-                    writable: process.stdout
-                })
             }
             else {
                 /*  file I/O  */
                 if (this.params.type === "audio") {
                     this.stream = Stream.Duplex.from({
-                        readable: fs.createReadStream(this.params.path),
-                        writable: fs.createWriteStream(this.params.path)
+                        readable: fs.createReadStream(this.params.path,
+                            { highWaterMark: highWaterMarkAudio }),
+                        writable: fs.createWriteStream(this.params.path,
+                            { highWaterMark: highWaterMarkAudio })
                     })
                 }
                 else {
                     this.stream = Stream.Duplex.from({
-                        readable: fs.createReadStream(this.params.path,
-                            { encoding: this.config.textEncoding }),
-                        writable: fs.createWriteStream(this.params.path,
-                            { encoding: this.config.textEncoding })
+                        readable: fs.createReadStream(this.params.path, {
+                            highWaterMark: highWaterMarkText,
+                            encoding: this.config.textEncoding
+                        }),
+                        writable: fs.createWriteStream(this.params.path, {
+                            highWaterMark: highWaterMarkText,
+                            encoding: this.config.textEncoding
+                        })
                     })
                 }
             }
 
             /*  convert regular stream into object-mode stream  */
             const wrapper1 = utils.createTransformStreamForWritableSide()
-            const wrapper2 = utils.createTransformStreamForReadableSide(this.params.type, () => this.timeZero)
+            const wrapper2 = utils.createTransformStreamForReadableSide(
+                this.params.type, () => this.timeZero)
             this.stream = Stream.compose(wrapper1, this.stream, wrapper2)
         }
         else if (this.params.mode === "r") {
             if (this.params.path === "-") {
                 /*  standard I/O  */
-                if (this.params.type === "audio")
+                let chunker: Stream.PassThrough
+                if (this.params.type === "audio") {
                     process.stdin.setEncoding()
-                else
+                    chunker = new Stream.PassThrough({ highWaterMark: highWaterMarkAudio })
+                }
+                else {
                     process.stdin.setEncoding(this.config.textEncoding)
-                this.stream = process.stdin
+                    chunker = new Stream.PassThrough({ highWaterMark: highWaterMarkText })
+                }
+                const wrapper = utils.createTransformStreamForReadableSide(
+                    this.params.type, () => this.timeZero)
+                this.stream = Stream.compose(process.stdin, chunker, wrapper)
             }
             else {
                 /*  file I/O  */
+                let readable: Stream.Readable
                 if (this.params.type === "audio")
-                    this.stream = fs.createReadStream(this.params.path)
+                    readable = fs.createReadStream(this.params.path,
+                        { highWaterMark: highWaterMarkAudio })
                 else
-                    this.stream = fs.createReadStream(this.params.path,
-                        { encoding: this.config.textEncoding })
+                    readable = fs.createReadStream(this.params.path,
+                        { highWaterMark: highWaterMarkText, encoding: this.config.textEncoding })
+                const wrapper = utils.createTransformStreamForReadableSide(
+                    this.params.type, () => this.timeZero)
+                this.stream = Stream.compose(readable, wrapper)
             }
-
-            /*  convert regular stream into object-mode stream  */
-            const wrapper = utils.createTransformStreamForReadableSide(this.params.type, () => this.timeZero)
-            this.stream.pipe(wrapper)
-            this.stream = wrapper
         }
         else if (this.params.mode === "w") {
             if (this.params.path === "-") {
                 /*  standard I/O  */
-                if (this.params.type === "audio")
+                let chunker: Stream.PassThrough
+                if (this.params.type === "audio") {
                     process.stdout.setEncoding()
-                else
+                    chunker = new Stream.PassThrough({ highWaterMark: highWaterMarkAudio })
+                }
+                else {
                     process.stdout.setEncoding(this.config.textEncoding)
-                this.stream = process.stdout
+                    chunker = new Stream.PassThrough({ highWaterMark: highWaterMarkText })
+                }
+                const wrapper = utils.createTransformStreamForWritableSide()
+                this.stream = Stream.compose(wrapper, chunker, process.stdout)
             }
             else {
                 /*  file I/O  */
+                let writable: Stream.Writable
                 if (this.params.type === "audio")
-                    this.stream = fs.createWriteStream(this.params.path)
+                    writable = fs.createWriteStream(this.params.path,
+                        { highWaterMark: highWaterMarkAudio })
                 else
-                    this.stream = fs.createWriteStream(this.params.path,
-                        { encoding: this.config.textEncoding })
+                    writable = fs.createWriteStream(this.params.path,
+                        { highWaterMark: highWaterMarkText, encoding: this.config.textEncoding })
+                const wrapper = utils.createTransformStreamForWritableSide()
+                this.stream = Stream.compose(wrapper, writable)
             }
-
-            /*  convert regular stream into object-mode stream  */
-            const wrapper = utils.createTransformStreamForWritableSide()
-            wrapper.pipe(this.stream as Stream.Writable)
-            this.stream = wrapper
         }
         else
             throw new Error(`invalid file mode "${this.params.mode}"`)

package/src/speechflow-node-xio-mqtt.ts

@@ -5,11 +5,11 @@
 */
 
 /*  standard dependencies  */
-import Stream                          from "node:stream"
+import Stream from "node:stream"
 
 /*  external dependencies  */
-import MQTT                            from "mqtt"
-import UUID                            from "pure-uuid"
+import MQTT   from "mqtt"
+import UUID   from "pure-uuid"
 
 /*  internal dependencies  */
 import SpeechFlowNode, { SpeechFlowChunk } from "./speechflow-node"
@@ -39,12 +39,6 @@ export default class SpeechFlowNodeMQTT extends SpeechFlowNode {
             type:       { type: "string", pos: 6, val: "text", match: /^(?:audio|text)$/ }
         })
 
-        /*  logical parameter sanity check  */
-        if ((this.params.mode === "w" || this.params.mode === "rw") && this.params.topicWrite === "")
-            throw new Error("writing to MQTT requires a topicWrite parameter")
-        if ((this.params.mode === "r" || this.params.mode === "rw") && this.params.topicRead === "")
-            throw new Error("reading from MQTT requires a topicRead parameter")
-
         /*  declare node input/output format  */
         if (this.params.mode === "rw") {
             this.input  = this.params.type
@@ -62,6 +56,14 @@ export default class SpeechFlowNodeMQTT extends SpeechFlowNode {
 
     /*  open node  */
     async open () {
+        /*  logical parameter sanity check  */
+        if (this.params.url === "")
+            throw new Error("required parameter \"url\" has to be given")
+        if ((this.params.mode === "w" || this.params.mode === "rw") && this.params.topicWrite === "")
+            throw new Error("writing to MQTT requires a topicWrite parameter")
+        if ((this.params.mode === "r" || this.params.mode === "rw") && this.params.topicRead === "")
+            throw new Error("reading from MQTT requires a topicRead parameter")
+
         /*  connect remotely to a MQTT broker  */
         this.broker = MQTT.connect(this.params.url, {
             protocolId:      "MQTT",

package/src/speechflow-node-xio-websocket.ts

@@ -5,7 +5,7 @@
 */
 
 /*  standard dependencies  */
-import Stream                          from "node:stream"
+import Stream from "node:stream"
 
 /*  external dependencies  */
 import ws                              from "ws"
@@ -36,12 +36,6 @@ export default class SpeechFlowNodeWebsocket extends SpeechFlowNode {
             type:    { type: "string", val: "text", match: /^(?:audio|text)$/ }
         })
 
-        /*  sanity check usage  */
-        if (this.params.listen !== "" && this.params.connect !== "")
-            throw new Error("Websocket node cannot listen and connect at the same time")
-        else if (this.params.listen === "" && this.params.connect === "")
-            throw new Error("Websocket node requires either listen or connect mode")
-
         /*  declare node input/output format  */
         if (this.params.mode === "rw") {
             this.input  = this.params.type
@@ -59,6 +53,12 @@ export default class SpeechFlowNodeWebsocket extends SpeechFlowNode {
 
     /*  open node  */
     async open () {
+        /*  sanity check usage  */
+        if (this.params.listen !== "" && this.params.connect !== "")
+            throw new Error("Websocket node cannot listen and connect at the same time")
+        else if (this.params.listen === "" && this.params.connect === "")
+            throw new Error("Websocket node requires either listen or connect mode")
+
         if (this.params.listen !== "") {
             /*  listen locally on a Websocket port  */
             const url = new URL(this.params.listen)

package/src/speechflow-node.ts

@@ -7,6 +7,8 @@
 /*  standard dependencies  */
 import Events from "node:events"
 import Stream from "node:stream"
+
+/*  external dependencies  */
 import { DateTime, Duration } from "luxon"
 
 /*  the definition of a single payload chunk passed through the SpeechFlow nodes  */
@@ -16,7 +18,8 @@ export class SpeechFlowChunk {
         public timestampEnd:   Duration,
         public kind:           "intermediate" | "final",
         public type:           "audio" | "text",
-        public payload:        Buffer | string
+        public payload:        Buffer | string,
+        public meta            = new Map<string, any>()
     ) {}
     clone () {
         let payload: Buffer | string
@@ -29,7 +32,8 @@ export class SpeechFlowChunk {
             Duration.fromMillis(this.timestampEnd.toMillis()),
             this.kind,
             this.type,
-            payload
+            payload,
+            new Map(this.meta)
         )
     }
 }
@@ -168,6 +172,11 @@ export default class SpeechFlowNode extends Events.EventEmitter {
         this.emit("log", level, msg, data)
     }
 
+    /*  default implementation for status operation  */
+    async status (): Promise<{ [ key: string ]: string | number }> {
+        return {}
+    }
+
     /*  default implementation for open/close operations  */
     async open  () {}
     async close () {}

package/src/speechflow-utils.ts

@@ -4,14 +4,17 @@
 **  Licensed under GPL 3.0 <https://spdx.org/licenses/GPL-3.0-only>
 */
 
-/*  external dependencies  */
+/*  standard dependencies  */
 import Stream                 from "node:stream"
 import { EventEmitter }       from "node:events"
+
+/*  external dependencies  */
 import { DateTime, Duration } from "luxon"
 import CBOR                   from "cbor2"
+import * as IntervalTree      from "node-interval-tree"
 
 /*  internal dependencies  */
-import { SpeechFlowChunk } from "./speechflow-node"
+import { SpeechFlowChunk }    from "./speechflow-node"
 
 /*  calculate duration of an audio buffer  */
 export function audioBufferDuration (
@@ -68,6 +71,10 @@ export function createTransformStreamForWritableSide () {
         transform (chunk: SpeechFlowChunk, encoding, callback) {
             this.push(chunk.payload)
             callback()
+        },
+        final (callback) {
+            this.push(null)
+            callback()
         }
     })
 }
@@ -90,6 +97,10 @@ export function createTransformStreamForReadableSide (type: "text" | "audio", ge
             const obj = new SpeechFlowChunk(start, end, "final", type, chunk)
             this.push(obj)
             callback()
+        },
+        final (callback) {
+            this.push(null)
+            callback()
         }
     })
 }
@@ -252,6 +263,7 @@ export class QueuePointer<T extends QueueElement> extends EventEmitter {
         private queue: Queue<T>
     ) {
         super()
+        this.setMaxListeners(100)
     }
 
     /*  positioning operations  */
@@ -379,6 +391,10 @@ export class QueuePointer<T extends QueueElement> extends EventEmitter {
 export class Queue<T extends QueueElement> extends EventEmitter {
     public elements: T[] = []
     private pointers = new Map<string, QueuePointer<T>>()
+    constructor () {
+        super()
+        this.setMaxListeners(100)
+    }
     pointerUse (name: string): QueuePointer<T> {
         if (!this.pointers.has(name))
             this.pointers.set(name, new QueuePointer<T>(name, this))
@@ -405,3 +421,66 @@ export class Queue<T extends QueueElement> extends EventEmitter {
     }
 }
 
+/*  utility class for wrapping a custom stream into a regular Transform stream  */
+export class StreamWrapper extends Stream.Transform {
+    private foreignStream: any
+    constructor (foreignStream: any, options: Stream.TransformOptions = {}) {
+        options.readableObjectMode = true
+        options.writableObjectMode = true
+        super(options)
+        this.foreignStream = foreignStream
+        this.foreignStream.on("data", (chunk: any) => {
+            this.push(chunk)
+        })
+        this.foreignStream.on("error", (err: Error) => {
+            this.emit("error", err)
+        })
+        this.foreignStream.on("end", () => {
+            this.push(null)
+        })
+    }
+    _transform (chunk: any, encoding: BufferEncoding, callback: Stream.TransformCallback): void {
+        try {
+            const canContinue = this.foreignStream.write(chunk)
+            if (canContinue)
+                callback()
+            else
+                this.foreignStream.once("drain", callback)
+        }
+        catch (err) {
+            callback(err as Error)
+        }
+    }
+    _flush (callback: Stream.TransformCallback): void {
+        try {
+            if (typeof this.foreignStream.end === "function")
+                this.foreignStream.end()
+            callback()
+        }
+        catch (err) {
+            callback(err as Error)
+        }
+    }
+}
+
+/*  meta store  */
+interface TimeStoreInterval<T> extends IntervalTree.Interval {
+    item: T
+}
+export class TimeStore<T> extends EventEmitter {
+    private tree = new IntervalTree.IntervalTree<TimeStoreInterval<T>>()
+    store (start: Duration, end: Duration, item: T): void {
+        this.tree.insert({ low: start.toMillis(), high: end.toMillis(), item })
+    }
+    fetch (start: Duration, end: Duration): T[] {
+        const intervals = this.tree.search(start.toMillis(), end.toMillis())
+        return intervals.map((interval) => interval.item)
+    }
+    prune (_before: Duration): void {
+        const before = _before.toMillis()
+        const intervals = this.tree.search(0, before - 1)
+        for (const interval of intervals)
+            if (interval.low < before && interval.high < before)
+                this.tree.remove(interval)
+    }
+}