speechflow 0.9.9 → 1.0.0

package/src/speechflow-utils.ts

@@ -31,6 +31,33 @@ export function audioBufferDuration (
     return totalSamples / sampleRate
 }
 
+/*  calculate duration of an audio array  */
+export function audioArrayDuration (
+    arr: Float32Array,
+    sampleRate   = 48000,
+    channels     = 1
+) {
+    const totalSamples = arr.length / channels
+    return totalSamples / sampleRate
+}
+
+/*  helper function: convert Buffer in PCM/I16 to Float32Array in PCM/F32 format  */
+export function convertBufToF32 (buf: Buffer, littleEndian = true) {
+    const dataView = new DataView(buf.buffer)
+    const arr = new Float32Array(buf.length / 2)
+    for (let i = 0; i < arr.length; i++)
+        arr[i] = dataView.getInt16(i * 2, littleEndian) / 32768
+    return arr
+}
+
+/*  helper function: convert Float32Array in PCM/F32 to Buffer in PCM/I16 format  */
+export function convertF32ToBuf (arr: Float32Array) {
+    const int16Array = new Int16Array(arr.length)
+    for (let i = 0; i < arr.length; i++)
+        int16Array[i] = Math.max(-32768, Math.min(32767, Math.round(arr[i] * 32768)))
+    return Buffer.from(int16Array.buffer)
+}
+
 /*  create a Duplex/Transform stream which has
     object-mode on Writable side and buffer/string-mode on Readable side  */
 export function createTransformStreamForWritableSide () {
@@ -210,3 +237,171 @@ export class DoubleQueue<T0, T1> extends EventEmitter {
         })
     }
 }
+
+/*  queue element */
+export type QueueElement = { type: string }
+
+/*  queue pointer  */
+export class QueuePointer<T extends QueueElement> extends EventEmitter {
+    /*  internal state  */
+    private index = 0
+
+    /*  construction  */
+    constructor (
+        private name: string,
+        private queue: Queue<T>
+    ) {
+        super()
+    }
+
+    /*  positioning operations  */
+    maxPosition () {
+        return this.queue.elements.length
+    }
+    position (index?: number): number {
+        if (index !== undefined) {
+            this.index = index
+            if (this.index < 0)
+                this.index = 0
+            else if (this.index >= this.queue.elements.length)
+                this.index = this.queue.elements.length
+            this.emit("position", this.index)
+        }
+        return this.index
+    }
+    walk (num: number) {
+        if (num > 0) {
+            for (let i = 0; i < num && this.index < this.queue.elements.length; i++)
+                this.index++
+            this.emit("position", { start: this.index })
+        }
+        else if (num < 0) {
+            for (let i = 0; i < Math.abs(num) && this.index > 0; i++)
+                this.index--
+            this.emit("position", { start: this.index })
+        }
+    }
+    walkForwardUntil (type: T["type"]) {
+        while (this.index < this.queue.elements.length
+            && this.queue.elements[this.index].type !== type)
+            this.index++
+        this.emit("position", { start: this.index })
+    }
+    walkBackwardUntil (type: T["type"]) {
+        while (this.index > 0
+            && this.queue.elements[this.index].type !== type)
+            this.index--
+        this.emit("position", { start: this.index })
+    }
+
+    /*  search operations  */
+    searchForward (type: T["type"]) {
+        let position = this.index
+        while (position < this.queue.elements.length
+            && this.queue.elements[position].type !== type)
+            position++
+        this.emit("search", { start: this.index, end: position })
+        return position
+    }
+    searchBackward (type: T["type"]) {
+        let position = this.index
+        while (position > 0
+            && this.queue.elements[position].type !== type)
+            position--
+        this.emit("search", { start: position, end: this.index })
+        return position
+    }
+
+    /*  reading operations  */
+    peek (position?: number) {
+        if (position === undefined)
+            position = this.index
+        else {
+            if (position < 0)
+                position = 0
+            else if (position > this.queue.elements.length)
+                position = this.queue.elements.length
+        }
+        const element = this.queue.elements[position]
+        this.queue.emit("read", { start: position, end: position })
+        return element
+    }
+    read () {
+        const element = this.queue.elements[this.index]
+        if (this.index < this.queue.elements.length)
+            this.index++
+        this.queue.emit("read", { start: this.index - 1, end: this.index - 1 })
+        return element
+    }
+    slice (size?: number) {
+        let slice: T[]
+        const start = this.index
+        if (size !== undefined) {
+            if (size < 0)
+                size = 0
+            else if (size > this.queue.elements.length - this.index)
+                size = this.queue.elements.length - this.index
+            slice = this.queue.elements.slice(this.index, size)
+            this.index += size
+        }
+        else {
+            slice = this.queue.elements.slice(this.index)
+            this.index = this.queue.elements.length
+        }
+        this.queue.emit("read", { start, end: this.index })
+        return slice
+    }
+
+    /*  writing operations  */
+    touch () {
+        if (this.index >= this.queue.elements.length)
+            throw new Error("cannot touch after last element")
+        this.queue.emit("write", { start: this.index, end: this.index + 1 })
+    }
+    append (element: T) {
+        this.queue.elements.push(element)
+        this.index = this.queue.elements.length
+        this.queue.emit("write", { start: this.index - 1, end: this.index - 1 })
+    }
+    insert (element: T) {
+        this.queue.elements.splice(this.index++, 0, element)
+        this.queue.emit("write", { start: this.index - 1, end: this.index })
+    }
+    delete () {
+        if (this.index >= this.queue.elements.length)
+            throw new Error("cannot delete after last element")
+        this.queue.elements.splice(this.index, 1)
+        this.queue.emit("write", { start: this.index, end: this.index })
+    }
+}
+
+/*  queue  */
+export class Queue<T extends QueueElement> extends EventEmitter {
+    public elements: T[] = []
+    private pointers = new Map<string, QueuePointer<T>>()
+    pointerUse (name: string): QueuePointer<T> {
+        if (!this.pointers.has(name))
+            this.pointers.set(name, new QueuePointer<T>(name, this))
+        return this.pointers.get(name)!
+    }
+    pointerDelete (name: string): void {
+        if (!this.pointers.has(name))
+            throw new Error("pointer not exists")
+        this.pointers.delete(name)
+    }
+    trim (): void {
+        /*  determine minimum pointer position  */
+        let min = this.elements.length
+        for (const pointer of this.pointers.values())
+            if (min > pointer.position())
+                min = pointer.position()
+
+        /*  trim the maximum amount of first elements  */
+        this.elements.splice(0, min)
+
+        /*  shift all pointers  */
+        for (const pointer of this.pointers.values())
+            pointer.position(pointer.position() - min)
+    }
+}
+

package/src/speechflow.ts

@@ -8,6 +8,11 @@
 import path                     from "node:path"
 import Stream                   from "node:stream"
 import { EventEmitter }         from "node:events"
+import http                     from "node:http"
+import * as HAPI                from "@hapi/hapi"
+import WebSocket                from "ws"
+import HAPIWebSocket            from "hapi-plugin-websocket"
+import HAPIHeader               from "hapi-plugin-header"
 
 /*  external dependencies  */
 import { DateTime }             from "luxon"
@@ -20,6 +25,7 @@ import objectPath               from "object-path"
 import installedPackages        from "installed-packages"
 import dotenvx                  from "@dotenvx/dotenvx"
 import syspath                  from "syspath"
+import * as arktype             from "arktype"
 
 /*  internal dependencies  */
 import SpeechFlowNode           from "./speechflow-node"
@@ -28,6 +34,15 @@ import pkg                      from "../package.json"
 /*  central CLI context  */
 let cli: CLIio | null = null
 
+type wsPeerCtx = {
+    peer: string
+}
+type wsPeerInfo = {
+    ctx:        wsPeerCtx
+    ws:         WebSocket
+    req:        http.IncomingMessage
+}
+
 /*  establish asynchronous environment  */
 ;(async () => {
     /*  determine system paths  */
@@ -45,6 +60,8 @@ let cli: CLIio | null = null
             "[-h|--help] " +
             "[-V|--version] " +
             "[-v|--verbose <level>] " +
+            "[-a|--address <ip-address>] " +
+            "[-p|--port <tcp-port>] " +
             "[-C|--cache <directory>] " +
             "[-e|--expression <expression>] " +
             "[-f|--file <file>] " +
@@ -68,6 +85,24 @@ let cli: CLIio | null = null
             default:  "warning",
             describe: "level for verbose logging ('none', 'error', 'warning', 'info', 'debug')"
         })
+        .option("a", {
+            alias:    "address",
+            type:     "string",
+            array:    false,
+            coerce,
+            nargs:    1,
+            default:  "0.0.0.0",
+            describe: "IP address for REST/WebSocket API"
+        })
+        .option("p", {
+            alias:    "port",
+            type:     "number",
+            array:    false,
+            coerce,
+            nargs:    1,
+            default:  8484,
+            describe: "TCP port for REST/WebSocket API"
+        })
         .option("C", {
             alias:    "cache",
             type:     "string",
@@ -186,6 +221,9 @@ let cli: CLIio | null = null
     const pkgsI = [
         "./speechflow-node-a2a-ffmpeg.js",
         "./speechflow-node-a2a-wav.js",
+        "./speechflow-node-a2a-mute.js",
+        "./speechflow-node-a2a-meter.js",
+        "./speechflow-node-a2a-vad.js",
         "./speechflow-node-a2t-deepgram.js",
         "./speechflow-node-t2a-elevenlabs.js",
         "./speechflow-node-t2a-kokoro.js",
@@ -193,7 +231,6 @@ let cli: CLIio | null = null
         "./speechflow-node-t2t-openai.js",
         "./speechflow-node-t2t-ollama.js",
         "./speechflow-node-t2t-transformers.js",
-        "./speechflow-node-t2t-opus.js",
         "./speechflow-node-t2t-subtitle.js",
         "./speechflow-node-t2t-format.js",
         "./speechflow-node-x2x-trace.js",
@@ -237,9 +274,9 @@ let cli: CLIio | null = null
             cli!.log("debug", msg)
         }
     })
-    let nodenum = 1
     const variables = { argv: args._, env: process.env }
     const graphNodes = new Set<SpeechFlowNode>()
+    const nodeNums = new Map<typeof SpeechFlowNode, number>()
     const cfg = {
         audioChannels:     1,
         audioBitDepth:     16,
@@ -275,17 +312,19 @@ let cli: CLIio | null = null
                     throw new Error(`unknown node "${id}"`)
                 let node: SpeechFlowNode
                 try {
-                    node = new nodes[id](`${id}[${nodenum}]`, cfg, opts, args)
+                    let num = nodeNums.get(nodes[id]) ?? 0
+                    nodeNums.set(nodes[id], ++num)
+                    const name = num === 1 ? id : `${id}:${num}`
+                    node = new nodes[id](name, cfg, opts, args)
                 }
                 catch (err) {
                     /*  fatal error  */
                     if (err instanceof Error)
-                        cli!.log("error", `creation of "${id}[${nodenum}]" node failed: ${err.message}`)
+                        cli!.log("error", `creation of <${id}> node failed: ${err.message}`)
                     else
-                        cli!.log("error", `creation of "${id}"[${nodenum}] node failed: ${err}`)
+                        cli!.log("error", `creation of <${id}> node failed: ${err}`)
                     process.exit(1)
                 }
-                nodenum++
                 const params = Object.keys(node.params)
                     .map((key) => `${key}: ${JSON.stringify(node.params[key])}`).join(", ")
                 cli!.log("info", `create node "${node.id}" (${params})`)
@@ -403,6 +442,150 @@ let cli: CLIio | null = null
         })
     }
 
+    /*  define external request/response structure  */
+    const requestValidator = arktype.type({
+        request: "string",
+        node:    "string",
+        args:    "unknown[]"
+    })
+
+    /*  forward external request to target node in graph  */
+    const consumeExternalRequest = async (_req: any) => {
+        const req = requestValidator(_req)
+        if (req instanceof arktype.type.errors)
+            throw new Error(`invalid request: ${req.summary}`)
+        if (req.request !== "COMMAND")
+            throw new Error("invalid external request (command expected)")
+        const name = req.node as string
+        const args = req.args as any[]
+        const foundNode = Array.from(graphNodes).find((node) => node.id === name)
+        if (foundNode === undefined)
+            cli!.log("warning", `external request failed: no such node <${name}>`)
+        else {
+            await foundNode.receiveRequest(args).catch((err: Error) => {
+                cli!.log("warning", `external request to node <${name}> failed: ${err}`)
+                throw new Error(`external request to node <${name}> failed: ${err}`)
+            })
+        }
+    }
+
+    /*  establish REST/WebSocket API  */
+    const wsPeers = new Map<string, wsPeerInfo>()
+    const hapi = new HAPI.Server({
+        address: args.a,
+        port:    args.p
+    })
+    await hapi.register({ plugin: HAPIHeader, options: { Server: `${pkg.name}/${pkg.version}` } })
+    await hapi.register({ plugin: HAPIWebSocket })
+    hapi.events.on("response", (request: HAPI.Request) => {
+        let protocol = `HTTP/${request.raw.req.httpVersion}`
+        const ws = request.websocket()
+        if (ws.mode === "websocket") {
+            const wsVersion = (ws.ws as any).protocolVersion ??
+                request.headers["sec-websocket-version"] ?? "13?"
+            protocol = `WebSocket/${wsVersion}+${protocol}`
+        }
+        const msg =
+            "remote="   + request.info.remoteAddress + ", " +
+            "method="   + request.method.toUpperCase() + ", " +
+            "url="      + request.url.pathname + ", " +
+            "protocol=" + protocol + ", " +
+            "response=" + ("statusCode" in request.response ? request.response.statusCode : "<unknown>")
+        cli!.log("info", `HAPI: request: ${msg}`)
+    })
+    hapi.events.on({ name: "request", channels: [ "error" ] }, (request: HAPI.Request, event: HAPI.RequestEvent, tags: { [key: string]: true }) => {
+        if (event.error instanceof Error)
+            cli!.log("error", `HAPI: request-error: ${event.error.message}`)
+        else
+            cli!.log("error", `HAPI: request-error: ${event.error}`)
+    })
+    hapi.events.on("log", (event: HAPI.LogEvent, tags: { [key: string]: true }) => {
+        if (tags.error) {
+            const err = event.error
+            if (err instanceof Error)
+                cli!.log("error", `HAPI: log: ${err.message}`)
+            else
+                cli!.log("error", `HAPI: log: ${err}`)
+        }
+    })
+    hapi.route({
+        method: "GET",
+        path:   "/api/{req}/{node}/{params*}",
+        options: {
+        },
+        handler: (request: HAPI.Request, h: HAPI.ResponseToolkit) => {
+            const peer = request.info.remoteAddress
+            const req = {
+                request: request.params.req,
+                node:    request.params.node,
+                args:    (request.params.params as string ?? "").split("/").filter((seg) => seg !== "")
+            }
+            cli!.log("info", `HAPI: peer ${peer}: GET: ${JSON.stringify(req)}`)
+            return consumeExternalRequest(req).then(() => {
+                return h.response({ response: "OK" }).code(200)
+            }).catch((err) => {
+                return h.response({ response: "ERROR", data: err.message }).code(417)
+            })
+        }
+    })
+    hapi.route({
+        method: "POST",
+        path:   "/api",
+        options: {
+            payload: {
+                output: "data",
+                parse:  true,
+                allow:  "application/json"
+            },
+            plugins: {
+                websocket: {
+                    autoping: 30 * 1000,
+                    connect: (args: any) => {
+                        const ctx: wsPeerCtx            = args.ctx
+                        const ws:  WebSocket            = args.ws
+                        const req: http.IncomingMessage = args.req
+                        const peer = `${req.socket.remoteAddress}:${req.socket.remotePort}`
+                        ctx.peer = peer
+                        wsPeers.set(peer, { ctx, ws, req })
+                        cli!.log("info", `HAPI: WebSocket: connect: peer ${peer}`)
+                    },
+                    disconnect: (args: any) => {
+                        const ctx: wsPeerCtx = args.ctx
+                        const peer = ctx.peer
+                        wsPeers.delete(peer)
+                        cli!.log("info", `HAPI: WebSocket: disconnect: peer ${peer}`)
+                    }
+                }
+            }
+        },
+        handler: (request: HAPI.Request, h: HAPI.ResponseToolkit) => {
+            /*  on WebSocket message transfer  */
+            const peer = request.info.remoteAddress
+            const req = requestValidator(request.payload)
+            if (req instanceof arktype.type.errors)
+                return h.response({ response: "ERROR", data: `invalid request: ${req.summary}` }).code(417)
+            cli!.log("info", `HAPI: peer ${peer}: POST: ${JSON.stringify(req)}`)
+            return consumeExternalRequest(req).then(() => {
+                return h.response({ response: "OK" }).code(200)
+            }).catch((err: Error) => {
+                return h.response({ response: "ERROR", data: err.message }).code(417)
+            })
+        }
+    })
+    await hapi.start()
+    cli!.log("info", `HAPI: started REST/WebSocket network service: http://${args.address}:${args.port}`)
+
+    /*  hook for sendResponse method of nodes  */
+    for (const node of graphNodes) {
+        node.on("send-response", (args: any[]) => {
+            const data = JSON.stringify({ response: "NOTIFY", node: node.id, args })
+            for (const [ peer, info ] of wsPeers.entries()) {
+                cli!.log("info", `HAPI: peer ${peer}: ${data}`)
+                info.ws.send(data)
+            }
+        })
+    }
+
     /*  start of internal stream processing  */
     cli!.log("info", "**** everything established -- stream processing in SpeechFlow graph starts ****")
 
@@ -417,6 +600,10 @@ let cli: CLIio | null = null
         else
             cli!.log("warning", `**** received signal ${signal} -- shutting down service ****`)
 
+        /*  shutdown HAPI service  */
+        cli!.log("info", `HAPI: stopping REST/WebSocket network service: http://${args.address}:${args.port}`)
+        await hapi.stop()
+
         /*  graph processing: PASS 1: disconnect node streams  */
         for (const node of graphNodes) {
             if (node.stream === null) {

package/dst/speechflow-node-deepgram.d.ts

@@ -1,12 +0,0 @@
-import SpeechFlowNode from "./speechflow-node";
-export default class SpeechFlowNodeDeepgram extends SpeechFlowNode {
-    static name: string;
-    private dg;
-    constructor(id: string, cfg: {
-        [id: string]: any;
-    }, opts: {
-        [id: string]: any;
-    }, args: any[]);
-    open(): Promise<void>;
-    close(): Promise<void>;
-}