speechflow 1.7.1 → 2.0.1

package/speechflow-cli/src/speechflow-node-t2a-amazon.ts

@@ -9,6 +9,7 @@ import Stream from "node:stream"
 
 /*  external dependencies  */
 import { getStreamAsBuffer } from "get-stream"
+import { Duration }          from "luxon"
 import SpeexResampler        from "speex-resampler"
 import {
     PollyClient, SynthesizeSpeechCommand,
@@ -25,9 +26,9 @@ export default class SpeechFlowNodeT2AAmazon extends SpeechFlowNode {
     public static name = "t2a-amazon"
 
     /*  internal state  */
-    private client: PollyClient | null = null
-    private closing = false
+    private client:    PollyClient    | null = null
     private resampler: SpeexResampler | null = null
+    private closing                          = false
 
     /*  construct node  */
     constructor (id: string, cfg: { [ id: string ]: any }, opts: { [ id: string ]: any }, args: any[]) {
@@ -129,22 +130,43 @@ export default class SpeechFlowNodeT2AAmazon extends SpeechFlowNode {
                 }
                 if (Buffer.isBuffer(chunk.payload))
                     callback(new Error("invalid chunk payload type"))
-                else if (chunk.payload.length > 0) {
+                else if (chunk.payload === "")
+                    callback()
+                else {
+                    let processTimeout: ReturnType<typeof setTimeout> | null = setTimeout(() => {
+                        processTimeout = null
+                        callback(new Error("AWS Polly API timeout"))
+                    }, 60 * 1000)
+                    const clearProcessTimeout = () => {
+                        if (processTimeout !== null) {
+                            clearTimeout(processTimeout)
+                            processTimeout = null
+                        }
+                    }
                     self.log("debug", `send data (${chunk.payload.length} bytes): "${chunk.payload}"`)
                     textToSpeech(chunk.payload as string).then((buffer) => {
-                        if (self.closing)
-                            throw new Error("stream destroyed during processing")
+                        if (self.closing) {
+                            clearProcessTimeout()
+                            callback(new Error("stream destroyed during processing"))
+                            return
+                        }
+                        /*  calculate actual audio duration from PCM buffer size  */
+                        const durationMs = util.audioBufferDuration(buffer,
+                            self.config.audioSampleRate, self.config.audioBitDepth) * 1000
+
+                        /*  create new chunk with recalculated timestamps  */
                         const chunkNew = chunk.clone()
-                        chunkNew.type = "audio"
-                        chunkNew.payload = buffer
+                        chunkNew.type         = "audio"
+                        chunkNew.payload      = buffer
+                        chunkNew.timestampEnd = Duration.fromMillis(chunkNew.timestampStart.toMillis() + durationMs)
+                        clearProcessTimeout()
                         this.push(chunkNew)
                         callback()
                     }).catch((error: unknown) => {
-                        callback(util.ensureError(error, "failed to send to AWS Polly"))
+                        clearProcessTimeout()
+                        callback(util.ensureError(error, "AWS Polly processing failed"))
                     })
                 }
-                else
-                    callback()
             },
             final (callback) {
                 callback()

package/speechflow-cli/src/speechflow-node-t2a-elevenlabs.ts

@@ -24,8 +24,8 @@ export default class SpeechFlowNodeT2AElevenlabs extends SpeechFlowNode {
 
     /*  internal state  */
     private elevenlabs: ElevenLabs.ElevenLabsClient | null = null
-    private closing = false
-    private resampler: SpeexResampler | null = null
+    private resampler:  SpeexResampler              | null = null
+    private closing                                        = false
 
     /*  construct node  */
     constructor (id: string, cfg: { [ id: string ]: any }, opts: { [ id: string ]: any }, args: any[]) {
@@ -131,8 +131,8 @@ export default class SpeechFlowNodeT2AElevenlabs extends SpeechFlowNode {
             })
         }
 
-        /*  establish resampler from ElevenLabs's maximum 24Khz
-            output to our standard audio sample rate (48KHz)  */
+        /*  establish resampler from ElevenLabs's tier-dependent
+            output sample rate to our standard audio sample rate (48KHz)  */
         this.resampler = new SpeexResampler(1, maxSampleRate, this.config.audioSampleRate, 7)
 
         /*  create transform stream and connect it to the ElevenLabs API  */
@@ -147,6 +147,8 @@ export default class SpeechFlowNodeT2AElevenlabs extends SpeechFlowNode {
                     callback(new Error("stream already destroyed"))
                 else if (Buffer.isBuffer(chunk.payload))
                     callback(new Error("invalid chunk payload type"))
+                else if (chunk.payload === "")
+                    callback()
                 else {
                     let processTimeout: ReturnType<typeof setTimeout> | null = setTimeout(() => {
                         processTimeout = null

package/speechflow-cli/src/speechflow-node-t2a-google.ts

@@ -0,0 +1,203 @@
+/*
+**  SpeechFlow - Speech Processing Flow Graph
+**  Copyright (c) 2024-2025 Dr. Ralf S. Engelschall <rse@engelschall.com>
+**  Licensed under GPL 3.0 <https://spdx.org/licenses/GPL-3.0-only>
+*/
+
+/*  standard dependencies  */
+import Stream from "node:stream"
+
+/*  external dependencies  */
+import * as GoogleTTS from "@google-cloud/text-to-speech"
+import { Duration }   from "luxon"
+import SpeexResampler from "speex-resampler"
+import * as arktype   from "arktype"
+
+/*  internal dependencies  */
+import SpeechFlowNode, { SpeechFlowChunk } from "./speechflow-node"
+import * as util                           from "./speechflow-util"
+
+/*  SpeechFlow node for Google Cloud text-to-speech conversion  */
+export default class SpeechFlowNodeT2AGoogle extends SpeechFlowNode {
+    /*  declare official node name  */
+    public static name = "t2a-google"
+
+    /*  internal state  */
+    private client:    GoogleTTS.TextToSpeechClient | null = null
+    private resampler: SpeexResampler               | null = null
+    private closing                                        = false
+
+    /*  construct node  */
+    constructor (id: string, cfg: { [ id: string ]: any }, opts: { [ id: string ]: any }, args: any[]) {
+        super(id, cfg, opts, args)
+
+        /*  declare node configuration parameters  */
+        this.configure({
+            key:      { type: "string",         val: process.env.SPEECHFLOW_GOOGLE_KEY ?? "" },
+            voice:    { type: "string", pos: 0, val: "en-US-Neural2-J" },
+            language: { type: "string", pos: 1, val: "en-US" },
+            speed:    { type: "number", pos: 2, val: 1.0, match: (n: number) => n >=  0.25 && n <=  4.0 },
+            pitch:    { type: "number", pos: 3, val: 0.0, match: (n: number) => n >= -20.0 && n <= 20.0 }
+        })
+
+        /*  validate API key  */
+        if (this.params.key === "")
+            throw new Error("Google Cloud API credentials JSON key is required")
+
+        /*  declare node input/output format  */
+        this.input  = "text"
+        this.output = "audio"
+    }
+
+    /*  one-time status of node  */
+    async status () {
+        return {}
+    }
+
+    /*  open node  */
+    async open () {
+        /*  clear destruction flag  */
+        this.closing = false
+
+        /*  instantiate Google TTS client  */
+        const data = util.run("Google Cloud API credentials key", () =>
+            JSON.parse(this.params.key))
+        const credentials = util.importObject("Google Cloud API credentials key",
+            data,
+            arktype.type({
+                project_id:   "string",
+                private_key:  "string",
+                client_email: "string"
+            })
+        )
+        this.client = new GoogleTTS.TextToSpeechClient({
+            credentials: {
+                private_key:  credentials.private_key,
+                client_email: credentials.client_email
+            },
+            projectId: credentials.project_id
+        })
+
+        /*  establish resampler from Google TTS's output sample rate
+            to our standard audio sample rate (48KHz)  */
+        const googleSampleRate = 24000 /*  Google TTS outputs 24kHz for LINEAR16  */
+        this.resampler = new SpeexResampler(1, googleSampleRate, this.config.audioSampleRate, 7)
+
+        /*  perform text-to-speech operation with Google Cloud TTS API  */
+        const textToSpeech = async (text: string) => {
+            this.log("info", `Google TTS: send text "${text}"`)
+            const [ response ] = await this.client!.synthesizeSpeech({
+                input: { text },
+                voice: {
+                    languageCode: this.params.language,
+                    name:         this.params.voice
+                },
+                audioConfig: {
+                    audioEncoding:   "LINEAR16",
+                    sampleRateHertz: googleSampleRate,
+                    speakingRate:    this.params.speed,
+                    pitch:           this.params.pitch
+                }
+            })
+            if (!response.audioContent)
+                throw new Error("no audio content returned from Google TTS")
+
+            /*  convert response to buffer  */
+            const buffer = Buffer.isBuffer(response.audioContent)
+                ? response.audioContent
+                : Buffer.from(response.audioContent)
+            this.log("info", `Google TTS: received audio (buffer length: ${buffer.byteLength})`)
+
+            /*  resample from Google's sample rate to our standard rate  */
+            const bufferResampled = this.resampler!.processChunk(buffer)
+            this.log("info", `Google TTS: forwarding resampled audio (buffer length: ${bufferResampled.byteLength})`)
+            return bufferResampled
+        }
+
+        /*  create transform stream and connect it to the Google TTS API  */
+        const self = this
+        this.stream = new Stream.Transform({
+            writableObjectMode: true,
+            readableObjectMode: true,
+            decodeStrings:      false,
+            highWaterMark:      1,
+            async transform (chunk: SpeechFlowChunk, encoding, callback) {
+                if (self.closing)
+                    callback(new Error("stream already destroyed"))
+                else if (Buffer.isBuffer(chunk.payload))
+                    callback(new Error("invalid chunk payload type"))
+                else if (chunk.payload === "")
+                    callback()
+                else {
+                    let processTimeout: ReturnType<typeof setTimeout> | null = setTimeout(() => {
+                        processTimeout = null
+                        callback(new Error("Google TTS API timeout"))
+                    }, 60 * 1000)
+                    const clearProcessTimeout = () => {
+                        if (processTimeout !== null) {
+                            clearTimeout(processTimeout)
+                            processTimeout = null
+                        }
+                    }
+                    try {
+                        if (self.closing) {
+                            clearProcessTimeout()
+                            callback(new Error("stream destroyed during processing"))
+                            return
+                        }
+                        const buffer = await textToSpeech(chunk.payload as string)
+                        if (self.closing) {
+                            clearProcessTimeout()
+                            callback(new Error("stream destroyed during processing"))
+                            return
+                        }
+
+                        /*  calculate actual audio duration from PCM buffer size  */
+                        const durationMs = util.audioBufferDuration(buffer,
+                            self.config.audioSampleRate, self.config.audioBitDepth) * 1000
+
+                        /*  create new chunk with recalculated timestamps  */
+                        const chunkNew = chunk.clone()
+                        chunkNew.type         = "audio"
+                        chunkNew.payload      = buffer
+                        chunkNew.timestampEnd = Duration.fromMillis(chunkNew.timestampStart.toMillis() + durationMs)
+                        clearProcessTimeout()
+                        this.push(chunkNew)
+                        callback()
+                    }
+                    catch (error) {
+                        clearProcessTimeout()
+                        callback(util.ensureError(error, "Google TTS processing failed"))
+                    }
+                }
+            },
+            final (callback) {
+                callback()
+            }
+        })
+    }
+
+    /*  close node  */
+    async close () {
+        /*  indicate closing  */
+        this.closing = true
+
+        /*  shutdown stream  */
+        if (this.stream !== null) {
+            await util.destroyStream(this.stream)
+            this.stream = null
+        }
+
+        /*  destroy resampler  */
+        if (this.resampler !== null)
+            this.resampler = null
+
+        /*  destroy Google TTS client  */
+        if (this.client !== null) {
+            await this.client.close().catch((error) => {
+                this.log("warning", `error closing Google TTS client: ${error}`)
+            })
+            this.client = null
+        }
+    }
+}

package/speechflow-cli/src/speechflow-node-t2a-kokoro.ts

@@ -9,6 +9,7 @@ import Stream from "node:stream"
 
 /*  external dependencies  */
 import { KokoroTTS }  from "kokoro-js"
+import { Duration }   from "luxon"
 import SpeexResampler from "speex-resampler"
 
 /*  internal dependencies  */
@@ -21,9 +22,9 @@ export default class SpeechFlowNodeT2AKokoro extends SpeechFlowNode {
     public static name = "t2a-kokoro"
 
     /*  internal state  */
-    private kokoro: KokoroTTS | null = null
-    private closing = false
+    private kokoro:    KokoroTTS      | null = null
     private resampler: SpeexResampler | null = null
+    private closing                          = false
 
     /*  construct node  */
     constructor (id: string, cfg: { [ id: string ]: any }, opts: { [ id: string ]: any }, args: any[]) {
@@ -122,9 +123,7 @@ export default class SpeechFlowNodeT2AKokoro extends SpeechFlowNode {
             }
 
             /*  resample audio samples from PCM/I16/24Khz to PCM/I16/48KHz  */
-            const buffer2 = this.resampler!.processChunk(buffer1)
-
-            return buffer2
+            return this.resampler!.processChunk(buffer1)
         }
 
         /*  create transform stream and connect it to the Kokoro API  */
@@ -139,18 +138,42 @@ export default class SpeechFlowNodeT2AKokoro extends SpeechFlowNode {
                     callback(new Error("stream already destroyed"))
                 else if (Buffer.isBuffer(chunk.payload))
                     callback(new Error("invalid chunk payload type"))
+                else if (chunk.payload === "")
+                    callback()
                 else {
+                    let processTimeout: ReturnType<typeof setTimeout> | null = setTimeout(() => {
+                        processTimeout = null
+                        callback(new Error("Kokoro TTS timeout"))
+                    }, 60 * 1000)
+                    const clearProcessTimeout = () => {
+                        if (processTimeout !== null) {
+                            clearTimeout(processTimeout)
+                            processTimeout = null
+                        }
+                    }
                     text2speech(chunk.payload).then((buffer) => {
-                        if (self.closing)
-                            throw new Error("stream destroyed during processing")
+                        if (self.closing) {
+                            clearProcessTimeout()
+                            callback(new Error("stream destroyed during processing"))
+                            return
+                        }
                         self.log("info", `Kokoro: received audio (buffer length: ${buffer.byteLength})`)
+
+                        /*  calculate actual audio duration from PCM buffer size  */
+                        const durationMs = util.audioBufferDuration(buffer,
+                            self.config.audioSampleRate, self.config.audioBitDepth) * 1000
+
+                        /*  create new chunk with recalculated timestamps  */
                         const chunkNew = chunk.clone()
-                        chunkNew.type = "audio"
-                        chunkNew.payload = buffer
+                        chunkNew.type         = "audio"
+                        chunkNew.payload      = buffer
+                        chunkNew.timestampEnd = Duration.fromMillis(chunkNew.timestampStart.toMillis() + durationMs)
+                        clearProcessTimeout()
                         this.push(chunkNew)
                         callback()
                     }).catch((error: unknown) => {
-                        callback(util.ensureError(error))
+                        clearProcessTimeout()
+                        callback(util.ensureError(error, "Kokoro processing failed"))
                     })
                 }
             },

package/speechflow-cli/src/speechflow-node-t2a-openai.ts

@@ -0,0 +1,176 @@
+/*
+**  SpeechFlow - Speech Processing Flow Graph
+**  Copyright (c) 2024-2025 Dr. Ralf S. Engelschall <rse@engelschall.com>
+**  Licensed under GPL 3.0 <https://spdx.org/licenses/GPL-3.0-only>
+*/
+
+/*  standard dependencies  */
+import Stream from "node:stream"
+
+/*  external dependencies  */
+import OpenAI         from "openai"
+import { Duration }   from "luxon"
+import SpeexResampler from "speex-resampler"
+
+/*  internal dependencies  */
+import SpeechFlowNode, { SpeechFlowChunk } from "./speechflow-node"
+import * as util                           from "./speechflow-util"
+
+/*  SpeechFlow node for OpenAI text-to-speech conversion  */
+export default class SpeechFlowNodeT2AOpenAI extends SpeechFlowNode {
+    /*  declare official node name  */
+    public static name = "t2a-openai"
+
+    /*  internal state  */
+    private openai:    OpenAI         | null = null
+    private resampler: SpeexResampler | null = null
+    private closing                          = false
+
+    /*  construct node  */
+    constructor (id: string, cfg: { [ id: string ]: any }, opts: { [ id: string ]: any }, args: any[]) {
+        super(id, cfg, opts, args)
+
+        /*  declare node configuration parameters  */
+        this.configure({
+            key:      { type: "string", val: process.env.SPEECHFLOW_OPENAI_KEY },
+            api:      { type: "string", val: "https://api.openai.com/v1", match: /^https?:\/\/.+/ },
+            voice:    { type: "string", val: "alloy",  pos: 0, match: /^(?:alloy|echo|fable|onyx|nova|shimmer)$/ },
+            model:    { type: "string", val: "tts-1",  pos: 1, match: /^(?:tts-1|tts-1-hd)$/ },
+            speed:    { type: "number", val: 1.0,      pos: 2, match: (n: number) => n >= 0.25 && n <= 4.0 }
+        })
+
+        /*  sanity check parameters  */
+        if (!this.params.key)
+            throw new Error("OpenAI API key not configured")
+
+        /*  declare node input/output format  */
+        this.input  = "text"
+        this.output = "audio"
+    }
+
+    /*  one-time status of node  */
+    async status () {
+        return {}
+    }
+
+    /*  open node  */
+    async open () {
+        /*  clear destruction flag  */
+        this.closing = false
+
+        /*  establish OpenAI API connection  */
+        this.openai = new OpenAI({
+            baseURL: this.params.api,
+            apiKey:  this.params.key,
+            timeout: 60000
+        })
+
+        /*  establish resampler from OpenAI's 24Khz PCM output
+            to our standard audio sample rate (48KHz)  */
+        this.resampler = new SpeexResampler(1, 24000, this.config.audioSampleRate, 7)
+
+        /*  perform text-to-speech operation with OpenAI API  */
+        const textToSpeech = async (text: string) => {
+            this.log("info", `OpenAI TTS: send text "${text}"`)
+            const response = await this.openai!.audio.speech.create({
+                model:           this.params.model,
+                voice:           this.params.voice,
+                input:           text,
+                response_format: "pcm",
+                speed:           this.params.speed
+            })
+
+            /*  convert response to buffer (PCM 24kHz, 16-bit, little-endian)  */
+            const arrayBuffer = await response.arrayBuffer()
+            const buffer = Buffer.from(arrayBuffer)
+            this.log("info", `OpenAI TTS: received audio (buffer length: ${buffer.byteLength})`)
+
+            /*  resample from 24kHz to 48kHz  */
+            const bufferResampled = this.resampler!.processChunk(buffer)
+            this.log("info", `OpenAI TTS: forwarding resampled audio (buffer length: ${bufferResampled.byteLength})`)
+            return bufferResampled
+        }
+
+        /*  create transform stream and connect it to the OpenAI API  */
+        const self = this
+        this.stream = new Stream.Transform({
+            writableObjectMode: true,
+            readableObjectMode: true,
+            decodeStrings:      false,
+            highWaterMark:      1,
+            async transform (chunk: SpeechFlowChunk, encoding, callback) {
+                if (self.closing)
+                    callback(new Error("stream already destroyed"))
+                else if (Buffer.isBuffer(chunk.payload))
+                    callback(new Error("invalid chunk payload type"))
+                else if (chunk.payload === "")
+                    callback()
+                else {
+                    let processTimeout: ReturnType<typeof setTimeout> | null = setTimeout(() => {
+                        processTimeout = null
+                        callback(new Error("OpenAI TTS API timeout"))
+                    }, 60 * 1000)
+                    const clearProcessTimeout = () => {
+                        if (processTimeout !== null) {
+                            clearTimeout(processTimeout)
+                            processTimeout = null
+                        }
+                    }
+                    try {
+                        if (self.closing) {
+                            clearProcessTimeout()
+                            callback(new Error("stream destroyed during processing"))
+                            return
+                        }
+                        const buffer = await textToSpeech(chunk.payload as string)
+                        if (self.closing) {
+                            clearProcessTimeout()
+                            callback(new Error("stream destroyed during processing"))
+                            return
+                        }
+
+                        /*  calculate actual audio duration from PCM buffer size  */
+                        const durationMs = util.audioBufferDuration(buffer,
+                            self.config.audioSampleRate, self.config.audioBitDepth) * 1000
+
+                        /*  create new chunk with recalculated timestamps  */
+                        const chunkNew = chunk.clone()
+                        chunkNew.type         = "audio"
+                        chunkNew.payload      = buffer
+                        chunkNew.timestampEnd = Duration.fromMillis(chunkNew.timestampStart.toMillis() + durationMs)
+                        clearProcessTimeout()
+                        this.push(chunkNew)
+                        callback()
+                    }
+                    catch (error) {
+                        clearProcessTimeout()
+                        callback(util.ensureError(error, "OpenAI TTS processing failed"))
+                    }
+                }
+            },
+            final (callback) {
+                callback()
+            }
+        })
+    }
+
+    /*  close node  */
+    async close () {
+        /*  indicate closing  */
+        this.closing = true
+
+        /*  shutdown stream  */
+        if (this.stream !== null) {
+            await util.destroyStream(this.stream)
+            this.stream = null
+        }
+
+        /*  destroy resampler  */
+        if (this.resampler !== null)
+            this.resampler = null
+
+        /*  destroy OpenAI API  */
+        if (this.openai !== null)
+            this.openai = null
+    }
+}