speechflow 0.9.4 → 0.9.7

package/dst/speechflow-node-whisper.js

@@ -0,0 +1,604 @@
+"use strict";
+/*
+**  SpeechFlow - Speech Processing Flow Graph
+**  Copyright (c) 2024-2025 Dr. Ralf S. Engelschall <rse@engelschall.com>
+**  Licensed under GPL 3.0 <https://spdx.org/licenses/GPL-3.0-only>
+*/
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+/*  standard dependencies  */
+const node_os_1 = __importDefault(require("node:os"));
+const node_path_1 = __importDefault(require("node:path"));
+const node_events_1 = require("node:events");
+const node_stream_1 = __importDefault(require("node:stream"));
+const node_worker_threads_1 = require("node:worker_threads");
+const wavefile = __importStar(require("wavefile"));
+const vad_node_realtime_1 = require("@ericedouard/vad-node-realtime");
+/*  internal dependencies  */
+const speechflow_node_1 = __importDefault(require("./speechflow-node"));
+/*  audio stream queue pointer  */
+class AudioQueuePointer extends node_events_1.EventEmitter {
+    name;
+    queue;
+    /*  internal state  */
+    index = 0;
+    /*  construction  */
+    constructor(name, queue) {
+        super();
+        this.name = name;
+        this.queue = queue;
+    }
+    /*  positioning operations  */
+    maxPosition() {
+        return this.queue.elements.length;
+    }
+    position(index) {
+        if (index !== undefined) {
+            this.index = index;
+            if (this.index < 0)
+                this.index = 0;
+            else if (this.index >= this.queue.elements.length)
+                this.index = this.queue.elements.length;
+            this.emit("position", this.index);
+        }
+        return this.index;
+    }
+    walk(num) {
+        if (num > 0) {
+            for (let i = 0; i < num && this.index < this.queue.elements.length; i++)
+                this.index++;
+            this.emit("position", { start: this.index });
+        }
+        else if (num < 0) {
+            for (let i = 0; i < Math.abs(num) && this.index > 0; i++)
+                this.index--;
+            this.emit("position", { start: this.index });
+        }
+    }
+    walkForwardUntil(type) {
+        while (this.index < this.queue.elements.length
+            && this.queue.elements[this.index].type !== type)
+            this.index++;
+        this.emit("position", { start: this.index });
+    }
+    walkBackwardUntil(type) {
+        while (this.index > 0
+            && this.queue.elements[this.index].type !== type)
+            this.index--;
+        this.emit("position", { start: this.index });
+    }
+    /*  search operations  */
+    searchForward(type) {
+        let position = this.index;
+        while (position < this.queue.elements.length
+            && this.queue.elements[position].type !== type)
+            position++;
+        this.emit("search", { start: this.index, end: position });
+        return position;
+    }
+    searchBackward(type) {
+        let position = this.index;
+        while (position > 0
+            && this.queue.elements[position].type !== type)
+            position--;
+        this.emit("search", { start: position, end: this.index });
+    }
+    /*  reading operations  */
+    peek(position) {
+        if (position === undefined)
+            position = this.index;
+        else {
+            if (position < 0)
+                position = 0;
+            else if (position >= this.queue.elements.length)
+                position = this.queue.elements.length;
+        }
+        const element = this.queue.elements[position];
+        this.queue.emit("read", { start: position, end: position });
+        return element;
+    }
+    read() {
+        const element = this.queue.elements[this.index];
+        if (this.index < this.queue.elements.length)
+            this.index++;
+        this.queue.emit("read", { start: this.index - 1, end: this.index - 1 });
+        return element;
+    }
+    slice(size) {
+        let slice;
+        const start = this.index;
+        if (size !== undefined) {
+            slice = this.queue.elements.slice(this.index, size);
+            this.index += size;
+        }
+        else {
+            slice = this.queue.elements.slice(this.index);
+            this.index = this.queue.elements.length;
+        }
+        this.queue.emit("read", { start, end: this.index });
+        return slice;
+    }
+    /*  writing operations  */
+    append(element) {
+        this.queue.elements.push(element);
+        this.index = this.queue.elements.length;
+        this.queue.emit("write", { start: this.index - 1, end: this.index - 1 });
+    }
+    insert(element) {
+        this.queue.elements.splice(this.index++, 0, element);
+        this.queue.emit("write", { start: this.index - 1, end: this.index });
+    }
+    delete() {
+        if (this.index >= this.queue.elements.length)
+            throw new Error("cannot delete after last element");
+        this.queue.elements.splice(this.index, 1);
+        this.queue.emit("write", { start: this.index, end: this.index });
+    }
+}
+/*  audio stream queue  */
+class AudioQueue extends node_events_1.EventEmitter {
+    elements = [];
+    pointers = new Map();
+    pointerUse(name) {
+        if (!this.pointers.has(name))
+            this.pointers.set(name, new AudioQueuePointer(name, this));
+        return this.pointers.get(name);
+    }
+    pointerDelete(name) {
+        if (!this.pointers.has(name))
+            throw new Error("pointer not exists");
+        this.pointers.delete(name);
+    }
+    trim() {
+        /*  determine minimum pointer position  */
+        let min = this.elements.length;
+        for (const pointer of this.pointers.values())
+            if (min > pointer.position())
+                min = pointer.position();
+        /*  trim the maximum amount of first elements  */
+        this.elements.splice(0, min);
+        /*  shift all pointers  */
+        for (const pointer of this.pointers.values())
+            pointer.position(pointer.position() - min);
+    }
+}
+/*  transcription queue  */
+class TranscriptionQueue extends node_events_1.EventEmitter {
+    cacheDir;
+    model;
+    runtime;
+    log;
+    tasks = [];
+    timer = null;
+    busy = false;
+    worker = null;
+    constructor(cacheDir, model, runtime, log) {
+        super();
+        this.cacheDir = cacheDir;
+        this.model = model;
+        this.runtime = runtime;
+        this.log = log;
+        if (this.runtime === "auto") {
+            const platform = node_os_1.default.platform();
+            if (platform === "win32")
+                this.runtime = "onnx";
+            else if (platform === "darwin")
+                this.runtime = "ggml";
+            else
+                this.runtime = "onnx";
+        }
+    }
+    enqueue(task) {
+        /*  destroy previous tasks of same id  */
+        while (this.tasks.length > 0
+            && this.tasks[this.tasks.length - 1].id === task.id) {
+            this.log(`dropping existing queued request for ${task.type} transcription task #${task.id}`);
+            this.tasks.splice(this.tasks.length - 1, 1);
+        }
+        /*  add task  */
+        this.log(`enqueue request for ${task.type} transcription task #${task.id}`);
+        this.tasks.push(task);
+        this.dequeue();
+    }
+    dequeue() {
+        if (this.tasks.length === 0)
+            return;
+        if (!this.busy && this.worker !== null) {
+            this.busy = true;
+            const task = this.tasks.shift();
+            if (task !== undefined) {
+                this.log(`dequeue and send request for ${task.type} transcription task #${task.id}`);
+                this.worker.postMessage({ type: "task-request", task });
+            }
+        }
+    }
+    async start() {
+        this.log("start transcription service worker: BEGIN");
+        if (this.runtime === "ggml") {
+            const basedir = node_path_1.default.dirname(await require.resolve("smart-whisper/package.json"));
+            process.env.GGML_METAL_PATH_RESOURCES = node_path_1.default.resolve(basedir, "whisper.cpp/ggml/src");
+            console.log(process.env.GGML_METAL_PATH_RESOURCES);
+        }
+        const script = node_path_1.default.resolve(__dirname, `speechflow-node-whisper-${this.runtime}.js`);
+        this.worker = new node_worker_threads_1.Worker(script, { env: { ...process.env } });
+        this.worker.postMessage({
+            type: "open",
+            cacheDir: this.cacheDir,
+            model: this.model
+        });
+        this.worker.on("message", (response) => {
+            if (response.type === "log")
+                this.log(response.message);
+        });
+        await new Promise((resolve, reject) => {
+            let cb = null;
+            const cleanResolve = () => {
+                this.worker.off("message", cb);
+                resolve();
+            };
+            const cleanReject = (error) => {
+                this.worker.off("message", cb);
+                reject(error);
+            };
+            cb = (response) => {
+                if (response.type === "ok")
+                    cleanResolve();
+                else if (response.type === "error")
+                    cleanReject(new Error(response.message));
+            };
+            this.worker.on("message", cb);
+        });
+        this.worker.on("message", (response) => {
+            this.busy = false;
+            if (response.type === "error")
+                this.emit("error", response.message);
+            else if (response.type === "task-response") {
+                console.log(`receive response for task #${response.task.id}`);
+                this.emit("task", response.task);
+            }
+            this.dequeue();
+        });
+        if (this.timer !== null)
+            clearTimeout(this.timer);
+        this.timer = setInterval(() => {
+            this.dequeue();
+        }, 10);
+        this.log("start transcription service worker: END");
+    }
+    async stop() {
+        this.log("stop transcription service worker: BEGIN");
+        if (this.timer !== null) {
+            clearTimeout(this.timer);
+            this.timer = null;
+        }
+        if (this.worker !== null) {
+            this.worker.postMessage({ type: "close" });
+            await this.worker.terminate();
+            this.worker = null;
+        }
+        this.busy = false;
+        this.log("stop transcription service worker: END");
+    }
+}
+/*  SpeechFlow node for Whisper speech-to-text conversion  */
+class SpeechFlowNodeWhisper extends speechflow_node_1.default {
+    /*  declare official node name  */
+    static name = "whisper";
+    /*  OpenAI Whisper https://github.com/openai/whisper/  */
+    models = {
+        "v1-tiny": { version: "v1", released: "2022-09", paramsM: 39, vramGB: 1, speed: 10 },
+        "v1-base": { version: "v1", released: "2022-09", paramsM: 74, vramGB: 1, speed: 7 },
+        "v1-small": { version: "v1", released: "2022-09", paramsM: 244, vramGB: 2, speed: 4 },
+        "v1-medium": { version: "v1", released: "2022-09", paramsM: 769, vramGB: 5, speed: 2 },
+        "v2-large": { version: "v2", released: "2022-12", paramsM: 1550, vramGB: 10, speed: 1 },
+        "v3-large": { version: "v3", released: "2023-11", paramsM: 1550, vramGB: 10, speed: 1 },
+        "v3-large-turbo": { version: "v3", released: "2024-09", paramsM: 798, vramGB: 6, speed: 8 }
+    };
+    /*  internal state  */
+    transcriber = null;
+    vad = null;
+    queue = new AudioQueue();
+    queueRecv = this.queue.pointerUse("recv");
+    queueVAD = this.queue.pointerUse("vad");
+    queueSTT = this.queue.pointerUse("stt");
+    tqueue = null;
+    /*  construct node  */
+    constructor(id, cfg, opts, args) {
+        super(id, cfg, opts, args);
+        /*  declare node configuration parameters  */
+        const validModels = new RegExp(`^(?:${Object.keys(this.models).join("|")})$`);
+        this.configure({
+            language: { type: "string", val: "en", pos: 0, match: /^(?:en|de)$/ },
+            model: { type: "string", val: "v3-large-turbo", pos: 1, match: validModels },
+            runtime: { type: "string", val: "auto", pos: 2, match: /^(?:auto|onnx|ggml)$/ }
+        });
+        /*  sanity check model  */
+        if (this.models[this.params.model] === undefined)
+            throw new Error(`invalid OpenAI Whisper model "${this.params.model}`);
+        /*  declare node input/output format  */
+        this.input = "audio";
+        this.output = "text";
+    }
+    /*  open node  */
+    async open() {
+        /*  sanity check situation  */
+        if (this.config.audioBitDepth !== 16 || !this.config.audioLittleEndian)
+            throw new Error("Whisper node currently supports PCM-S16LE audio only");
+        /*  pass-through logging  */
+        const log = (level, msg) => {
+            this.log(level, msg);
+        };
+        /*  create queue for results  */
+        const queueOutput = new node_events_1.EventEmitter();
+        /*  internal processing constants  */
+        const sampleRateTarget = 16000;
+        const samplesPerVADFrame = 512; /* required for VAD v5 */
+        const minFramesPerSecond = Math.trunc(sampleRateTarget / samplesPerVADFrame) + 1;
+        /*  initialize the transcription pipeline  */
+        const model = this.models[this.params.model];
+        this.log("info", `loading OpenAI Whisper ${this.params.model} ` +
+            `(version: ${model.version}, released: ${model.released}, parameters: ${model.paramsM})`);
+        /*  transcribe a chunk of audio  */
+        this.tqueue = new TranscriptionQueue(this.config.cacheDir, this.params.model, this.params.runtime, (msg) => { this.log("info", msg); });
+        await this.tqueue.start();
+        this.tqueue.on("task", async (task) => {
+            // if (task.type === "intermediate")
+            //     return
+            this.log("info", `received ${task.type} transcription #${task.id}: "${task.text}"`);
+            // DEBUG
+            // const wav = new wavefile.WaveFile()
+            // wav.fromScratch(1, sampleRateTarget, "32f", task.audio)
+            // const data = wav.toBuffer()
+            // fs.writeFileSync(`chunk-out-${n++}.wav`, data)
+        });
+        /*  track audio queue element changes  */
+        let speechActive = false;
+        let speechStart = -1;
+        let speechEnd = -1;
+        let speechMinSeconds = 2;
+        this.queue.on("write", () => {
+            if (!speechActive) {
+                const position = this.queueSTT.searchForward("speech-start");
+                const element = this.queueSTT.peek(position);
+                if (element !== undefined && element.type === "speech-start") {
+                    this.queueSTT.position(position + 1);
+                    speechActive = true;
+                    speechStart = this.queueSTT.position();
+                    speechEnd = speechStart;
+                    speechMinSeconds = 2;
+                }
+            }
+            else {
+                speechEnd = this.queueSTT.searchForward("speech-end");
+                /*   determine number of speech and fill frames  */
+                let framesSpeech = 0;
+                for (let f = speechStart; f < speechEnd; f++) {
+                    const element = this.queueSTT.peek(f);
+                    if (element.type === "audio-frame")
+                        framesSpeech++;
+                }
+                let framesFilled = minFramesPerSecond - framesSpeech;
+                if (framesFilled < 0)
+                    framesFilled = 0;
+                /*  assemble all speech and fill frames  */
+                const assembleFrames = () => {
+                    const speech = new Float32Array((framesSpeech + framesFilled) * samplesPerVADFrame);
+                    let i = 0;
+                    for (let f = speechStart; f < speechEnd; f++) {
+                        const element = this.queueSTT.peek(f);
+                        if (element.type === "audio-frame")
+                            speech.set(element.data, samplesPerVADFrame * i++);
+                    }
+                    if (framesFilled > 0)
+                        speech.fill(0.0, i * samplesPerVADFrame, (i + framesFilled) * samplesPerVADFrame);
+                    // DEBUG
+                    // const wav = new wavefile.WaveFile()
+                    // wav.fromScratch(1, sampleRateTarget, "32f", speech)
+                    // const data = wav.toBuffer()
+                    // fs.writeFileSync(`chunk-speech-${m++}.wav`, data)
+                    return speech;
+                };
+                if (speechEnd === this.queueSTT.maxPosition()) {
+                    /*  intermediate transcription  */
+                    const duration = ((framesSpeech + framesFilled) * samplesPerVADFrame) / sampleRateTarget;
+                    if (duration >= speechMinSeconds) {
+                        /*  intermediate transcription of at least the next required minimum seconds  */
+                        const samples = assembleFrames();
+                        this.log("info", `trigger intermediate transcription (duration: ${duration.toFixed(1)}s)`);
+                        this.tqueue.enqueue({ id: speechStart, type: "intermediate", audio: samples, language: this.params.language });
+                        speechMinSeconds++;
+                    }
+                }
+                else {
+                    /*  final transcription  */
+                    const duration = ((framesSpeech + framesFilled) * samplesPerVADFrame) / sampleRateTarget;
+                    if (duration >= 1.0) {
+                        const samples = assembleFrames();
+                        this.log("info", `trigger final transcription (duration: ${duration.toFixed(1)}s)`);
+                        this.tqueue.enqueue({ id: speechStart, type: "final", audio: samples, language: this.params.language });
+                        this.queueSTT.position(speechEnd + 1);
+                    }
+                    else
+                        this.log("info", `skipping final transcription -- too short (duration: ${duration.toFixed(1)}s)`);
+                    speechActive = false;
+                }
+            }
+        });
+        /*  Voice Activity Detection (VAD)  */
+        this.vad = await vad_node_realtime_1.RealTimeVAD.new({
+            onSpeechStart: () => {
+                this.log("info", "VAD: speech start");
+                this.queueVAD.insert({ type: "speech-start" });
+            },
+            onSpeechEnd: (audio) => {
+                this.log("info", `VAD: speech end (samples: ${audio.length})`);
+                this.queueVAD.insert({ type: "speech-end", short: false });
+            },
+            onVADMisfire: () => {
+                this.log("info", "VAD: speech end (segment too short)");
+                this.queueVAD.insert({ type: "speech-end", short: true });
+            },
+            onFrameProcessed: () => {
+                this.queueVAD.walk(+1);
+            },
+            sampleRate: 16000,
+            model: "v5",
+            frameSamples: samplesPerVADFrame, /* (= 32ms: 512 frameSamples / 16000 sampleSize) */
+            positiveSpeechThreshold: 0.50,
+            negativeSpeechThreshold: 0.35,
+            minSpeechFrames: 4, /* (= 128ms: 4 x 512 frameSamples) */
+            redemptionFrames: 8, /* (= 256ms: 8 x 512 frameSamples) */
+            preSpeechPadFrames: 1, /* (= 32ms:  1 x 512 frameSamples) */
+        });
+        this.vad.start();
+        /*  provide Duplex stream and internally attach to Ollama API  */
+        const vad = this.vad;
+        const cfg = this.config;
+        const queueRecv = this.queueRecv;
+        let carrySamples = new Float32Array();
+        let endOfStream = false;
+        this.stream = new node_stream_1.default.Duplex({
+            writableObjectMode: true,
+            readableObjectMode: true,
+            decodeStrings: false,
+            /*  receive audio samples  */
+            write(chunk, encoding, callback) {
+                if (!Buffer.isBuffer(chunk))
+                    callback(new Error("expected audio input as Buffer chunks"));
+                else if (chunk.byteLength === 0)
+                    callback();
+                else {
+                    /*  convert audio samples from PCM/I16/48KHz to PCM/F32/16KHz  */
+                    const bufferToInt16Array = (buf) => {
+                        const dataView = new DataView(buf.buffer);
+                        const result = new Int16Array(buf.length / 2);
+                        for (let i = 0; i < result.length; i++)
+                            result[i] = dataView.getInt16(i * 2, cfg.audioLittleEndian);
+                        return result;
+                    };
+                    const wav = new wavefile.WaveFile();
+                    wav.fromScratch(cfg.audioChannels, cfg.audioSampleRate, String(cfg.audioBitDepth), bufferToInt16Array(chunk));
+                    wav.toBitDepth("32f");
+                    wav.toSampleRate(16000, { method: "cubic" });
+                    let data = wav.getSamples(false, Float32Array);
+                    /*  merge previous carry samples  */
+                    if (carrySamples.length > 0) {
+                        const merged = new Float32Array(carrySamples.length + data.length);
+                        merged.set(carrySamples);
+                        merged.set(data, carrySamples.length);
+                        data = merged;
+                        carrySamples = new Float32Array();
+                    }
+                    /*  DEBUG  */
+                    // const wav2 = new wavefile.WaveFile()
+                    // wav2.fromScratch(1, sampleRateTarget, "32f", data)
+                    // const data2 = wav.toBuffer()
+                    // fs.writeFileSync(`chunk-in-${k++}.wav`, data2)
+                    /*  queue audio samples as individual VAD-sized frames
+                        and in parallel send it into the Voice Activity Detection (VAD)  */
+                    const chunks = Math.trunc(data.length / samplesPerVADFrame);
+                    for (let i = 0; i < chunks; i++) {
+                        const frame = data.slice(i * samplesPerVADFrame, (i + 1) * samplesPerVADFrame);
+                        queueRecv.append({ type: "audio-frame", data: frame });
+                        vad.processAudio(frame);
+                    }
+                    /*  remember new carry samples  */
+                    const bulkLen = chunks * samplesPerVADFrame;
+                    carrySamples = data.slice(bulkLen);
+                    callback();
+                }
+            },
+            /*  send transcription texts  */
+            read(size) {
+                if (endOfStream)
+                    this.push(null);
+                else {
+                    queueOutput.once("text", (text) => {
+                        log("info", `Whisper: receive data (${text.length} bytes)`);
+                        this.push(text, cfg.textEncoding);
+                    });
+                }
+            },
+            /*  react on end of input  */
+            final(callback) {
+                if (carrySamples.length > 0) {
+                    /*  flush pending audio samples  */
+                    if (carrySamples.length < samplesPerVADFrame) {
+                        const merged = new Float32Array(samplesPerVADFrame);
+                        merged.set(carrySamples);
+                        merged.fill(0.0, carrySamples.length, samplesPerVADFrame);
+                        carrySamples = merged;
+                    }
+                    queueRecv.append({ type: "audio-frame", data: carrySamples });
+                    vad.processAudio(carrySamples);
+                    /*  give the processing a chance to still process the remaining samples  */
+                    setTimeout(() => {
+                        endOfStream = true;
+                        this.push(null);
+                        callback();
+                    }, 2000);
+                }
+                else {
+                    endOfStream = true;
+                    this.push(null);
+                    callback();
+                }
+            }
+        });
+    }
+    /*  close node  */
+    async close() {
+        /*  close stream  */
+        if (this.stream !== null) {
+            this.stream.destroy();
+            this.stream = null;
+        }
+        /*  close VAD  */
+        if (this.vad !== null) {
+            await this.vad.flush();
+            this.vad.destroy();
+            this.vad = null;
+        }
+        /*  close transcription queue  */
+        if (this.tqueue !== null) {
+            await this.tqueue.stop();
+            this.tqueue = null;
+        }
+    }
+}
+exports.default = SpeechFlowNodeWhisper;

package/dst/speechflow-node-x2x-trace.d.ts

@@ -0,0 +1,11 @@
+import SpeechFlowNode from "./speechflow-node";
+export default class SpeechFlowNodeTrace extends SpeechFlowNode {
+    static name: string;
+    constructor(id: string, cfg: {
+        [id: string]: any;
+    }, opts: {
+        [id: string]: any;
+    }, args: any[]);
+    open(): Promise<void>;
+    close(): Promise<void>;
+}