use-voice-control 0.1.0 → 0.1.2

package/speech/docs/README.md

@@ -0,0 +1,167 @@
+# Speech Library
+
+Unified text-to-speech API supporting Kokoro (default) and Deepgram providers.
+
+## Features
+
+- **Kokoro** (default): Fast, natural-sounding voices, runs on Node CPU
+- **Deepgram**: Requires Cloudflare AI binding, returns MP3
+
+## Usage
+
+### API Endpoint
+
+**POST** `/api/agent/voice`
+
+```json
+{
+  "text": "Hello world",
+  "provider": "kokoro",
+  "voice": "af_heart"
+}
+```
+
+**Request Body:**
+- `text` (required): Text to convert to speech (max 5000 chars)
+- `provider` (optional): `"kokoro"` (default) or `"deepgram"`
+- `voice` (optional): Voice ID (see below)
+
+**Response:** Audio file (WAV for Kokoro, MP3 for Deepgram)
+
+### Direct Library Usage
+
+```ts
+import { generateSpeech } from "@/lib/speech";
+
+// Kokoro (default)
+const audio = await generateSpeech({
+  text: "Hello world",
+  voice: "af_heart"
+});
+
+// Deepgram
+const audio = await generateSpeech({
+  text: "Hello world",
+  provider: "deepgram",
+  voice: "angus"
+});
+
+// Returns
+{
+  audio: ArrayBuffer,
+  contentType: "audio/wav" | "audio/mpeg"
+}
+```
+
+## Voices
+
+### Kokoro Voices (Default)
+
+**Female:**
+- `af_heart` - Default, warm and natural
+- `af_alloy` - Clear and professional
+- `af_aoede` - Soft and expressive
+- `af_bella` - Bright and friendly
+- `af_jessica` - Calm and steady
+- `af_nicole` - Smooth and confident
+- `af_river` - Cool and relaxed
+- `af_sarah` - Warm and approachable
+- `af_sky` - Light and airy
+
+**Male:**
+- `am_adam` - Strong and authoritative
+- `am_echo` - Deep and resonant
+- `am_fable` - Storytelling quality
+- `am_fenrir` - Bold and commanding
+- `am_liam` - Friendly and casual
+- `am_michael` - Professional and clear
+- `am_onyx` - Smooth and sophisticated
+
+### Deepgram Aura Voices
+
+- `angus` (default), `asteria`, `arcas`, `orion`, `orpheus`, `athena`
+- `luna`, `zeus`, `perseus`, `helios`, `hera`, `stella`
+
+## Architecture
+
+```
+lib/speech/
+├── index.ts       # Main API
+├── types.ts       # Shared types
+├── kokoro.ts      # Kokoro provider
+└── deepgram.ts    # Deepgram provider
+```
+
+### Model Loading
+
+Kokoro model is lazy-loaded once per server instance:
+- Model: `onnx-community/Kokoro-82M-v1.0-ONNX`
+- Quantization: `q8` (8-bit)
+- Device: `cpu`
+- First request triggers download (~82MB)
+- Subsequent requests reuse loaded model
+
+### Rate Limiting
+
+- Guests: 10 requests per day
+- Authenticated users: Unlimited
+- Enforced at API route level
+
+## Performance
+
+**Kokoro:**
+- Fast CPU inference
+- Natural prosody and intonation
+- WAV output (higher quality)
+- ~100-300ms generation time
+
+**Deepgram:**
+- Cloudflare Workers AI
+- Lower latency in edge locations
+- MP3 output (smaller size)
+- Requires CF binding
+
+## Error Handling
+
+```ts
+try {
+  const audio = await generateSpeech({ text: "..." });
+} catch (error) {
+  // "Text is required"
+  // "Unknown TTS provider: ..."
+  // "Cloudflare AI binding not available"
+}
+```
+
+## Migration from Old API
+
+**Before:**
+```json
+{
+  "text": "Hello",
+  "speaker": "angus"
+}
+```
+
+**After (same behavior):**
+```json
+{
+  "text": "Hello",
+  "provider": "deepgram",
+  "voice": "angus"
+}
+```
+
+**Now (Kokoro default):**
+```json
+{
+  "text": "Hello",
+  "voice": "af_heart"
+}
+```
+
+## References
+
+- [kokoro-js npm](https://www.npmjs.com/package/kokoro-js)
+- [Kokoro ONNX Model](https://huggingface.co/onnx-community/Kokoro-82M-v1.0-ONNX)
+- [Deepgram Aura](https://developers.cloudflare.com/workers-ai/models/deepgram-aura/)

package/speech/index.ts

@@ -0,0 +1,54 @@
+/**
+ * @fileoverview Unified text-to-speech API supporting Kokoro (default) and Deepgram
+ *
+ * Kokoro: Faster, more natural, runs on Node CPU
+ * Deepgram: Requires Cloudflare AI binding, MP3 output
+ */
+import type { TTSOptions, TTSResult } from "./types/types";
+import { generateKokoroSpeech } from "./core/kokoro";
+import { generateDeepgramSpeech } from "./core/deepgram";
+
+export * from "./types/types";
+
+/**
+ * Generate speech from text using the specified provider
+ *
+ * @param options - TTS configuration
+ * @returns Audio buffer and content type
+ *
+ * @example
+ * ```ts
+ * // Use Kokoro (default)
+ * const audio = await generateSpeech({
+ *   text: "Hello world",
+ *   voice: "af_heart"
+ * });
+ *
+ * // Use Deepgram
+ * const audio = await generateSpeech({
+ *   text: "Hello world",
+ *   provider: "deepgram",
+ *   voice: "angus"
+ * });
+ * ```
+ */
+export async function generateSpeech(
+  options: TTSOptions
+): Promise<TTSResult> {
+  const { text, provider = "kokoro", voice = "af_heart" } = options;
+
+  if (!text || typeof text !== "string" || text.trim().length === 0) {
+    throw new Error("Text is required");
+  }
+
+  switch (provider) {
+    case "kokoro":
+      return generateKokoroSpeech(text, voice);
+
+    case "deepgram":
+      return generateDeepgramSpeech(text, voice);
+
+    default:
+      throw new Error(`Unknown TTS provider: ${provider}`);
+  }
+}

package/speech/legacy/conversation.js

@@ -0,0 +1,150 @@
+import { SpeechToText } from "./stt.js";
+import { textToSpeech, ttsModelReadyPromise } from "./tts.js";
+import { processStreamingText } from "./sentence-detector.js";
+import { displayConversation } from "./ui.js";
+
+const $ = document.querySelector.bind(document);
+
+export class Conversation {
+    constructor() {
+        this.modelsReady = false;
+        this.speechToText = new SpeechToText();
+        this.initModels();
+        this.conversationHistory = [
+            {
+                role: "system",
+                content: "placeholder"
+            }
+        ];
+    }
+    async initModels() {
+        try {
+            await Promise.all([
+                this.speechToText.modelReadyPromise,
+                ttsModelReadyPromise
+            ]);
+            this.modelsReady = true;
+            const toggleButton = document.getElementById('toggleRecording');
+            toggleButton.disabled = false;
+            toggleButton.textContent = 'Start Recording';
+            const recordingStatus = document.getElementById('recordingStatus');
+            recordingStatus.textContent = 'Models loaded. Click "Start Recording" to begin';
+        } catch (error) {
+            console.error('Error initializing models:', error);
+            const recordingStatus = document.getElementById('recordingStatus');
+            recordingStatus.textContent = 'Error loading models: ' + error.message;
+        }
+    }
+
+    startRecording() {
+        if (this.modelsReady) {
+            this.speechToText.startRecording();
+        } else {
+            console.warn('Cannot start recording: models are not yet loaded');
+            const recordingStatus = document.getElementById('recordingStatus');
+            recordingStatus.textContent = 'Please wait for models to finish loading...';
+        }
+    }
+
+    async stopRecording() {
+        let text = await this.speechToText.stopRecording();
+        console.log('Transcription:', text)
+        $('#transcriptionStatus').textContent = text;
+        this.conversationHistory.push({
+            role: "user",
+            content: text
+        });
+        await this.sendConversationHistory();
+    }
+
+    async sendConversationHistory() {
+        let serverUrl = $('#serverUrl').value;
+        let system_prompt = $('#systemPrompt').value;
+
+        this.conversationHistory[0].content = system_prompt;
+
+        const response = await fetch(serverUrl, {
+            method: "POST",
+            headers: { "Content-Type": "application/json" },
+            body: JSON.stringify({
+                stream: true,
+                "messages": this.conversationHistory
+            })
+        });
+
+        const decoder = new TextDecoder("utf-8");
+        const reader = response.body.getReader();
+        let accumulatedText = "";
+        const voiceSelect = document.getElementById('voiceSelect');
+        const voiceId = (voiceSelect && voiceSelect.value) ? voiceSelect.value : "af_heart";
+
+
+        while (true) {
+            const { value, done } = await reader.read();
+            if (done) break;
+            const chunk = decoder.decode(value, { stream: true });
+            //console.log(chunk);
+
+            for (const line of chunk.split("\n")) {
+                if (line.startsWith("data: ")) {
+                    const payload = line.slice(6).trim();
+                    if (payload === "[DONE]") {
+                        if (accumulatedText.trim().length > 0) {
+                            textToSpeech(accumulatedText, voiceId);
+                            this.conversationHistory.push({
+                                "role": "assistant",
+                                "content": accumulatedText
+                            });
+                        }
+                        
+                        console.log("\n[Stream complete]");
+                        displayConversation(this.conversationHistory);
+                        console.log("response", accumulatedText);
+                        return;
+                    }
+                    const json = JSON.parse(payload);
+                    const content = json.choices[0].delta.content || "";
+                    const result = processStreamingText(accumulatedText, content);
+
+                    // If we have complete sentences, speak them
+                    if (result.sentences.length > 0) {
+                        result.sentences.forEach(sentence => {
+                            textToSpeech(sentence, voiceId);
+                            this.conversationHistory.push({
+                                "role": "assistant",
+                                "content": sentence
+                            });
+                        });
+
+                        accumulatedText = result.remainder;
+                    } else {
+                        accumulatedText = result.remainder;
+                    }
+                }
+            }
+        }
+        // This code will never be reached when streaming is enabled
+        // because of the return statement in the "[DONE]" handling block
+        console.timeEnd('LLM Processing');
+
+        // This section is for non-streaming mode
+        try {
+            const data = await response.json();
+            const response_text = data.choices[0].message.content;
+            this.conversationHistory.push({
+                "role": "assistant",
+                "content": response_text
+            });
+            console.log("response", response_text);
+        } catch (error) {
+            console.log("Error parsing response as JSON, likely already processed as a stream.");
+        }
+
+
+        // This was the original call, but now we're streaming per sentence
+        // textToSpeech(reponse, "af_heart");
+    }
+}
+
+
+

package/speech/legacy/main.js

@@ -0,0 +1,56 @@
+import { AudioPlayer } from "./AudioPlayer.js";
+
+const my_worker = new Worker(new URL("./worker.js", import.meta.url), { type: "module" });
+
+let audioPlayer = new AudioPlayer();
+
+// Create a promise that will resolve when the TTS model is ready
+export const ttsModelReadyPromise = new Promise((resolve) => {
+    window.ttsModelReadyResolve = resolve;
+});
+
+const onMessageReceived = async (e) => {
+    switch (e.data.status) {
+        case "ready":
+            console.log("TTS model loaded successfully");
+            // Resolve the promise to indicate the TTS model is ready
+            window.ttsModelReadyResolve();
+            break;
+
+        case "device":
+            console.log(e.data);
+            break;
+
+        case "progress":
+            break;
+
+        case "stream":
+            console.log("audioPlayer.queueAudio", e.data);
+            audioPlayer.queueAudio(e.data.audio);
+            break;
+    }
+};
+
+const onErrorReceived = (e) => { console.error("Worker error:", e); };
+
+my_worker.addEventListener("message", onMessageReceived);
+my_worker.addEventListener("error", onErrorReceived);
+
+export function textToSpeech(text, voice) {
+    if (!text || !voice) {
+        console.error("Text and voice parameters are required for text-to-speech.");
+        return;
+    }
+
+    text = text.replaceAll("*", "");
+
+    // Remove any special Markdown formatting for better speech
+    text = text.replace(/\*\*(.*?)\*\*/g, '$1'); // Bold
+    text = text.replace(/\*(.*?)\*/g, '$1');     // Italic
+    text = text.replace(/`(.*?)`/g, '$1');       // Code
+    text = text.replace(/~~(.*?)~~/g, '$1');     // Strikethrough
+
+    my_worker.postMessage({ type: "generate", text: text, voice: voice });
+
+    return text;
+}

package/speech/legacy/stt.js

@@ -0,0 +1,161 @@
+import { pipeline } from "https://cdn.jsdelivr.net/npm/@huggingface/transformers@3.5.1/dist/transformers.min.js";
+
+import { convertAudioBufferToWav, resampleAudio, applyAudioGain } from "./audio-utils.js";
+
+let mediaRecorder;
+let audioChunks = [];
+let mode;
+let wav;
+
+async function detectWebGPU() {
+    try {
+        const adapter = await navigator.gpu.requestAdapter();
+        return !!adapter;
+    } catch (e) {
+        return false;
+    }
+}
+
+export class SpeechToText {
+
+    constructor() {
+        // Create a promise that will resolve when the model is loaded
+        this.modelReadyPromise = new Promise((resolve, reject) => {
+            this._modelReadyResolve = resolve;
+            this._modelReadyReject = reject;
+        });
+        this.initialize();
+    }
+
+
+    async initialize() {
+        try {
+            const isWebGPUSupported = await detectWebGPU();
+            const device = isWebGPUSupported ? "webgpu" : "wasm";
+            const dtype = isWebGPUSupported ? "fp32" : "q8";
+            const options = {
+                device: device,
+                dtype: dtype,
+                quantized: !isWebGPUSupported,
+            };
+
+            this.transcriber = await pipeline(
+                'automatic-speech-recognition',
+                'onnx-community/moonshine-base-ONNX', // 'onnx-community/whisper-large-v3-turbo', 
+                options
+            );
+            console.log('Speech-to-text model loaded successfully');
+            // Resolve the promise to indicate the model is ready
+            this._modelReadyResolve();
+        } catch (error) {
+            console.error('Error loading speech-to-text model:', error);
+            this._modelReadyReject(error);
+        }
+    }
+
+
+    startRecording() {
+        navigator.mediaDevices.getUserMedia({ audio: true })
+            .then(stream => {
+                mode = "WAV";
+                const options = { mimeType: 'audio/wav' };
+                try {
+                    mediaRecorder = new MediaRecorder(stream, options);
+                } catch (e) {
+                    //console.info('WAV format not supported, using default format.');
+                    mediaRecorder = new MediaRecorder(stream);
+                    mode = "OGG";
+                }
+                audioChunks = [];
+                mediaRecorder.ondataavailable = event => {
+                    audioChunks.push(event.data);
+                };
+                mediaRecorder.start();
+            })
+            .catch(error => {
+                console.error('Error accessing microphone:', error);
+                recordingStatus.textContent = 'Error accessing microphone: ' + error.message;
+            });
+
+    }
+
+
+    async stopRecording() {
+        return new Promise((resolve, reject) => {
+            if (mediaRecorder && mediaRecorder.state !== 'inactive') {
+                mediaRecorder.stop();
+                mediaRecorder.onstop = async () => {
+                    try {
+                        mediaRecorder.stream.getTracks().forEach(track => track.stop());
+                        // Create blob with WAV MIME type
+                        let type = { type: 'audio/webm;codecs=opus' }
+                        if (mode === "WAV") {
+                            type = { type: 'audio/wav' };
+                        }
+
+                        if (mode === "WAV") {
+                            console.log('WAV format is already selected.');
+                        } else {
+                            console.info('Converting audio to WAV format...');
+                            const audioContext = new AudioContext();
+                            let audioBlob = new Blob(audioChunks, { type: type });
+                            const arrayBuffer = await audioBlob.arrayBuffer();
+                            //console.log("arrayBuffer", arrayBuffer)
+                            const audioBuffer = await audioContext.decodeAudioData(arrayBuffer);
+                            //console.log('Audio buffer decoded:', audioBuffer);
+                            //wav = audioBuffer.getChannelData(0); // Float32Array of first channel
+                            //console.log(wav)
+                            wav = convertAudioBufferToWav(audioBuffer);
+                        }
+
+                        //wav = await resampleAudio(wav, 16000);
+                        //const output = await transcriber(wav);
+
+                        let wavBlob = new Blob([wav], { type: 'audio/wav' });
+                        let wavBlobUrl = URL.createObjectURL(wavBlob);
+
+                        const playbackStatus = document.getElementById('playbackStatus');
+                        const audioPlayback = document.getElementById('audioPlayback');
+                        audioPlayback.src = wavBlobUrl;
+                        audioPlayback.style.display = 'block';
+                        playbackStatus.textContent = 'Audio ready for playback:';
+
+                        let output = await this.transcriber(wavBlobUrl);                        
+                        if (output.text === undefined || output.text.length == 0) {
+                            console.log('Trying transcription again 1...');
+
+                            // Load audio and apply gain
+                            wav = await convertAudioBufferToWav(await (async () => {
+                                const audioContext = new AudioContext();
+                                const arrayBuffer = await (await fetch(wavBlobUrl)).arrayBuffer();
+                                const audioBuffer = await audioContext.decodeAudioData(arrayBuffer);
+                                // Use the applyAudioGain function to increase volume
+                                return applyAudioGain(audioBuffer, 1.5);
+                            })());
+                            wavBlob = new Blob([wav], { type: 'audio/wav' });
+                            wavBlobUrl = URL.createObjectURL(wavBlob);
+                            audioPlayback.src = wavBlobUrl;
+                            output = await this.transcriber(wavBlobUrl);
+                        }
+
+                        if (output.text === undefined || output.text.length == 0) {
+                            console.log('Trying transcription again 2...');
+                            output = await this.transcriber(wavBlobUrl);
+                        }
+
+                        console.log('Transcription output 1:', output);
+                        resolve(output.text);
+
+                    } catch (error) {
+                        console.error('Error during transcription:', error);
+                        reject(error);
+                    }
+                };
+            } else {
+                reject(new Error("MediaRecorder is not active"));
+            }
+        });
+    }
+}
+
+

package/speech/legacy/tts.js

@@ -0,0 +1,56 @@
+import { AudioPlayer } from "./AudioPlayer.js";
+
+const my_worker = new Worker(new URL("./worker.js", import.meta.url), { type: "module" });
+
+let audioPlayer = new AudioPlayer();
+
+// Create a promise that will resolve when the TTS model is ready
+export const ttsModelReadyPromise = new Promise((resolve) => {
+    window.ttsModelReadyResolve = resolve;
+});
+
+const onMessageReceived = async (e) => {
+    switch (e.data.status) {
+        case "ready":
+            console.log("TTS model loaded successfully");
+            // Resolve the promise to indicate the TTS model is ready
+            window.ttsModelReadyResolve();
+            break;
+
+        case "device":
+            console.log(e.data);
+            break;
+
+        case "progress":
+            break;
+
+        case "stream":
+            console.log("audioPlayer.queueAudio", e.data);
+            audioPlayer.queueAudio(e.data.audio);
+            break;
+    }
+};
+
+const onErrorReceived = (e) => { console.error("Worker error:", e); };
+
+my_worker.addEventListener("message", onMessageReceived);
+my_worker.addEventListener("error", onErrorReceived);
+
+export function textToSpeech(text, voice) {
+    if (!text || !voice) {
+        console.error("Text and voice parameters are required for text-to-speech.");
+        return;
+    }
+
+    text = text.replaceAll("*", "");
+    
+    // Remove any special Markdown formatting for better speech
+    text = text.replace(/\*\*(.*?)\*\*/g, '$1'); // Bold
+    text = text.replace(/\*(.*?)\*/g, '$1');     // Italic
+    text = text.replace(/`(.*?)`/g, '$1');       // Code
+    text = text.replace(/~~(.*?)~~/g, '$1');     // Strikethrough
+    
+    my_worker.postMessage({ type: "generate", text: text, voice: voice });
+    
+    return text;
+}