use-voice-control 0.1.0 → 0.1.2

package/speech/legacy/worker.js

@@ -0,0 +1,71 @@
+import { KokoroTTS } from "./kokoro.js";
+import { splitTextSmart } from "./semantic-split.js";
+
+async function detectWebGPU() {
+    try {
+        const adapter = await navigator.gpu.requestAdapter();
+        return !!adapter;
+    } catch (e) {
+        return false;
+    }
+}
+
+const device = (await detectWebGPU()) ? "webgpu" : "wasm";
+self.postMessage({ status: "device", device });
+
+let model_id = "onnx-community/Kokoro-82M-v1.0-ONNX";
+
+const chunkQueue = [];
+let isProcessing = false;
+
+async function processQueue() {
+    if (isProcessing || chunkQueue.length === 0) return;
+    
+    isProcessing = true;
+    const { chunk, voice } = chunkQueue.shift();
+    
+    try {
+        console.log("Processing chunk", chunk);
+        const audio = await tts.generate(chunk, { voice });
+        let ab = audio.audio.buffer;
+        console.log("generate done");
+        self.postMessage({ status: "stream", audio: ab, text: chunk }, [ab]);
+    } catch (error) {
+        console.error("Error processing chunk:", error);
+        self.postMessage({ status: "error", error: error.message });
+    }
+    
+    isProcessing = false;
+    
+    if (chunkQueue.length > 0) {
+        processQueue();
+    } else {
+        self.postMessage({ status: "complete" });
+    }
+}
+
+const tts = await KokoroTTS.from_pretrained(model_id, {
+    dtype: device === "wasm" ? "q8" : "fp32",
+    device,
+    progress_callback: (progress) => {
+        // Report progress to main thread
+        self.postMessage({ status: "progress", progress });
+    }
+}).catch((e) => {
+    self.postMessage({ status: "error", error: e.message });
+    throw e;
+});
+
+self.postMessage({ status: "ready", voices: tts.voices, device });
+
+self.addEventListener("message", async (e) => {
+    const { text, voice } = e.data;
+    let chunks = splitTextSmart(text, 600);
+
+    for (const chunk of chunks) {
+        chunkQueue.push({ chunk, voice });
+    }
+    
+    processQueue();
+});
+

package/speech/types/types.ts

@@ -0,0 +1,34 @@
+/**
+ * @fileoverview Type definitions for text-to-speech providers
+ */
+
+export type TTSProvider = "kokoro" | "deepgram";
+
+export interface TTSOptions {
+  text: string;
+  provider?: TTSProvider;
+  voice?: string;
+}
+
+export interface TTSResult {
+  audio: ArrayBuffer;
+  contentType: string;
+}
+
+// Kokoro voices from the model
+export const KOKORO_VOICES = [
+  "af_heart", "af_alloy", "af_aoede", "af_bella",
+  "af_jessica", "af_nicole", "af_river", "af_sarah", "af_sky",
+  "am_adam", "am_echo", "am_fable", "am_fenrir",
+  "am_liam", "am_michael", "am_onyx"
+] as const;
+
+export type KokoroVoice = (typeof KOKORO_VOICES)[number];
+
+// Deepgram Aura speakers
+export const DEEPGRAM_SPEAKERS = [
+  "angus", "asteria", "arcas", "orion", "orpheus", "athena",
+  "luna", "zeus", "perseus", "helios", "hera", "stella",
+] as const;
+
+export type DeepgramSpeaker = (typeof DEEPGRAM_SPEAKERS)[number];

package/speech/ui/AudioPlayer.js

@@ -0,0 +1,84 @@
+const SAMPLE_RATE = 24000;
+
+export class AudioPlayer {
+
+  constructor() {
+    this.audioContext = new AudioContext();
+    this.audioQueue = [];
+    this.isPlaying = false;
+    this.currentSource = null; // Track current audio source for stopping
+  }
+
+  queueAudio(audioData) {
+    const audioData2 = new Float32Array(audioData);
+    const audioBuffer = this.audioContext.createBuffer(1, audioData2.length, SAMPLE_RATE);
+    audioBuffer.getChannelData(0).set(audioData2);
+    this.audioQueue.push(audioBuffer);
+    this.playAudioQueue();
+  }
+
+  async playAudioQueue() {
+    if (this.isPlaying || this.audioQueue.length === 0) return;
+
+    this.isPlaying = true;
+    try {
+      while (this.audioQueue.length > 0) {
+        const source = this.audioContext.createBufferSource();
+        this.currentSource = source; // Store current source for stopping
+        source.buffer = this.audioQueue.shift();
+        source.connect(this.audioContext.destination);
+
+        if (this.audioContext.state === "suspended") {
+          await this.audioContext.resume();
+          console.log("AudioContext resumed.");
+        }
+
+        console.log("Playing audio buffer");
+        await new Promise((resolve) => {
+          source.onended = () => {
+            this.currentSource = null; // Clear reference when playback ends
+            resolve();
+          };
+          source.start();
+        });
+
+        console.log("Audio playback finished.");
+
+       
+      }
+    } catch (error) {
+      console.error("Error during audio playback:", error);
+    } finally {
+      this.isPlaying = false;
+    }
+  }
+
+  // Stop audio playback and clear the queue
+  stop() {
+    console.log("Stopping audio playback");
+    
+    // Stop the currently playing source if any
+    if (this.currentSource) {
+      try {
+        this.currentSource.stop();
+        this.currentSource = null;
+      } catch (error) {
+        console.error("Error stopping current source:", error);
+      }
+    }
+    
+    this.audioQueue = [];
+    this.isPlaying = false;
+    
+  }
+
+  close() {
+    if (this.audioContext && this.audioContext.state !== "closed") {
+      this.audioContext.close();
+    }
+  }
+
+  getAudioContext() {
+    return this.audioContext;
+  }
+}

package/speech/ui/ui.js

@@ -0,0 +1,109 @@
+import { Conversation } from './conversation.js';
+import { initVoiceSelector } from './voice-selector.js';
+
+
+function updateTimer(recordingStartTime, recordingTimer) {
+    const elapsed = new Date() - recordingStartTime;
+    const seconds = Math.floor((elapsed / 1000) % 60).toString().padStart(2, '0');
+    const minutes = Math.floor((elapsed / 1000 / 60) % 60).toString().padStart(2, '0');
+    recordingTimer.textContent = `${minutes}:${seconds}`;
+}
+
+export function displayConversation(conversationHistory) {
+    let transcriptionResult = document.getElementById('transcriptionResult');
+    let conversationHTML = '';
+    // Skip the system message at index 0
+    for (let i = 1; i < conversationHistory.length; i++) {
+        const message = conversationHistory[i];
+        const roleClass = message.role === 'user' ? 'user-message' : 'assistant-message';
+        const roleLabel = message.role === 'user' ? 'You' : 'Assistant';
+        conversationHTML += `<div class="${roleClass}">
+            <strong>${roleLabel}:</strong> ${message.content}
+        </div>`;
+    }
+    transcriptionResult.innerHTML = conversationHTML;
+}
+
+function displayTranscriptionError(transcriptionStatus, transcriptionResult, error) {
+    transcriptionStatus.textContent = 'Error during transcription:';
+    transcriptionResult.innerHTML = `<p class="error">${error.message}</p>`;
+}
+
+function setupTabNavigation() {
+    const tabButtons = document.querySelectorAll('.tab-button');
+    const tabContents = document.querySelectorAll('.tab-content');
+    
+    tabButtons.forEach(button => {
+        button.addEventListener('click', () => {
+            const tabId = button.getAttribute('data-tab');
+            
+            // Update active button
+            tabButtons.forEach(btn => btn.classList.remove('active'));
+            button.classList.add('active');
+            
+            // Show selected tab content
+            tabContents.forEach(content => {
+                content.classList.remove('active');
+                if (content.id === tabId) {
+                    content.classList.add('active');
+                }
+            });
+        });
+    });
+}
+
+document.addEventListener('DOMContentLoaded', async function () {
+    const toggleButton = document.getElementById('toggleRecording');
+    const recordingStatus = document.getElementById('recordingStatus');
+    const recordingIndicator = document.getElementById('recordingIndicator');
+    const recordingTimer = document.getElementById('recordingTimer');
+    const transcriptionStatus = document.getElementById('transcriptionStatus');
+    const transcriptionResult = document.getElementById('transcriptionResult');
+
+    let recordingStartTime;
+    let timerInterval;
+    let isRecording = false;
+    
+    // Disable the button initially and show loading state
+    toggleButton.disabled = true;
+    toggleButton.textContent = 'Loading Models...';
+    recordingStatus.textContent = 'Loading speech recognition and synthesis models...';
+    
+    setupTabNavigation();
+    initVoiceSelector();
+
+    let conversation = new Conversation();
+
+    document.addEventListener('keydown', function (event) {
+        if (event.key === 'F8') {
+            conversation.conversationHistory.push({
+                role: "user",
+                content: "please continue."
+            });
+            conversation.sendConversationHistory();
+        }
+    });
+
+    toggleButton.addEventListener('click', function () {
+        if (!isRecording) {
+            isRecording = true;
+            toggleButton.textContent = 'Stop Recording';
+            recordingStatus.textContent = 'Recording...';
+            recordingIndicator.style.display = 'block';
+            transcriptionStatus.textContent = 'Recording in progress...';
+            transcriptionResult.textContent = '';
+            recordingStartTime = new Date();
+            timerInterval = setInterval(() => updateTimer(recordingStartTime, recordingTimer), 1000);
+            updateTimer(recordingStartTime, recordingTimer);
+            conversation.startRecording();
+        } else {
+            isRecording = false;
+            toggleButton.textContent = 'Start Recording';
+            recordingStatus.textContent = 'Recording stopped. Processing...';
+            transcriptionStatus.textContent = 'Transcribing audio...';
+            conversation.stopRecording();
+            clearInterval(timerInterval);
+            recordingIndicator.style.display = 'none';
+        }
+    });
+});

package/speech/ui/voice-selector.js

@@ -0,0 +1,77 @@
+import { VOICES } from './voices.js';
+
+export function initVoiceSelector() {
+    const voiceSelect = document.getElementById('voiceSelect');
+    
+    voiceSelect.innerHTML = '';
+    
+    const femaleVoices = [];
+    const maleVoices = [];
+    const otherVoices = [];
+    
+    Object.entries(VOICES).forEach(([id, voice]) => {
+        const option = {
+            id,
+            name: voice.name,
+            gender: voice.gender,
+            traits: voice.traits || '',
+            grade: voice.overallGrade || 'N/A'
+        };
+        
+        if (voice.gender === 'Female') {
+            femaleVoices.push(option);
+        } else if (voice.gender === 'Male') {
+            maleVoices.push(option);
+        } else {
+            otherVoices.push(option);
+        }
+    });
+    
+    const sortByGrade = (a, b) => {
+        const gradeA = a.grade.charAt(0);
+        const gradeB = b.grade.charAt(0);
+        return gradeA.localeCompare(gradeB);
+    };
+    
+    femaleVoices.sort(sortByGrade);
+    maleVoices.sort(sortByGrade);
+    otherVoices.sort(sortByGrade);
+    
+    const femaleGroup = document.createElement('optgroup');
+    femaleGroup.label = 'Female Voices';
+    
+    const maleGroup = document.createElement('optgroup');
+    maleGroup.label = 'Male Voices';
+    
+    const otherGroup = document.createElement('optgroup');
+    otherGroup.label = 'Other Voices';
+    
+    femaleVoices.forEach(voice => {
+        const option = document.createElement('option');
+        option.value = voice.id;
+        option.textContent = `${voice.name} (${voice.grade})`;
+        femaleGroup.appendChild(option);
+    });
+    
+    maleVoices.forEach(voice => {
+        const option = document.createElement('option');
+        option.value = voice.id;
+        option.textContent = `${voice.name} (${voice.grade})`;
+        maleGroup.appendChild(option);
+    });
+    
+    otherVoices.forEach(voice => {
+        const option = document.createElement('option');
+        option.value = voice.id;
+        option.textContent = `${voice.name} (${voice.grade})`;
+        otherGroup.appendChild(option);
+    });
+    
+    if (femaleVoices.length > 0) voiceSelect.appendChild(femaleGroup);
+    if (maleVoices.length > 0) voiceSelect.appendChild(maleGroup);
+    if (otherVoices.length > 0) voiceSelect.appendChild(otherGroup);
+    
+    if (VOICES.af_heart) {
+        voiceSelect.value = 'af_heart';
+    }
+}

package/speech/ui/voices.js

@@ -0,0 +1,259 @@
+export const VOICES = Object.freeze({
+  af_heart: {
+    name: "Heart",
+    language: "en-us",
+    gender: "Female",
+    traits: "❤️",
+    targetQuality: "A",
+    overallGrade: "A",
+  },
+  af_alloy: {
+    name: "Alloy",
+    language: "en-us",
+    gender: "Female",
+    targetQuality: "B",
+    overallGrade: "C",
+  },
+  af_aoede: {
+    name: "Aoede",
+    language: "en-us",
+    gender: "Female",
+    targetQuality: "B",
+    overallGrade: "C+",
+  },
+  af_bella: {
+    name: "Bella",
+    language: "en-us",
+    gender: "Female",
+    traits: "🔥",
+    targetQuality: "A",
+    overallGrade: "A-",
+  },
+  af_jessica: {
+    name: "Jessica",
+    language: "en-us",
+    gender: "Female",
+    targetQuality: "C",
+    overallGrade: "D",
+  },
+  af_kore: {
+    name: "Kore",
+    language: "en-us",
+    gender: "Female",
+    targetQuality: "B",
+    overallGrade: "C+",
+  },
+  af_nicole: {
+    name: "Nicole",
+    language: "en-us",
+    gender: "Female",
+    traits: "🎧",
+    targetQuality: "B",
+    overallGrade: "B-",
+  },
+  af_nova: {
+    name: "Nova",
+    language: "en-us",
+    gender: "Female",
+    targetQuality: "B",
+    overallGrade: "C",
+  },
+  af_river: {
+    name: "River",
+    language: "en-us",
+    gender: "Female",
+    targetQuality: "C",
+    overallGrade: "D",
+  },
+  af_sarah: {
+    name: "Sarah",
+    language: "en-us",
+    gender: "Female",
+    targetQuality: "B",
+    overallGrade: "C+",
+  },
+  af_sky: {
+    name: "Sky",
+    language: "en-us",
+    gender: "Female",
+    targetQuality: "B",
+    overallGrade: "C-",
+  },
+  am_adam: {
+    name: "Adam",
+    language: "en-us",
+    gender: "Male",
+    targetQuality: "D",
+    overallGrade: "F+",
+  },
+  am_echo: {
+    name: "Echo",
+    language: "en-us",
+    gender: "Male",
+    targetQuality: "C",
+    overallGrade: "D",
+  },
+  am_eric: {
+    name: "Eric",
+    language: "en-us",
+    gender: "Male",
+    targetQuality: "C",
+    overallGrade: "D",
+  },
+  am_fenrir: {
+    name: "Fenrir",
+    language: "en-us",
+    gender: "Male",
+    targetQuality: "B",
+    overallGrade: "C+",
+  },
+  am_liam: {
+    name: "Liam",
+    language: "en-us",
+    gender: "Male",
+    targetQuality: "C",
+    overallGrade: "D",
+  },
+  am_michael: {
+    name: "Michael",
+    language: "en-us",
+    gender: "Male",
+    targetQuality: "B",
+    overallGrade: "C+",
+  },
+  am_onyx: {
+    name: "Onyx",
+    language: "en-us",
+    gender: "Male",
+    targetQuality: "C",
+    overallGrade: "D",
+  },
+  am_puck: {
+    name: "Puck",
+    language: "en-us",
+    gender: "Male",
+    targetQuality: "B",
+    overallGrade: "C+",
+  },
+  am_santa: {
+    name: "Santa",
+    language: "en-us",
+    gender: "Male",
+    targetQuality: "C",
+    overallGrade: "D-",
+  },
+  bf_emma: {
+    name: "Emma",
+    language: "en-gb",
+    gender: "Female",
+    traits: "🚺",
+    targetQuality: "B",
+    overallGrade: "B-",
+  },
+  bf_isabella: {
+    name: "Isabella",
+    language: "en-gb",
+    gender: "Female",
+    targetQuality: "B",
+    overallGrade: "C",
+  },
+  bm_george: {
+    name: "George",
+    language: "en-gb",
+    gender: "Male",
+    targetQuality: "B",
+    overallGrade: "C",
+  },
+  bm_lewis: {
+    name: "Lewis",
+    language: "en-gb",
+    gender: "Male",
+    targetQuality: "C",
+    overallGrade: "D+",
+  },
+  bf_alice: {
+    name: "Alice",
+    language: "en-gb",
+    gender: "Female",
+    traits: "🚺",
+    targetQuality: "C",
+    overallGrade: "D",
+  },
+  bf_lily: {
+    name: "Lily",
+    language: "en-gb",
+    gender: "Female",
+    traits: "🚺",
+    targetQuality: "C",
+    overallGrade: "D",
+  },
+  bm_daniel: {
+    name: "Daniel",
+    language: "en-gb",
+    gender: "Male",
+    traits: "🚹",
+    targetQuality: "C",
+    overallGrade: "D",
+  },
+  bm_fable: {
+    name: "Fable",
+    language: "en-gb",
+    gender: "Male",
+    traits: "🚹",
+    targetQuality: "B",
+    overallGrade: "C",
+  },
+});
+
+const VOICE_DATA_URL = "https://huggingface.co/onnx-community/Kokoro-82M-v1.0-ONNX/resolve/main/voices";
+
+/**
+ *
+ * @param {keyof typeof VOICES} id
+ * @returns {Promise<ArrayBufferLike>}
+ */
+async function getVoiceFile(id) {
+  const url = `${VOICE_DATA_URL}/${id}.bin`;
+
+  let cache;
+  try {
+    cache = await caches.open("kokoro-voices");
+    const cachedResponse = await cache.match(url);
+    if (cachedResponse) {
+      return await cachedResponse.arrayBuffer();
+    }
+  } catch (e) {
+    console.warn("Unable to open cache", e);
+  }
+
+  // No cache, or cache failed to open. Fetch the file.
+  const response = await fetch(url);
+  const buffer = await response.arrayBuffer();
+
+  if (cache) {
+    try {
+      // NOTE: We use `new Response(buffer, ...)` instead of `response.clone()` to handle LFS files
+      await cache.put(
+        url,
+        new Response(buffer, {
+          headers: response.headers,
+        }),
+      );
+    } catch (e) {
+      console.warn("Unable to cache file", e);
+    }
+  }
+
+  return buffer;
+}
+
+const VOICE_CACHE = new Map();
+export async function getVoiceData(voice) {
+  if (VOICE_CACHE.has(voice)) {
+    return VOICE_CACHE.get(voice);
+  }
+
+  const buffer = new Float32Array(await getVoiceFile(voice));
+  VOICE_CACHE.set(voice, buffer);
+  return buffer;
+}