voicecc 1.1.36 → 1.2.0

package/server/voice/tts-elevenlabs.ts

@@ -1,397 +0,0 @@
-/**
- * ElevenLabs TTS provider via streaming HTTP API.
- *
- * Calls the ElevenLabs text-to-speech streaming endpoint to generate audio,
- * then writes raw PCM chunks to the speaker stream for playback. No subprocess
- * is needed -- audio is fetched over HTTP and piped directly into the pipeline.
- *
- * Responsibilities:
- * - POST text to the ElevenLabs TTS streaming API and receive chunked PCM audio
- * - Buffer streaming text deltas into sentences via shared bufferSentences utility
- * - Write PCM audio to the speaker stream with backpressure handling
- * - Track playback timing and wait for audio to finish before resolving
- * - Support interruption by cancelling in-flight requests and clearing playback
- */
-
-import type { Writable } from "stream";
-import type { TtsPlayer, TextChunk } from "./types.js";
-
-// ============================================================================
-// CONSTANTS
-// ============================================================================
-
-/** ElevenLabs TTS streaming API base URL */
-const ELEVENLABS_TTS_BASE_URL = "https://api.elevenlabs.io/v1/text-to-speech";
-
-/** PCM output sample rate in Hz (must match speaker pipeline) */
-const TTS_SAMPLE_RATE = 24000;
-
-/** Speaker audio bit depth */
-const SPEAKER_BIT_DEPTH = 16;
-
-/** Speaker channel count */
-const SPEAKER_CHANNELS = 1;
-
-/** Bytes per second of PCM audio at 24kHz 16-bit mono */
-const BYTES_PER_SECOND = TTS_SAMPLE_RATE * (SPEAKER_BIT_DEPTH / 8) * SPEAKER_CHANNELS;
-
-/** Interval (ms) for checking the interrupt flag during playback wait */
-const INTERRUPT_CHECK_INTERVAL_MS = 50;
-
-// ============================================================================
-// INTERFACES
-// ============================================================================
-
-/**
- * Configuration for the ElevenLabs TTS provider.
- */
-export interface ElevenlabsTtsConfig {
-  /** ElevenLabs API key for authentication */
-  apiKey: string;
-  /** ElevenLabs voice ID to use for generation */
-  voiceId: string;
-  /** ElevenLabs model ID (e.g. "eleven_monolingual_v1") */
-  modelId: string;
-  /** Writable stream for PCM audio output */
-  speakerInput: Writable;
-  /** Callback to clear the playback buffer on interruption */
-  interruptPlayback: () => void;
-  /** Callback to resume playback after an interrupt */
-  resumePlayback: () => void;
-}
-
-// ============================================================================
-// MAIN HANDLERS
-// ============================================================================
-
-/**
- * Create a TtsPlayer that uses the ElevenLabs streaming TTS API.
- *
- * Sends text to the ElevenLabs API and receives raw PCM audio at 24kHz 16-bit
- * mono, which is written directly to the speaker stream. No format conversion
- * is needed since the output matches the speaker pipeline exactly.
- *
- * @param config - ElevenLabs TTS configuration (API key, voice, model, speaker stream)
- * @returns A TtsPlayer instance ready for playback
- */
-export async function createElevenlabsTts(config: ElevenlabsTtsConfig): Promise<TtsPlayer> {
-  const { apiKey, voiceId, modelId, speakerInput, interruptPlayback, resumePlayback } = config;
-
-  let destroyed = false;
-  let speaking = false;
-  let interruptFlag = false;
-  let wasInterrupted = false;
-
-  /**
-   * POST text to the ElevenLabs TTS streaming endpoint and stream PCM chunks
-   * to the speaker. Returns the total number of PCM bytes written.
-   *
-   * @param text - The text to synthesize
-   * @returns Total PCM bytes written to the speaker stream
-   */
-  async function streamTtsToSpeaker(text: string): Promise<number> {
-    const url = `${ELEVENLABS_TTS_BASE_URL}/${voiceId}/stream?output_format=pcm_24000`;
-
-    const response = await fetch(url, {
-      method: "POST",
-      headers: {
-        "Content-Type": "application/json",
-        "xi-api-key": apiKey,
-      },
-      body: JSON.stringify({ text, model_id: modelId }),
-    });
-
-    if (!response.ok) {
-      const errorText = await response.text().catch(() => "unknown error");
-      throw new Error(`ElevenLabs TTS API error ${response.status}: ${errorText}`);
-    }
-
-    let totalBytes = 0;
-
-    for await (const chunk of readResponseChunks(response)) {
-      if (interruptFlag) break;
-
-      const pcmBuffer = Buffer.from(chunk);
-      totalBytes += pcmBuffer.length;
-      await writePcm(speakerInput, pcmBuffer);
-    }
-
-    return totalBytes;
-  }
-
-  /**
-   * Wait for the estimated remaining playback time, allowing interruption to cancel.
-   *
-   * @param remainingMs - Milliseconds to wait for playback to finish
-   */
-  function waitForPlayback(remainingMs: number): Promise<void> {
-    return new Promise<void>((resolve) => {
-      const timer = setTimeout(resolve, remainingMs);
-
-      // Poll the interrupt flag to allow early cancellation
-      const check = setInterval(() => {
-        if (interruptFlag) {
-          clearTimeout(timer);
-          clearInterval(check);
-          resolve();
-        }
-      }, INTERRUPT_CHECK_INTERVAL_MS);
-
-      // Clean up interval when timer fires naturally
-      setTimeout(() => clearInterval(check), remainingMs + 100);
-    });
-  }
-
-  /**
-   * Generate audio for a single text string via ElevenLabs API and play it.
-   * @param text - The text to speak
-   */
-  async function speak(text: string): Promise<void> {
-    if (destroyed) throw new Error("TtsPlayer has been destroyed");
-
-    interruptFlag = false;
-    speaking = true;
-
-    if (wasInterrupted) {
-      resumePlayback();
-      wasInterrupted = false;
-    }
-
-    try {
-      await streamTtsToSpeaker(text);
-    } finally {
-      speaking = false;
-    }
-  }
-
-  /**
-   * Stream text chunks into TTS for pipelined playback.
-   * Buffers text deltas into sentences, generates audio per sentence via
-   * the ElevenLabs API, and writes PCM to the speaker stream.
-   * @param texts - Async iterable of text chunks from the narrator
-   */
-  async function speakStream(texts: AsyncIterable<TextChunk>): Promise<void> {
-    if (destroyed) throw new Error("TtsPlayer has been destroyed");
-
-    const t0 = Date.now();
-    let firstTextLogged = false;
-    let chunkIndex = 0;
-    let playbackFinishAt = 0;
-
-    interruptFlag = false;
-    speaking = true;
-
-    if (wasInterrupted) {
-      resumePlayback();
-      wasInterrupted = false;
-    }
-
-    try {
-      for await (const sentence of bufferSentences(texts)) {
-        if (interruptFlag) break;
-
-        if (!firstTextLogged) {
-          console.log(`[tts-elevenlabs] first sentence at +${Date.now() - t0}ms: "${sentence.slice(0, 50)}${sentence.length > 50 ? "..." : ""}"`);
-          firstTextLogged = true;
-        }
-
-        const sentAt = Date.now();
-
-        // Fetch streamed PCM from ElevenLabs
-        const url = `${ELEVENLABS_TTS_BASE_URL}/${voiceId}/stream?output_format=pcm_24000`;
-        const response = await fetch(url, {
-          method: "POST",
-          headers: {
-            "Content-Type": "application/json",
-            "xi-api-key": apiKey,
-          },
-          body: JSON.stringify({ text: sentence, model_id: modelId }),
-        });
-
-        if (!response.ok) {
-          const errorText = await response.text().catch(() => "unknown error");
-          throw new Error(`ElevenLabs TTS API error ${response.status}: ${errorText}`);
-        }
-
-        if (interruptFlag) break;
-
-        // Read chunked PCM from the response body
-        for await (const chunk of readResponseChunks(response)) {
-          if (interruptFlag) break;
-
-          const pcmBuffer = Buffer.from(chunk);
-          const now = Date.now() - t0;
-          const audioDurationMs = (pcmBuffer.length / BYTES_PER_SECOND) * 1000;
-          const genMs = Date.now() - sentAt;
-
-          chunkIndex++;
-
-          await writePcm(speakerInput, pcmBuffer);
-
-          // Track estimated playback end. If the speaker buffer drained during a
-          // gap (e.g. tool call), new audio starts from now, not after previous audio.
-          playbackFinishAt = Math.max(playbackFinishAt, Date.now()) + audioDurationMs;
-        }
-
-        if (interruptFlag) break;
-      }
-
-      // Wait for buffered audio to finish playing through the speakers
-      if (!interruptFlag && playbackFinishAt > 0) {
-        const remainingMs = playbackFinishAt - Date.now();
-        if (remainingMs > 0) {
-          console.log(`[tts-elevenlabs] waiting ${(remainingMs / 1000).toFixed(1)}s for playback to finish`);
-          await waitForPlayback(remainingMs);
-        }
-      }
-    } finally {
-      speaking = false;
-    }
-  }
-
-  /**
-   * Interrupt current playback and cancel in-flight generation.
-   * Clears the playback buffer and sets the interrupt flag.
-   */
-  function interrupt(): void {
-    if (destroyed) return;
-    interruptFlag = true;
-    wasInterrupted = true;
-    interruptPlayback();
-  }
-
-  /**
-   * Check whether TTS is currently active.
-   * @returns true if a speak/speakStream call is in progress
-   */
-  function checkIsSpeaking(): boolean {
-    return speaking;
-  }
-
-  /**
-   * Free all resources and prevent further usage.
-   */
-  function destroyPlayer(): void {
-    if (destroyed) return;
-    destroyed = true;
-    interrupt();
-  }
-
-  return {
-    speak,
-    speakStream,
-    interrupt,
-    isSpeaking: checkIsSpeaking,
-    destroy: destroyPlayer,
-  };
-}
-
-// ============================================================================
-// HELPER FUNCTIONS
-// ============================================================================
-
-/**
- * Write a PCM buffer to the speaker stream, respecting backpressure.
- * @param stream - The speaker writable stream
- * @param pcmBuffer - Raw PCM bytes to write
- */
-function writePcm(stream: Writable, pcmBuffer: Buffer): Promise<void> {
-  return new Promise<void>((resolve, reject) => {
-    const ok = stream.write(pcmBuffer, (err: Error | null | undefined) => {
-      if (err) reject(err);
-    });
-    if (ok) {
-      resolve();
-    } else {
-      stream.once("drain", () => resolve());
-    }
-  });
-}
-
-/** Sentence-ending punctuation pattern: .!? followed by whitespace or end */
-const SENTENCE_END_RE = /[.!?][\s]+/;
-
-/** Minimum sentence length before we'll split on punctuation */
-const MIN_SENTENCE_LENGTH = 20;
-
-/**
- * Buffer streaming text deltas into complete sentences for TTS generation.
- * Chunks tagged with { flush: true } are yielded immediately (e.g. tool narration).
- * Plain string chunks are buffered and split on sentence-ending punctuation.
- * @param texts - Async iterable of TextChunk from the narrator
- * @yields Complete sentences ready for TTS
- */
-async function* bufferSentences(texts: AsyncIterable<TextChunk>): AsyncGenerator<string> {
-  let buffer = "";
-
-  for await (const raw of texts) {
-    if (typeof raw !== "string") {
-      if (buffer.trim()) {
-        yield buffer.trim();
-        buffer = "";
-      }
-      yield raw.text;
-      continue;
-    }
-
-    buffer += raw;
-
-    while (buffer.length >= MIN_SENTENCE_LENGTH) {
-      const match = SENTENCE_END_RE.exec(buffer.slice(MIN_SENTENCE_LENGTH - 1));
-      if (!match) break;
-
-      const splitIndex = MIN_SENTENCE_LENGTH - 1 + match.index + match[0].length;
-      const sentence = buffer.slice(0, splitIndex).trim();
-      buffer = buffer.slice(splitIndex);
-
-      if (sentence) yield sentence;
-    }
-  }
-
-  const remaining = buffer.trim();
-  if (remaining) yield remaining;
-}
-
-/**
- * Read chunks from a fetch Response body as an async iterable, ensuring each
- * yielded chunk is aligned to 16-bit sample boundaries (even byte count).
- *
- * HTTP streaming splits the byte stream at arbitrary TCP packet boundaries.
- * A chunk with an odd byte count splits a 16-bit PCM sample in half. Downstream
- * consumers (browser WebSocket -> Int16Array) interpret each chunk independently,
- * so a misaligned chunk corrupts all its samples (heard as hiss/static).
- *
- * @param response - The fetch Response to read from
- * @yields Buffer chunks of sample-aligned raw PCM audio data
- */
-async function* readResponseChunks(response: Response): AsyncGenerator<Buffer> {
-  const body = response.body;
-  if (!body) throw new Error("ElevenLabs TTS response has no body");
-
-  const reader = body.getReader();
-  let leftover: Buffer | null = null;
-
-  try {
-    while (true) {
-      const { done, value } = await reader.read();
-      if (done) break;
-      if (!value) continue;
-
-      let chunk: Buffer = leftover ? Buffer.concat([leftover, value]) : Buffer.from(value);
-      leftover = null;
-
-      // Hold back the last byte if odd length (split sample)
-      if (chunk.byteLength % 2 !== 0) {
-        leftover = Buffer.from(chunk.subarray(chunk.byteLength - 1));
-        chunk = chunk.subarray(0, chunk.byteLength - 1);
-      }
-
-      if (chunk.byteLength > 0) yield chunk;
-    }
-
-    // Flush any remaining byte (only happens with malformed PCM)
-    if (leftover && leftover.byteLength > 0) yield leftover;
-  } finally {
-    reader.releaseLock();
-  }
-}

package/server/voice/tts-provider.ts

@@ -1,155 +0,0 @@
-/**
- * TTS provider factory and readiness checks.
- *
- * Routes TTS creation to the ElevenLabs provider implementation.
- * Checks provider readiness (API keys) for dashboard status.
- *
- * Responsibilities:
- * - Create a TtsPlayer for the configured provider
- * - Check provider readiness (API keys set)
- * - Provide static metadata about available TTS providers
- */
-
-import { createElevenlabsTts } from "./tts-elevenlabs.js";
-import { readEnv } from "../services/env.js";
-
-import type { Writable } from "stream";
-import type { TtsPlayer, TtsProviderType, TtsProviderConfig, ProviderStatus } from "./types.js";
-
-// ============================================================================
-// CONSTANTS
-// ============================================================================
-
-/** ElevenLabs API base URL */
-const ELEVENLABS_API_BASE = "https://api.elevenlabs.io/v1";
-
-// ============================================================================
-// INTERFACES
-// ============================================================================
-
-/**
- * A voice option returned by listVoicesForProvider.
- */
-export interface VoiceOption {
-  id: string;
-  name: string;
-}
-
-/**
- * Metadata about a TTS provider for display in the dashboard.
- */
-export interface TtsProviderInfo {
-  /** Provider type identifier */
-  type: TtsProviderType;
-  /** Human-readable provider name */
-  name: string;
-  /** Short description of the provider */
-  description: string;
-  /** Environment variable name for the API key (undefined = no key needed) */
-  requiresApiKey?: string;
-}
-
-/**
- * Options for creating a TTS player via the provider factory.
- */
-export interface CreateTtsOptions {
-  /** Provider configuration (which provider + per-provider settings) */
-  providerConfig: TtsProviderConfig;
-  /** Writable stream for PCM audio output */
-  speakerInput: Writable;
-  /** Callback to clear the playback buffer on interruption */
-  interruptPlayback: () => void;
-  /** Callback to resume playback after an interrupt */
-  resumePlayback: () => void;
-}
-
-// ============================================================================
-// MAIN HANDLERS
-// ============================================================================
-
-/**
- * Create a TtsPlayer for the configured provider.
- *
- * @param options - Provider config, speaker stream, and playback callbacks
- * @returns A TtsPlayer instance ready for playback
- * @throws Error if the provider is not implemented
- */
-export async function createTtsForProvider(options: CreateTtsOptions): Promise<TtsPlayer> {
-  const { providerConfig, speakerInput, interruptPlayback, resumePlayback } = options;
-
-  switch (providerConfig.provider) {
-    case "elevenlabs":
-      return createElevenlabsTts({
-        apiKey: providerConfig.elevenlabs.apiKey,
-        voiceId: providerConfig.elevenlabs.voiceId,
-        modelId: providerConfig.elevenlabs.modelId,
-        speakerInput,
-        interruptPlayback,
-        resumePlayback,
-      });
-
-    default:
-      throw new Error(`Unknown TTS provider: ${providerConfig.provider}`);
-  }
-}
-
-/**
- * Check whether a TTS provider is ready to use.
- * Checks ELEVENLABS_API_KEY is set in .env.
- *
- * @param providerType - The provider to check
- * @returns Readiness status with reason if not ready
- */
-export async function getTtsProviderStatus(providerType: TtsProviderType): Promise<ProviderStatus> {
-  switch (providerType) {
-    case "elevenlabs": {
-      const env = await readEnv();
-      if (!env.ELEVENLABS_API_KEY) {
-        return { ready: false, reason: "missing_api_key", detail: "ELEVENLABS_API_KEY is not set in .env" };
-      }
-      return { ready: true };
-    }
-
-    default:
-      throw new Error(`Unknown TTS provider: ${providerType}`);
-  }
-}
-
-/**
- * List available voices for a TTS provider.
- * Fetches available voices from the ElevenLabs API.
- *
- * @param providerType - The provider to list voices for
- * @returns Array of voice options
- */
-export async function listVoicesForProvider(
-  providerType: TtsProviderType,
-): Promise<VoiceOption[]> {
-  switch (providerType) {
-    case "elevenlabs": {
-      const res = await fetch(`${ELEVENLABS_API_BASE}/voices`);
-      if (!res.ok) throw new Error(`ElevenLabs API error: ${res.status}`);
-      const data = (await res.json()) as { voices: Array<{ voice_id: string; name: string }> };
-      return data.voices.map((v) => ({ id: v.voice_id, name: v.name }));
-    }
-
-    default:
-      throw new Error(`Unknown TTS provider: ${providerType}`);
-  }
-}
-
-/**
- * Get the list of all known TTS providers with metadata.
- *
- * @returns Static array of TTS provider info
- */
-export function getAvailableTtsProviders(): TtsProviderInfo[] {
-  return [
-    {
-      type: "elevenlabs",
-      name: "ElevenLabs",
-      description: "Cloud TTS via ElevenLabs streaming API",
-      requiresApiKey: "ELEVENLABS_API_KEY",
-    },
-  ];
-}