voicecc 1.0.7

package/sidecar/narration.ts

@@ -0,0 +1,204 @@
+/**
+ * Processes Claude's streaming output into TTS-friendly text.
+ *
+ * Two modes of operation:
+ * - Response mode: passes text_delta content through immediately for streaming
+ *   TTS. Text is buffered into sentences downstream in the TTS module.
+ * - Long-task mode: emits periodic template-based summaries during tool use
+ *   (e.g. "Running Bash...", "Still working on Bash...").
+ *
+ * Responsibilities:
+ * - Pass through streaming text deltas immediately for low-latency TTS
+ * - Track tool execution and emit periodic spoken summaries
+ * - Flush remaining text on result/error events
+ */
+
+import type { ClaudeStreamEvent, NarrationConfig } from "./types.js";
+
+/** Strip markdown syntax so text reads naturally when spoken. */
+function stripMarkdown(text: string): string {
+  return text
+    .replace(/\*+/g, "")       // bold/italic asterisks
+    .replace(/#+\s*/g, "")     // heading markers
+    .replace(/`+/g, "")        // inline code / code fences
+    .replace(/\[([^\]]*)\]\([^)]*\)/g, "$1") // [text](url) → text
+    .replace(/^-\s+/gm, "")   // unordered list markers
+    .replace(/^\d+\.\s+/gm, ""); // ordered list markers
+}
+
+// ============================================================================
+// INTERFACES
+// ============================================================================
+
+/**
+ * Narrator instance that processes Claude stream events into speakable text.
+ */
+export interface Narrator {
+  /**
+   * Process a single Claude stream event and return any text ready to be spoken.
+   * @param event - The Claude stream event to process
+   * @returns Array of strings to speak (often empty, sometimes 1-2 sentences)
+   */
+  processEvent(event: ClaudeStreamEvent): string[];
+
+  /**
+   * Flush any remaining buffered text that hasn't been emitted yet.
+   * @returns Array of remaining text strings to speak
+   */
+  flush(): string[];
+
+  /**
+   * Reset all internal state for a new conversation turn.
+   */
+  reset(): void;
+}
+
+// ============================================================================
+// MAIN HANDLERS
+// ============================================================================
+
+/**
+ * Create a new Narrator instance that converts Claude stream events into
+ * TTS-friendly sentence chunks.
+ * @param config - Narration configuration (summaryIntervalMs controls long-task summary frequency)
+ * @returns A Narrator instance
+ */
+export function createNarrator(config: NarrationConfig, onEmit?: (text: string) => void): Narrator {
+  // -- internal state --
+  let currentToolName: string | null = null;
+  let summaryTimer: NodeJS.Timeout | null = null;
+  let inLongTask = false;
+
+  /**
+   * Process a single Claude stream event.
+   * @param event - The streaming event from Claude
+   * @returns Array of strings to speak
+   */
+  function processEvent(event: ClaudeStreamEvent): string[] {
+    switch (event.type) {
+      case "text_delta":
+        return handleTextDelta(event);
+      case "tool_start":
+        return handleToolStart(event);
+      case "tool_end":
+        return handleToolEnd();
+      case "result":
+      case "error":
+        return handleTerminal();
+      default:
+        return [];
+    }
+  }
+
+  /**
+   * Flush any remaining text in the buffer.
+   * @returns Array of remaining text strings
+   */
+  function flush(): string[] {
+    return [];
+  }
+
+  /**
+   * Reset all state for a new conversation turn.
+   */
+  function reset(): void {
+    currentToolName = null;
+    clearSummaryTimer();
+    inLongTask = false;
+  }
+
+  return { processEvent, flush, reset };
+
+  // ============================================================================
+  // HELPER FUNCTIONS
+  // ============================================================================
+
+  /**
+   * Handle a text_delta event: pass through immediately, exit long-task mode.
+   * Text chunking for TTS is handled downstream by TextSplitterStream.
+   * @param event - The text_delta event
+   * @returns Array containing the delta text
+   */
+  function handleTextDelta(event: ClaudeStreamEvent): string[] {
+    // Text arriving means Claude is responding directly -- leave long-task mode
+    if (inLongTask) {
+      clearSummaryTimer();
+      inLongTask = false;
+      currentToolName = null;
+    }
+
+    const results: string[] = [];
+    if (event.content) {
+      const clean = stripMarkdown(event.content);
+      if (clean) results.push(clean);
+    }
+    return results;
+  }
+
+  /**
+   * Handle a tool_start event: enter long-task mode, start the summary timer,
+   * and emit an initial "Running {toolName}..." message.
+   * @param event - The tool_start event (must have toolName)
+   * @returns Array containing the initial tool message
+   */
+  function handleToolStart(event: ClaudeStreamEvent): string[] {
+    const toolName = event.toolName ?? "unknown tool";
+    currentToolName = toolName;
+    inLongTask = true;
+
+    // Clear any existing timer before starting a new one
+    clearSummaryTimer();
+    startSummaryTimer();
+
+    return [`Running ${toolName}...`];
+  }
+
+  /**
+   * Handle a tool_end event: clear current tool context but stay in long-task
+   * mode since more tools might follow.
+   * @returns Empty array
+   */
+  function handleToolEnd(): string[] {
+    currentToolName = null;
+    return [];
+  }
+
+  /**
+   * Handle result or error events: flush all remaining text and reset state.
+   * @returns Array of any remaining text
+   */
+  function handleTerminal(): string[] {
+    const remaining = flush();
+
+    // Full reset for next turn
+    clearSummaryTimer();
+    currentToolName = null;
+    inLongTask = false;
+
+    return remaining;
+  }
+
+  /**
+   * Start the periodic summary timer for long-task mode.
+   * Emits "Still working on {toolName}..." at the configured interval.
+   */
+  function startSummaryTimer(): void {
+    summaryTimer = setInterval(() => {
+      const name = currentToolName ?? "the task";
+      const summary = `Still working on ${name}...`;
+      if (onEmit) {
+        onEmit(summary);
+      }
+    }, config.summaryIntervalMs);
+  }
+
+  /**
+   * Clear the summary timer if one is active.
+   */
+  function clearSummaryTimer(): void {
+    if (summaryTimer !== null) {
+      clearInterval(summaryTimer);
+      summaryTimer = null;
+    }
+  }
+}

package/sidecar/scripts/generate-startup-audio.py

@@ -0,0 +1,79 @@
+"""
+One-time script to generate the startup audio greeting.
+
+Uses mlx_audio's Kokoro model (same API as tts-server.py) to synthesize a short
+spoken greeting and writes it as raw 24kHz 16-bit signed mono PCM to
+sidecar/assets/startup.pcm.
+
+Usage:
+  cd sidecar
+  .venv/bin/python3 scripts/generate-startup-audio.py
+"""
+
+import os
+import sys
+import numpy as np
+
+# ============================================================================
+# CONSTANTS
+# ============================================================================
+
+MODEL_ID = "prince-canuma/Kokoro-82M"
+VOICE = "af_heart"
+STARTUP_TEXT = "Hi there! I'm Voice CC. How can I help you today?"
+OUTPUT_DIR = os.path.join(os.path.dirname(__file__), "..", "assets")
+OUTPUT_FILE = os.path.join(OUTPUT_DIR, "startup.pcm")
+
+# ============================================================================
+# MAIN ENTRYPOINT
+# ============================================================================
+
+def main():
+    """Load the Kokoro model, generate startup audio, and save as raw PCM."""
+    from mlx_audio.tts.utils import load_model
+
+    print(f"Loading model: {MODEL_ID}")
+    model = load_model(MODEL_ID)
+    print(f"Model loaded (sample_rate={model.sample_rate})")
+
+    print(f"Generating: \"{STARTUP_TEXT}\"")
+    chunks = []
+    try:
+        for result in model.generate(text=STARTUP_TEXT, voice=VOICE, stream=True):
+            audio = np.array(result.audio, copy=False)
+            chunks.append(audio)
+            print(f"  chunk {len(chunks)}: {audio.shape}")
+    except Exception as e:
+        print(f"ERROR during generation: {e}", file=sys.stderr)
+        import traceback
+        traceback.print_exc()
+        sys.exit(1)
+
+    combined = np.concatenate(chunks)
+    pcm = float32_to_int16_pcm(combined)
+
+    os.makedirs(OUTPUT_DIR, exist_ok=True)
+    with open(OUTPUT_FILE, "wb") as f:
+        f.write(pcm)
+
+    duration_s = len(combined) / model.sample_rate
+    print(f"Wrote {len(pcm)} bytes ({duration_s:.1f}s) to {OUTPUT_FILE}")
+
+# ============================================================================
+# HELPER FUNCTIONS
+# ============================================================================
+
+def float32_to_int16_pcm(audio: np.ndarray) -> bytes:
+    """
+    Convert float32 audio samples (-1.0..1.0) to 16-bit signed PCM bytes.
+
+    @param audio - numpy array of float32 samples
+    @returns Raw bytes of int16 little-endian PCM
+    """
+    clamped = np.clip(audio, -1.0, 1.0)
+    int16 = (clamped * 32767).astype(np.int16)
+    return int16.tobytes()
+
+
+if __name__ == "__main__":
+    main()

package/sidecar/session-lock.ts

@@ -0,0 +1,123 @@
+/**
+ * Cross-process session limiter using PID-based lock files.
+ *
+ * Ensures the total number of active voice sessions (local mic + Twilio combined)
+ * does not exceed MAX_CONCURRENT_SESSIONS. Stale lock files from crashed processes
+ * are automatically cleaned up on every acquire.
+ *
+ * Responsibilities:
+ * - Acquire a session slot by creating a PID lock file in ~/.claude-voice-sessions/
+ * - Validate existing lock files by checking if their PIDs are still alive
+ * - Clean up stale lock files from dead processes
+ * - Release the lock file on session stop or process exit
+ */
+
+import { mkdirSync, readdirSync, readFileSync, writeFileSync, unlinkSync } from "fs";
+import { join } from "path";
+import { homedir } from "os";
+import { randomUUID } from "crypto";
+
+// ============================================================================
+// CONSTANTS
+// ============================================================================
+
+/** Directory where PID lock files are stored */
+const LOCK_DIR = join(homedir(), ".claude-voice-sessions");
+
+// ============================================================================
+// INTERFACES
+// ============================================================================
+
+/**
+ * Handle returned by acquireSessionLock. Call release() to free the session slot.
+ */
+export interface SessionLock {
+  /** Release the session lock (deletes the lock file) */
+  release: () => void;
+}
+
+// ============================================================================
+// MAIN HANDLERS
+// ============================================================================
+
+/**
+ * Acquire a session lock slot. Throws if the maximum number of concurrent
+ * sessions has been reached.
+ *
+ * Cleans up stale lock files (dead PIDs) on every call. Creates a new lock
+ * file containing the current PID. Registers a process.on('exit') handler
+ * as a safety net to release on shutdown.
+ *
+ * @param maxSessions - Maximum number of concurrent sessions allowed
+ * @returns A SessionLock handle with a release() method
+ * @throws Error if maxSessions has been reached
+ */
+export function acquireSessionLock(maxSessions: number): SessionLock {
+  // Ensure lock directory exists
+  mkdirSync(LOCK_DIR, { recursive: true });
+
+  // List existing lock files and validate their PIDs
+  const files = readdirSync(LOCK_DIR).filter((f) => f.endsWith(".lock"));
+  let activeCount = 0;
+
+  for (const file of files) {
+    const filePath = join(LOCK_DIR, file);
+    try {
+      const pid = parseInt(readFileSync(filePath, "utf-8").trim(), 10);
+      if (isNaN(pid) || !isProcessAlive(pid)) {
+        // Stale lock file -- process is dead, clean it up
+        unlinkSync(filePath);
+      } else {
+        activeCount++;
+      }
+    } catch {
+      // File disappeared between readdir and read, or parse error -- skip
+      try { unlinkSync(filePath); } catch { /* already gone */ }
+    }
+  }
+
+  if (activeCount >= maxSessions) {
+    throw new Error(
+      `Session limit reached (${activeCount}/${maxSessions}). ` +
+      `Cannot start another voice session.`
+    );
+  }
+
+  // Create a new lock file with the current PID
+  const lockFile = join(LOCK_DIR, `${randomUUID()}.lock`);
+  writeFileSync(lockFile, String(process.pid), "utf-8");
+
+  let released = false;
+
+  /** Delete the lock file if it hasn't been released yet */
+  function release(): void {
+    if (released) return;
+    released = true;
+    try { unlinkSync(lockFile); } catch { /* already gone */ }
+  }
+
+  // Safety net: release on process exit
+  process.on("exit", release);
+
+  return { release };
+}
+
+// ============================================================================
+// HELPER FUNCTIONS
+// ============================================================================
+
+/**
+ * Check if a process with the given PID is still alive.
+ * Uses signal 0 which does not kill the process -- it only checks existence.
+ *
+ * @param pid - The process ID to check
+ * @returns true if the process is alive, false otherwise
+ */
+export function isProcessAlive(pid: number): boolean {
+  try {
+    process.kill(pid, 0);
+    return true;
+  } catch {
+    return false;
+  }
+}

package/sidecar/sherpa-onnx-node.d.ts

@@ -0,0 +1,4 @@
+declare module "sherpa-onnx-node" {
+  const sherpa: any;
+  export default sherpa;
+}

package/sidecar/stt.ts

@@ -0,0 +1,199 @@
+/**
+ * Local speech-to-text via sherpa-onnx with Whisper ONNX model (offline/batch).
+ *
+ * Whisper models in sherpa-onnx are offline-only (not streaming). Audio is
+ * accumulated during speech (SPEECH_START to SPEECH_END), then batch-transcribed
+ * on SPEECH_END using `createOfflineRecognizer`.
+ *
+ * Responsibilities:
+ * - Load the sherpa-onnx offline recognizer with a Whisper ONNX model
+ * - Accumulate audio samples during speech into an internal buffer
+ * - Batch-transcribe the accumulated buffer on demand
+ * - Manage buffer and recognizer lifecycle
+ */
+
+import { existsSync } from "fs";
+import type { TranscriptionResult } from "./types.js";
+
+// ============================================================================
+// INTERFACES
+// ============================================================================
+
+/** Internal interface for the STT processor returned by createStt. */
+interface SttProcessor {
+  /**
+   * Appends audio samples to the internal buffer.
+   * Call continuously during speech (between SPEECH_START and SPEECH_END).
+   *
+   * @param samples - Float32Array of audio samples (16kHz, normalized -1.0 to 1.0)
+   */
+  accumulate(samples: Float32Array): void;
+
+  /**
+   * Batch-transcribes the accumulated audio buffer using the offline recognizer.
+   * Creates an offline stream, feeds the accumulated audio, decodes, and returns
+   * the result. Clears the buffer afterward.
+   *
+   * @returns Transcription result with text, isFinal flag, and timestamp
+   */
+  transcribe(): Promise<TranscriptionResult>;
+
+  /**
+   * Clears the accumulated audio buffer without transcribing.
+   * Use on interruption or when discarding a speech segment.
+   */
+  clearBuffer(): void;
+
+  /**
+   * Frees the underlying recognizer resources.
+   * Call on shutdown to prevent resource leaks.
+   */
+  destroy(): void;
+}
+
+// ============================================================================
+// CONSTANTS
+// ============================================================================
+
+/** Sample rate expected by the Whisper model */
+const SAMPLE_RATE = 16000;
+
+/** Default model file prefix (sherpa-onnx naming convention: "small.en", "tiny.en", etc.) */
+const DEFAULT_MODEL_PREFIX = "small.en";
+
+/** Required model file suffixes within the model directory */
+const REQUIRED_SUFFIXES = ["-encoder.int8.onnx", "-decoder.int8.onnx", "-tokens.txt"];
+
+// ============================================================================
+// MAIN HANDLERS
+// ============================================================================
+
+/**
+ * Loads the sherpa-onnx offline recognizer with the Whisper model at the given
+ * path and returns an SttProcessor.
+ *
+ * @param modelPath - Path to directory containing encoder.onnx, decoder.onnx, and tokens.txt
+ * @returns Promise resolving to an SttProcessor instance
+ * @throws Error if any required model files are missing
+ */
+async function createStt(modelPath: string): Promise<SttProcessor> {
+  validateModelFiles(modelPath);
+
+  // Dynamic import to avoid ONNX runtime conflict with kokoro-js.
+  // Both sherpa-onnx-node and kokoro-js bundle native ONNX runtimes that
+  // crash if loaded simultaneously via static imports.
+  const sherpa = (await import("sherpa-onnx-node")).default;
+
+  const prefix = DEFAULT_MODEL_PREFIX;
+  const config = {
+    modelConfig: {
+      whisper: {
+        encoder: `${modelPath}/${prefix}-encoder.int8.onnx`,
+        decoder: `${modelPath}/${prefix}-decoder.int8.onnx`,
+      },
+      tokens: `${modelPath}/${prefix}-tokens.txt`,
+    },
+  };
+
+  const recognizer = new sherpa.OfflineRecognizer(config);
+
+  // Buffer stored as array of chunks to avoid repeated copying during accumulation
+  let audioChunks: Float32Array[] = [];
+
+  return {
+    accumulate(samples: Float32Array): void {
+      audioChunks.push(samples);
+    },
+
+    async transcribe(): Promise<TranscriptionResult> {
+      const combinedSamples = concatenateChunks(audioChunks);
+      audioChunks = [];
+
+      if (combinedSamples.length === 0) {
+        return { text: "", isFinal: true, timestamp: Date.now() };
+      }
+
+      // Create a fresh stream, feed audio, decode
+      const stream = recognizer.createStream();
+      stream.acceptWaveform({ sampleRate: SAMPLE_RATE, samples: combinedSamples });
+      recognizer.decode(stream);
+
+      const result = recognizer.getResult(stream);
+      const text = result.text.trim();
+
+      return { text, isFinal: true, timestamp: Date.now() };
+    },
+
+    clearBuffer(): void {
+      audioChunks = [];
+    },
+
+    destroy(): void {
+      audioChunks = [];
+      // recognizer cleanup is handled by sherpa-onnx-node garbage collection
+    },
+  };
+}
+
+// ============================================================================
+// HELPER FUNCTIONS
+// ============================================================================
+
+/**
+ * Validates that all required model files exist in the given directory.
+ *
+ * @param modelPath - Path to the model directory
+ * @throws Error with details about which files are missing
+ */
+function validateModelFiles(modelPath: string): void {
+  if (!existsSync(modelPath)) {
+    throw new Error(
+      `STT model directory not found: ${modelPath}. ` +
+        `Download a Whisper ONNX model and place encoder.onnx, decoder.onnx, and tokens.txt in this directory.`
+    );
+  }
+
+  const expectedFiles = REQUIRED_SUFFIXES.map((suffix) => `${DEFAULT_MODEL_PREFIX}${suffix}`);
+  const missingFiles = expectedFiles.filter(
+    (file) => !existsSync(`${modelPath}/${file}`)
+  );
+
+  if (missingFiles.length > 0) {
+    throw new Error(
+      `Missing STT model files in ${modelPath}: ${missingFiles.join(", ")}. ` +
+        `Required files: ${expectedFiles.join(", ")}.`
+    );
+  }
+}
+
+/**
+ * Concatenates an array of Float32Array chunks into a single Float32Array.
+ * Avoids repeated copying during accumulation by deferring concatenation
+ * until transcription time.
+ *
+ * @param chunks - Array of Float32Array audio chunks
+ * @returns Single concatenated Float32Array
+ */
+function concatenateChunks(chunks: Float32Array[]): Float32Array {
+  if (chunks.length === 0) {
+    return new Float32Array(0);
+  }
+
+  if (chunks.length === 1) {
+    return chunks[0];
+  }
+
+  const totalLength = chunks.reduce((sum, chunk) => sum + chunk.length, 0);
+  const result = new Float32Array(totalLength);
+
+  let offset = 0;
+  for (const chunk of chunks) {
+    result.set(chunk, offset);
+    offset += chunk.length;
+  }
+
+  return result;
+}
+
+export { createStt };
+export type { SttProcessor };