voicecc 1.0.7

package/sidecar/claude-session.ts

@@ -0,0 +1,295 @@
+/**
+ * Claude session via the @anthropic-ai/claude-code SDK.
+ *
+ * Keeps a single persistent Claude Code process alive across turns using
+ * streaming I/O (AsyncIterable<SDKUserMessage> input). This eliminates the
+ * ~2-3s process spawn overhead on each turn.
+ *
+ * Responsibilities:
+ * - Start a persistent query() on createClaudeSession (process spawns once)
+ * - Push user messages into the live session via an async queue
+ * - Extract streaming text deltas from SDKPartialAssistantMessage events
+ * - Map tool_use content blocks to tool_start / tool_end events
+ * - Support interruption via query.interrupt()
+ * - Provide clean session teardown
+ */
+
+import { query as claudeQuery, type Query, type Options, type SDKMessage, type SDKUserMessage } from "@anthropic-ai/claude-code";
+import type { ClaudeSessionConfig, ClaudeStreamEvent } from "./types.js";
+
+/** Injectable query function signature for testing. Matches the SDK query() contract. */
+export type QueryFn = (params: { prompt: AsyncIterable<SDKUserMessage>; options: Options }) => Query;
+
+// ============================================================================
+// ASYNC QUEUE
+// ============================================================================
+
+/** Simple async iterable backed by a push queue. */
+class AsyncQueue<T> implements AsyncIterable<T> {
+  private buf: T[] = [];
+  private resolve: ((r: IteratorResult<T>) => void) | null = null;
+  private done = false;
+
+  push(item: T) {
+    if (this.resolve) {
+      const r = this.resolve;
+      this.resolve = null;
+      r({ value: item, done: false });
+    } else {
+      this.buf.push(item);
+    }
+  }
+
+  close() {
+    this.done = true;
+    if (this.resolve) {
+      const r = this.resolve;
+      this.resolve = null;
+      r({ value: undefined as any, done: true });
+    }
+  }
+
+  /** Discard all buffered items. Used to clear stale events after interruption. */
+  drain(): void {
+    this.buf.length = 0;
+  }
+
+  /** Read one item (used by sendMessage to drain the event channel). */
+  async next(): Promise<T | undefined> {
+    if (this.buf.length > 0) return this.buf.shift()!;
+    if (this.done) return undefined;
+    const result = await new Promise<IteratorResult<T>>((r) => { this.resolve = r; });
+    return result.done ? undefined : result.value;
+  }
+
+  [Symbol.asyncIterator](): AsyncIterator<T> {
+    return {
+      next: (): Promise<IteratorResult<T>> => {
+        if (this.buf.length > 0) {
+          return Promise.resolve({ value: this.buf.shift()!, done: false as const });
+        }
+        if (this.done) {
+          return Promise.resolve({ value: undefined as any, done: true as const });
+        }
+        return new Promise<IteratorResult<T>>((r) => { this.resolve = r; });
+      },
+    };
+  }
+}
+
+// ============================================================================
+// INTERFACES
+// ============================================================================
+
+/** Session object returned by createClaudeSession. */
+interface ClaudeSession {
+  sendMessage(text: string): AsyncIterable<ClaudeStreamEvent>;
+  interrupt(): void;
+  close(): Promise<void>;
+}
+
+// ============================================================================
+// CONSTANTS
+// ============================================================================
+
+const CLAUDE_BIN = "/Users/Focus/.local/bin/claude";
+
+const DEFAULT_SYSTEM_PROMPT =
+  "Respond concisely. You are in voice mode -- your responses will be spoken aloud. Keep answers conversational and brief.";
+
+// ============================================================================
+// MAIN HANDLERS
+// ============================================================================
+
+async function createClaudeSession(
+  config: ClaudeSessionConfig,
+  queryOverride?: QueryFn,
+): Promise<ClaudeSession> {
+  const systemPrompt = config.systemPrompt || DEFAULT_SYSTEM_PROMPT;
+  let sessionId = "";
+  let closed = false;
+  let lastTurnCompletedCleanly = true;
+
+  // Persistent input stream — user messages are pushed here across turns
+  const userMessages = new AsyncQueue<SDKUserMessage>();
+
+  // Event channel — SDK events are routed here for sendMessage to consume
+  const sdkEvents = new AsyncQueue<SDKMessage>();
+
+  const options: Options = {
+    pathToClaudeCodeExecutable: CLAUDE_BIN,
+    includePartialMessages: true,
+    maxThinkingTokens: 10000,
+    appendSystemPrompt: systemPrompt,
+    permissionMode: config.permissionMode as Options["permissionMode"],
+    stderr: (data: string) => {
+      const msg = data.trim();
+      if (msg) console.error(`[claude-stderr] ${msg}`);
+    },
+  };
+
+  // Start persistent query — process spawns once and stays alive.
+  // NOTE: with AsyncIterable<SDKUserMessage> input, the SDK won't yield
+  // events until the first user message is consumed, so we don't block
+  // waiting for a system init event here. Session ID is captured when
+  // the system event arrives during the first turn.
+  const queryFn = queryOverride ?? claudeQuery;
+  const q = queryFn({ prompt: userMessages, options });
+
+  // Background: pump SDK events into our channel
+  (async () => {
+    try {
+      for await (const msg of q) {
+        if (msg.type === "system" && !sessionId) {
+          sessionId = msg.session_id;
+          console.log(`[claude] session ready (id=${sessionId})`);
+        }
+        sdkEvents.push(msg);
+      }
+    } catch (err) {
+      console.error("[claude] SDK pump error:", err);
+    } finally {
+      sdkEvents.close();
+    }
+  })();
+
+  console.log("[claude] persistent process started");
+
+  return {
+    async *sendMessage(text: string): AsyncIterable<ClaudeStreamEvent> {
+      if (closed) {
+        throw new Error("Session is closed.");
+      }
+
+      if (!text.trim()) {
+        throw new Error("Cannot send empty message.");
+      }
+
+      // If the previous turn was interrupted, consume remaining events until its result
+      if (!lastTurnCompletedCleanly) {
+        while (true) {
+          const msg = await sdkEvents.next();
+          if (!msg || msg.type === "result") break;
+        }
+      }
+      sdkEvents.drain();
+      lastTurnCompletedCleanly = false;
+
+      const t0 = Date.now();
+      let hasStreamedContent = false;
+      const toolUseBlocks = new Set<number>();
+      const thinkingBlocks = new Set<number>();
+
+      // Push user message into the live session
+      userMessages.push({
+        type: "user",
+        message: { content: text, role: "user" },
+        parent_tool_use_id: null,
+        session_id: sessionId,
+      });
+
+      // Read events for this turn until result
+      while (true) {
+        const msg = await sdkEvents.next();
+        if (!msg) break; // channel closed (process died)
+
+        // Streaming events (token-level deltas)
+        if (msg.type === "stream_event") {
+          const event = msg.event;
+
+          if (event.type === "content_block_start") {
+            if (event.content_block.type === "tool_use") {
+              hasStreamedContent = true;
+              toolUseBlocks.add(event.index);
+              yield { type: "tool_start", content: "", toolName: event.content_block.name };
+            }
+            if (event.content_block.type === "thinking") {
+              hasStreamedContent = true;
+              thinkingBlocks.add(event.index);
+              console.log(`[claude] thinking started at +${Date.now() - t0}ms`);
+              yield { type: "text_delta", content: "Thinking... " };
+            }
+            continue;
+          }
+
+          if (event.type === "content_block_delta") {
+            if (event.delta.type === "text_delta") {
+              if (!hasStreamedContent) {
+                console.log(`[claude] first delta at +${Date.now() - t0}ms`);
+              }
+              hasStreamedContent = true;
+              yield { type: "text_delta", content: event.delta.text };
+            }
+            continue;
+          }
+
+          if (event.type === "content_block_stop") {
+            if (thinkingBlocks.has(event.index)) {
+              thinkingBlocks.delete(event.index);
+              console.log(`[claude] thinking ended at +${Date.now() - t0}ms`);
+            }
+            if (toolUseBlocks.has(event.index)) {
+              toolUseBlocks.delete(event.index);
+              yield { type: "tool_end", content: "" };
+            }
+            continue;
+          }
+
+          continue;
+        }
+
+        // Full assistant message — fallback if streaming didn't produce deltas
+        if (msg.type === "assistant") {
+          if (hasStreamedContent) {
+            console.log(`[claude] full message at +${Date.now() - t0}ms (skipped, already streamed)`);
+          } else {
+            console.log(`[claude] full message at +${Date.now() - t0}ms (no streaming, using fallback)`);
+            const blocks = msg.message.content;
+            if (Array.isArray(blocks)) {
+              for (const block of blocks) {
+                if (block.type === "text") {
+                  yield { type: "text_delta", content: block.text };
+                }
+                if (block.type === "tool_use") {
+                  yield { type: "tool_start", content: "", toolName: block.name };
+                }
+              }
+            }
+          }
+          toolUseBlocks.clear();
+          continue;
+        }
+
+        // Skip system events and synthetic user messages (tool results)
+        if (msg.type === "system" || msg.type === "user") {
+          continue;
+        }
+
+        // Result — turn complete
+        if (msg.type === "result") {
+          lastTurnCompletedCleanly = true;
+          console.log(`[claude] result at +${Date.now() - t0}ms (streamed=${hasStreamedContent})`);
+          if (msg.is_error) {
+            yield { type: "error", content: msg.subtype === "success" ? String((msg as any).result) : msg.subtype };
+          }
+          break;
+        }
+      }
+
+      yield { type: "result", content: "" };
+    },
+
+    interrupt(): void {
+      q.interrupt();
+    },
+
+    async close(): Promise<void> {
+      closed = true;
+      userMessages.close();
+      await q.interrupt();
+    },
+  };
+}
+
+export { createClaudeSession };
+export type { ClaudeSession };

package/sidecar/endpointing.ts

@@ -0,0 +1,163 @@
+/**
+ * Endpointing module -- determines when the user is done speaking.
+ *
+ * Uses a two-tier approach to decide turn completion:
+ * - Fast path: VAD silence duration + sufficient word count (0ms latency)
+ * - Slow path: Haiku semantic check for short/ambiguous utterances (~200ms)
+ * - Timeout path: Forces completion after extended silence regardless of content
+ *
+ * Responsibilities:
+ * - Track silence duration from VAD events
+ * - Apply fast-path completion for longer utterances
+ * - Call Haiku API for semantic turn-completion on short utterances
+ * - Force timeout after extended silence
+ * - Reset state between turns
+ */
+
+import Anthropic from "@anthropic-ai/sdk";
+import type { EndpointDecision, EndpointingConfig, VadEvent } from "./types.js";
+
+// ============================================================================
+// CONSTANTS
+// ============================================================================
+
+const HAIKU_MODEL = "claude-haiku-4-5-20251001";
+const HAIKU_MAX_TOKENS = 10;
+
+// ============================================================================
+// INTERFACES
+// ============================================================================
+
+/**
+ * Endpointer that processes VAD events and decides when the user is done speaking.
+ */
+export interface Endpointer {
+  /**
+   * Process a VAD event and determine if the user's turn is complete.
+   * @param event - The VAD event from the voice activity detector
+   * @param currentTranscript - The accumulated transcript so far
+   * @returns Decision on whether the user has finished speaking
+   */
+  onVadEvent(event: VadEvent, currentTranscript: string): Promise<EndpointDecision>;
+
+  /**
+   * Reset internal state for a new turn.
+   */
+  reset(): void;
+}
+
+// ============================================================================
+// MAIN ENTRYPOINT
+// ============================================================================
+
+/**
+ * Create an endpointer instance with the given configuration.
+ * @param config - Endpointing thresholds and feature flags
+ * @returns A configured Endpointer
+ */
+export function createEndpointer(config: EndpointingConfig): Endpointer {
+  const anthropicClient = config.enableHaikuFallback ? new Anthropic() : null;
+
+  return {
+    onVadEvent(event: VadEvent, currentTranscript: string): Promise<EndpointDecision> {
+      return handleVadEvent(event, currentTranscript, config, anthropicClient);
+    },
+
+    reset(): void {
+      // No internal state to reset -- completion is evaluated per SPEECH_END event.
+    },
+  };
+}
+
+// ============================================================================
+// MAIN LOGIC
+// ============================================================================
+
+/**
+ * Handle a single VAD event and produce an endpoint decision.
+ * @param event - The VAD event to process
+ * @param transcript - Current accumulated transcript
+ * @param config - Endpointing configuration
+ * @param client - Anthropic client for Haiku calls (null if disabled)
+ * @returns The endpoint decision
+ */
+async function handleVadEvent(
+  event: VadEvent,
+  transcript: string,
+  config: EndpointingConfig,
+  client: Anthropic | null,
+): Promise<EndpointDecision> {
+  // Active speech -- not complete
+  if (event.type === "SPEECH_START" || event.type === "SPEECH_CONTINUE") {
+    return { isComplete: false, transcript, method: "vad_fast" };
+  }
+
+  // Speech ended -- evaluate completion immediately.
+  // avr-vad's SPEECH_END fires after internal debouncing (redemptionFrames),
+  // so silence has already been confirmed by the VAD. No need to wait for
+  // separate SILENCE events (avr-vad doesn't emit them).
+  if (event.type === "SPEECH_END") {
+    const wordCount = countWords(transcript);
+
+    // Fast path: sufficient words, complete immediately
+    if (wordCount >= config.minWordCountForFastPath) {
+      return { isComplete: true, transcript, method: "vad_fast" };
+    }
+
+    // Short utterance: ask Haiku for semantic turn-completion check
+    if (config.enableHaikuFallback && client !== null) {
+      const isComplete = await checkTurnCompletionWithHaiku(client, transcript);
+      return { isComplete, transcript, method: "haiku_semantic" };
+    }
+
+    // Haiku disabled, treat as complete
+    return { isComplete: true, transcript, method: "vad_fast" };
+  }
+
+  // Unknown event type -- not complete
+  return { isComplete: false, transcript, method: "vad_fast" };
+}
+
+// ============================================================================
+// HELPER FUNCTIONS
+// ============================================================================
+
+/**
+ * Count the number of words in a transcript string.
+ * @param text - The transcript text
+ * @returns Number of whitespace-separated words
+ */
+function countWords(text: string): number {
+  const trimmed = text.trim();
+  if (trimmed.length === 0) {
+    return 0;
+  }
+  return trimmed.split(/\s+/).length;
+}
+
+/**
+ * Call Haiku to determine if a short transcript represents a complete user turn.
+ * @param client - The Anthropic SDK client
+ * @param transcript - The short transcript to evaluate
+ * @returns True if Haiku considers the turn complete
+ */
+async function checkTurnCompletionWithHaiku(client: Anthropic, transcript: string): Promise<boolean> {
+  const response = await client.messages.create({
+    model: HAIKU_MODEL,
+    max_tokens: HAIKU_MAX_TOKENS,
+    messages: [
+      {
+        role: "user",
+        content: `Is this a complete user turn? Answer only "yes" or "no".\n\nTranscript: "${transcript}"`,
+      },
+    ],
+  });
+
+  const firstBlock = response.content[0];
+  if (firstBlock.type !== "text") {
+    throw new Error(`Unexpected Haiku response block type: ${firstBlock.type}`);
+  }
+
+  const answer = firstBlock.text.trim().toLowerCase();
+  return answer.startsWith("yes");
+}

package/sidecar/index.ts

@@ -0,0 +1,83 @@
+/**
+ * Entry point for the Claude Code voice sidecar.
+ *
+ * Thin wrapper that creates a local audio adapter and voice session.
+ * All voice loop logic lives in voice-session.ts.
+ *
+ * Responsibilities:
+ * - Load .env configuration via dotenv
+ * - Create a local AudioAdapter (VPIO echo cancellation)
+ * - Create a voice session with default config
+ * - Handle SIGINT/SIGTERM for clean shutdown
+ */
+
+import "dotenv/config";
+
+import { homedir } from "os";
+import { join } from "path";
+
+import { createLocalAudioAdapter } from "./local-audio.js";
+import { createVoiceSession } from "./voice-session.js";
+
+// ============================================================================
+// CONSTANTS
+// ============================================================================
+
+/** Mic capture sample rate in Hz (must match VAD/STT expectations) */
+const MIC_SAMPLE_RATE = 16000;
+
+/** TTS output sample rate in Hz -- must match tts-server.py output format */
+const TTS_SAMPLE_RATE = 24000;
+
+/** Default configuration for the voice session */
+const DEFAULT_CONFIG = {
+  stopPhrase: "stop listening",
+  sttModelPath: join(homedir(), ".claude-voice-models", "whisper-small"),
+  ttsModel: "prince-canuma/Kokoro-82M",
+  ttsVoice: "af_heart",
+  modelCacheDir: join(homedir(), ".claude-voice-models"),
+  interruptionThresholdMs: 1500,
+  endpointing: {
+    silenceThresholdMs: 700,
+    maxSilenceBeforeTimeoutMs: 1200,
+    minWordCountForFastPath: 2,
+    enableHaikuFallback: false,
+  },
+  narration: {
+    summaryIntervalMs: 12000,
+  },
+  claudeSession: {
+    allowedTools: [] as string[],
+    permissionMode: "bypassPermissions",
+    systemPrompt:
+      "Respond concisely. You are in voice mode -- your responses will be spoken aloud. Keep answers conversational and brief.",
+  },
+};
+
+// ============================================================================
+// ENTRY POINT
+// ============================================================================
+
+/**
+ * Main entry point. Creates the local audio adapter and voice session,
+ * then waits for shutdown via stop phrase or signal.
+ */
+async function main(): Promise<void> {
+  const adapter = await createLocalAudioAdapter(MIC_SAMPLE_RATE, TTS_SAMPLE_RATE);
+
+  const session = await createVoiceSession(adapter, {
+    ...DEFAULT_CONFIG,
+    onSessionEnd: () => process.exit(0),
+  });
+
+  const signalHandler = () => {
+    session.stop().then(() => process.exit(0));
+  };
+  process.on("SIGINT", signalHandler);
+  process.on("SIGTERM", signalHandler);
+}
+
+main().catch((err) => {
+  console.error(`Voice loop failed: ${err}`);
+  process.exit(1);
+});

package/sidecar/local-audio.ts

@@ -0,0 +1,126 @@
+/**
+ * Local audio adapter wrapping the VPIO-based audio-capture module.
+ *
+ * Implements the AudioAdapter interface for the laptop mic path using macOS
+ * Voice Processing IO (VPIO) with echo cancellation. Delegates all low-level
+ * audio I/O to audio-capture.ts (singleton module).
+ *
+ * Responsibilities:
+ * - Start the VPIO binary via startCapture()
+ * - Wire VPIO stdout through bufferToFloat32 to the onAudio callback
+ * - Write PCM audio to VPIO stdin with backpressure handling
+ * - Send SIGUSR1/SIGUSR2 signals for interrupt/resume
+ * - Play the macOS system chime via afplay
+ */
+
+import { spawn } from "child_process";
+
+import { startCapture, stopCapture, interruptPlayback, resumePlayback, bufferToFloat32 } from "./audio-capture.js";
+
+import type { Writable } from "stream";
+import type { AudioAdapter } from "./audio-adapter.js";
+
+// ============================================================================
+// CONSTANTS
+// ============================================================================
+
+/** macOS system sound played when the agent finishes speaking and starts listening */
+const READY_CHIME_PATH = "/System/Library/Sounds/Glass.aiff";
+
+// ============================================================================
+// MAIN ENTRYPOINT
+// ============================================================================
+
+/**
+ * Create a local AudioAdapter backed by the VPIO echo-cancelling audio process.
+ *
+ * Starts the mic-vpio binary, wires stdout through bufferToFloat32 to the
+ * onAudio callback, and returns an AudioAdapter.
+ *
+ * @param micRate - Mic output sample rate in Hz (e.g. 16000 for VAD/STT)
+ * @param speakerRate - Speaker input sample rate in Hz (e.g. 24000 for TTS)
+ * @returns An AudioAdapter for local mic I/O
+ * @throws Error if VPIO binary fails to start
+ */
+export async function createLocalAudioAdapter(micRate: number, speakerRate: number): Promise<AudioAdapter> {
+  const audioIO = await startCapture(micRate, speakerRate);
+  const micStream = audioIO.micStream;
+  const speakerInput: Writable = audioIO.speakerInput;
+
+  let audioCallback: ((samples: Float32Array) => void) | null = null;
+
+  /**
+   * Subscribe to incoming audio chunks from the VPIO mic stream.
+   * Converts each Buffer chunk to Float32Array and invokes the callback.
+   *
+   * @param callback - Called with each audio chunk as Float32Array
+   */
+  function onAudio(callback: (samples: Float32Array) => void): void {
+    audioCallback = callback;
+
+    micStream.on("data", (chunk: Buffer) => {
+      const samples = bufferToFloat32(chunk);
+      audioCallback?.(samples);
+    });
+
+    micStream.on("error", (err: Error) => {
+      console.error(`Mic stream error: ${err.message}`);
+    });
+  }
+
+  /**
+   * Write PCM audio to the VPIO speaker stream with backpressure handling.
+   *
+   * @param pcm - Raw PCM buffer (16-bit signed, 24kHz mono)
+   * @returns Resolves when the write completes
+   */
+  function writeSpeaker(pcm: Buffer): Promise<void> {
+    return new Promise<void>((resolve, reject) => {
+      const ok = speakerInput.write(pcm, (err: Error | null | undefined) => {
+        if (err) reject(err);
+      });
+      if (ok) {
+        resolve();
+      } else {
+        speakerInput.once("drain", () => resolve());
+      }
+    });
+  }
+
+  /**
+   * Clear the VPIO playback buffer immediately (sends SIGUSR1).
+   */
+  function interrupt(): void {
+    interruptPlayback();
+  }
+
+  /**
+   * Resume VPIO stdin processing after an interrupt (sends SIGUSR2).
+   */
+  function resume(): void {
+    resumePlayback();
+  }
+
+  /**
+   * Play the macOS system ready chime. Fire-and-forget.
+   */
+  function playChime(): void {
+    spawn("afplay", ["--volume", "6", READY_CHIME_PATH]).on("error", () => {});
+  }
+
+  /**
+   * Stop the VPIO process and free all resources.
+   */
+  function destroy(): void {
+    stopCapture();
+  }
+
+  return {
+    onAudio,
+    writeSpeaker,
+    interrupt,
+    resume,
+    playChime,
+    destroy,
+  };
+}