use-voice-control 0.1.37 → 0.1.39

package/speech/client/read-aloud.ts

@@ -0,0 +1,305 @@
+/**
+ * @fileoverview Browser-side "read aloud" playback engine.
+ *
+ * `ReadAloudController` takes a block of text, chunks it along sentence/paragraph
+ * boundaries with `splitTextSmart`, synthesizes each chunk (Kokoro by default, via
+ * the package's `/api/speech/tts` route) and plays the chunks back-to-back while
+ * pre-fetching the next one, so playback starts after the first short chunk rather
+ * than after the whole document. Pause/resume/stop are supported throughout, and
+ * each chunk is reported to `onChunk` so callers can highlight what is being spoken.
+ *
+ * When no TTS endpoint is reachable — a host app that has not mounted the route, or
+ * an offline build — the controller falls back to the browser's built-in
+ * `speechSynthesis` rather than failing, so the feature still works everywhere.
+ */
+import type { TTSProvider } from "../types/types";
+import { splitTextSmart } from "../utils/semantic-split.js";
+
+export type ReadAloudState = "idle" | "loading" | "speaking" | "paused";
+
+export interface ReadAloudChunk {
+  /** The text of the chunk currently being spoken. */
+  text: string;
+  /** Zero-based position of this chunk in the queue. */
+  index: number;
+  /** Total number of chunks queued for this utterance. */
+  total: number;
+}
+
+/** Turns a chunk of text into playable audio. Return `null` to defer to `speechSynthesis`. */
+export type SynthesizeFn = (
+  text: string,
+  signal: AbortSignal
+) => Promise<Blob | null>;
+
+export interface ReadAloudOptions {
+  /** TTS provider passed to the endpoint. Default `kokoro`. */
+  provider?: TTSProvider;
+  /** Provider-specific voice id. Default `af_heart`. */
+  voice?: string;
+  /** HTTP route that synthesizes text. Default `/api/speech/tts`. */
+  endpoint?: string;
+  /**
+   * Target chunk size in characters. Smaller starts talking sooner but makes the
+   * seams between chunks more audible. Default 240.
+   */
+  maxChunkLength?: number;
+  /** Override synthesis entirely (tests, a bring-your-own-TTS host app). */
+  synthesize?: SynthesizeFn;
+  /** Called as each chunk starts playing. */
+  onChunk?: (chunk: ReadAloudChunk) => void;
+  /** Called on every state transition. */
+  onStateChange?: (state: ReadAloudState) => void;
+  /** Called when playback ends, either by running out of chunks or by `stop()`. */
+  onEnd?: (reason: "finished" | "stopped") => void;
+  /** Called when synthesis or playback fails irrecoverably. */
+  onError?: (error: Error) => void;
+}
+
+const DEFAULT_ENDPOINT = "/api/speech/tts";
+const DEFAULT_VOICE = "af_heart";
+const DEFAULT_MAX_CHUNK = 240;
+
+/** Resolves once `signal` aborts. Used to race playback against `stop()`. */
+function abortPromise(signal: AbortSignal): Promise<void> {
+  if (signal.aborted) return Promise.resolve();
+  return new Promise((resolve) => {
+    signal.addEventListener("abort", () => resolve(), { once: true });
+  });
+}
+
+export class ReadAloudController {
+  private options: ReadAloudOptions;
+  private state: ReadAloudState = "idle";
+  private abortController: AbortController | null = null;
+  private audio: HTMLAudioElement | null = null;
+  private objectUrl: string | null = null;
+  /** Set once the endpoint has proved unreachable, so later chunks skip the retry. */
+  private endpointUnavailable = false;
+
+  constructor(options: ReadAloudOptions = {}) {
+    this.options = options;
+  }
+
+  /** Replace the options (voice, callbacks, …) without discarding playback state. */
+  setOptions(options: ReadAloudOptions): void {
+    this.options = options;
+  }
+
+  getState(): ReadAloudState {
+    return this.state;
+  }
+
+  isActive(): boolean {
+    return this.state === "speaking" || this.state === "paused" || this.state === "loading";
+  }
+
+  /**
+   * Speak `text`, cancelling anything already playing. Resolves when playback
+   * finishes or is stopped — it never rejects; failures go to `onError`.
+   */
+  async speak(text: string): Promise<void> {
+    this.stop();
+
+    const maxChunkLength = this.options.maxChunkLength ?? DEFAULT_MAX_CHUNK;
+    const chunks = splitTextSmart(text ?? "", maxChunkLength)
+      .map((chunk) => chunk.trim())
+      .filter((chunk) => chunk.length > 0);
+
+    if (chunks.length === 0) return;
+
+    const abortController = new AbortController();
+    this.abortController = abortController;
+    const { signal } = abortController;
+    this.setState("loading");
+
+    try {
+      // Kick off the first chunk, then always keep exactly one chunk in flight
+      // ahead of the one playing.
+      let upcoming = this.synthesize(chunks[0], signal);
+
+      for (let index = 0; index < chunks.length; index += 1) {
+        const current = upcoming;
+        upcoming =
+          index + 1 < chunks.length
+            ? this.synthesize(chunks[index + 1], signal)
+            : Promise.resolve(null);
+
+        const blob = await current;
+        if (signal.aborted) return;
+
+        // Announce the state before the chunk so a listener reacting to
+        // `onChunk` sees the controller already speaking.
+        this.setState("speaking");
+        this.options.onChunk?.({ text: chunks[index], index, total: chunks.length });
+        if (signal.aborted) return;
+        await this.playChunk(blob, chunks[index], signal);
+        if (signal.aborted) return;
+      }
+
+      this.cleanup();
+      this.setState("idle");
+      this.options.onEnd?.("finished");
+    } catch (error) {
+      if (signal.aborted) return;
+      this.cleanup();
+      this.setState("idle");
+      this.options.onError?.(
+        error instanceof Error ? error : new Error(String(error))
+      );
+      this.options.onEnd?.("stopped");
+    } finally {
+      if (this.abortController === abortController) {
+        this.abortController = null;
+      }
+    }
+  }
+
+  pause(): void {
+    if (this.state !== "speaking") return;
+    if (this.audio) {
+      this.audio.pause();
+    } else if (typeof speechSynthesis !== "undefined") {
+      speechSynthesis.pause();
+    }
+    this.setState("paused");
+  }
+
+  resume(): void {
+    if (this.state !== "paused") return;
+    if (this.audio) {
+      void this.audio.play().catch(() => undefined);
+    } else if (typeof speechSynthesis !== "undefined") {
+      speechSynthesis.resume();
+    }
+    this.setState("speaking");
+  }
+
+  /** Stop playback and drop any queued chunks. Safe to call when already idle. */
+  stop(): void {
+    const wasActive = this.isActive();
+    this.abortController?.abort();
+    this.abortController = null;
+    this.cleanup();
+    if (wasActive) {
+      this.setState("idle");
+      this.options.onEnd?.("stopped");
+    }
+  }
+
+  private setState(state: ReadAloudState): void {
+    if (this.state === state) return;
+    this.state = state;
+    this.options.onStateChange?.(state);
+  }
+
+  private cleanup(): void {
+    if (this.audio) {
+      this.audio.pause();
+      this.audio.src = "";
+      this.audio = null;
+    }
+    if (this.objectUrl) {
+      URL.revokeObjectURL(this.objectUrl);
+      this.objectUrl = null;
+    }
+    if (typeof speechSynthesis !== "undefined") {
+      speechSynthesis.cancel();
+    }
+  }
+
+  private synthesize(text: string, signal: AbortSignal): Promise<Blob | null> {
+    const promise = this.options.synthesize
+      ? this.options.synthesize(text, signal)
+      : this.fetchAudio(text, signal);
+    // Pre-fetched chunks are awaited a beat later; mark them handled now so a
+    // stop() mid-flight does not surface as an unhandled rejection.
+    promise.catch(() => undefined);
+    return promise;
+  }
+
+  private async fetchAudio(text: string, signal: AbortSignal): Promise<Blob | null> {
+    if (this.endpointUnavailable) return null;
+
+    const endpoint = this.options.endpoint ?? DEFAULT_ENDPOINT;
+    try {
+      const response = await fetch(endpoint, {
+        method: "POST",
+        headers: { "Content-Type": "application/json" },
+        body: JSON.stringify({
+          text,
+          provider: this.options.provider ?? "kokoro",
+          voice: this.options.voice ?? DEFAULT_VOICE,
+        }),
+        signal,
+      });
+
+      if (!response.ok) throw new Error(`TTS failed: ${response.status}`);
+
+      const contentType = response.headers.get("Content-Type") || "audio/wav";
+      return new Blob([await response.arrayBuffer()], { type: contentType });
+    } catch (error) {
+      if (signal.aborted) throw error;
+      // No usable endpoint in this host app — remember it and let every
+      // remaining chunk go straight to the browser's own synthesizer.
+      this.endpointUnavailable = true;
+      return null;
+    }
+  }
+
+  private playChunk(
+    blob: Blob | null,
+    text: string,
+    signal: AbortSignal
+  ): Promise<void> {
+    return blob
+      ? this.playAudioBlob(blob, signal)
+      : this.playWithSpeechSynthesis(text, signal);
+  }
+
+  private async playAudioBlob(blob: Blob, signal: AbortSignal): Promise<void> {
+    const url = URL.createObjectURL(blob);
+    const audio = new Audio(url);
+    this.audio = audio;
+    this.objectUrl = url;
+
+    const finished = new Promise<void>((resolve, reject) => {
+      audio.addEventListener("ended", () => resolve(), { once: true });
+      audio.addEventListener(
+        "error",
+        () => reject(new Error("Audio playback failed")),
+        { once: true }
+      );
+    });
+
+    try {
+      await audio.play();
+      await Promise.race([finished, abortPromise(signal)]);
+    } finally {
+      if (this.audio === audio) this.audio = null;
+      if (this.objectUrl === url) this.objectUrl = null;
+      audio.pause();
+      URL.revokeObjectURL(url);
+    }
+  }
+
+  private async playWithSpeechSynthesis(
+    text: string,
+    signal: AbortSignal
+  ): Promise<void> {
+    if (typeof speechSynthesis === "undefined" || typeof SpeechSynthesisUtterance === "undefined") {
+      throw new Error("No speech synthesis available in this browser");
+    }
+
+    const utterance = new SpeechSynthesisUtterance(text);
+    const finished = new Promise<void>((resolve, reject) => {
+      utterance.onend = () => resolve();
+      // `cancel()` fires an error event; a stop is not a failure.
+      utterance.onerror = () =>
+        signal.aborted ? resolve() : reject(new Error("Speech synthesis failed"));
+    });
+
+    speechSynthesis.speak(utterance);
+    await Promise.race([finished, abortPromise(signal)]);
+  }
+}

package/speech/core/KokoroTTS.js

@@ -1,3 +1,10 @@
+/**
+ * @fileoverview Standalone `KokoroTTS` class generating audio from text via a Kokoro model loaded from Hugging Face, using transformers.js pulled from a CDN URL.
+ *
+ * Near-identical unused duplicate of `core/kokoro.js` (same CDN-based implementation,
+ * missing that file's commented-out package-import alternative); no other file in this
+ * package imports `KokoroTTS.js` directly.
+ */
 import { StyleTextToSpeech2Model, AutoTokenizer, Tensor, RawAudio } from "https://cdn.jsdelivr.net/npm/@huggingface/transformers@3.5.1/dist/transformers.min.js";
 
 import { phonemize } from "./phonemize.js";

package/speech/core/kokoro.js

@@ -1,3 +1,11 @@
+/**
+ * @fileoverview Browser/worker `KokoroTTS` class that generates audio from text using a Kokoro model loaded via Hugging Face transformers.
+ *
+ * Loads `@huggingface/transformers` from a jsdelivr CDN URL (rather than the npm package)
+ * so it can run inside the Web Worker in `legacy/worker.js` without a bundler resolving the
+ * dependency. Kept alongside the near-duplicate `KokoroTTS.js` in this directory; this is the
+ * file actually imported by `legacy/worker.js`.
+ */
 import { StyleTextToSpeech2Model, AutoTokenizer, Tensor, RawAudio } from "https://cdn.jsdelivr.net/npm/@huggingface/transformers@3.5.1/dist/transformers.min.js";
 //import { StyleTextToSpeech2Model, AutoTokenizer, Tensor, RawAudio } from "@huggingface/transformers";
 

package/speech/legacy/conversation.js

@@ -1,3 +1,10 @@
+/**
+ * @fileoverview Legacy `Conversation` class driving the full voice loop: record speech, transcribe it, stream an LLM chat completion, and speak the response sentence-by-sentence.
+ *
+ * Pre-refactor, DOM-coupled implementation (queries elements like `#serverUrl` and
+ * `#systemPrompt` directly) that wires together `SpeechToText`, the streaming chat fetch,
+ * `processStreamingText` for incremental sentence detection, and `textToSpeech` playback.
+ */
 import { SpeechToText } from "./stt.js";
 import { textToSpeech, ttsModelReadyPromise } from "./tts.js";
 import { processStreamingText } from "./sentence-detector.js";

package/speech/legacy/stt.js

@@ -1,3 +1,10 @@
+/**
+ * @fileoverview Legacy `SpeechToText` class performing in-browser speech recognition via a Hugging Face transformers.js ASR pipeline.
+ *
+ * Detects WebGPU support to pick device/dtype, records microphone audio with
+ * `MediaRecorder`, converts it to WAV (with a gain-boosted retry if transcription comes
+ * back empty), and transcribes it with the `onnx-community/moonshine-base-ONNX` model.
+ */
 import { pipeline } from "https://cdn.jsdelivr.net/npm/@huggingface/transformers@3.5.1/dist/transformers.min.js";
 
 import { convertAudioBufferToWav, resampleAudio, applyAudioGain } from "./audio-utils.js";

package/speech/legacy/tts.js

@@ -1,3 +1,10 @@
+/**
+ * @fileoverview Legacy `textToSpeech` entry point that offloads Kokoro TTS generation to a Web Worker and streams the resulting audio to an `AudioPlayer`.
+ *
+ * Exposes `ttsModelReadyPromise` (resolved once the worker reports its model is loaded) and
+ * `textToSpeech(text, voice)`, which strips Markdown formatting before posting the request
+ * to `worker.js`.
+ */
 import { AudioPlayer } from "./AudioPlayer.js";
 
 const my_worker = new Worker(new URL("./worker.js", import.meta.url), { type: "module" });

package/speech/legacy/worker.js

@@ -1,3 +1,10 @@
+/**
+ * @fileoverview Web Worker (legacy) that loads the Kokoro TTS model off the main thread and generates audio for queued text chunks, streaming each result back via `postMessage`.
+ *
+ * Detects WebGPU, loads `KokoroTTS` from `../core/kokoro.js`, splits incoming text into
+ * chunks with `splitTextSmart`, and processes them one at a time through a simple queue,
+ * transferring each generated audio buffer back to the main thread.
+ */
 import { KokoroTTS } from "./kokoro.js";
 import { splitTextSmart } from "./semantic-split.js";
 

package/speech/react/SpokenPhraseOverlay.tsx

@@ -0,0 +1,126 @@
+/**
+ * @fileoverview Full-screen display of the phrase that was just spoken.
+ *
+ * While dictating, the words land in whatever input has focus — which is easy to
+ * lose track of when you are looking away from the screen. This overlay echoes the
+ * latest phrase in large type at the centre of the viewport and fades out two
+ * seconds after the last update, so a glance confirms you were heard correctly.
+ *
+ * It renders into `document.body` through a portal with pointer events disabled, so
+ * it never intercepts clicks or gets clipped by the host app's layout, and it is
+ * styled inline so it looks the same in every app that mounts it regardless of the
+ * CSS framework in play.
+ */
+import { useEffect, useRef, useState, type CSSProperties } from "react";
+import { createPortal } from "react-dom";
+
+export interface SpokenPhraseOverlayProps {
+  /** The words to display. An empty string hides the overlay. */
+  phrase: string;
+  /**
+   * Bump this whenever `phrase` is refreshed to restart the hide timer, including
+   * when the same words are repeated. `useLiveTranscription` supplies it.
+   */
+  phraseId?: number;
+  /** How long the phrase stays up after the last update. Default 2000ms. */
+  durationMs?: number;
+  /** Set false to suppress the overlay entirely (e.g. once dictation stops). */
+  visible?: boolean;
+  /** Stacking order. Default 2147483000, above typical app chrome and modals. */
+  zIndex?: number;
+  className?: string;
+}
+
+const HIDE_AFTER_MS = 2000;
+
+export function SpokenPhraseOverlay({
+  phrase,
+  phraseId,
+  durationMs = HIDE_AFTER_MS,
+  visible = true,
+  zIndex = 2147483000,
+  className,
+}: SpokenPhraseOverlayProps) {
+  const [mounted, setMounted] = useState(false);
+  const [shown, setShown] = useState(false);
+  // Held separately from `phrase` so the text stays put while fading out rather
+  // than blanking the moment the transcript clears.
+  const [displayed, setDisplayed] = useState("");
+  const timerRef = useRef<ReturnType<typeof setTimeout> | null>(null);
+
+  useEffect(() => setMounted(true), []);
+
+  useEffect(() => {
+    const text = phrase.trim();
+    if (!visible || !text) {
+      if (!visible) setShown(false);
+      return;
+    }
+
+    setDisplayed(text);
+    setShown(true);
+
+    if (timerRef.current) clearTimeout(timerRef.current);
+    timerRef.current = setTimeout(() => setShown(false), durationMs);
+
+    return () => {
+      if (timerRef.current) clearTimeout(timerRef.current);
+    };
+  }, [phrase, phraseId, visible, durationMs]);
+
+  useEffect(() => {
+    return () => {
+      if (timerRef.current) clearTimeout(timerRef.current);
+    };
+  }, []);
+
+  if (!mounted || !displayed) return null;
+
+  const containerStyle: CSSProperties = {
+    position: "fixed",
+    inset: 0,
+    display: "flex",
+    alignItems: "center",
+    justifyContent: "center",
+    pointerEvents: "none",
+    padding: "6vw",
+    zIndex,
+    opacity: shown ? 1 : 0,
+    transition: "opacity 320ms ease-out",
+  };
+
+  const textStyle: CSSProperties = {
+    maxWidth: "min(1100px, 90vw)",
+    textAlign: "center",
+    // Scales with the viewport so a short phrase fills the screen on a laptop
+    // and still fits on a phone.
+    fontSize: "clamp(2rem, 7vw, 6rem)",
+    fontWeight: 700,
+    lineHeight: 1.1,
+    letterSpacing: "-0.02em",
+    color: "#ffffff",
+    // A dark pill keeps the words readable over light and dark pages alike.
+    background: "rgba(15, 23, 42, 0.82)",
+    borderRadius: "1.5rem",
+    padding: "0.6em 0.8em",
+    boxShadow: "0 25px 80px rgba(0, 0, 0, 0.45)",
+    backdropFilter: "blur(8px)",
+    transform: shown ? "scale(1)" : "scale(0.96)",
+    transition: "transform 320ms ease-out",
+    overflowWrap: "anywhere",
+  };
+
+  return createPortal(
+    <div
+      aria-hidden="true"
+      className={className}
+      data-spoken-phrase-overlay=""
+      style={containerStyle}
+    >
+      <div style={textStyle}>{displayed}</div>
+    </div>,
+    document.body
+  );
+}
+
+export default SpokenPhraseOverlay;

package/speech/react/index.ts

@@ -0,0 +1,27 @@
+/**
+ * @fileoverview Public entry point for the React layer: the read-aloud and live
+ * dictation hooks, plus the centred overlay that echoes the phrase just spoken.
+ */
+export { useReadAloud, type UseReadAloudOptions, type UseReadAloudReturn } from "./useReadAloud";
+
+export {
+  useLiveTranscription,
+  type UseLiveTranscriptionOptions,
+  type UseLiveTranscriptionReturn,
+} from "./useLiveTranscription";
+
+export {
+  SpokenPhraseOverlay,
+  type SpokenPhraseOverlayProps,
+} from "./SpokenPhraseOverlay";
+
+export {
+  ReadAloudController,
+  LiveTranscriber,
+  isTranscriptionSupported,
+  type ReadAloudChunk,
+  type ReadAloudOptions,
+  type ReadAloudState,
+  type LiveTranscriberOptions,
+  type TranscriberEngine,
+} from "../client";

package/speech/react/useLiveTranscription.ts

@@ -0,0 +1,142 @@
+/**
+ * @fileoverview React binding for `LiveTranscriber`.
+ *
+ * Exposes the two streams a dictation UI needs: `partial`, which changes on every
+ * word while a phrase is in progress, and the committed phrases delivered through
+ * `onCommit`. `lastPhrase` carries whatever was most recently heard — partial or
+ * committed — along with a monotonically increasing `phraseId` so a display can
+ * re-trigger its own timeout even when the same words are said twice in a row.
+ */
+import { useCallback, useEffect, useMemo, useRef, useState } from "react";
+
+import {
+  LiveTranscriber,
+  isTranscriptionSupported,
+  type LiveTranscriberOptions,
+} from "../client/live-transcriber";
+
+export interface UseLiveTranscriptionOptions
+  extends Omit<LiveTranscriberOptions, "onStateChange"> {}
+
+export interface UseLiveTranscriptionReturn {
+  isListening: boolean;
+  /** True when this browser can dictate at all. */
+  isSupported: boolean;
+  /** The in-progress phrase, updating as it is spoken. Empty between phrases. */
+  partial: string;
+  /** The most recent thing heard, partial or committed. */
+  lastPhrase: string;
+  /** Increments on every `lastPhrase` update, including repeats of the same words. */
+  phraseId: number;
+  error: Error | null;
+  start: () => Promise<void>;
+  stop: () => Promise<void>;
+  toggle: () => Promise<void>;
+}
+
+export function useLiveTranscription(
+  options: UseLiveTranscriptionOptions = {}
+): UseLiveTranscriptionReturn {
+  const [isListening, setIsListening] = useState(false);
+  const [isSupported, setIsSupported] = useState(false);
+  const [partial, setPartial] = useState("");
+  const [lastPhrase, setLastPhrase] = useState("");
+  const [phraseId, setPhraseId] = useState(0);
+  const [error, setError] = useState<Error | null>(null);
+
+  const optionsRef = useRef(options);
+  optionsRef.current = options;
+
+  // Support depends on `window`, so it can only be settled after hydration.
+  useEffect(() => setIsSupported(isTranscriptionSupported()), []);
+
+  const transcriber = useMemo(
+    () =>
+      new LiveTranscriber({
+        onStateChange: setIsListening,
+        onPartial: (text) => {
+          setPartial(text);
+          if (text) {
+            setLastPhrase(text);
+            setPhraseId((id) => id + 1);
+          }
+          optionsRef.current.onPartial?.(text);
+        },
+        onCommit: (text) => {
+          setPartial("");
+          setLastPhrase(text);
+          setPhraseId((id) => id + 1);
+          optionsRef.current.onCommit?.(text);
+        },
+        onError: (err) => {
+          setError(err);
+          optionsRef.current.onError?.(err);
+        },
+      }),
+    []
+  );
+
+  // Engine/language changes take effect on the next `start()`.
+  useEffect(() => {
+    transcriber.setOptions({
+      engine: options.engine,
+      language: options.language,
+      model: options.model,
+      onStateChange: setIsListening,
+      onPartial: (text) => {
+        setPartial(text);
+        if (text) {
+          setLastPhrase(text);
+          setPhraseId((id) => id + 1);
+        }
+        optionsRef.current.onPartial?.(text);
+      },
+      onCommit: (text) => {
+        setPartial("");
+        setLastPhrase(text);
+        setPhraseId((id) => id + 1);
+        optionsRef.current.onCommit?.(text);
+      },
+      onError: (err) => {
+        setError(err);
+        optionsRef.current.onError?.(err);
+      },
+    });
+  }, [transcriber, options.engine, options.language, options.model]);
+
+  useEffect(() => {
+    return () => {
+      void transcriber.stop();
+    };
+  }, [transcriber]);
+
+  const start = useCallback(async () => {
+    setError(null);
+    await transcriber.start();
+  }, [transcriber]);
+
+  const stop = useCallback(async () => {
+    await transcriber.stop();
+    setPartial("");
+  }, [transcriber]);
+
+  const toggle = useCallback(async () => {
+    if (transcriber.isListening()) {
+      await stop();
+    } else {
+      await start();
+    }
+  }, [transcriber, start, stop]);
+
+  return {
+    isListening,
+    isSupported,
+    partial,
+    lastPhrase,
+    phraseId,
+    error,
+    start,
+    stop,
+    toggle,
+  };
+}