speech-to-speech 0.1.0

package/dist/stt/index.mjs

@@ -0,0 +1,664 @@
+import { getDefaultRealTimeVADOptions, MicVAD } from '@ricky0123/vad-web';
+
+// src/stt/reset-stt-logic.ts
+var ResetSTTLogic = class {
+  constructor(options = {}) {
+    this.partialTranscript = "";
+    this.maxSilenceMs = options.maxSilenceMs ?? 2e3;
+    this.maxUtteranceMs = options.maxUtteranceMs ?? 15e3;
+    this.onReset = options.onReset;
+    this.now = options.now ?? (() => Date.now());
+    const start = this.now();
+    this.utteranceStartedAt = start;
+    this.lastActivityAt = start;
+  }
+  recordSpeechActivity(timestamp) {
+    const now = timestamp ?? this.now();
+    this.lastActivityAt = now;
+    if (!this.utteranceStartedAt) {
+      this.utteranceStartedAt = now;
+    }
+  }
+  updatePartialTranscript(partial, timestamp) {
+    this.partialTranscript = partial;
+    this.recordSpeechActivity(timestamp);
+  }
+  shouldReset(timestamp) {
+    const now = timestamp ?? this.now();
+    const silenceElapsed = now - this.lastActivityAt;
+    const utteranceElapsed = now - this.utteranceStartedAt;
+    if (silenceElapsed >= this.maxSilenceMs) {
+      return "silence";
+    }
+    if (utteranceElapsed >= this.maxUtteranceMs) {
+      return "utterance-complete";
+    }
+    return null;
+  }
+  maybeReset(timestamp) {
+    const reason = this.shouldReset(timestamp);
+    if (reason) {
+      this.reset(reason, timestamp);
+    }
+    return reason;
+  }
+  forceReset(reason = "manual", timestamp) {
+    this.reset(reason, timestamp);
+  }
+  reset(reason, timestamp) {
+    const now = timestamp ?? this.now();
+    const stats = {
+      utteranceStartedAt: this.utteranceStartedAt,
+      lastActivityAt: this.lastActivityAt,
+      partialTranscript: this.partialTranscript
+    };
+    this.utteranceStartedAt = now;
+    this.lastActivityAt = now;
+    this.partialTranscript = "";
+    if (this.onReset) {
+      this.onReset(reason, stats);
+    }
+  }
+};
+var VADController = class {
+  constructor(options) {
+    this.vad = null;
+    this.voiceStartListeners = /* @__PURE__ */ new Set();
+    this.voiceStopListeners = /* @__PURE__ */ new Set();
+    this.running = false;
+    this.options = options;
+  }
+  async start() {
+    if (this.running && this.vad) {
+      if (!this.vad.listening) {
+        await this.vad.start();
+      }
+      return;
+    }
+    if (typeof navigator === "undefined" || !navigator.mediaDevices || !navigator.mediaDevices.getUserMedia) {
+      throw new Error("Microphone access is not available.");
+    }
+    try {
+      const ortAny = window.ort;
+      if (ortAny && ortAny.env && ortAny.env.wasm) {
+        ortAny.env.wasm.wasmPaths = "/ort/";
+      }
+      if (!this.vad) {
+        const defaultOptions = getDefaultRealTimeVADOptions("v5");
+        this.vad = await MicVAD.new({
+          ...defaultOptions,
+          startOnLoad: false,
+          onSpeechStart: () => {
+            this.emitVoiceStart();
+          },
+          onSpeechEnd: (audio) => {
+            this.emitVoiceStop();
+          },
+          onVADMisfire: () => {
+          },
+          minSpeechMs: this.options?.minSpeechMs || 150,
+          positiveSpeechThreshold: 0.5,
+          negativeSpeechThreshold: 0.35,
+          redemptionMs: this.options?.minSilenceMs || 450,
+          preSpeechPadMs: 50,
+          processorType: "ScriptProcessor",
+          onnxWASMBasePath: "/ort/",
+          baseAssetPath: "/vad/",
+          workletOptions: {}
+        });
+      }
+      if (!this.vad.listening) {
+        await this.vad.start();
+      }
+      this.running = true;
+    } catch (error) {
+      this.running = false;
+      throw new Error(
+        error?.message || "Failed to initialize voice activity detector"
+      );
+    }
+  }
+  stop() {
+    if (!this.running || !this.vad) return;
+    try {
+      this.vad.pause();
+      this.running = false;
+    } catch (error) {
+    }
+  }
+  destroy() {
+    this.stop();
+    if (this.vad) {
+      try {
+        this.vad.destroy();
+      } catch (error) {
+      }
+      this.vad = null;
+    }
+    this.voiceStartListeners.clear();
+    this.voiceStopListeners.clear();
+  }
+  isActive() {
+    return this.running && this.vad !== null && this.vad.listening;
+  }
+  onVoiceStart(listener) {
+    this.voiceStartListeners.add(listener);
+    return () => this.voiceStartListeners.delete(listener);
+  }
+  onVoiceStop(listener) {
+    this.voiceStopListeners.add(listener);
+    return () => this.voiceStopListeners.delete(listener);
+  }
+  emitVoiceStart() {
+    this.voiceStartListeners.forEach((listener) => {
+      try {
+        listener();
+      } catch (error) {
+        console.error("Error in voice start listener:", error);
+      }
+    });
+  }
+  emitVoiceStop() {
+    this.voiceStopListeners.forEach((listener) => {
+      try {
+        listener();
+      } catch (error) {
+        console.error("Error in voice stop listener:", error);
+      }
+    });
+  }
+};
+
+// src/stt/stt-logic.ts
+var ResetSTTLogic2 = class {
+  constructor(onLog, onTranscript, options = {}) {
+    this.isListening = false;
+    this.fullTranscript = "";
+    this.heardWords = [];
+    this.onWordsUpdate = null;
+    this.onMicTimeUpdate = null;
+    this.onRestartMetrics = null;
+    this.micOnTime = 0;
+    this.sessionDuration = 3e4;
+    this.lastTickTime = 0;
+    this.micTimeInterval = null;
+    this.restartCount = 0;
+    this.isRestarting = false;
+    this.isRecognitionRunning = false;
+    this.lastInterimTranscript = "";
+    this.lastInterimSaveTime = 0;
+    this.interimSaveInterval = 1e3;
+    this.lastInterimResultTime = 0;
+    this.lastSavedLength = 0;
+    this.transcriptBeforeRestart = "";
+    this.sessionStartTranscript = "";
+    this.sessionId = 0;
+    this.awaitingRestartFirstResultId = null;
+    this.lastWasFinal = false;
+    this.restartMetrics = {};
+    this.isAutoRestarting = false;
+    this.onLog = onLog;
+    this.onTranscript = onTranscript;
+    this.options = {
+      sessionDurationMs: options.sessionDurationMs ?? 3e4,
+      interimSaveIntervalMs: options.interimSaveIntervalMs ?? 5e3,
+      preserveTranscriptOnStart: options.preserveTranscriptOnStart ?? false
+    };
+    this.sessionDuration = this.options.sessionDurationMs;
+    this.interimSaveInterval = this.options.interimSaveIntervalMs;
+    const SpeechRecognitionAPI = window.SpeechRecognition || window.webkitSpeechRecognition;
+    if (!SpeechRecognitionAPI) {
+      this.onLog("Speech Recognition API not supported", "error");
+      throw new Error("Speech Recognition API not available");
+    }
+    this.recognition = new SpeechRecognitionAPI();
+    this.setupRecognition();
+  }
+  setWordsUpdateCallback(callback) {
+    this.onWordsUpdate = callback;
+  }
+  setMicTimeUpdateCallback(callback) {
+    this.onMicTimeUpdate = callback;
+  }
+  setRestartMetricsCallback(callback) {
+    this.onRestartMetrics = callback;
+  }
+  setVadCallbacks(onSpeechStart, onSpeechEnd) {
+    this.onUserSpeechStart = onSpeechStart || void 0;
+    this.onUserSpeechEnd = onSpeechEnd || void 0;
+  }
+  getSessionDurationMs() {
+    return this.sessionDuration;
+  }
+  isInAutoRestart() {
+    return this.isAutoRestarting;
+  }
+  getFullTranscript() {
+    if (this.transcriptBeforeRestart.length > 0) {
+      if (this.fullTranscript.length > 0) {
+        return (this.transcriptBeforeRestart + " " + this.fullTranscript).trim();
+      }
+      return this.transcriptBeforeRestart;
+    }
+    return this.fullTranscript;
+  }
+  clearTranscript() {
+    this.fullTranscript = "";
+    this.transcriptBeforeRestart = "";
+    this.sessionStartTranscript = "";
+    this.heardWords = [];
+  }
+  setupRecognition() {
+    this.recognition.lang = "en-US";
+    this.recognition.interimResults = true;
+    this.recognition.continuous = true;
+    this.recognition.maxAlternatives = 1;
+    this.resultHandler = (event) => {
+      const speechEvent = event;
+      let completeTranscript = "";
+      for (let i = 0; i < speechEvent.results.length; i++) {
+        completeTranscript += speechEvent.results[i][0].transcript + " ";
+      }
+      completeTranscript = completeTranscript.trim();
+      const isFinal = speechEvent.results[speechEvent.results.length - 1].isFinal;
+      completeTranscript = this.collapseRepeats(completeTranscript);
+      this.lastInterimTranscript = completeTranscript;
+      this.lastInterimResultTime = Date.now();
+      if (this.awaitingRestartFirstResultId != null) {
+        const rid = this.awaitingRestartFirstResultId;
+        if (this.restartMetrics[rid] && !this.restartMetrics[rid].firstResultAt) {
+          this.restartMetrics[rid].firstResultAt = Date.now();
+          const delta = this.restartMetrics[rid].firstResultAt - this.restartMetrics[rid].requestedAt;
+          this.onLog(
+            `\u{1F514} First result after restart #${rid} in ${delta}ms`,
+            "info"
+          );
+          this.awaitingRestartFirstResultId = null;
+        }
+      }
+      this.onLog(
+        `[${isFinal ? "FINAL" : "INTERIM"}] "${completeTranscript}"`,
+        isFinal ? "info" : "warning"
+      );
+      if (!isFinal && this.lastWasFinal) {
+        try {
+          this.onUserSpeechStart?.();
+        } catch {
+        }
+      }
+      this.lastWasFinal = isFinal;
+      if (isFinal) {
+        this.fullTranscript = (this.sessionStartTranscript + " " + completeTranscript).trim();
+        this.fullTranscript = this.collapseRepeats(this.fullTranscript);
+        this.heardWords = this.fullTranscript.split(/\s+/).filter((word) => word.length > 0);
+        this.onTranscript(this.getFullTranscript());
+        this.lastSavedLength = this.fullTranscript.length;
+        if (this.onWordsUpdate) this.onWordsUpdate(this.heardWords);
+        this.lastInterimTranscript = "";
+        if (this.awaitingRestartFirstResultId != null) {
+          const rid = this.awaitingRestartFirstResultId;
+          if (this.restartMetrics[rid] && !this.restartMetrics[rid].firstResultAt) {
+            this.restartMetrics[rid].firstResultAt = Date.now();
+            this.restartMetrics[rid].startedAt || this.restartMetrics[rid].startAttemptAt || Date.now();
+            const firstResultDelta = this.restartMetrics[rid].firstResultAt - this.restartMetrics[rid].requestedAt;
+            this.onLog(
+              `\u{1F514} First result after restart #${rid} in ${firstResultDelta}ms`,
+              "info"
+            );
+            this.awaitingRestartFirstResultId = null;
+          }
+        }
+      }
+    };
+    this.recognition.addEventListener("result", this.resultHandler);
+    this.errorHandler = (event) => {
+      const errorEvent = event;
+      if (errorEvent.error === "aborted" && this.isRestarting) {
+        this.onLog("Aborted during restart (ignored)", "info");
+        this.isRecognitionRunning = false;
+        return;
+      }
+      this.onLog(`Error: ${errorEvent.error}`, "error");
+      if (errorEvent.error === "no-speech" || errorEvent.error === "audio-capture" || errorEvent.error === "network") {
+        setTimeout(() => {
+          if (this.isListening && !this.isRestarting && !this.isRecognitionRunning) {
+            try {
+              this.recognition.start();
+              this.isRecognitionRunning = true;
+              this.sessionId++;
+            } catch (e) {
+              this.onLog(`Failed restart after error: ${e}`, "error");
+            }
+          }
+        }, 500);
+      } else {
+        this.onLog(
+          `Unhandled SpeechRecognition error: ${errorEvent.error}`,
+          "warning"
+        );
+      }
+    };
+    this.recognition.addEventListener("error", this.errorHandler);
+    this.endHandler = () => {
+      this.isRecognitionRunning = false;
+      if (this.isListening && !this.isRestarting) {
+        setTimeout(() => {
+          if (this.isListening && !this.isRestarting) {
+            try {
+              this.recognition.start();
+              this.isRecognitionRunning = true;
+              this.sessionId++;
+              this.onLog(
+                `\u{1F501} Auto-resumed recognition after end (session ${this.sessionId})`,
+                "info"
+              );
+            } catch (e) {
+              this.onLog(`Failed to auto-start after end: ${e}`, "error");
+            }
+          }
+        }, 100);
+      }
+    };
+    this.recognition.addEventListener("end", this.endHandler);
+    this.startHandler = () => {
+      this.isRecognitionRunning = true;
+      const rid = this.awaitingRestartFirstResultId;
+      if (rid != null && this.restartMetrics[rid]) {
+        if (!this.restartMetrics[rid].startedAt) {
+          this.restartMetrics[rid].startedAt = Date.now();
+          this.onLog(
+            `\u25B6\uFE0F Restart #${rid} recognition started in ${this.restartMetrics[rid].startedAt - this.restartMetrics[rid].requestedAt}ms`,
+            "info"
+          );
+        }
+      }
+    };
+    this.recognition.addEventListener("start", this.startHandler);
+  }
+  waitForEventOnce(eventName, timeoutMs) {
+    return new Promise((resolve) => {
+      let timer = null;
+      const handler = (ev) => {
+        if (timer !== null) clearTimeout(timer);
+        this.recognition.removeEventListener(eventName, handler);
+        resolve(ev);
+      };
+      this.recognition.addEventListener(eventName, handler);
+      timer = window.setTimeout(() => {
+        this.recognition.removeEventListener(eventName, handler);
+        resolve(null);
+      }, timeoutMs);
+    });
+  }
+  startMicTimer() {
+    this.lastTickTime = Date.now();
+    this.lastInterimSaveTime = Date.now();
+    this.micTimeInterval = window.setInterval(() => {
+      if (this.isListening) {
+        const now = Date.now();
+        const elapsed = now - this.lastTickTime;
+        this.micOnTime += elapsed;
+        this.lastTickTime = now;
+        if (now - this.lastInterimSaveTime >= this.interimSaveInterval) {
+          this.saveInterimToFinal();
+          this.lastInterimSaveTime = now;
+        }
+        if (this.micOnTime >= this.sessionDuration) {
+          if (!this.isRestarting) this.performRestart();
+        }
+        if (this.onMicTimeUpdate) this.onMicTimeUpdate(this.micOnTime);
+      }
+    }, 100);
+  }
+  stopMicTimer() {
+    if (this.micTimeInterval) {
+      clearInterval(this.micTimeInterval);
+      this.micTimeInterval = null;
+    }
+  }
+  saveInterimToFinal() {
+    if (!this.lastInterimTranscript) return;
+    const now = Date.now();
+    if (now - this.lastInterimResultTime > this.interimSaveInterval && this.lastInterimTranscript.length > this.lastSavedLength) {
+      this.fullTranscript = (this.fullTranscript + " " + this.lastInterimTranscript).trim();
+      this.fullTranscript = this.collapseRepeats(this.fullTranscript);
+      this.lastSavedLength = this.fullTranscript.length;
+      if (this.onWordsUpdate) {
+        const words = this.fullTranscript.split(/\s+/).filter((w) => w.length > 0);
+        this.onWordsUpdate(words);
+      }
+      this.onTranscript(this.getFullTranscript());
+    }
+  }
+  getSuffixToAppend(base, current) {
+    if (!base || base.length === 0) return current;
+    if (!current || current.length === 0) return "";
+    base = base.trim();
+    current = current.trim();
+    if (current.startsWith(base)) {
+      return current.slice(base.length).trim();
+    }
+    const maxOverlap = Math.min(base.length, current.length);
+    for (let overlap = maxOverlap; overlap > 0; overlap--) {
+      if (base.endsWith(current.slice(0, overlap))) {
+        return current.slice(overlap).trim();
+      }
+    }
+    return current;
+  }
+  collapseRepeats(text) {
+    if (!text || text.trim().length === 0) return text.trim();
+    let normalized = text.replace(/\s+/g, " ").trim();
+    const n = normalized.length;
+    const lps = new Array(n).fill(0);
+    for (let i = 1; i < n; i++) {
+      let j = lps[i - 1];
+      while (j > 0 && normalized[i] !== normalized[j]) j = lps[j - 1];
+      if (normalized[i] === normalized[j]) j++;
+      lps[i] = j;
+    }
+    const period = n - lps[n - 1];
+    if (period < n && n % period === 0) {
+      return normalized.slice(0, period).trim();
+    }
+    const words = normalized.split(" ");
+    for (let block = Math.min(20, Math.floor(words.length / 2)); block >= 1; block--) {
+      let i = 0;
+      while (i + 2 * block <= words.length) {
+        let blockA = words.slice(i, i + block).join(" ");
+        let blockB = words.slice(i + block, i + 2 * block).join(" ");
+        if (blockA === blockB) {
+          words.splice(i + block, block);
+        } else {
+          i++;
+        }
+      }
+    }
+    const collapsedWords = [];
+    for (const w of words) {
+      if (collapsedWords.length === 0 || collapsedWords[collapsedWords.length - 1] !== w)
+        collapsedWords.push(w);
+    }
+    return collapsedWords.join(" ").trim();
+  }
+  performRestart() {
+    if (!this.isListening || this.isRestarting) return;
+    const restartStartTime = Date.now();
+    this.restartCount++;
+    this.isRestarting = true;
+    this.isAutoRestarting = true;
+    const rid = ++this.sessionId;
+    this.awaitingRestartFirstResultId = rid;
+    this.restartMetrics[rid] = { requestedAt: restartStartTime };
+    this.onLog(
+      `\u{1F504} [AUTO-RESTART] Session ${rid} - buffering transcript, waiting for silence...`,
+      "warning"
+    );
+    if (this.lastInterimTranscript.trim().length > 0) {
+      this.saveInterimToFinal();
+    }
+    this.transcriptBeforeRestart = this.getFullTranscript();
+    this.fullTranscript = "";
+    this.sessionStartTranscript = "";
+    this.lastInterimTranscript = "";
+    this.heardWords = [];
+    this.stopMicTimer();
+    const stopTimeout = 600;
+    const startTimeout = 1e3;
+    const firstResultTimeout = 2e3;
+    const stopNow = async () => {
+      try {
+        if (this.isRecognitionRunning) {
+          this.recognition.stop();
+        } else {
+          this.onLog("Recognition not running at stop attempt", "warning");
+        }
+      } catch (err) {
+        this.onLog(`Stop threw: ${err}`, "warning");
+      }
+      const endEvent = await this.waitForEventOnce("end", stopTimeout);
+      if (!endEvent) {
+        try {
+          this.recognition.abort();
+        } catch (err) {
+          this.onLog(`Abort also failed: ${err}`, "error");
+        }
+        await this.waitForEventOnce("end", 300);
+      }
+      this.restartMetrics[rid].stopAt = Date.now();
+    };
+    (async () => {
+      await stopNow();
+      this.restartMetrics[rid].startAttemptAt = Date.now();
+      try {
+        if (!this.isRecognitionRunning) {
+          this.sessionId = rid;
+          this.recognition.start();
+        } else {
+          this.onLog(
+            "Recognition already running at restart time; skipping start.",
+            "warning"
+          );
+        }
+      } catch (e) {
+        this.onLog(`Failed to start recognition after restart: ${e}`, "error");
+      }
+      const startEv = await this.waitForEventOnce("start", startTimeout);
+      if (startEv) {
+        this.restartMetrics[rid].startedAt = Date.now();
+      } else {
+        this.onLog(
+          `Restart #${rid} did not produce start event within ${startTimeout}ms`,
+          "warning"
+        );
+      }
+      const resEv = await this.waitForEventOnce("result", firstResultTimeout);
+      if (resEv) {
+        if (this.restartMetrics[rid])
+          this.restartMetrics[rid].firstResultAt = Date.now();
+        const firstResultDelta = (this.restartMetrics[rid].firstResultAt || Date.now()) - (this.restartMetrics[rid].requestedAt || Date.now());
+        this.onLog(
+          `\u{1F514} First result after restart #${rid} in ${firstResultDelta}ms`,
+          "info"
+        );
+      } else {
+        this.onLog(
+          `Restart #${rid} produced no result within ${firstResultTimeout}ms`,
+          "warning"
+        );
+      }
+      const startedAt = this.restartMetrics[rid].startedAt || this.restartMetrics[rid].startAttemptAt || Date.now();
+      const restartDuration = startedAt - this.restartMetrics[rid].requestedAt;
+      if (this.onRestartMetrics)
+        this.onRestartMetrics(this.restartCount, restartDuration);
+      this.onLog(
+        `\u2705 Session ${rid} restarted in ${restartDuration}ms - resuming from silence gate`,
+        "info"
+      );
+      this.startMicTimer();
+      this.isRestarting = false;
+      this.isAutoRestarting = false;
+    })();
+  }
+  start() {
+    if (this.isListening) return;
+    try {
+      this.isListening = true;
+      if (!this.options.preserveTranscriptOnStart) {
+        this.fullTranscript = "";
+        this.heardWords = [];
+        this.transcriptBeforeRestart = "";
+        this.sessionStartTranscript = "";
+      } else {
+        this.sessionStartTranscript = this.fullTranscript;
+      }
+      this.micOnTime = 0;
+      this.restartCount = 0;
+      this.lastSavedLength = 0;
+      this.lastInterimTranscript = "";
+      this.lastWasFinal = false;
+      if (!this.isRecognitionRunning) {
+        this.sessionId++;
+        this.recognition.start();
+        this.isRecognitionRunning = true;
+      }
+      this.startMicTimer();
+      this.onLog(
+        "Listening started (auto-restart every 30s of mic time)",
+        "info"
+      );
+    } catch (error) {
+      this.isListening = false;
+      this.onLog(`Failed to start: ${error}`, "error");
+    }
+  }
+  stop() {
+    if (!this.isListening) return;
+    try {
+      this.isListening = false;
+      this.isAutoRestarting = false;
+      this.stopMicTimer();
+      this.recognition.stop();
+      this.isRecognitionRunning = false;
+      this.onLog(
+        `Stopped listening (total mic time: ${(this.micOnTime / 1e3).toFixed(
+          1
+        )}s, restarts: ${this.restartCount})`,
+        "info"
+      );
+    } catch (error) {
+      this.onLog(`Failed to stop: ${error}`, "error");
+    }
+  }
+  destroy() {
+    this.isListening = false;
+    this.stopMicTimer();
+    try {
+      this.recognition.abort?.();
+    } catch (e) {
+    }
+    try {
+      if (this.resultHandler)
+        this.recognition.removeEventListener("result", this.resultHandler);
+      if (this.errorHandler)
+        this.recognition.removeEventListener("error", this.errorHandler);
+      if (this.endHandler)
+        this.recognition.removeEventListener(
+          "end",
+          this.endHandler
+        );
+      if (this.startHandler)
+        this.recognition.removeEventListener(
+          "start",
+          this.startHandler
+        );
+    } catch (e) {
+    }
+  }
+};
+var STTLogic = class extends ResetSTTLogic2 {
+};
+
+export { ResetSTTLogic, STTLogic, VADController };
+//# sourceMappingURL=index.mjs.map
+//# sourceMappingURL=index.mjs.map