webtalk 1.0.0

package/tts-models.js

@@ -0,0 +1,97 @@
+const fs = require('fs');
+const path = require('path');
+const { createDownloadLock, resolveDownloadLock, rejectDownloadLock, getDownloadPromise, isDownloading } = require('./download-lock');
+const { downloadFile, ensureDir, isFileCorrupted } = require('./whisper-models');
+
+const TTS_FILES = [
+  { name: 'mimi_encoder.onnx', size: '73MB', minBytes: 73 * 1024 * 1024 * 0.8 },
+  { name: 'text_conditioner.onnx', size: '16MB', minBytes: 16 * 1024 * 1024 * 0.8 },
+  { name: 'flow_lm_main_int8.onnx', size: '76MB', minBytes: 76 * 1024 * 1024 * 0.8 },
+  { name: 'flow_lm_flow_int8.onnx', size: '10MB', minBytes: 10 * 1024 * 1024 * 0.8 },
+  { name: 'mimi_decoder_int8.onnx', size: '23MB', minBytes: 23 * 1024 * 1024 * 0.8 },
+  { name: 'tokenizer.model', size: '59KB', minBytes: 59 * 1024 * 0.8 }
+];
+
+const TTS_WEB_FILES = [
+  { name: 'sentencepiece.js', url: 'https://cdn.jsdelivr.net/npm/@facebookresearch/fasttext@1.0.2/dist/fasttext.js', size: '4MB' }
+];
+
+async function checkTTSModelExists(config) {
+  if (!fs.existsSync(config.ttsModelsDir)) return false;
+
+  const mainFiles = [
+    { name: 'mimi_encoder.onnx', minBytes: 73 * 1024 * 1024 * 0.8 },
+    { name: 'flow_lm_main_int8.onnx', minBytes: 76 * 1024 * 1024 * 0.8 },
+    { name: 'mimi_decoder_int8.onnx', minBytes: 23 * 1024 * 1024 * 0.8 }
+  ];
+
+  for (const file of mainFiles) {
+    const filePath = path.join(config.ttsModelsDir, file.name);
+    if (!fs.existsSync(filePath) || isFileCorrupted(filePath, file.minBytes)) {
+      return false;
+    }
+  }
+  return true;
+}
+
+async function downloadTTSModels(config) {
+  ensureDir(config.ttsModelsDir);
+
+  let downloadedCount = 0;
+
+  for (const file of TTS_FILES) {
+    const destPath = path.join(config.ttsModelsDir, file.name);
+
+    if (fs.existsSync(destPath)) {
+      if (isFileCorrupted(destPath, file.minBytes)) {
+        fs.unlinkSync(destPath);
+      } else {
+        continue;
+      }
+    }
+
+    const url = config.ttsBaseUrl + file.name;
+
+    try {
+      await downloadFile(url, destPath, 3);
+      downloadedCount++;
+    } catch (err) {}
+  }
+}
+
+async function downloadTTSWebFiles(config) {
+  ensureDir(config.ttsDir);
+  for (const file of TTS_WEB_FILES) {
+    const destPath = path.join(config.ttsDir, file.name);
+    if (fs.existsSync(destPath)) continue;
+    try { await downloadFile(file.url, destPath); } catch (err) {}
+  }
+}
+
+async function ensureTTSModels(config) {
+  const lockKey = 'tts-models';
+
+  if (isDownloading(lockKey)) {
+    return getDownloadPromise(lockKey);
+  }
+
+  const downloadPromise = (async () => {
+    try {
+      const exists = await checkTTSModelExists(config);
+      if (!exists) {
+        await downloadTTSModels(config);
+      }
+
+      await downloadTTSWebFiles(config);
+      resolveDownloadLock(lockKey, true);
+    } catch (err) {
+      rejectDownloadLock(lockKey, err);
+      throw err;
+    }
+  })();
+
+  createDownloadLock(lockKey);
+  return downloadPromise;
+}
+
+module.exports = { ensureTTSModels, checkTTSModelExists };

package/tts-utils.js

@@ -0,0 +1,52 @@
+const SAMPLE_RATE = 24000;
+
+function resample(data, fromRate, toRate) {
+  const ratio = fromRate / toRate;
+  const newLen = Math.round(data.length / ratio);
+  const out = new Float32Array(newLen);
+  for (let i = 0; i < newLen; i++) {
+    const idx = i * ratio;
+    const lo = Math.floor(idx);
+    const hi = Math.min(lo + 1, data.length - 1);
+    const frac = idx - lo;
+    out[i] = data[lo] * (1 - frac) + data[hi] * frac;
+  }
+  return out;
+}
+
+function encodeWAV(audioBuffer) {
+  const totalLen = audioBuffer.reduce((s, b) => s + b.length, 0);
+  const merged = new Float32Array(totalLen);
+  let off = 0;
+  for (const buf of audioBuffer) {
+    merged.set(buf, off);
+    off += buf.length;
+  }
+
+  const wavBuf = new ArrayBuffer(44 + merged.length * 2);
+  const view = new DataView(wavBuf);
+  const writeStr = (o, s) => {
+    for (let i = 0; i < s.length; i++) view.setUint8(o + i, s.charCodeAt(i));
+  };
+  writeStr(0, 'RIFF');
+  view.setUint32(4, 36 + merged.length * 2, true);
+  writeStr(8, 'WAVE');
+  writeStr(12, 'fmt ');
+  view.setUint32(16, 16, true);
+  view.setUint16(20, 1, true);
+  view.setUint16(22, 1, true);
+  view.setUint32(24, SAMPLE_RATE, true);
+  view.setUint32(28, SAMPLE_RATE * 2, true);
+  view.setUint16(32, 2, true);
+  view.setUint16(34, 16, true);
+  writeStr(36, 'data');
+  view.setUint32(40, merged.length * 2, true);
+  for (let i = 0; i < merged.length; i++) {
+    const s = Math.max(-1, Math.min(1, merged[i]));
+    view.setInt16(44 + i * 2, s < 0 ? s * 0x8000 : s * 0x7FFF, true);
+  }
+
+  return wavBuf;
+}
+
+export { SAMPLE_RATE, resample, encodeWAV };

package/tts.js

@@ -0,0 +1,167 @@
+import { SAMPLE_RATE, resample, encodeWAV } from './tts-utils.js';
+
+const STATUS = {
+  LOADING: 'loading',
+  READY: 'ready',
+  SPEAKING: 'speaking',
+  ERROR: 'error'
+};
+
+class TTS {
+  constructor(options = {}) {
+    this.voice = options.voice || null;
+    this.onAudioReady = options.onAudioReady || null;
+    this.onStatus = options.onStatus || null;
+    this.onVoicesLoaded = options.onVoicesLoaded || null;
+    this.onMetrics = options.onMetrics || null;
+    this.onAudioChunk = options.onAudioChunk || null;
+    this.basePath = options.basePath || '';
+    this.apiBasePath = options.apiBasePath || '';
+    this.ttsWorkerFile = options.ttsWorkerFile || 'inference-worker.js';
+    this.worker = null;
+    this.audioContext = null;
+    this.audioBuffer = [];
+    this.currentAudioUrl = null;
+    this.startTime = 0;
+    this.firstChunkTime = null;
+    this.ready = false;
+  }
+
+  async init() {
+    const statusUrl = this.apiBasePath + '/api/tts-status';
+    const statusResponse = await fetch(statusUrl);
+    const statusData = await statusResponse.json();
+    if (!statusData.available) throw new Error('TTS models not available');
+
+    this.audioContext = new (globalThis.AudioContext || globalThis.webkitAudioContext)({ sampleRate: SAMPLE_RATE });
+
+    return new Promise((resolve, reject) => {
+      try {
+        const workerPath = this.basePath + '/tts/' + this.ttsWorkerFile;
+        this.worker = new Worker(workerPath, { type: 'module' });
+        this.worker.onmessage = (e) => this._handleMessage(e.data, resolve);
+        this.worker.onerror = (e) => reject(e);
+        this.worker.postMessage({ type: 'load' });
+      } catch (err) {
+        reject(err);
+      }
+    });
+  }
+
+  _handleMessage(msg, initResolve) {
+    switch (msg.type) {
+      case 'status':
+        this.onStatus?.(msg.status || msg.data?.status, msg.state || msg.data?.state);
+        break;
+      case 'voices_loaded':
+        this.onVoicesLoaded?.(msg.voices, msg.defaultVoice);
+        break;
+      case 'loaded':
+        this.ready = true;
+        initResolve?.();
+        this.onStatus?.(STATUS.READY, 'ready');
+        break;
+      case 'audio_chunk':
+        this.audioBuffer.push(new Float32Array(msg.data));
+        this.onAudioChunk?.();
+        if (!this.firstChunkTime) {
+          this.firstChunkTime = performance.now();
+          this.onMetrics?.({ ttfb: this.firstChunkTime - this.startTime });
+        }
+        if (msg.metrics) {
+          const elapsed = (performance.now() - this.startTime) / 1000;
+          const audioDur = this.audioBuffer.reduce((s, b) => s + b.length, 0) / SAMPLE_RATE;
+          if (elapsed > 0) this.onMetrics?.({ rtfx: audioDur / elapsed });
+        }
+        break;
+      case 'stream_ended':
+        this._finalize();
+        break;
+      case 'error':
+        this.onStatus?.(STATUS.ERROR, 'error');
+        this._generateReject?.(new Error(msg.error));
+        this._generateReject = null;
+        this._generateResolve = null;
+        break;
+    }
+  }
+
+  generate(text, voice) {
+    return new Promise((resolve, reject) => {
+      if (!this.worker || !this.ready) return reject(new Error('TTS not initialized'));
+      this.audioBuffer = [];
+      this.startTime = performance.now();
+      this.firstChunkTime = null;
+      if (this.currentAudioUrl) {
+        URL.revokeObjectURL(this.currentAudioUrl);
+        this.currentAudioUrl = null;
+      }
+      this._generateResolve = resolve;
+      this._generateReject = reject;
+      const v = voice || this.voice;
+      if (v && v !== 'custom') {
+        this.worker.postMessage({ type: 'set_voice', data: { voiceName: v } });
+      }
+      this.worker.postMessage({ type: 'generate', data: { text, voice: v } });
+    });
+  }
+
+  stop() {
+    this.worker?.postMessage({ type: 'stop' });
+  }
+
+  async uploadVoice(file) {
+    if (!this.worker || !this.audioContext) return;
+    const arrayBuffer = await file.arrayBuffer();
+    const audioBuffer = await this.audioContext.decodeAudioData(arrayBuffer);
+    let audioData = audioBuffer.getChannelData(0);
+    if (audioBuffer.sampleRate !== SAMPLE_RATE) {
+      audioData = resample(audioData, audioBuffer.sampleRate, SAMPLE_RATE);
+    }
+    this.worker.postMessage({ type: 'encode_voice', data: { audio: audioData } });
+  }
+
+  _finalize() {
+    if (this.audioBuffer.length === 0) {
+      this._generateResolve?.(null);
+      this._generateResolve = null;
+      return;
+    }
+    const wavBuf = encodeWAV(this.audioBuffer);
+    if (this.currentAudioUrl) URL.revokeObjectURL(this.currentAudioUrl);
+    this.currentAudioUrl = URL.createObjectURL(new Blob([wavBuf], { type: 'audio/wav' }));
+    this.onAudioReady?.(this.currentAudioUrl);
+    this._generateResolve?.(this.currentAudioUrl);
+    this._generateResolve = null;
+  }
+
+  downloadAudio() {
+    if (this.currentAudioUrl) {
+      const a = document.createElement('a');
+      a.href = this.currentAudioUrl;
+      a.download = 'tts-output.wav';
+      a.click();
+    }
+  }
+
+  getStatus() {
+    return {
+      ready: this.ready,
+      voice: this.voice,
+      workerFile: this.ttsWorkerFile,
+      hasWorker: !!this.worker,
+      hasAudioContext: !!this.audioContext,
+      audioBuffered: this.audioBuffer.length
+    };
+  }
+
+  destroy() {
+    this.worker?.terminate();
+    this.worker = null;
+    this.ready = false;
+    if (this.currentAudioUrl) URL.revokeObjectURL(this.currentAudioUrl);
+    this.audioContext?.close();
+  }
+}
+
+export { TTS };

package/whisper-models.js

@@ -0,0 +1,161 @@
+const fs = require('fs');
+const path = require('path');
+const https = require('https');
+const { createDownloadLock, resolveDownloadLock, rejectDownloadLock, getDownloadPromise, isDownloading } = require('./download-lock');
+
+const WHISPER_REQUIRED_FILES = [
+  'config.json',
+  'preprocessor_config.json',
+  'tokenizer.json',
+  'tokenizer_config.json',
+  'vocab.json',
+  'merges.txt',
+  'model_quantized.onnx',
+  'onnx/encoder_model.onnx',
+  'onnx/decoder_model_merged_q4.onnx',
+  'onnx/decoder_model_merged.onnx'
+];
+
+function ensureDir(dir) {
+  if (!fs.existsSync(dir)) {
+    fs.mkdirSync(dir, { recursive: true });
+  }
+}
+
+function downloadFile(url, dest, maxRetries = 3, attempt = 0) {
+  return new Promise((resolve, reject) => {
+    ensureDir(path.dirname(dest));
+    const file = fs.createWriteStream(dest);
+    https.get(url, (response) => {
+      if (response.statusCode === 302 || response.statusCode === 301 || response.statusCode === 307 || response.statusCode === 308) {
+        file.close();
+        if (fs.existsSync(dest)) fs.unlinkSync(dest);
+        downloadFile(response.headers.location, dest, maxRetries, attempt).then(resolve).catch(reject);
+        return;
+      }
+      if (response.statusCode !== 200) {
+        file.close();
+        if (fs.existsSync(dest)) fs.unlinkSync(dest);
+        const error = new Error(`Failed to download: ${response.statusCode}`);
+        if (attempt < maxRetries - 1) {
+          const delayMs = Math.pow(2, attempt) * 1000;
+          setTimeout(() => downloadFile(url, dest, maxRetries, attempt + 1).then(resolve).catch(reject), delayMs);
+        } else {
+          reject(error);
+        }
+        return;
+      }
+
+      let downloaded = 0;
+      response.on('data', (chunk) => {
+        downloaded += chunk.length;
+        process.stdout.write(`\r  ↓ ${path.basename(dest)} ... ${(downloaded / 1024 / 1024).toFixed(2)}MB`);
+      });
+
+      response.pipe(file);
+      file.on('finish', () => {
+        file.close();
+        process.stdout.write(' ✓\n');
+        resolve();
+      });
+    }).on('error', (err) => {
+      if (fs.existsSync(dest)) fs.unlinkSync(dest);
+      if (attempt < maxRetries - 1) {
+        const delayMs = Math.pow(2, attempt) * 1000;
+        setTimeout(() => downloadFile(url, dest, maxRetries, attempt + 1).then(resolve).catch(reject), delayMs);
+      } else {
+        reject(err);
+      }
+    });
+  });
+}
+
+function isFileCorrupted(filePath, minSizeBytes = null) {
+  try {
+    const stats = fs.statSync(filePath);
+    if (minSizeBytes !== null && stats.size < minSizeBytes) {
+      return true;
+    }
+    return false;
+  } catch (err) {
+    return true;
+  }
+}
+
+async function checkWhisperModelExists(modelName, config) {
+  const modelDir = path.join(config.modelsDir, modelName);
+  if (!fs.existsSync(modelDir)) return false;
+
+  const encoderPath = path.join(modelDir, 'onnx', 'encoder_model.onnx');
+  const decoderPath = path.join(modelDir, 'onnx', 'decoder_model_merged_q4.onnx');
+  const decoderFallback = path.join(modelDir, 'onnx', 'decoder_model_merged.onnx');
+
+  const hasEncoder = fs.existsSync(encoderPath);
+  const hasDecoder = fs.existsSync(decoderPath) || fs.existsSync(decoderFallback);
+
+  if (!hasEncoder || !hasDecoder) return false;
+
+  const encoderValid = !isFileCorrupted(encoderPath, 40 * 1024 * 1024);
+  const decoderValid = isFileCorrupted(decoderPath, 100 * 1024 * 1024) === false ||
+                       isFileCorrupted(decoderFallback, 100 * 1024 * 1024) === false;
+
+  return encoderValid && decoderValid;
+}
+
+async function downloadWhisperModel(modelName, config) {
+  const modelDir = path.join(config.modelsDir, modelName);
+  ensureDir(modelDir);
+
+  const baseUrl = `${config.whisperBaseUrl}${modelName}/resolve/main/`;
+
+  let downloadedCount = 0;
+  let failedCount = 0;
+
+  for (const file of WHISPER_REQUIRED_FILES) {
+    const destPath = path.join(modelDir, file);
+
+    if (fs.existsSync(destPath)) {
+      if (isFileCorrupted(destPath)) {
+        fs.unlinkSync(destPath);
+      } else {
+        continue;
+      }
+    }
+
+    const url = baseUrl + file;
+
+    try {
+      await downloadFile(url, destPath, 3);
+      downloadedCount++;
+    } catch (err) {
+      failedCount++;
+      if (fs.existsSync(destPath)) fs.unlinkSync(destPath);
+    }
+  }
+}
+
+async function ensureModel(modelName, config) {
+  const lockKey = `whisper-${modelName}`;
+
+  if (isDownloading(lockKey)) {
+    return getDownloadPromise(lockKey);
+  }
+
+  const downloadPromise = (async () => {
+    try {
+      const exists = await checkWhisperModelExists(modelName, config);
+      if (!exists) {
+        await downloadWhisperModel(modelName, config);
+      }
+      resolveDownloadLock(lockKey, true);
+    } catch (err) {
+      rejectDownloadLock(lockKey, err);
+      throw err;
+    }
+  })();
+
+  createDownloadLock(lockKey);
+  return downloadPromise;
+}
+
+module.exports = { ensureModel, downloadFile, ensureDir, isFileCorrupted };

package/worker-patch.js

@@ -0,0 +1,32 @@
+const fs = require('fs');
+const path = require('path');
+
+const FETCH_PATCH = `const originalFetch = self.fetch;
+self.fetch = function(input, init) {
+  let url = typeof input === 'string' ? input : input.url;
+  if (url.includes('huggingface.co') && url.includes('/resolve/main/')) {
+    const match = url.match(/huggingface\\.co\\/([^\\/]+\\/[^\\/]+)\\/resolve\\/main\\/(.*)/);
+    if (match) {
+      const [, modelName, filePath] = match;
+      return originalFetch('/models/' + modelName + '/' + filePath, init);
+    }
+  }
+  return originalFetch(input, init);
+};
+`;
+
+function patchWorker(config) {
+  const workerPath = path.join(config.assetsDir, config.workerFile);
+  const backupPath = path.join(config.assetsDir, config.workerBackup);
+
+  try {
+    if (fs.existsSync(backupPath)) {
+      fs.copyFileSync(backupPath, workerPath);
+    }
+    const content = fs.readFileSync(workerPath, 'utf8');
+    if (content.includes('originalFetch')) return;
+    fs.writeFileSync(workerPath, FETCH_PATCH + content);
+  } catch (err) {}
+}
+
+module.exports = { patchWorker };