voxflow 1.14.0 → 1.15.1

package/lib/commands/dub.js

@@ -0,0 +1,542 @@
+/**
+ * VoxFlow CLI — Dub command
+ *
+ * Video dubbing: SRT subtitle → per-sentence TTS → timeline-aligned audio.
+ * Supports multi-speaker voice mapping, dynamic speed compensation,
+ * video merging, BGM ducking, and patch mode for partial re-synthesis.
+ *
+ * Pipeline: SRT → parse captions → TTS each → timeline align → WAV/MP4
+ */
+
+const fs = require('fs');
+const path = require('path');
+const { DUB_DEFAULTS } = require('../core/config');
+const { ApiError } = require('../core/http');
+const { buildWav } = require('../core/audio');
+const { parseSrt } = require('../core/srt');
+const { buildTimelineAudio, msToBytes, BYTES_PER_MS } = require('../core/timeline');
+const { synthesizeTTS } = require('../core/tts-synthesizer');
+
+// ─── Voice Map Parsing ──────────────────────────────────────────────────────
+
+/**
+ * Parse a voices.json file for multi-speaker mapping.
+ * Format: { "SpeakerName": "voiceId", ... }
+ * @param {string} filePath
+ * @returns {object} Speaker → voiceId map
+ */
+function parseVoicesMap(filePath) {
+  if (!fs.existsSync(filePath)) {
+    throw new Error(`Voices map file not found: ${filePath}`);
+  }
+
+  let raw;
+  try {
+    raw = JSON.parse(fs.readFileSync(filePath, 'utf8'));
+  } catch (err) {
+    throw new Error(`Invalid JSON in voices map: ${err.message}`);
+  }
+
+  if (typeof raw !== 'object' || raw === null || Array.isArray(raw)) {
+    throw new Error('Voices map must be a JSON object: { "SpeakerName": "voiceId", ... }');
+  }
+
+  for (const [speaker, voiceId] of Object.entries(raw)) {
+    if (typeof voiceId !== 'string' || voiceId.trim().length === 0) {
+      throw new Error(`Invalid voice ID for speaker "${speaker}": must be a non-empty string`);
+    }
+  }
+
+  return raw;
+}
+
+// ─── TTS Synthesis ──────────────────────────────────────────────────────────
+
+/**
+ * Synthesize a single caption text via TTS API.
+ * @returns {{ audio: Buffer, quota: object, durationMs: number }}
+ */
+async function synthesizeCaption(apiBase, token, text, voiceId, speed, format, index, total) {
+  const result = await synthesizeTTS({
+    apiBase, token, text, voiceId,
+    speed: speed ?? 1.0,
+    format: format || 'pcm',
+    index, total,
+  });
+
+  // Calculate duration from PCM buffer size: bytes / (sampleRate * bytesPerSample) * 1000
+  const durationMs = result.audio.length / BYTES_PER_MS;
+
+  console.log(` OK (${(result.audio.length / 1024).toFixed(0)} KB, ${(durationMs / 1000).toFixed(1)}s)`);
+  return { audio: result.audio, quota: result.quota, durationMs };
+}
+
+// ─── Public API ─────────────────────────────────────────────────────────────
+
+/**
+ * Dub: convert SRT subtitles to timeline-aligned TTS audio.
+ *
+ * @param {object} opts
+ * @param {string} opts.token       - JWT token
+ * @param {string} opts.api         - API base URL
+ * @param {string} opts.srt         - Path to SRT subtitle file
+ * @param {string} [opts.voice]     - Default TTS voice ID
+ * @param {string} [opts.voicesMap] - Path to voices.json for multi-speaker
+ * @param {number} [opts.speed]     - Default TTS speed (0.5-2.0)
+ * @param {boolean} [opts.speedAuto] - Enable dynamic speed compensation
+ * @param {string} [opts.video]     - Path to video file (merge after dubbing)
+ * @param {string} [opts.bgm]       - Path to BGM audio file
+ * @param {number} [opts.ducking]   - BGM ducking level (0-1)
+ * @param {number} [opts.patch]     - Patch mode: re-synthesize only this caption ID
+ * @param {string} [opts.output]    - Output file path
+ * @returns {Promise<{outputPath: string, duration: number, quotaUsed: number, segmentCount: number, warnings: string[]}>}
+ */
+async function dub(opts) {
+  let isExiting = false;
+
+  const sigintHandler = () => {
+    if (isExiting) return;
+    isExiting = true;
+    console.log('\n\nDubbing cancelled.');
+    process.exit(130);
+  };
+  process.on('SIGINT', sigintHandler);
+
+  try {
+    return await _dub(opts);
+  } finally {
+    process.removeListener('SIGINT', sigintHandler);
+  }
+}
+
+async function _dub(opts) {
+  const srtPath = opts.srt;
+  if (!srtPath) {
+    throw new Error('No SRT file provided. Usage: voxflow dub --srt <file.srt>');
+  }
+
+  const resolvedSrt = path.resolve(srtPath);
+  if (!fs.existsSync(resolvedSrt)) {
+    throw new Error(`SRT file not found: ${resolvedSrt}`);
+  }
+
+  const srtContent = fs.readFileSync(resolvedSrt, 'utf8');
+  const captions = parseSrt(srtContent);
+  if (captions.length === 0) {
+    throw new Error('SRT file contains no valid captions');
+  }
+
+  const defaultVoice = opts.voice || DUB_DEFAULTS.voice;
+  const defaultSpeed = opts.speed ?? DUB_DEFAULTS.speed;
+  const speedAuto = opts.speedAuto || false;
+  const toleranceMs = DUB_DEFAULTS.toleranceMs;
+  const api = opts.api;
+  const token = opts.token;
+  const patchId = opts.patch;
+  const warnings = [];
+
+  // Load multi-speaker voice map if provided
+  let voicesMap = null;
+  if (opts.voicesMap) {
+    voicesMap = parseVoicesMap(path.resolve(opts.voicesMap));
+  }
+
+  // Determine output path
+  let outputPath = opts.output;
+  const hasVideo = !!opts.video;
+  const defaultExt = hasVideo ? '.mp4' : '.wav';
+  if (!outputPath) {
+    const ts = new Date().toISOString().replace(/[:.]/g, '-').slice(0, 19);
+    outputPath = path.resolve(`dub-${ts}${defaultExt}`);
+  }
+
+  // Print header
+  console.log('\n=== VoxFlow Dub ===');
+  console.log(`SRT:      ${srtPath} (${captions.length} captions)`);
+  console.log(`Voice:    ${defaultVoice}${voicesMap ? ` + voices map (${Object.keys(voicesMap).length} speakers)` : ''}`);
+  console.log(`Speed:    ${defaultSpeed}${speedAuto ? ' (auto-compensate)' : ''}`);
+  if (hasVideo) console.log(`Video:    ${opts.video}`);
+  if (opts.bgm) console.log(`BGM:      ${opts.bgm} (ducking: ${opts.ducking ?? DUB_DEFAULTS.ducking})`);
+  if (patchId != null) console.log(`Patch:    caption #${patchId}`);
+  console.log(`Output:   ${outputPath}`);
+
+  // ─── Patch mode: only re-synthesize one caption ───────────────────────
+
+  if (patchId != null) {
+    return _dubPatch(opts, captions, outputPath, warnings, voicesMap);
+  }
+
+  // ─── Full synthesis ───────────────────────────────────────────────────
+
+  console.log(`\n[1/2] Synthesizing TTS audio (${captions.length} captions)...`);
+
+  const segments = []; // { startMs, endMs, audioBuffer }
+  let lastQuota = null;
+  let quotaUsed = 0;
+
+  for (let i = 0; i < captions.length; i++) {
+    const cap = captions[i];
+    const targetMs = cap.endMs - cap.startMs;
+
+    // Determine voice for this caption
+    let voiceId = defaultVoice;
+    if (voicesMap && cap.speakerId && voicesMap[cap.speakerId]) {
+      voiceId = voicesMap[cap.speakerId];
+    }
+
+    // First synthesis
+    let result = await synthesizeCaption(api, token, cap.text, voiceId, defaultSpeed, 'pcm', i, captions.length);
+    quotaUsed++;
+    lastQuota = result.quota;
+
+    // Dynamic speed compensation
+    if (speedAuto && result.durationMs > targetMs + toleranceMs) {
+      const alpha = result.durationMs / targetMs;
+
+      if (alpha <= 2.0) {
+        const newSpeed = Math.min(defaultSpeed * alpha, 2.0);
+        process.stdout.write(`    ↳ Re-synth #${cap.id} (${(result.durationMs / 1000).toFixed(1)}s > ${(targetMs / 1000).toFixed(1)}s, speed: ${newSpeed.toFixed(2)})...`);
+
+        result = await synthesizeCaption(api, token, cap.text, voiceId, newSpeed, 'pcm', i, captions.length);
+        quotaUsed++;
+        lastQuota = result.quota;
+      } else {
+        // Cannot fit even at max speed
+        const msg = `Caption #${cap.id}: audio too long (${(result.durationMs / 1000).toFixed(1)}s for ${(targetMs / 1000).toFixed(1)}s slot, alpha=${alpha.toFixed(1)}). Consider shortening text.`;
+        warnings.push(msg);
+        console.log(`    ⚠ OVERFLOW: ${msg}`);
+
+        // Try with max speed anyway
+        const result2 = await synthesizeCaption(api, token, cap.text, voiceId, 2.0, 'pcm', i, captions.length);
+        quotaUsed++;
+        lastQuota = result2.quota;
+        result = result2;
+      }
+    }
+
+    segments.push({
+      startMs: cap.startMs,
+      endMs: cap.endMs,
+      audioBuffer: result.audio,
+    });
+  }
+
+  // ─── Build timeline audio ─────────────────────────────────────────────
+
+  console.log('\n[2/2] Building timeline audio...');
+  const { wav, duration } = buildTimelineAudio(segments);
+
+  // Determine intermediate WAV path (always .wav, never the user's target .mp3/.mp4)
+  const outputExt = path.extname(outputPath).toLowerCase();
+  const needsAudioConvert = !hasVideo && outputExt !== '.wav';
+  const wavOutputPath = hasVideo
+    ? outputPath.replace(/\.[^.]+$/, '.tmp.wav') // temp WAV for video merge
+    : needsAudioConvert
+      ? outputPath.slice(0, -outputExt.length) + '.tmp.wav' // temp WAV for format conversion
+      : outputPath;
+
+  const outDir = path.dirname(wavOutputPath);
+  fs.mkdirSync(outDir, { recursive: true });
+  fs.writeFileSync(wavOutputPath, wav);
+
+  // Write transcript file (derive from final outputPath, not temp wav)
+  const textPath = outputPath.replace(/\.[^.]+$/, '.txt');
+  const textContent = captions.map((cap) => {
+    const speaker = cap.speakerId ? `|${cap.speakerId}` : '';
+    const voiceId = voicesMap && cap.speakerId && voicesMap[cap.speakerId]
+      ? `|${voicesMap[cap.speakerId]}`
+      : '';
+    return `[${cap.id}${speaker}${voiceId}] ${cap.text}`;
+  }).join('\n\n');
+  fs.writeFileSync(textPath, textContent, 'utf8');
+
+  // ─── Post-processing: BGM mixing and/or video merge ─────────────────
+
+  const needsFfmpeg = hasVideo || opts.bgm || needsAudioConvert;
+  if (needsFfmpeg) {
+    const { checkFfmpeg, mergeAudioVideo, mixWithBgm, convertAudioFormat } = require('../core/ffmpeg');
+    const ffmpegInfo = await checkFfmpeg();
+    if (!ffmpegInfo.available) {
+      throw new Error(
+        'ffmpeg is required for BGM mixing / video merging / audio format conversion. Install it:\n' +
+        '  macOS: brew install ffmpeg\n' +
+        '  Ubuntu: sudo apt install ffmpeg\n' +
+        '  Windows: https://ffmpeg.org/download.html'
+      );
+    }
+
+    let currentAudio = wavOutputPath;
+
+    // Mix with BGM if provided
+    if (opts.bgm) {
+      const bgmOutputPath = wavOutputPath.replace('.tmp.wav', '-mixed.tmp.wav').replace(/(?<!\.tmp)\.wav$/, '-mixed.wav');
+      console.log(`    Mixing BGM (ducking: ${opts.ducking ?? DUB_DEFAULTS.ducking})...`);
+      await mixWithBgm(wavOutputPath, opts.bgm, bgmOutputPath, {
+        ducking: opts.ducking ?? DUB_DEFAULTS.ducking,
+      });
+      currentAudio = bgmOutputPath;
+
+      // Replace the work WAV with the mixed version
+      if (!hasVideo) {
+        fs.copyFileSync(currentAudio, wavOutputPath);
+        try { fs.unlinkSync(currentAudio); } catch {}
+        currentAudio = wavOutputPath;
+      }
+    }
+
+    // Merge with video if provided
+    if (hasVideo) {
+      console.log('    Merging with video...');
+      await mergeAudioVideo(opts.video, currentAudio, outputPath);
+
+      // Clean up temp WAV files
+      try {
+        if (wavOutputPath !== outputPath) fs.unlinkSync(wavOutputPath);
+        if (opts.bgm) {
+          const mixed = wavOutputPath.replace('.tmp.wav', '-mixed.tmp.wav').replace(/(?<!\.tmp)\.wav$/, '-mixed.wav');
+          if (fs.existsSync(mixed)) fs.unlinkSync(mixed);
+        }
+      } catch {
+        // ignore cleanup errors
+      }
+    }
+
+    // Convert audio format if needed (e.g. WAV → MP3 for audio-only output)
+    if (needsAudioConvert && !hasVideo) {
+      await convertAudioFormat(wavOutputPath, outputPath);
+      try { fs.unlinkSync(wavOutputPath); } catch { /* ignore */ }
+    }
+  }
+
+  // ─── Summary ──────────────────────────────────────────────────────────
+
+  console.log(`\n=== Done ===`);
+  console.log(`Output:     ${outputPath} (${(fs.statSync(outputPath).size / 1024).toFixed(1)} KB)`);
+  console.log(`Duration:   ${duration.toFixed(1)}s`);
+  console.log(`Transcript: ${textPath}`);
+  console.log(`Captions:   ${captions.length}`);
+  console.log(`Quota:      ${quotaUsed} used, ${lastQuota?.remaining ?? '?'} remaining`);
+
+  if (warnings.length > 0) {
+    console.log(`\nWarnings (${warnings.length}):`);
+    for (const w of warnings) {
+      console.log(`  ⚠ ${w}`);
+    }
+  }
+
+  return { outputPath, textPath, duration, quotaUsed, segmentCount: captions.length, warnings };
+}
+
+// ─── Patch Mode ─────────────────────────────────────────────────────────────
+
+async function _dubPatch(opts, captions, outputPath, warnings, voicesMap) {
+  const patchId = opts.patch;
+  const api = opts.api;
+  const token = opts.token;
+  const defaultVoice = opts.voice || DUB_DEFAULTS.voice;
+  const defaultSpeed = opts.speed ?? DUB_DEFAULTS.speed;
+
+  // Find the caption to patch
+  const capIndex = captions.findIndex((c) => c.id === patchId);
+  if (capIndex === -1) {
+    throw new Error(`Caption #${patchId} not found in SRT. Available IDs: ${captions.map((c) => c.id).join(', ')}`);
+  }
+  const cap = captions[capIndex];
+
+  // Check that existing output exists
+  const existingWav = outputPath.replace(/\.[^.]+$/, '.wav');
+  if (!fs.existsSync(existingWav)) {
+    throw new Error(
+      `Patch mode requires an existing output file. ` +
+      `Run a full dub first, then use --patch to update individual captions.`
+    );
+  }
+
+  // Determine voice
+  let voiceId = defaultVoice;
+  if (voicesMap && cap.speakerId && voicesMap[cap.speakerId]) {
+    voiceId = voicesMap[cap.speakerId];
+  }
+
+  console.log(`\n[Patch] Re-synthesizing caption #${patchId}: "${cap.text.slice(0, 40)}..."`);
+
+  const result = await synthesizeCaption(api, token, cap.text, voiceId, defaultSpeed, 'pcm', 0, 1);
+
+  // Read existing WAV, extract PCM, replace the segment
+  const existingBuf = fs.readFileSync(existingWav);
+  const pcmData = existingBuf.subarray(44); // skip WAV header
+
+  const startByte = msToBytes(cap.startMs);
+  const endByte = msToBytes(cap.endMs);
+
+  // Clear the old segment region
+  pcmData.fill(0, startByte, Math.min(endByte, pcmData.length));
+
+  // Copy new audio into the region
+  const copyLen = Math.min(result.audio.length, endByte - startByte, pcmData.length - startByte);
+  if (copyLen > 0) {
+    result.audio.copy(pcmData, startByte, 0, copyLen);
+  }
+
+  // Rebuild WAV
+  const { wav } = buildWav([pcmData], 0);
+  fs.writeFileSync(existingWav, wav);
+
+  console.log(`\n=== Patch Done ===`);
+  console.log(`Updated:  caption #${patchId} in ${existingWav}`);
+  console.log(`Quota:    1 used, ${result.quota?.remaining ?? '?'} remaining`);
+
+  return {
+    outputPath: existingWav,
+    textPath: existingWav.replace(/\.wav$/i, '.txt'),
+    duration: pcmData.length / (BYTES_PER_MS * 1000),
+    quotaUsed: 1,
+    segmentCount: 1,
+    warnings,
+  };
+}
+
+// ─── CLI Handler ────────────────────────────────────────────────────────────
+
+async function handle(args) {
+  const { parseFlag, parseIntFlag, parseFloatFlag, parseBoolFlag, validateSpeed, runWithRetry } = require('../core/args');
+  const { getToken, getTokenInfo } = require('../core/auth');
+  const { API_BASE, getConfigDir } = require('../core/config');
+  const { warnIfMissingFfmpeg } = require('../core/ffmpeg');
+
+  // Check FFmpeg availability (one-time hint)
+  await warnIfMissingFfmpeg(getConfigDir(), 'dub');
+
+  const api = parseFlag(args, '--api') || API_BASE;
+  const explicitToken = parseFlag(args, '--token');
+
+  // Parse and validate before auth
+  const srt = parseFlag(args, '--srt');
+  const video = parseFlag(args, '--video');
+  const output = parseFlag(args, '--output', '-o');
+  const speed = parseFloatFlag(args, '--speed');
+  const ducking = parseFloatFlag(args, '--ducking');
+  const patch = parseIntFlag(args, '--patch');
+
+  // Validate SRT file
+  if (!srt && !parseBoolFlag(args, '--help')) {
+    console.error('Error: --srt <file> is required. Usage: voxflow dub --srt <file.srt>');
+    process.exit(1);
+  }
+
+  if (srt) {
+    const fs = require('fs');
+    const path = require('path');
+    const resolved = path.resolve(srt);
+    if (!fs.existsSync(resolved)) {
+      console.error(`Error: SRT file not found: ${resolved}`);
+      process.exit(1);
+    }
+  }
+
+  // Validate video file
+  if (video) {
+    const fs = require('fs');
+    const path = require('path');
+    const resolved = path.resolve(video);
+    if (!fs.existsSync(resolved)) {
+      console.error(`Error: Video file not found: ${resolved}`);
+      process.exit(1);
+    }
+  }
+
+  // Validate voices map
+  const voicesMap = parseFlag(args, '--voices');
+  if (voicesMap) {
+    const fs = require('fs');
+    const path = require('path');
+    const resolved = path.resolve(voicesMap);
+    if (!fs.existsSync(resolved)) {
+      console.error(`Error: Voices map file not found: ${resolved}`);
+      process.exit(1);
+    }
+  }
+
+  // Validate BGM file
+  const bgm = parseFlag(args, '--bgm');
+  if (bgm) {
+    const fs = require('fs');
+    const path = require('path');
+    const resolved = path.resolve(bgm);
+    if (!fs.existsSync(resolved)) {
+      console.error(`Error: BGM file not found: ${resolved}`);
+      process.exit(1);
+    }
+  }
+
+  validateSpeed(args, speed);
+
+  // Validate output extension
+  if (output) {
+    const validExts = video ? ['.mp4', '.mkv', '.mov'] : ['.wav', '.mp3'];
+    const hasValidExt = validExts.some(ext => output.toLowerCase().endsWith(ext));
+    if (!hasValidExt) {
+      const extList = validExts.join(', ');
+      console.error(`Error: --output path must end with ${extList}`);
+      process.exit(1);
+    }
+  }
+
+  if (ducking !== undefined) {
+    if (isNaN(ducking) || ducking < 0 || ducking > 1.0) {
+      console.error(`Error: --ducking must be between 0 and 1.0 (got: "${parseFlag(args, '--ducking')}")`);
+      process.exit(1);
+    }
+  }
+
+  // Authenticate after validation
+  let token;
+  if (explicitToken) {
+    token = explicitToken;
+  } else {
+    token = await getToken({ api });
+    const info = getTokenInfo();
+    if (info) {
+      console.log(`\x1b[32mLogged in as ${info.email}\x1b[0m`);
+    }
+  }
+
+  const opts = {
+    token, api, srt, video, output, speed, patch,
+    voice: parseFlag(args, '--voice'),
+    voicesMap,
+    speedAuto: parseBoolFlag(args, '--speed-auto'),
+    bgm,
+    ducking,
+  };
+
+  await runWithRetry(dub, opts, api, explicitToken);
+}
+
+const meta = {
+  dub: {
+    usage: '[opts]',
+    description: 'Dub video/audio from SRT subtitles (timeline-aligned TTS)',
+    options: [
+      `--srt <file>         SRT subtitle file (required)`,
+      `--video <file>       Video file — merge dubbed audio into video`,
+      `--voice <id>         Default TTS voice ID (default: ${DUB_DEFAULTS.voice})`,
+      `--voices <file>      JSON speaker→voiceId map for multi-speaker dubbing`,
+      `--speed <n>          TTS speed 0.5-2.0 (default: ${DUB_DEFAULTS.speed})`,
+      `--speed-auto         Auto-adjust speed when audio overflows time slot`,
+      `--bgm <file>         Background music file to mix in`,
+      `--ducking <n>        BGM volume ducking 0-1.0 (default: ${DUB_DEFAULTS.ducking})`,
+      `--patch <id>         Re-synthesize a single caption by ID (patch mode)`,
+      `--output <path>      Output file path (default: ./dub-<timestamp>.wav)`,
+    ],
+    examples: [
+      'voxflow dub --srt subtitles.srt',
+      'voxflow dub --srt subtitles.srt --video input.mp4 --output dubbed.mp4',
+      'voxflow dub --srt subtitles.srt --voices speakers.json --speed-auto',
+      'voxflow dub --srt subtitles.srt --bgm music.mp3 --ducking 0.3',
+      'voxflow dub --srt subtitles.srt --patch 5 --output dub-existing.wav',
+    ],
+  },
+};
+
+module.exports = { dub, handle, meta, ApiError, _test: { parseVoicesMap } };