voxflow 1.15.0 → 1.15.2

package/lib/commands/video-translate.js

@@ -0,0 +1,577 @@
+/**
+ * VoxFlow CLI — Video Translate command
+ *
+ * End-to-end video translation pipeline:
+ *   1. Extract audio from video (FFmpeg)
+ *   2. Transcribe audio → SRT (ASR, forced flash mode for fine-grained timestamps)
+ *   3. Translate SRT → target language (LLM)
+ *   4. TTS dub + burn subtitles + merge back into video (Dub + FFmpeg)
+ *
+ * Chains existing asr(), translate(), dub() functions programmatically.
+ * All intermediate files go in a temp directory — user only sees the final MP4.
+ *
+ * Cost: ~2-4 quota (1 ASR + 1 translate batch + 1-N TTS per caption)
+ */
+
+const fs = require('fs');
+const path = require('path');
+const os = require('os');
+const { checkFfmpeg, extractAudio } = require('../core/ffmpeg');
+const { asr } = require('./asr');
+const { translate } = require('./translate');
+const { dub } = require('./dub');
+const { detectLanguage } = require('../core/llm-client');
+const { parseSrt } = require('../core/srt');
+const { API_BASE, VIDEO_TRANSLATE_DEFAULTS } = require('../core/config');
+
+// ─── Language → ASR engine mapping ──────────────────────────────────────────
+
+/**
+ * Map short language codes (used by --from) to Tencent ASR engine types.
+ * Falls back to 16k_zh (Chinese) if not found.
+ */
+const LANG_TO_ASR_ENGINE = {
+  zh: '16k_zh',
+  en: '16k_en',
+  ja: '16k_ja',
+  ko: '16k_ko',
+  // Mixed modes
+  'zh-en': '16k_zh_en',
+};
+
+function resolveAsrLang(fromCode, explicitAsrLang) {
+  // Explicit --asr-lang takes priority
+  if (explicitAsrLang) return explicitAsrLang;
+  // Map from --from code
+  if (fromCode && LANG_TO_ASR_ENGINE[fromCode]) return LANG_TO_ASR_ENGINE[fromCode];
+  // Default to Chinese
+  return '16k_zh';
+}
+
+function looksUntranslated(translatedText, sourceText, targetLang) {
+  if (translatedText === sourceText) return true;
+  const latinTargets = ['en', 'fr', 'de', 'es', 'pt', 'it', 'vi'];
+  if (latinTargets.includes(targetLang)) {
+    const cjkChars = (translatedText.match(/[一-鿿぀-ヿ가-힯]/g) || []).length;
+    const totalChars = translatedText.replace(/\s/g, '').length;
+    if (totalChars > 0 && cjkChars / totalChars > 0.5) return true;
+  }
+  return false;
+}
+
+// ─── Main command ────────────────────────────────────────────────────────────
+
+/**
+ * Video-translate command entry point (with SIGINT guard).
+ *
+ * @param {object} opts
+ * @param {string} opts.token           - JWT auth token
+ * @param {string} [opts.api]           - Backend base URL
+ * @param {string} opts.input           - Input video file path
+ * @param {string} [opts.from]          - Source language code (auto-detect if omitted)
+ * @param {string} opts.to              - Target language code (required)
+ * @param {string} [opts.voice]         - TTS voice ID
+ * @param {string} [opts.voicesMap]     - Voice mapping JSON file path
+ * @param {boolean}[opts.realign]       - Adjust subtitle timing for target language
+ * @param {string} [opts.output]        - Output MP4 path
+ * @param {boolean}[opts.keepIntermediates] - Keep temp files after completion
+ * @param {number} [opts.batchSize]     - Translation batch size
+ * @param {number} [opts.speed]         - TTS speed (0.5-2.0)
+ * @param {string} [opts.asrMode]       - Override ASR mode: sentence, flash, file, auto
+ * @param {string} [opts.asrLang]       - Override ASR engine type: 16k_zh, 16k_en, etc.
+ * @param {string} [opts.engine]        - ASR engine: auto, local, cloud (default: auto)
+ * @param {string} [opts.model]         - Whisper model for local engine: tiny, base, small, medium, large
+ * @returns {Promise<object>} Result with outputPath and stage details
+ */
+async function videoTranslate(opts) {
+  const sigintHandler = () => {
+    console.log('\n\nVideo translation cancelled.');
+    process.exit(130);
+  };
+  process.on('SIGINT', sigintHandler);
+
+  // Background refresh: video-translate routinely outlives the 1h Supabase
+  // JWT (long video → ASR poll → batch translate → per-caption TTS). Refresh
+  // proactively at expiresAt-5min so the user doesn't see "Token expired,
+  // refreshing..." mid-pipeline.
+  const { startBackgroundRefresh } = require('../core/auth');
+  const stopRefresh = startBackgroundRefresh({ api: opts.api });
+
+  try {
+    return await _videoTranslate(opts);
+  } finally {
+    stopRefresh();
+    process.removeListener('SIGINT', sigintHandler);
+  }
+}
+
+async function _videoTranslate(opts) {
+  const {
+    token,
+    api = API_BASE,
+    input,
+    from,
+    to,
+    voice,
+    voicesMap,
+    realign = false,
+    output: userOutput,
+    keepIntermediates = false,
+    batchSize = VIDEO_TRANSLATE_DEFAULTS.batchSize,
+    speed = VIDEO_TRANSLATE_DEFAULTS.speed,
+    asrMode,
+    asrLang,
+    engine = 'auto',
+    model,
+  } = opts;
+
+  const resolvedInput = path.resolve(input);
+  const inputBasename = path.basename(resolvedInput, path.extname(resolvedInput));
+
+  console.log('\n=== VoxFlow Video Translate ===');
+  console.log(`Input:  ${path.basename(resolvedInput)}`);
+  console.log(`Target: ${to}`);
+  console.log('');
+
+  // Create temp directory for intermediate files
+  const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'voxflow-vtranslate-'));
+
+  let totalQuota = 0;
+  const stages = {};
+
+  try {
+    // ─── Stage 1: Check FFmpeg ──────────────────────────────────────────
+    process.stdout.write('[1/5] Checking FFmpeg... ');
+    const ffmpegInfo = await checkFfmpeg();
+    if (!ffmpegInfo.available) {
+      throw new Error('FFmpeg is required for video-translate. Install: https://ffmpeg.org/download.html');
+    }
+    console.log(`OK (${ffmpegInfo.version})`);
+
+    // ─── Stage 2: Extract audio + ASR ───────────────────────────────────
+    process.stdout.write('[2/5] Transcribing audio... ');
+    const audioPath = path.join(tmpDir, 'extracted-audio.wav');
+    await extractAudio(resolvedInput, audioPath);
+
+    const asrOutputPath = path.join(tmpDir, 'source.srt');
+    const effectiveAsrLang = resolveAsrLang(from, asrLang);
+    const asrOpts = {
+      token,
+      api,
+      input: audioPath,
+      format: 'srt',
+      output: asrOutputPath,
+      lang: effectiveAsrLang,
+      engine,
+      // Force 'flash' mode for video-translate: ensures fine-grained per-sentence
+      // timestamps (one caption per sentence) instead of 'sentence' mode which
+      // dumps everything into a single giant block.
+      mode: asrMode || 'flash',
+    };
+    if (model) asrOpts.model = model;
+
+    let asrResult;
+    try {
+      asrResult = await asr(asrOpts);
+    } catch (asrErr) {
+      // Flash mode can fail on some Tencent backend issues — fallback to azure
+      if (!asrMode && engine !== 'local' && asrErr.message && asrErr.message.includes('500')) {
+        console.log(`\n  ⚠ Flash ASR failed, falling back to Azure...`);
+        asrOpts.engine = 'azure';
+        asrOpts.mode = undefined;
+        // Map Tencent lang codes to Azure BCP-47
+        const azureLangMap = { '16k_zh': 'zh-CN', '16k_en': 'en-US', '16k_ja': 'ja-JP', '16k_ko': 'ko-KR', '16k_zh_en': 'auto' };
+        asrOpts.lang = azureLangMap[effectiveAsrLang] || 'auto';
+        asrResult = await asr(asrOpts);
+      } else {
+        throw asrErr;
+      }
+    }
+
+    if (asrResult.captionCount === 0) {
+      throw new Error('ASR produced no captions. The video may have no audible speech.');
+    }
+
+    stages.asr = {
+      mode: asrResult.mode,
+      duration: asrResult.duration,
+      captionCount: asrResult.captionCount,
+      quotaUsed: asrResult.quotaUsed,
+    };
+    totalQuota += asrResult.quotaUsed;
+    console.log(`${asrResult.captionCount} captions (${asrResult.mode} mode)`);
+
+    // ─── Detect source language if not specified ────────────────────────
+    let fromCode = from;
+    if (!fromCode) {
+      const srtContent = fs.readFileSync(asrOutputPath, 'utf8');
+      const captions = parseSrt(srtContent);
+      const sample = captions.slice(0, 3).map(c => c.text).join(' ');
+      fromCode = await detectLanguage({ apiBase: api, text: sample }) || 'auto';
+    }
+
+    // ─── Stage 3: Translate ─────────────────────────────────────────────
+    process.stdout.write(`[3/5] Translating (${fromCode} → ${to})... `);
+    const translatedSrtPath = path.join(tmpDir, `translated-${to}.srt`);
+    const translateResult = await translate({
+      token,
+      api,
+      srt: asrOutputPath,
+      from: fromCode,
+      to,
+      output: translatedSrtPath,
+      realign,
+      batchSize,
+    });
+
+    stages.translate = {
+      from: translateResult.from,
+      to: translateResult.to,
+      captionCount: translateResult.captionCount,
+      quotaUsed: translateResult.quotaUsed,
+    };
+    totalQuota += translateResult.quotaUsed;
+
+    // ─── Translation quality fix: re-translate untranslated captions ────
+    const translatedContent = fs.readFileSync(translatedSrtPath, 'utf8');
+    const translatedCaptions = parseSrt(translatedContent);
+    const sourceCaptions = parseSrt(fs.readFileSync(asrOutputPath, 'utf8'));
+    const { formatSrt } = require('../core/srt');
+    const { chatCompletion } = require('../core/llm-client');
+    const { TRANSLATE_DEFAULTS } = require('../core/config');
+
+    let fixedCount = 0;
+    for (let ci = 0; ci < translatedCaptions.length && ci < sourceCaptions.length; ci++) {
+      if (looksUntranslated(translatedCaptions[ci].text, sourceCaptions[ci].text, to)) {
+        // This caption was not translated — retry individually (up to 2 attempts)
+        const { LANG_MAP } = require('./translate');
+        const toLang = LANG_MAP[to] || to;
+        const fromLang = LANG_MAP[fromCode] || fromCode;
+
+        for (let attempt = 0; attempt < 2; attempt++) {
+          try {
+            if (attempt > 0) await new Promise(r => setTimeout(r, 2000)); // wait 2s before retry
+            const singleResult = await chatCompletion({
+              apiBase: api,
+              token,
+              messages: [
+                { role: 'system', content: `Translate this subtitle from ${fromLang} to ${toLang}. Return ONLY the translation, nothing else.` },
+                { role: 'user', content: sourceCaptions[ci].text },
+              ],
+              temperature: TRANSLATE_DEFAULTS.temperature,
+              maxTokens: TRANSLATE_DEFAULTS.maxTokens,
+            });
+            const fixed = singleResult.content.trim();
+            if (fixed && fixed !== sourceCaptions[ci].text) {
+              translatedCaptions[ci].text = fixed;
+              fixedCount++;
+              break; // success
+            }
+            totalQuota++;
+          } catch {
+            // Retry on next iteration or give up
+            totalQuota++;
+          }
+        }
+      }
+    }
+
+    // Write the fixed SRT back
+    if (fixedCount > 0) {
+      const fixedSrt = formatSrt(translatedCaptions);
+      fs.writeFileSync(translatedSrtPath, fixedSrt, 'utf8');
+    }
+
+    const finalUntranslated = translatedCaptions.filter((c, i) => i < sourceCaptions.length && looksUntranslated(c.text, sourceCaptions[i].text, to)).length;
+    if (fixedCount > 0) {
+      console.log(`${translateResult.captionCount} captions translated (${fixedCount} fixed on retry)`);
+    } else {
+      console.log(`${translateResult.captionCount} captions translated`);
+    }
+    if (finalUntranslated > 0) {
+      console.log(`  ⚠ ${finalUntranslated} caption(s) could not be translated`);
+    }
+
+    // ─── Stage 4: Dub + Video merge ─────────────────────────────────────
+    process.stdout.write('[4/5] Dubbing and merging video... ');
+
+    // Determine final output path (use temp for now; subtitle burn-in is stage 5)
+    // Pre-build filename — see scripts/check-no-asset-rewrite.js (ncc rewrite trap)
+    const defaultFname = inputBasename + '-' + to + '.mp4';
+    const outputPath = userOutput
+      ? path.resolve(userOutput)
+      : path.resolve(path.dirname(resolvedInput), defaultFname);
+
+    // Dub into a temp file first — subtitle burn-in happens in stage 5
+    const dubbedTempPath = path.join(tmpDir, 'dubbed-nosubs.mp4');
+
+    const dubResult = await dub({
+      token,
+      api,
+      srt: translatedSrtPath,
+      voice,
+      voicesMap,
+      speed,
+      speedAuto: true,   // Always enable speed-auto for video-translate
+      video: resolvedInput,
+      output: dubbedTempPath,
+    });
+
+    stages.dub = {
+      segmentCount: dubResult.segmentCount,
+      duration: dubResult.duration,
+      quotaUsed: dubResult.quotaUsed,
+      warnings: dubResult.warnings,
+    };
+    totalQuota += dubResult.quotaUsed;
+    console.log(`${dubResult.segmentCount} segments dubbed`);
+
+    // ─── Stage 5: Burn subtitles into video ─────────────────────────────
+    process.stdout.write('[5/5] Burning subtitles into video... ');
+
+    // Copy subtitle file to a simple path (avoids ffmpeg path parsing issues)
+    const subTempPath = path.join(tmpDir, 'subs.srt');
+    fs.copyFileSync(translatedSrtPath, subTempPath);
+
+    let subtitlesBurned = false;
+    try {
+      const { runCommand } = require('../core/ffmpeg');
+      await runCommand('ffmpeg', [
+        '-i', dubbedTempPath,
+        '-vf', `subtitles=${subTempPath}:force_style='FontSize=18,PrimaryColour=&Hffffff,OutlineColour=&H000000,Outline=2,Shadow=1,Alignment=2,Bold=1'`,
+        '-c:a', 'copy',
+        '-y',
+        outputPath,
+      ]);
+      subtitlesBurned = true;
+      console.log('OK');
+    } catch (err) {
+      // Subtitle burn-in failed (likely no libass) — fall back to video without subs
+      console.log(`skip (${err.message?.includes('subtitles') ? 'libass not available' : 'failed'})`);
+      // Move dubbed file as final output
+      fs.copyFileSync(dubbedTempPath, outputPath);
+      // Save SRT alongside output for external use
+      const srtOutputPath = outputPath.replace(/\.[^.]+$/, `.${to}.srt`);
+      fs.copyFileSync(translatedSrtPath, srtOutputPath);
+      console.log(`  → Subtitle file saved separately: ${srtOutputPath}`);
+    }
+
+    // ─── Copy intermediates if requested ────────────────────────────────
+    if (keepIntermediates) {
+      const intermediateDir = path.resolve(
+        path.dirname(outputPath),
+        `${inputBasename}-${to}-intermediates`,
+      );
+      fs.mkdirSync(intermediateDir, { recursive: true });
+
+      const filesToCopy = [
+        ['extracted-audio.wav', audioPath],
+        ['source.srt', asrOutputPath],
+        [`translated-${to}.srt`, translatedSrtPath],
+      ];
+
+      for (const [name, src] of filesToCopy) {
+        if (fs.existsSync(src)) {
+          fs.copyFileSync(src, path.join(intermediateDir, name));
+        }
+      }
+
+      console.log(`\nIntermediates saved: ${intermediateDir}`);
+    }
+
+    // ─── Always save SRT alongside output (useful for external players) ──
+    const srtOutputPath = outputPath.replace(/\.[^.]+$/, `.srt`);
+    fs.copyFileSync(translatedSrtPath, srtOutputPath);
+
+    // ─── Summary ────────────────────────────────────────────────────────
+    console.log('\n=== Done ===');
+    console.log(`Output:   ${outputPath}`);
+    console.log(`Subtitle: ${srtOutputPath}`);
+    console.log(`Language: ${fromCode} → ${to}`);
+    console.log(`Captions: ${translateResult.captionCount}`);
+    console.log(`Duration: ${dubResult.duration.toFixed(1)}s`);
+    console.log(`Quota:    ${totalQuota} used`);
+
+    if (stages.dub.warnings && stages.dub.warnings.length > 0) {
+      console.log(`\nWarnings:`);
+      for (const w of stages.dub.warnings) {
+        console.log(`  - ${w}`);
+      }
+    }
+
+    // ─── Auto-open the output video ─────────────────────────────────────
+    try {
+      const open = require('open');
+      await open(outputPath);
+    } catch {
+      // open() may fail in headless/CI environments — ignore silently
+    }
+
+    return {
+      outputPath,
+      srtOutputPath,
+      from: fromCode,
+      to,
+      captionCount: translateResult.captionCount,
+      quotaUsed: totalQuota,
+      stages,
+    };
+  } finally {
+    // Cleanup temp directory
+    if (!keepIntermediates) {
+      try {
+        fs.rmSync(tmpDir, { recursive: true, force: true });
+      } catch {
+        // Best-effort cleanup
+      }
+    }
+  }
+}
+
+// ─── Exports ─────────────────────────────────────────────────────────────────
+
+// ─── CLI Handler ────────────────────────────────────────────────────────────
+
+async function handle(args) {
+  const { parseFlag, parseIntFlag, parseFloatFlag, parseBoolFlag, validateSpeed, runWithRetry } = require('../core/args');
+  const { getToken, getTokenInfo } = require('../core/auth');
+  const { API_BASE } = require('../core/config');
+
+  const api = parseFlag(args, '--api') || API_BASE;
+  const explicitToken = parseFlag(args, '--token');
+
+  // Parse and validate before auth
+  const input = parseFlag(args, '--input');
+  const from = parseFlag(args, '--from');
+  const to = parseFlag(args, '--to');
+  const voice = parseFlag(args, '--voice');
+  const voicesMap = parseFlag(args, '--voices');
+  const output = parseFlag(args, '--output', '-o');
+  const realign = parseBoolFlag(args, '--realign');
+  const keepIntermediates = parseBoolFlag(args, '--keep-intermediates');
+  const batchSize = parseIntFlag(args, '--batch-size');
+  const speed = parseFloatFlag(args, '--speed');
+  const asrMode = parseFlag(args, '--asr-mode');
+  const asrLang = parseFlag(args, '--asr-lang');
+  const engine = parseFlag(args, '--engine');
+  const model = parseFlag(args, '--model');
+
+  // Validate --input is required
+  if (!input) {
+    console.error('Error: --input <video-file> is required. Example: voxflow video-translate --input video.mp4 --to en');
+    process.exit(1);
+  }
+
+  // Validate --to is required
+  if (!to) {
+    console.error('Error: --to <lang> is required. Example: voxflow video-translate --input video.mp4 --to en');
+    process.exit(1);
+  }
+
+  // Validate language codes
+  const validLangs = ['zh', 'en', 'ja', 'ko', 'fr', 'de', 'es', 'pt', 'ru', 'ar', 'th', 'vi', 'it'];
+  if (to && !validLangs.includes(to)) {
+    console.error(`Error: --to must be one of: ${validLangs.join(', ')} (got: "${to}")`);
+    process.exit(1);
+  }
+  if (from && !validLangs.includes(from) && from !== 'auto') {
+    console.error(`Error: --from must be one of: auto, ${validLangs.join(', ')} (got: "${from}")`);
+    process.exit(1);
+  }
+
+  // Validate input file exists
+  if (input) {
+    const fs = require('fs');
+    const path = require('path');
+    const resolved = path.resolve(input);
+    if (!fs.existsSync(resolved)) {
+      console.error(`Error: Video file not found: ${resolved}`);
+      process.exit(1);
+    }
+  }
+
+  // Validate speed
+  validateSpeed(args, speed);
+
+  // Validate batch size
+  if (batchSize !== undefined && (isNaN(batchSize) || batchSize < 1 || batchSize > 20)) {
+    console.error(`Error: --batch-size must be between 1 and 20 (got: "${parseFlag(args, '--batch-size')}")`);
+    process.exit(1);
+  }
+
+  // Validate ASR mode
+  const validAsrModes = ['auto', 'sentence', 'flash', 'file'];
+  if (asrMode && !validAsrModes.includes(asrMode)) {
+    console.error(`Error: --asr-mode must be one of: ${validAsrModes.join(', ')} (got: "${asrMode}")`);
+    process.exit(1);
+  }
+
+  // Validate engine
+  const validEngines = ['auto', 'local', 'cloud', 'whisper', 'tencent'];
+  if (engine && !validEngines.includes(engine)) {
+    console.error(`Error: --engine must be one of: ${validEngines.join(', ')} (got: "${engine}")`);
+    process.exit(1);
+  }
+
+  // Validate voices map file exists
+  if (voicesMap) {
+    const fs = require('fs');
+    const path = require('path');
+    const resolved = path.resolve(voicesMap);
+    if (!fs.existsSync(resolved)) {
+      console.error(`Error: Voices map file not found: ${resolved}`);
+      process.exit(1);
+    }
+  }
+
+  // Authenticate after all validation
+  let token;
+  if (explicitToken) {
+    token = explicitToken;
+  } else {
+    token = await getToken({ api });
+    const info = getTokenInfo();
+    if (info) {
+      console.log(`\x1b[32mLogged in as ${info.email}\x1b[0m`);
+    }
+  }
+
+  const opts = {
+    token, api, input, from, to, voice, voicesMap, output,
+    realign, keepIntermediates, batchSize, speed, asrMode, asrLang,
+    engine, model,
+  };
+
+  await runWithRetry(videoTranslate, opts, api, explicitToken);
+}
+
+const meta = {
+  'video-translate': {
+    usage: '[opts]',
+    description: 'Translate entire video: ASR → translate → dub → merge',
+    options: [
+      `--input <file>       Input video file (required)`,
+      `--to <lang>          Target language code (required)`,
+      `--from <lang>        Source language code (default: auto-detect)`,
+      `--voice <id>         TTS voice ID for dubbed audio`,
+      `--voices <file>      JSON speaker→voiceId map for multi-speaker dubbing`,
+      `--realign            Adjust subtitle timing for target language length`,
+      `--speed <n>          TTS speed 0.5-2.0 (default: ${require('../core/config').VIDEO_TRANSLATE_DEFAULTS.speed})`,
+      `--batch-size <n>     Translation batch size, 1-20 (default: ${require('../core/config').VIDEO_TRANSLATE_DEFAULTS.batchSize})`,
+      `--keep-intermediates Keep intermediate files (SRT, audio) for debugging`,
+      `--output <path>      Output MP4 path (default: <input>-<lang>.mp4)`,
+      `--asr-mode <mode>    Override ASR mode: auto, sentence, flash, file`,
+      `--asr-lang <engine>  Override ASR engine: 16k_zh, 16k_en, 16k_ja, 16k_ko, etc.`,
+      `--engine <engine>    ASR engine: auto, local, cloud (default: auto)`,
+      `--model <model>      Whisper model for local engine: tiny, base, small, medium, large`,
+    ],
+    examples: [
+      'voxflow video-translate --input video.mp4 --to en',
+      'voxflow video-translate --input video.mp4 --from zh --to en --realign',
+      'voxflow video-translate --input video.mp4 --to ja --voice v-male-Bk7vD3xP',
+      'voxflow video-translate --input video.mp4 --to en --engine local',
+    ],
+  },
+};
+
+module.exports = { videoTranslate, handle, meta };