voxflow 1.15.0 → 1.15.2

package/lib/commands/summarize.js

@@ -0,0 +1,532 @@
+/**
+ * VoxFlow CLI — Summarize command (V2)
+ *
+ * Video/audio → ASR → LLM structured summary → PPTX + Video
+ *
+ * Video pipeline (--video):
+ *   1. ASR with timestamps → segmented transcript
+ *   2. LLM extracts key points with time ranges from original video
+ *   3. Per-point: cut original video clip + overlay text + TTS narration
+ *   4. FFmpeg concat: title card + annotated clips + closing card → MP4
+ */
+
+const fs = require('fs');
+const path = require('path');
+const os = require('os');
+const { asr } = require('./asr');
+const { chatCompletion } = require('../core/llm-client');
+const { request, throwApiError, throwNetworkError } = require('../core/http');
+const { buildWav, createSilence } = require('../core/audio');
+const { API_BASE, SUMMARIZE_DEFAULTS } = require('../core/config');
+
+// ─── LLM Prompt: structured summary with time ranges ────────────────────────
+
+const SUMMARIZE_PROMPT = `You are a video content analyst. Given a transcript with timestamps, extract key points and structure them for a summary presentation.
+
+LANGUAGE RULE: Detect the language of the input content. All text fields (titles, bullets, narration) MUST be in the SAME language as the input.
+
+STRICT RULES:
+1. Output ONLY valid JSON, no markdown fences, no extra text.
+2. Extract 4-8 key points from the transcript.
+3. Each point MUST reference a time range from the original video.
+4. narration: a professional summary of this point, 2-3 sentences, as if narrating a recap video.
+5. Remove filler words and repetitions. Every bullet should be specific.
+6. Do NOT invent facts not in the transcript.
+7. title: a short slide title for the point (max 15 chars).
+8. bullets: 1-3 concise bullet points (max 25 chars each).
+
+OUTPUT FORMAT:
+{
+  "meta": { "title": "...", "language": "..." },
+  "points": [
+    {
+      "id": 1,
+      "title": "<point title>",
+      "bullets": ["<bullet 1>", "<bullet 2>"],
+      "startSec": 0,
+      "endSec": 25,
+      "narration": "<narrator summary of this point>"
+    }
+  ]
+}`;
+
+// ─── PPTX generation ────────────────────────────────────────────────────────
+
+async function generatePptx(summaryData, outputPath) {
+  let PptxGenJS;
+  try {
+    PptxGenJS = require('pptxgenjs');
+  } catch {
+    const jsonPath = outputPath.replace(/\.pptx$/, '.json');
+    fs.writeFileSync(jsonPath, JSON.stringify(summaryData, null, 2), 'utf8');
+    return { path: jsonPath, format: 'json', fallback: true };
+  }
+
+  const pptx = new PptxGenJS();
+  pptx.title = summaryData.meta?.title || 'Summary';
+  pptx.author = 'VoxFlow Studio';
+  pptx.layout = 'LAYOUT_WIDE';
+
+  const accent = '6366F1';
+  const bg = '0F172A';
+  const text = 'F1F5F9';
+  const muted = '94A3B8';
+
+  for (const point of summaryData.points) {
+    const s = pptx.addSlide();
+    s.background = { color: bg };
+    s.addText(point.title, {
+      x: 0.8, y: 0.5, w: '85%', fontSize: 32, bold: true,
+      color: text, fontFace: 'Arial',
+    });
+    s.addShape('rect', { x: 0.8, y: 1.3, w: 1.2, h: 0.04, fill: { color: accent } });
+
+    if (point.bullets?.length) {
+      const bulletText = point.bullets.map(b => ({
+        text: b,
+        options: { bullet: { code: '2022' }, fontSize: 18, color: text, breakLine: true },
+      }));
+      s.addText(bulletText, {
+        x: 0.8, y: 1.8, w: '85%', h: 3, fontFace: 'Arial', paraSpaceAfter: 10, valign: 'top',
+      });
+    }
+
+    // Time range indicator
+    const timeLabel = `${formatTime(point.startSec)} - ${formatTime(point.endSec)}`;
+    s.addText(timeLabel, {
+      x: 0.8, y: 4.8, w: 3, fontSize: 12, color: muted, fontFace: 'Arial',
+    });
+
+    if (point.narration) s.addNotes(point.narration);
+  }
+
+  const buffer = await pptx.write({ outputType: 'nodebuffer' });
+  fs.writeFileSync(outputPath, buffer);
+  return { path: outputPath, format: 'pptx', fallback: false };
+}
+
+function formatTime(sec) {
+  const m = Math.floor(sec / 60);
+  const s = Math.floor(sec % 60);
+  return `${m}:${s.toString().padStart(2, '0')}`;
+}
+
+// ─── Video composition via FFmpeg ───────────────────────────────────────────
+
+/**
+ * Compose a summary video with PiP layout:
+ * - Background: blurred frame from original video (per point)
+ * - PiP (bottom-left): original video clip (clear, with person speaking)
+ * - Audio: AI TTS narration replacing original audio
+ *
+ * Layout:
+ * ┌──────────────────────────┐
+ * │     (blurred bg)         │
+ * │                          │
+ * │  ┌────────┐              │
+ * │  │ 原视频  │              │
+ * │  │ (人物)  │              │
+ * │  └────────┘              │
+ * └──────────────────────────┘
+ */
+async function composeVideo(originalVideo, points, ttsWavs, outputPath) {
+  const {
+    generateTitleCard, concatVideos, normalizeVideo, runCommand,
+  } = require('../core/ffmpeg');
+
+  const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'voxflow-sumvideo-'));
+
+  // Probe original video resolution
+  let width = 1920, height = 1080;
+  try {
+    const { stdout } = await runCommand('ffprobe', [
+      '-v', 'quiet', '-select_streams', 'v:0',
+      '-show_entries', 'stream=width,height',
+      '-of', 'csv=p=0', path.resolve(originalVideo),
+    ]);
+    const [w, h] = stdout.trim().split(',').map(Number);
+    if (w && h) { width = w; height = h; }
+  } catch {}
+
+  const resolution = `${width}x${height}`;
+  const pipW = Math.round(width * 0.35);  // PiP = 35% of frame width
+  const pipPad = Math.round(width * 0.025); // 2.5% padding
+  const segmentPaths = [];
+
+  try {
+    // 1. Title card (darkened first frame)
+    process.stdout.write('    [1] Title card... ');
+    const titlePath = path.join(tmpDir, 'title.mp4');
+    await generateTitleCard(titlePath, '', {
+      duration: 3, resolution, sourceVideo: originalVideo,
+    });
+    const titleNorm = path.join(tmpDir, 'title-norm.mp4');
+    await normalizeVideo(titlePath, titleNorm, { resolution });
+    segmentPaths.push(titleNorm);
+    console.log('OK');
+
+    // 2. Each point: PiP layout (blurred bg + clear PiP + TTS audio)
+    for (let i = 0; i < points.length; i++) {
+      const point = points[i];
+      process.stdout.write(`    [${i + 2}] "${point.title}"... `);
+
+      const safeStart = Math.max(0, point.startSec);
+      const safeEnd = Math.min(point.endSec, point.startSec + 30);
+      const pipPath = path.join(tmpDir, `pip-${i}.mp4`);
+
+      // Build PiP: blurred full-screen background + clear small video in bottom-left
+      // Round pip dimensions to even numbers (required by libx264)
+      const pipH = Math.round(pipW * height / width / 2) * 2;
+      const evenPipW = Math.round(pipW / 2) * 2;
+      const pipY = height - pipH - pipPad;
+
+      const filterComplex = [
+        // Background: scale original to fill, heavy blur + darken
+        `[0:v]scale=${width}:${height},boxblur=25:5,eq=brightness=-0.3:saturation=0.5[bg]`,
+        // PiP: scale original to small size with rounded corners effect
+        `[0:v]scale=${evenPipW}:${pipH}[pip]`,
+        // Overlay PiP on blurred background (bottom-right)
+        `[bg][pip]overlay=${width - evenPipW - pipPad}:${pipY}[out]`,
+      ].join(';');
+
+      if (ttsWavs[i]) {
+        // With TTS: use TTS audio, ignore original audio
+        await runCommand('ffmpeg', [
+          '-ss', String(safeStart), '-to', String(safeEnd), '-i', path.resolve(originalVideo),
+          '-i', path.resolve(ttsWavs[i]),
+          '-filter_complex', filterComplex,
+          '-map', '[out]', '-map', '1:a:0',
+          '-c:v', 'libx264', '-preset', 'fast', '-crf', '23', '-pix_fmt', 'yuv420p',
+          '-c:a', 'aac', '-ar', '24000', '-ac', '1',
+          '-shortest',
+          '-y', pipPath,
+        ], { timeout: 300_000 });
+      } else {
+        // Without TTS: keep original audio
+        await runCommand('ffmpeg', [
+          '-ss', String(safeStart), '-to', String(safeEnd), '-i', path.resolve(originalVideo),
+          '-filter_complex', filterComplex,
+          '-map', '[out]', '-map', '0:a:0',
+          '-c:v', 'libx264', '-preset', 'fast', '-crf', '23', '-pix_fmt', 'yuv420p',
+          '-c:a', 'aac', '-ar', '24000', '-ac', '1',
+          '-shortest',
+          '-y', pipPath,
+        ], { timeout: 300_000 });
+      }
+
+      // Normalize for concat
+      const normPath = path.join(tmpDir, `norm-${i}.mp4`);
+      await normalizeVideo(pipPath, normPath, { resolution });
+      segmentPaths.push(normPath);
+      console.log('OK');
+    }
+
+    // 3. Closing card
+    process.stdout.write('    [last] Closing... ');
+    const closingPath = path.join(tmpDir, 'closing.mp4');
+    await generateTitleCard(closingPath, '', {
+      duration: 2, resolution, sourceVideo: originalVideo,
+    });
+    const closingNorm = path.join(tmpDir, 'closing-norm.mp4');
+    await normalizeVideo(closingPath, closingNorm, { resolution });
+    segmentPaths.push(closingNorm);
+    console.log('OK');
+
+    // 4. Concatenate
+    process.stdout.write('    Concatenating... ');
+    await concatVideos(segmentPaths, path.resolve(outputPath));
+    console.log('OK');
+
+  } finally {
+    try { fs.rmSync(tmpDir, { recursive: true, force: true }); } catch {}
+  }
+}
+
+// ─── TTS synthesis for a single text ────────────────────────────────────────
+
+async function synthesizeTts(apiBase, token, text, voiceId, speed) {
+  let status, data;
+  try {
+    ({ status, data } = await request(`${apiBase}/api/tts/synthesize`, {
+      method: 'POST',
+      headers: {
+        'Content-Type': 'application/json',
+        'Authorization': `Bearer ${token}`,
+      },
+    }, { text, voiceId, speed, format: 'pcm', sampleRate: 24000 }));
+  } catch (err) {
+    throwNetworkError(err, apiBase);
+  }
+
+  if (status !== 200 || data.code !== 'success') {
+    throwApiError(status, data, 'TTS');
+  }
+
+  return Buffer.from(data.audio, 'base64');
+}
+
+// ─── Main command ───────────────────────────────────────────────────────────
+
+async function summarize(opts) {
+  const sigintHandler = () => {
+    console.log('\n\nSummarize cancelled.');
+    process.exit(0);
+  };
+  process.on('SIGINT', sigintHandler);
+
+  try {
+    return await _summarize(opts);
+  } finally {
+    process.removeListener('SIGINT', sigintHandler);
+  }
+}
+
+async function _summarize(opts) {
+  const {
+    token,
+    api = API_BASE,
+    input,
+    text: inputText,
+    slideCount = SUMMARIZE_DEFAULTS.slides,
+    language = SUMMARIZE_DEFAULTS.language,
+    voice = SUMMARIZE_DEFAULTS.voice,
+    speed = SUMMARIZE_DEFAULTS.speed,
+    engine = SUMMARIZE_DEFAULTS.engine,
+    model,
+    tts = false,
+    video = false,
+    output: userOutput,
+  } = opts;
+
+  console.log('\n=== VoxFlow Summarize ===');
+
+  let transcript;
+  let totalQuota = 0;
+  const resolvedInput = input ? path.resolve(input) : null;
+
+  // ─── Step 1: Get transcript ──────────────────────────────────────────
+  if (inputText) {
+    console.log(`Input:  text (${inputText.length} chars)`);
+    transcript = inputText;
+  } else if (input) {
+    console.log(`Input:  ${path.basename(resolvedInput)}`);
+    process.stdout.write('[1] Transcribing audio... ');
+
+    const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'voxflow-summarize-'));
+    const asrOutputPath = path.join(tmpDir, 'transcript.txt');
+
+    const asrResult = await asr({
+      token, api, input: resolvedInput,
+      format: 'txt', output: asrOutputPath,
+      engine, model,
+      mode: 'file',
+    });
+
+    transcript = fs.readFileSync(asrOutputPath, 'utf8');
+    totalQuota += asrResult.quotaUsed;
+    console.log(`${asrResult.captionCount} segments (${asrResult.mode})`);
+    try { fs.rmSync(tmpDir, { recursive: true, force: true }); } catch {}
+  } else {
+    throw new Error('Either --input <file> or --text <text> is required.');
+  }
+
+  if (!transcript.trim()) {
+    throw new Error('Transcript is empty. No audible speech detected.');
+  }
+
+  // ─── Step 2: LLM → structured points ─────────────────────────────────
+  process.stdout.write('[2] Extracting key points... ');
+
+  const llmResult = await chatCompletion({
+    apiBase: api, token,
+    messages: [
+      { role: 'system', content: SUMMARIZE_PROMPT },
+      {
+        role: 'user',
+        content: `Extract ${slideCount} key points from this transcript. Language: ${language}.\n\nTRANSCRIPT:\n${transcript.slice(0, 8000)}\n\nOutput JSON only:`,
+      },
+    ],
+    temperature: 0.3,
+    maxTokens: 4000,
+  });
+  totalQuota += 1;
+
+  let summaryData;
+  try {
+    summaryData = JSON.parse(llmResult.content.trim());
+  } catch {
+    const m = llmResult.content.match(/```(?:json)?\s*([\s\S]*?)```/);
+    if (m?.[1]) summaryData = JSON.parse(m[1].trim());
+    else throw new Error('Failed to parse summary JSON from LLM');
+  }
+
+  const points = summaryData.points || [];
+  console.log(`${points.length} points`);
+
+  // ─── Step 3: Generate PPTX ────────────────────────────────────────────
+  process.stdout.write('[3] Generating PPTX... ');
+  const baseName = input ? path.basename(resolvedInput, path.extname(resolvedInput)) : 'summary';
+  const outputDir = input ? path.dirname(resolvedInput) : process.cwd();
+  const pptxPath = userOutput
+    ? path.resolve(userOutput)
+    : path.resolve(outputDir, `${baseName}-summary.pptx`);
+
+  const pptxResult = await generatePptx(summaryData, pptxPath);
+  console.log(`OK (${pptxResult.format})`);
+
+  // ─── Step 4: TTS narrations ───────────────────────────────────────────
+  const ttsWavPaths = [];
+  if (video || tts) {
+    console.log('[4] Synthesizing narrations...');
+    const allPcm = [];
+
+    for (let i = 0; i < points.length; i++) {
+      const p = points[i];
+      if (!p.narration?.trim()) {
+        ttsWavPaths.push(null);
+        allPcm.push(createSilence(1, 24000));
+        continue;
+      }
+
+      process.stdout.write(`    TTS [${i + 1}/${points.length}]... `);
+      const pcm = await synthesizeTts(api, token, p.narration, voice, speed);
+      totalQuota += 1;
+
+      // Save per-point WAV for video pipeline
+      if (video) {
+        const wavPath = path.join(os.tmpdir(), `voxflow-sum-tts-${i}.wav`);
+        const wavBuf = buildWav([pcm], 0);
+        fs.writeFileSync(wavPath, wavBuf.wav || wavBuf);
+        ttsWavPaths.push(wavPath);
+      }
+
+      allPcm.push(pcm);
+      allPcm.push(createSilence(0.5, 24000));
+      const durMs = (pcm.length / 48).toFixed(0);
+      console.log(`OK (${durMs}ms)`);
+    }
+
+    // Save combined narration WAV
+    if (tts) {
+      const narrationPath = pptxPath.replace(/\.(pptx|json)$/, '-narration.wav');
+      // allPcm already contains interleaved PCM + silence buffers, so use silenceSec=0
+      const wavBuf = buildWav(allPcm, 0);
+      fs.writeFileSync(narrationPath, wavBuf.wav || wavBuf);
+      console.log(`    Narration: ${narrationPath}`);
+    }
+  }
+
+  // ─── Step 5: Video composition ────────────────────────────────────────
+  let videoPath = null;
+  if (video && resolvedInput) {
+    console.log('[5] Composing video...');
+    videoPath = pptxPath.replace(/\.(pptx|json)$/, '.mp4');
+
+    try {
+      await composeVideo(resolvedInput, points, ttsWavPaths, videoPath, {
+        title: summaryData.meta?.title || 'Summary',
+      });
+    } catch (err) {
+      console.log(`    Video failed: ${err.message}`);
+      videoPath = null;
+    }
+
+    // Cleanup TTS temp files
+    ttsWavPaths.forEach(f => { if (f) try { fs.unlinkSync(f); } catch {} });
+  }
+
+  // ─── Summary ──────────────────────────────────────────────────────────
+  console.log('\n=== Done ===');
+  console.log(`Slides:  ${pptxResult.path} (${points.length} points)`);
+  if (videoPath) console.log(`Video:   ${videoPath}`);
+  console.log(`Quota:   ${totalQuota} used`);
+
+  return { pptxPath: pptxResult.path, videoPath, slideCount: points.length, quotaUsed: totalQuota, summaryData };
+}
+
+// ─── CLI Handler ────────────────────────────────────────────────────────────
+
+async function handle(args) {
+  const { parseFlag, parseIntFlag, parseFloatFlag, parseBoolFlag, runWithRetry } = require('../core/args');
+  const { getToken } = require('../core/auth');
+  const { API_BASE, SUMMARIZE_DEFAULTS: SUM_DEFS } = require('../core/config');
+
+  const fs = require('fs');
+  const path = require('path');
+  const api = parseFlag(args, '--api') || API_BASE;
+  const explicitToken = parseFlag(args, '--token');
+
+  const input = parseFlag(args, '--input');
+  const text = parseFlag(args, '--text');
+  const output = parseFlag(args, '--output', '-o');
+  const voice = parseFlag(args, '--voice') || SUM_DEFS.voice;
+  const speed = parseFloatFlag(args, '--speed') ?? SUM_DEFS.speed;
+  const slideCount = parseIntFlag(args, '--slides') ?? SUM_DEFS.slides;
+  const language = parseFlag(args, '--lang') || SUM_DEFS.language;
+  const engine = parseFlag(args, '--engine') || SUM_DEFS.engine;
+  const model = parseFlag(args, '--model');
+  const tts = parseBoolFlag(args, '--tts');
+  const video = parseBoolFlag(args, '--video');
+  const scheme = parseFlag(args, '--scheme') || 'aurora';
+
+  if (!input && !text) {
+    console.error('Error: --input <file> or --text <text> is required.\nExample: voxflow summarize --input lecture.mp4');
+    process.exit(1);
+  }
+
+  if (input) {
+    const resolved = path.resolve(input);
+    if (!fs.existsSync(resolved)) {
+      console.error(`Error: File not found: ${resolved}`);
+      process.exit(1);
+    }
+  }
+
+  if (speed !== undefined && (isNaN(speed) || speed < 0.5 || speed > 2.0)) {
+    console.error(`Error: --speed must be between 0.5 and 2.0`);
+    process.exit(1);
+  }
+
+  const token = explicitToken || await getToken();
+  if (!token) {
+    console.error('Not logged in. Run: voxflow login');
+    process.exit(1);
+  }
+
+  const opts = {
+    token, api, input, text, output, voice, speed, slideCount, language,
+    engine, model, tts, video, scheme,
+  };
+
+  await runWithRetry(summarize, opts, api, explicitToken);
+}
+
+const meta = {
+  summarize: {
+    usage: '<--input|--text> [opts]',
+    description: 'Summarize video/audio/text into presentation slides (PPTX)',
+    options: [
+      `--input <file>       Input video/audio file → ASR + summarize`,
+      `--text <text>        Direct text input (skip ASR)`,
+      `--slides <n>         Number of slides, 4-12 (default: ${SUMMARIZE_DEFAULTS.slides})`,
+      `--lang <code>        Output language: en, zh, ja, etc. (default: ${SUMMARIZE_DEFAULTS.language})`,
+      `--engine <engine>    ASR engine: auto, local, cloud (default: ${SUMMARIZE_DEFAULTS.engine})`,
+      `--model <model>      Whisper model for local ASR: tiny, base, small, medium, large`,
+      `--tts                Generate TTS narration audio for each slide`,
+      `--video              Also render an MP4 video (requires remotion-cards/ + Remotion)`,
+      `--scheme <name>      Video visual scheme: noir, neon, editorial, aurora (default), brutalist`,
+      `--voice <id>         TTS voice ID (default: ${SUMMARIZE_DEFAULTS.voice})`,
+      `--speed <n>          TTS speed 0.5-2.0 (default: ${SUMMARIZE_DEFAULTS.speed})`,
+      `--output <path>      Output PPTX path (default: <input>-summary.pptx)`,
+    ],
+    examples: [
+      'voxflow summarize --input lecture.mp4',
+      'voxflow summarize --input meeting.mp4 --lang zh --slides 10',
+      'voxflow summarize --input podcast.mp3 --engine local --tts',
+      'voxflow summarize --input lecture.mp4 --video --scheme aurora',
+      'voxflow summarize --text "长篇文章内容..." --slides 6 --lang zh',
+    ],
+  },
+};
+
+module.exports = { summarize, handle, meta };