voxflow 1.14.0 → 1.15.1

package/lib/commands/podcast/synthesize.js

@@ -0,0 +1,188 @@
+/**
+ * Podcast command — Multi-voice TTS synthesis and audio assembly.
+ *
+ * Per turn, two paths depending on the picked voice:
+ *
+ *   1. Native pause path (voice.nativePauseSupported === true):
+ *        Send the full marked-up text in ONE TTS call. TRTC AI TTS honors
+ *        <|break|> / <|s_break|> markers natively (~250-430ms inserted).
+ *        Pace is applied via TRTC's `Speed` param (composeTtsSpeed).
+ *
+ *   2. Splice path (false / unknown):
+ *        Split the turn on markers, synthesize each chunk separately, and
+ *        splice silence client-side. Same as PR #3329's behavior — kept as
+ *        fallback for voices we couldn't empirically verify (旁白 narration
+ *        voices typically have prosodic variance that hides the marker effect).
+ *
+ * Inter-turn silence (between speakers) is always client-side; the caller
+ * computes it via computeGapsMs.
+ */
+
+'use strict';
+
+const { getIntentParams } = require('../../core/intent-params');
+const { synthesizeTTS } = require('../../core/tts-synthesizer');
+const { ApiError } = require('../../core/http');
+const { splitOnPauseMarkers, stripPauseMarkers } = require('../../core/pause-markers');
+const { resolvePaceMultiplier, composeTtsSpeed } = require('../../core/podcast-pacing');
+const { createSilence, PCM_SAMPLE_RATE } = require('../../core/audio');
+
+const sleep = (ms) => new Promise((r) => setTimeout(r, ms));
+
+// Backend's TTS rate limit is 60 req/min. Even native-path podcasts (1 call
+// per turn) can spike with long scripts; splice-path voices stay 1.5-2× heavier.
+// Silently retrying once on a 429 with the server's retry-after hint avoids
+// surfacing a confusing fatal error mid-render.
+const MAX_RATE_LIMIT_RETRIES = 1;
+const MAX_RETRY_WAIT_SEC = 30;
+
+/**
+ * Synthesize a single TTS request, with automatic 429 retry.
+ */
+async function synthesizeSegment(apiBase, token, text, voiceId, speed, volume, index, total, speaker) {
+  let lastErr;
+  for (let attempt = 0; attempt <= MAX_RATE_LIMIT_RETRIES; attempt++) {
+    try {
+      const result = await synthesizeTTS({
+        apiBase, token, text, voiceId, speed, volume,
+        index, total,
+        label: speaker,
+      });
+      console.log(` OK (${(result.audio.length / 1024).toFixed(0)} KB)`);
+      return { pcm: result.audio, quota: result.quota };
+    } catch (err) {
+      lastErr = err;
+      if (!(err instanceof ApiError) || err.code !== 'rate_limited') throw err;
+      if (attempt === MAX_RATE_LIMIT_RETRIES) throw err;
+      const waitSec = Math.min(err.retryAfterSec ?? 5, MAX_RETRY_WAIT_SEC);
+      console.log(`    rate-limited; waiting ${waitSec}s before retry ${attempt + 2}/${MAX_RATE_LIMIT_RETRIES + 1}`);
+      await sleep(waitSec * 1000);
+    }
+  }
+  throw lastErr;
+}
+
+/**
+ * Decide whether a turn can be synthesized as a single native-pause call.
+ *
+ *   - voiceMapping has nativePauseSupported === true
+ *   - Voice override is NOT in play (override doesn't carry the catalog flag)
+ *
+ * Returns true to use native path, false to fall back to splice.
+ */
+function shouldUseNativePause(mapping, voiceOverride) {
+  if (voiceOverride) return false;             // we don't know if override voice supports markers
+  return mapping?.nativePauseSupported === true;
+}
+
+/**
+ * Plan the TTS calls for a single turn given the chosen path.
+ * Returns: { calls: [{text, pauseAfterMsAfter}], path: 'native'|'splice' }
+ *
+ *   native: 1 call with full text (markers preserved, TRTC inserts pauses)
+ *   splice: N calls (one per chunk between markers) with silence inserted
+ *           between chunks client-side
+ */
+function planTurnCalls(seg, useNative) {
+  if (useNative) {
+    return { calls: [{ text: seg.text, pauseAfterMsAfter: 0 }], path: 'native' };
+  }
+  const subs = splitOnPauseMarkers(seg.text);
+  if (subs.length === 0) {
+    // No markers in text — single clean call
+    return { calls: [{ text: stripPauseMarkers(seg.text) || seg.text, pauseAfterMsAfter: 0 }], path: 'splice' };
+  }
+  return {
+    calls: subs.map((s, i) => ({
+      text: s.text,
+      pauseAfterMsAfter: i < subs.length - 1 ? s.pauseAfterMs : 0,
+    })),
+    path: 'splice',
+  };
+}
+
+/**
+ * Synthesize all segments with multi-voice mapping.
+ *
+ * For each turn:
+ *   - Pick path via shouldUseNativePause()
+ *   - native: 1 TTS call sending full marked-up text + TRTC-side Speed param
+ *   - splice: N TTS calls + client-side silence between chunks
+ *   - Inter-turn silence is the caller's responsibility (computeGapsMs)
+ *
+ * Pace handling:
+ *   - Both paths apply pace via TRTC Speed (composeTtsSpeed) so speech rate
+ *     scales with pace, keeping mixed-voice podcasts coherent across speakers.
+ *   - Native path additionally relies on TRTC scaling the inserted marker
+ *     pause along with speech — no client-side splice needed.
+ *   - Splice path additionally scales chunk-to-chunk silences via
+ *     PACE_MULTIPLIERS (≈ 1/paceSpeed) so silence shrinks together with
+ *     the faster speech.
+ */
+async function synthesizeAll(apiBase, token, segments, voiceMapping, defaultSpeed, voiceOverride, opts = {}) {
+  const pace = opts.pace || 'natural';
+  const paceSilence = resolvePaceMultiplier(pace);
+
+  // Pre-plan all calls so we know totalCalls upfront for "[N/T]" progress display
+  const plans = segments.map(seg => {
+    const mapping = voiceMapping[seg.speaker] || {};
+    const useNative = shouldUseNativePause(mapping, voiceOverride);
+    return { seg, mapping, useNative, plan: planTurnCalls(seg, useNative) };
+  });
+  const totalCalls = plans.reduce((acc, p) => acc + p.plan.calls.length, 0);
+  const nativeTurns = plans.filter(p => p.useNative).length;
+
+  console.log(`\n[2/3] 合成 TTS 音频 (${segments.length} 段, 多角色, ${totalCalls} 个 TTS 调用` +
+    (nativeTurns > 0 ? `; ${nativeTurns} 个 turn 走原生 marker pause` : '') + `)...`);
+
+  const pcmBuffers = [];
+  let lastQuota = null;
+  let synthIndex = 0;
+
+  for (let i = 0; i < plans.length; i++) {
+    const { seg, mapping, useNative, plan } = plans[i];
+
+    const voiceId = voiceOverride || mapping.voiceId || 'v-female-R2s4N9qJ';
+    const baseSpeed = mapping.speed || defaultSpeed;
+    const intentParams = getIntentParams(seg.intent);
+
+    // Speed: pace ALWAYS goes through TRTC Speed so speech rate is consistent
+    // across native and splice voices in the same podcast. Splice path also
+    // scales its own client-side silence (PACE_MULTIPLIERS), so total feel
+    // stays coherent.
+    const ttsSpeed = composeTtsSpeed({ voiceSpeed: baseSpeed, intentSpeed: intentParams.speed, pace });
+    const volume = intentParams.volume;
+
+    const turnPcmParts = [];
+    for (let s = 0; s < plan.calls.length; s++) {
+      const sub = plan.calls[s];
+      synthIndex += 1;
+      const pathTag = plan.path === 'native' ? '🔹' : '✂';
+      const result = await synthesizeSegment(
+        apiBase, token, sub.text, voiceId, ttsSpeed, volume,
+        synthIndex - 1, totalCalls,
+        `${seg.speaker}${pathTag}${plan.calls.length > 1 ? `[${s + 1}/${plan.calls.length}]` : ''}`
+      );
+      turnPcmParts.push(result.pcm);
+      lastQuota = result.quota;
+
+      // Splice path only: insert silence after this chunk (skip last)
+      if (plan.path === 'splice' && sub.pauseAfterMsAfter > 0) {
+        const ms = Math.round(sub.pauseAfterMsAfter * paceSilence);
+        if (ms > 0) turnPcmParts.push(createSilence(ms / 1000, PCM_SAMPLE_RATE));
+      }
+    }
+
+    pcmBuffers.push(Buffer.concat(turnPcmParts));
+  }
+
+  return { pcmBuffers, quota: lastQuota, ttsCallCount: synthIndex, nativeTurnCount: nativeTurns };
+}
+
+module.exports = {
+  synthesizeSegment,
+  synthesizeAll,
+  // exports for test
+  shouldUseNativePause,
+  planTurnCalls,
+};

package/lib/commands/podcast.js

@@ -0,0 +1,11 @@
+/**
+ * VoxFlow CLI — Podcast command
+ *
+ * Thin dispatcher — delegates to submodules in ./podcast/.
+ * See podcast/index.js for the orchestrator, podcast/dialogue.js,
+ * podcast/generate.js, podcast/synthesize.js for pipelines.
+ */
+
+'use strict';
+
+module.exports = require('./podcast/index');