storyshot 0.7.2 → 0.8.0

package/src/voice.mjs

@@ -201,12 +201,116 @@ export function sentenceGaps(pcm, sents) {
   });
 }
 
+// The phrases of a line, for scenes that animate on the words: its sentences, each split after a
+// comma-like mark (, ; : ， 、 ； ：), which stays with the phrase before it.
+export const PHRASE_BREAK = /(?<=[,;:，、；：])\s*/;
+export function phrases(text) {
+  return sentences(text).flatMap((s) => s.split(PHRASE_BREAK).map((p) => p.trim()).filter(Boolean));
+}
+
+// When each phrase of a spoken line starts and ends, in seconds into the line. parts: speak()'s
+// sentences (exact: their pauses were cut and widened). Inside a sentence each comma's expected time,
+// from the letters spoken before it, takes the nearest quiet stretch of 60 ms or more; a comma the
+// voice runs straight through keeps its estimate.
+export function phraseTimes(pcm, parts) {
+  const n = Math.floor(pcm.length / 2 / WIN), env = new Float32Array(n);
+  let peak = 0;
+  for (let w = 0; w < n; w++) {
+    let e = 0;
+    for (let i = w * WIN; i < (w + 1) * WIN; i++) e = Math.max(e, Math.abs(pcm[2 * i]), Math.abs(pcm[2 * i + 1]));
+    env[w] = e; peak = Math.max(peak, e);
+  }
+  const thr = peak * 0.02, out = [];
+  for (const p of parts) {
+    // (a part can hold two sentences the voice ran together: they are still two phrases)
+    const ph = phrases(p.text);
+    if (ph.length < 2) { out.push({ text: p.text, t: p.start, end: p.end }); continue; }
+    const a = Math.max(1, Math.floor(p.start * 100)), b = Math.min(n - 1, Math.ceil(p.end * 100)), qs = [];
+    for (let w = a; w < b;) {
+      if (env[w] >= thr) { w++; continue; }
+      const q0 = w;
+      while (w < b && env[w] < thr) w++;
+      if (w - q0 >= 6) qs.push([q0 / 100, w / 100]);
+    }
+    const total = letters(p.text) || 1;
+    let before = 0, used = -1, t = p.start;
+    for (let k = 0; k < ph.length; k++) {
+      before += letters(ph[k]);
+      if (k === ph.length - 1) { out.push({ text: ph[k], t, end: p.end }); break; }
+      const est = p.start + (before / total) * (p.end - p.start);
+      let best = null;
+      qs.forEach((q, j) => {
+        const mid = (q[0] + q[1]) / 2;
+        if (j > used && Math.abs(mid - est) < 0.6 && (!best || Math.abs(mid - est) < Math.abs(best.mid - est))) best = { j, mid, q };
+      });
+      if (best) { used = best.j; out.push({ text: ph[k], t, end: best.q[0] }); t = best.q[1]; }
+      else { out.push({ text: ph[k], t, end: est }); t = est; }
+    }
+  }
+  return out.map((c) => ({ text: c.text, t: +c.t.toFixed(3), end: +Math.max(c.t, c.end).toFixed(3) }));
+}
+
 // A line as stereo Float32 PCM at 48 kHz, spoken in one go (so the voice keeps its flow from sentence
 // to sentence), with each pause between sentences widened to at least `voice.pause` seconds: voices
 // often run sentences together with barely a breath. parts: where each sentence is, in seconds (for
 // subtitles).
-export async function speak(cfg, line, { log = console.log } = {}) {
-  const raw = await speakLine(cfg, line, log), pause = cfg.voice.pause ?? 0;
+// A scene's narration can run to a paragraph, longer than a voice reads well in one take: it is spoken in
+// chunks of whole sentences (up to about 100 Chinese characters or 45 words each), joined with the line's
+// sentence pause. Short lines are one chunk.
+const CHUNK_CJK = 100, CHUNK_WORDS = 45;
+const weight = (t) => { const c = (t.match(/[\u3040-\u30ff\u3400-\u9fff\uac00-\ud7af]/g) ?? []).length; return c / CHUNK_CJK + t.replace(/[\u3040-\u30ff\u3400-\u9fff\uac00-\ud7af]/g, ' ').split(/\s+/).filter((w) => /\w/.test(w)).length / CHUNK_WORDS; };
+export function chunks(line) {
+  const out = [];
+  for (const s of sentences(line)) {
+    const cjk = /[\u3040-\u30ff\u3400-\u9fff\uac00-\ud7af]$/.test(out.at(-1) ?? '') || /^[\u3040-\u30ff\u3400-\u9fff\uac00-\ud7af]/.test(s);
+    if (out.length && weight(out.at(-1)) + weight(s) <= 1) out[out.length - 1] += (cjk ? '' : ' ') + s;
+    else out.push(s);
+  }
+  return out.length ? out : [line];
+}
+export async function speak(cfg, line, { log = console.log, chunk = false } = {}) {
+  const pieces = chunk ? chunks(line) : [line];
+  if (pieces.length < 2) return speakOne(cfg, line, { log });
+  const said = [];
+  for (const p of pieces) said.push(await speakOne(cfg, p, { log }));
+  const gap = Math.round(Math.max(cfg.voice.pause ?? 0, 0.3) * SR);
+  const n = said.reduce((a, x) => a + x.pcm.length / 2, 0) + gap * (said.length - 1);
+  const pcm = new Float32Array(2 * n), parts = [];
+  let at = 0;
+  for (const [k, x] of said.entries()) {
+    pcm.set(x.pcm, 2 * at);
+    for (const q of x.parts) parts.push({ text: q.text, start: q.start + at / SR, end: q.end + at / SR });
+    at += x.pcm.length / 2 + (k < said.length - 1 ? gap : 0);
+  }
+  return { pcm, parts };
+}
+// `voice.maxPause` (seconds): no pause inside a line lasts longer than this (some voices stop for a second
+// at every comma); the sentence pauses are then widened to `voice.pause` as usual.
+export function capPauses(pcm, maxPause) {
+  if (!(maxPause > 0)) return pcm;
+  const n = Math.floor(pcm.length / 2 / WIN), env = new Float32Array(n);
+  let peak = 0;
+  for (let w = 0; w < n; w++) {
+    let e = 0;
+    for (let i = w * WIN; i < (w + 1) * WIN; i++) e = Math.max(e, Math.abs(pcm[2 * i]), Math.abs(pcm[2 * i + 1]));
+    env[w] = e; peak = Math.max(peak, e);
+  }
+  const thr = peak * 0.02, keep = Math.round(maxPause * 100), cuts = [];
+  for (let w = 1; w < n;) {
+    if (env[w] >= thr) { w++; continue; }
+    const a = w;
+    while (w < n && env[w] < thr) w++;
+    if (w < n && w - a > keep) cuts.push([(a + Math.floor(keep / 2)) * WIN, (w - Math.ceil(keep / 2)) * WIN]);
+  }
+  if (!cuts.length) return pcm;
+  const out = new Float32Array(pcm.length - 2 * cuts.reduce((x, [a, b]) => x + b - a, 0));
+  let from = 0, to = 0;
+  for (const [a, b] of cuts) { out.set(pcm.subarray(2 * from, 2 * a), 2 * to); to += a - from; from = b; }
+  out.set(pcm.subarray(2 * from), 2 * to);
+  return out;
+}
+async function speakOne(cfg, line, { log = console.log } = {}) {
+  const raw = capPauses(await speakLine(cfg, line, log), cfg.voice.maxPause), pause = cfg.voice.pause ?? 0;
   const sents = pause > 0 ? sentences(line) : [];
   if (sents.length < 2) return { pcm: raw, parts: [{ text: line, start: 0, end: raw.length / 2 / SR }] };
   const gaps = sentenceGaps(raw, sents), want = Math.round(pause * SR);
@@ -242,18 +346,23 @@ export function tempo(pcm, factor) {
 // so `duration: auto` steps can be timed to the voice.
 export async function measureVoice(cfg, { log = console.log } = {}) {
   const seconds = ({ pcm }) => pcm.length / 2 / SR;
-  const lens = { steps: {}, endCard: null, intro: null, cards: {} };
+  const lens = { steps: {}, endCard: null, intro: null, cards: {}, cues: {} };
   if (!cfg.voice) return lens;
   if (cfg.voice.fallback) log(`note: ${cfg.voice.fallback}`);
   const zh = cfg.voice.provider === 'kokoro' && isZhVoice(cfg.voice.voice);
   if (zh || cfg.voice.provider === 'qwen') {
     // synthesize every uncached line in one batch, so Python (the G2P, or the Qwen model) starts once
-    const lines = [...cfg.steps.filter((s) => !s.hidden).flatMap((s) => [s.say, s.card?.say]), cfg.intro?.say, cfg.endCard?.say].filter(Boolean);
+    const lines = [...cfg.steps.filter((s) => !s.hidden).flatMap((s) => [...(s.scene && s.say ? chunks(s.say) : [s.say]), s.card?.say]), cfg.intro?.say, cfg.endCard?.say].filter(Boolean);
     const todo = new Map(lines.map((l) => cached(cfg, l)).filter((c) => !fs.existsSync(c.file)).map((c) => [c.file, c]));
     await (zh ? synthZh : synthQwen)(cfg.voice, [...todo.values()], { log });
   }
   for (const [i, s] of cfg.steps.entries()) {
-    if (s.say && !s.hidden) lens.steps[i] = seconds(await speak(cfg, s.say, { log }));
+    if (s.say && !s.hidden) {
+      const sp = await speak(cfg, s.say, { log, chunk: !!s.scene });
+      lens.steps[i] = seconds(sp);
+      // a scene animates on the words: when each phrase is spoken
+      if (s.scene) lens.cues[i] = phraseTimes(sp.pcm, sp.parts);
+    }
     if (s.card?.say && !s.hidden) lens.cards[i] = seconds(await speak(cfg, s.card.say, { log }));
   }
   if (cfg.intro?.say) lens.intro = seconds(await speak(cfg, cfg.intro.say, { log }));