storyshot 0.4.1 → 0.4.2

package/README.md

@@ -217,7 +217,7 @@ Everything the agent does is also a command:
 | `storyshot record videos/acme-promo.yaml` | Only record. |
 | `storyshot guide` | Print the playbook and the reference. |
 | `storyshot setup <agent>` (`cursor`, `claude`, `codex`, `gemini`, `vscode`, `windsurf`, `cline`) | Set up the MCP server and the skill in this project, and install or check what Storyshot needs. |
-| `storyshot doctor` | Check Node, ffmpeg, Chromium and voices; install Chromium if it's missing. |
+| `storyshot doctor` | Check Node, ffmpeg (and the filters and encoders Storyshot uses), Chromium and voices; install Chromium if it's missing. |
 | `storyshot mcp` | Run the MCP server. |
 
 Recordings, previews and voice lines are cached in `.storyshot/` next to the
@@ -256,7 +256,7 @@ captions: true                                  # false keeps caption pills off
 purpose: pitch                                  # pitch | tutorial | course | docs (sets the defaults below)
 highlight: false                                # spotlight targets before click/type (per step or per action too)
 stepNumbers: false                              # numbered caption badges
-subtitles: false                                # write <out>.srt (narration, else captions); `burn` also burns the narration into the video, each word lighting up as it is spoken (captions move to the top)
+subtitles: false                                # write <out>.srt (narration, else captions); `burn` also burns the narration into the video, each word lighting up as it is spoken (captions move to the top); uses ffmpeg's libass when it has it, else draws the words itself
 pace: slow                                      # fast (pitch) | normal (docs) | slow (tutorial, course): cursor and typing speed, a beat after each click, hold after narration, voice speed, how much of a wait is shown
 follow: true                                    # tutorial/course default: the camera zooms in on what each step touches and pans between steps (a step's own zoom wins; follow: false on a step skips it); or { scale: 1.8 }
 article: { title: Export a drawing, intro: … }  # docs: heading and first paragraph of <out>-steps/README.md

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "storyshot",
-  "version": "0.4.1",
+  "version": "0.4.2",
   "description": "Turn a short YAML storyboard into a polished product demo video of your web app — smooth cursor, auto zoom, captions, music.",
   "author": "boheling <boheling@gmail.com>",
   "repository": {

package/src/audio.mjs

@@ -137,8 +137,14 @@ export async function soundtrack(cfg, events, silentVideo, { log = console.log }
       let dur = pcm.length / 2 / SR;
       const start = Math.max(line.start, prevEnd + 0.12), room = line.until - start - 0.1;
       let sped = 1;
-      if (dur > room && dur / room <= 1.2) { sped = dur / room; pcm = tempo(pcm, sped); dur = pcm.length / 2 / SR; }
-      else if (dur > room) warn(`${line.where}: narration is ${dur.toFixed(1)}s but only ${Math.max(0, room).toFixed(1)}s until ${line === events.voice.at(-1) ? 'the video ends' : 'the next line'} — shorten "${line.text}" or give the step more duration`);
+      // a line pushed past its window by the one before has no room left (room <= 0): it plays late, at normal speed
+      if (dur > room && room > 0 && dur / room <= 1.2) { sped = dur / room; pcm = tempo(pcm, sped); dur = pcm.length / 2 / SR; }
+      else if (dur > room) warn(room > 0
+        ? `${line.where}: narration is ${dur.toFixed(1)}s but only ${room.toFixed(1)}s until ${line === events.voice.at(-1) ? 'the video ends' : 'the next line'} — shorten "${line.text}" or give the step more duration`
+        : `${line.where}: the narration before it runs ${(start - line.start).toFixed(1)}s past where this line should start, so "${line.text}" plays late — shorten the earlier lines or give their steps more duration`);
+      if (start + dur > events.total) warn(start >= events.total
+        ? `${line.where}: "${line.text}" is cut off: the narration before it pushes it past the end of the video (${events.total.toFixed(1)}s) — shorten the lines or give the steps more duration`
+        : `${line.where}: the last ${(start + dur - events.total).toFixed(1)}s of "${line.text}" is cut off at the end of the video — shorten it or give the steps more duration`);
       log(`  voice ${start.toFixed(2)}–${(start + dur).toFixed(2)}s  ${line.where}${sped > 1 ? `  (sped up x${sped.toFixed(2)})` : ''}  "${line.text}"`);
       let pk = 0;
       for (const v of pcm) pk = Math.max(pk, Math.abs(v));

package/src/build.mjs

@@ -12,7 +12,7 @@ export async function build(cfg, mode = 'render', { log = console.log } = {}) {
   const voiceLens = await measureVoice(cfg, { log });
   const { silent, clean, events, preview, warnings, captions, cards, segs } = await compose(cfg, { log, voiceLens });
   const audio = await soundtrack(cfg, events, silent, { log });
-  if (cfg.subtitles === 'burn') burnSubtitles(cfg, audio.narration ?? [], { log });
+  if (cfg.subtitles === 'burn') await burnSubtitles(cfg, audio.narration ?? [], { log });
   const extras = await writeOutputs(cfg, { captions, cards, segs, clean, narration: audio.narration, total: events.total }, { log });
   return { out: audio.out, extras, preview, warnings: [...warnings, ...audio.warnings], duration: events.total, seconds: Math.round((Date.now() - t0) / 1000) };
 }

package/src/check.mjs

@@ -138,6 +138,22 @@ export function check(file, { draft = false } = {}) {
     const says = visible.filter((s) => s.say).length + (cfg.endCard?.say ? 1 : 0) + (cfg.intro?.say ? 1 : 0);
     if (cfg.voice && !says) warnings.push('voice is set but no step has a `say:` line');
     if (!cfg.voice && says) warnings.push('steps have `say:` lines but there is no `voice:` block, so they will not be spoken');
+    // fixed-length steps: a line that clearly can't be spoken before the next one starts (a rough
+    // estimate from its length; the render measures the real voice)
+    if (cfg.voice) {
+      const words = (t) => { const x = String(t).replace(/\*\*/g, ''); const c = (x.match(/[\u3040-\u30ff\u3400-\u9fff\uac00-\ud7af]/g) ?? []).length; return { c, w: x.replace(/[\u3040-\u30ff\u3400-\u9fff\uac00-\ud7af]/g, ' ').split(/\s+/).filter(Boolean).length }; };
+      const secs = (t) => { const { c, w } = words(t); return (c / 4.5 + w / 2.8) / (cfg.voice.speed ?? 1); };
+      const vis = visible.map((s) => ({ s, i: cfg.steps.indexOf(s) }));
+      for (const [k, { s, i }] of vis.entries()) {
+        if (!s.say) continue;
+        const nextK = vis.findIndex((v, j) => j > k && v.s.say);
+        const span = vis.slice(k, nextK < 0 ? undefined : nextK).map((v) => v.s.duration);
+        if (!span.every((d) => typeof d === 'number')) continue;
+        const room = span.reduce((a, d) => a + d, 0) - (s.sayAt ?? 0.15) + (nextK < 0 && cfg.endCard ? 2.5 : 0);
+        const need = secs(s.say);
+        if (need > room * 1.4 && need - room > 1) warnings.push(`step ${i + 1}: the say: line takes about ${need.toFixed(1)}s to speak but has ${room.toFixed(1)}s before ${nextK < 0 ? 'the video ends' : 'the next line'} — shorten it or raise duration (the lines after it would play late)`);
+      }
+    }
     if (!draft && cfg.voice && ['elevenlabs', 'openai'].includes(cfg.voice.provider)) {
       const env = cfg.voice.provider === 'elevenlabs' ? 'ELEVENLABS_API_KEY' : 'OPENAI_API_KEY';
       if (!process.env[env]) errors.push(`voice provider ${cfg.voice.provider} needs ${env} (environment or a .env file next to the storyboard) — or render with draft mode to use the free local voice`);

package/src/doctor.mjs

@@ -5,6 +5,7 @@ import { chromium } from 'playwright';
 import { execFileSync } from 'node:child_process';
 import { createRequire } from 'node:module';
 import path from 'node:path';
+import { hasFilter, hasEncoder, NEEDED, OPTIONAL } from './ffmpeg.mjs';
 
 const has = (cmd, args = ['--version']) => { try { execFileSync(cmd, args, { stdio: 'ignore' }); return true; } catch { return false; } };
 
@@ -47,7 +48,15 @@ export async function doctor({ log = console.log } = {}) {
 
   try {
     const v = execFileSync('ffmpeg', ['-version']).toString().match(/ffmpeg version (\S+)/)?.[1];
-    lines.push(`✔ ffmpeg ${v ?? ''}`.trim());
+    // a build can lack pieces Storyshot uses: say which, and what that affects
+    const missing = [...NEEDED.filters.filter((f) => !hasFilter(f)), ...NEEDED.encoders.filter((e) => !hasEncoder(e))];
+    if (missing.length) {
+      lines.push(`✖ ffmpeg ${v ?? ''} is missing ${missing.join(', ')}`);
+      todo.push(`install an ffmpeg build with ${missing.join(', ')} (${process.platform === 'darwin' ? 'e.g. brew install ffmpeg-full, then put "$(brew --prefix ffmpeg-full)/bin" first on your PATH, as it is keg-only; or a static build from https://ffmpeg.org/download.html' : 'e.g. a static build from https://ffmpeg.org/download.html'})`);
+    } else lines.push(`✔ ffmpeg ${v ?? ''}`.trim());
+    for (const o of OPTIONAL) {
+      if (o.filter ? !hasFilter(o.filter) : !hasEncoder(o.encoder)) lines.push(`· ffmpeg has no ${o.filter ? `${o.filter} filter` : `${o.encoder} encoder`} — ${o.for}`);
+    }
   } catch {
     lines.push('✖ ffmpeg not found');
     todo.push(ffmpegInstallCommand());

package/src/ffmpeg.mjs

@@ -0,0 +1,27 @@
+// What the installed ffmpeg can do: builds differ (Homebrew's default ffmpeg has no libass, so no
+// `ass` filter for burned-in subtitles).
+import { execFileSync } from 'node:child_process';
+
+const cache = {};
+const list = (kind) => {
+  if (!cache[kind]) {
+    try {
+      const out = execFileSync('ffmpeg', ['-hide_banner', `-${kind}`], { stdio: ['ignore', 'pipe', 'ignore'] }).toString();
+      // " .. zoompan  V->V  ..." (filters) or " V....D libx264  ..." (encoders): the name is the second column
+      cache[kind] = new Set(out.split('\n').map((l) => l.trim().split(/\s+/)[1]).filter(Boolean));
+    } catch { cache[kind] = new Set(); }
+  }
+  return cache[kind];
+};
+export const hasFilter = (name) => list('filters').has(name);
+export const hasEncoder = (name) => list('encoders').has(name);
+
+// what Storyshot needs from ffmpeg, and what each optional piece is for
+export const NEEDED = {
+  filters: ['zoompan', 'overlay', 'alphamerge', 'gblur', 'fade', 'tpad', 'loudnorm', 'alimiter', 'aresample', 'atempo', 'silenceremove', 'areverse', 'palettegen', 'paletteuse'],
+  encoders: ['libx264', 'aac'],
+};
+export const OPTIONAL = [
+  { filter: 'ass', for: '`subtitles: burn` uses its own renderer instead (the word highlight steps word by word rather than sweeping)' },
+  { encoder: 'libvpx-vp9', for: 'webm clips (clips: webm)' },
+];

package/src/outputs.mjs

@@ -2,6 +2,8 @@
 import { execFileSync } from 'node:child_process';
 import fs from 'node:fs';
 import path from 'node:path';
+import { chromium } from 'playwright';
+import { hasFilter } from './ffmpeg.mjs';
 
 const plain = (s) => String(s).replace(/\*\*/g, '');
 const srtTime = (t) => {
@@ -80,7 +82,7 @@ export async function writeOutputs(cfg, { captions, cards = [], segs, clean, nar
 
 // `subtitles: burn`: the narration burned into the video, a few words at a time, each word lighting
 // up as it is spoken (karaoke). Word timing is estimated from word length within each spoken line.
-export function burnSubtitles(cfg, narration, { log = console.log } = {}) {
+export async function burnSubtitles(cfg, narration, { log = console.log, renderer = hasFilter('ass') ? 'ass' : 'png' } = {}) {
   if (!narration.length) return log('note: subtitles: burn needs narration (say: lines and a voice); nothing burned in');
   const { width: W, height: H } = cfg.output;
   const bgr = (hex, a = '00') => { const h = hex.replace('#', ''); return `&H${a}${h.slice(4, 6)}${h.slice(2, 4)}${h.slice(0, 2)}`.toUpperCase(); };
@@ -90,7 +92,7 @@ export function burnSubtitles(cfg, narration, { log = console.log } = {}) {
   const cjk = cfg.lang === 'zh' || narration.some((l) => /[\u3040-\u30ff\u3400-\u9fff\uac00-\ud7af]/.test(l.text));
   const size = Math.round(H * (cjk ? 0.042 : 0.048)), edge = Math.max(2, Math.round(size * (cjk ? 0.07 : 0.09)));
   const fontName = cjk && cfg.theme.font === 'Inter' ? (process.platform === 'darwin' ? 'PingFang SC' : 'Noto Sans CJK SC') : cfg.theme.font;
-  const events = [];
+  const cues = []; // { words: [{ w, start, d }] }: one on-screen line each
   for (const line of narration) {
     // CJK: one token per character (Latin words inside stay whole), punctuation stays with the one before
     const words = cjk
@@ -104,8 +106,7 @@ export function burnSubtitles(cfg, narration, { log = console.log } = {}) {
     let chunk = [];
     const flush = () => {
       if (!chunk.length) return;
-      const s0 = chunk[0].start, e0 = chunk.at(-1).start + chunk.at(-1).d;
-      events.push(`Dialogue: 0,${t(s0)},${t(e0 + 0.05)},Sub,,0,0,0,,${chunk.map((x) => `{\\kf${Math.round(x.d * 100)}}${x.w.replace(/[{}]/g, '')}`).join(cjk ? '' : ' ')}`);
+      cues.push({ words: chunk });
       chunk = [];
     };
     const len = () => chunk.reduce((a, x) => a + x.w.length, 0);
@@ -117,6 +118,64 @@ export function burnSubtitles(cfg, narration, { log = console.log } = {}) {
     }
     flush();
   }
+  const tmp = cfg.out.replace(/(\.[^.]+)$/, '.burn$1');
+  const encode = ['-c:v', 'libx264', '-preset', 'slow', '-crf', '18', '-pix_fmt', 'yuv420p', '-c:a', 'copy', '-movflags', '+faststart', tmp];
+  const edgeCol = cfg.theme.ink, sweep = cfg.theme.accent;
+  if (renderer === 'png') {
+    // no libass in this ffmpeg: draw each state of each line (the words spoken so far in the accent
+    // colour) in Chromium, string them into one transparent track and overlay it
+    const dir = path.join(cfg.work, 'subs');
+    fs.rmSync(dir, { recursive: true, force: true });
+    fs.mkdirSync(dir, { recursive: true });
+    const esc = (x) => x.replace(/&/g, '&amp;').replace(/</g, '&lt;').replace(/>/g, '&gt;');
+    const font = `'${fontName}',${cjk ? "'PingFang SC','Hiragino Sans GB','Noto Sans CJK SC','Microsoft YaHei'," : ''}-apple-system,'Segoe UI','Helvetica Neue',sans-serif`;
+    const head = cjk ? '' : `<link rel="stylesheet" href="https://fonts.googleapis.com/css2?family=${encodeURIComponent(fontName)}:wght@400;700&display=block">`;
+    const browser = await chromium.launch();
+    const pg = await browser.newPage({ viewport: { width: W, height: H } });
+    const shot = async (html, file) => {
+      await pg.setContent(`<html><head>${head}</head><body style="margin:0;background:transparent">${html}</body></html>`, { waitUntil: 'networkidle' }).catch(() => {});
+      await pg.evaluate(() => document.fonts.ready);
+      await pg.screenshot({ path: file, omitBackground: true });
+    };
+    const list = ['ffconcat version 1.0'];
+    let at = 0, n = 0;
+    const push = (file, until) => { if (until - at > 0.001) { list.push(`file '${file}'`, `duration ${(until - at).toFixed(3)}`); at = until; } };
+    await shot('', path.join(dir, 'blank.png'));
+    // libass sizes a font by its line height (ascent + descent), CSS by its em: match libass
+    await pg.setContent(`<html><head>${head}</head><body></body></html>`, { waitUntil: 'networkidle' }).catch(() => {});
+    const em = await pg.evaluate(async (font) => {
+      await document.fonts.load(`100px ${font}`);
+      const m = document.createElement('canvas').getContext('2d');
+      m.font = `100px ${font}`;
+      const r = m.measureText('Hg中');
+      return 100 / ((r.fontBoundingBoxAscent + r.fontBoundingBoxDescent) || 100);
+    }, font);
+    const px = Math.round(size * em);
+    for (const cue of cues) {
+      const end = cue.words.at(-1).start + cue.words.at(-1).d + 0.05;
+      if (cue.words[0].start > at) push('blank.png', cue.words[0].start);
+      for (const [j, x] of cue.words.entries()) {
+        const file = `s${String(n++).padStart(5, '0')}.png`;
+        const text = cue.words.map((y, k) => `<span style="color:${k <= j ? sweep : '#fff'}">${esc(y.w)}</span>`).join(cjk ? '' : ' ');
+        await shot(`<div style="position:absolute;left:${Math.round(W * 0.08)}px;right:${Math.round(W * 0.08)}px;bottom:${Math.round(H * 0.06)}px;text-align:center;
+          font-family:${font};font-size:${px}px;font-weight:${cjk ? 400 : 700};line-height:1.2;
+          -webkit-text-stroke:${edge * 2}px ${edgeCol};paint-order:stroke fill;text-shadow:${Math.round(edge * 0.8)}px ${Math.round(edge * 0.8)}px 0 rgba(0,0,0,.5)">${text}</div>`, path.join(dir, file));
+        push(file, j + 1 < cue.words.length ? cue.words[j + 1].start : end);
+      }
+    }
+    await browser.close();
+    push('blank.png', at + 1);
+    list.push("file 'blank.png'"); // the last entry's duration only counts when a file follows it
+    fs.writeFileSync(path.join(dir, 'list.txt'), list.join('\n') + '\n');
+    execFileSync('ffmpeg', ['-y', '-loglevel', 'error', '-i', cfg.out, '-f', 'concat', '-safe', '0', '-i', path.join(dir, 'list.txt'),
+      '-filter_complex', '[1:v]format=rgba[s];[0:v][s]overlay=0:0:eof_action=pass,format=yuv420p[v]', '-map', '[v]', '-map', '0:a?', ...encode]);
+    fs.renameSync(tmp, cfg.out);
+    return log(`subtitles: burned in ${cues.length} lines with word-by-word highlight (drawn by Storyshot: this ffmpeg has no libass)`);
+  }
+  const events = cues.map(({ words }) => {
+    const s0 = words[0].start, e0 = words.at(-1).start + words.at(-1).d;
+    return `Dialogue: 0,${t(s0)},${t(e0 + 0.05)},Sub,,0,0,0,,${words.map((x) => `{\\kf${Math.round(x.d * 100)}}${x.w.replace(/[{}]/g, '')}`).join(cjk ? '' : ' ')}`;
+  });
   const ass = `[Script Info]
 ScriptType: v4.00+
 PlayResX: ${W}
@@ -125,17 +184,17 @@ WrapStyle: 0
 
 [V4+ Styles]
 Format: Name, Fontname, Fontsize, PrimaryColour, SecondaryColour, OutlineColour, BackColour, Bold, Italic, Underline, StrikeOut, ScaleX, ScaleY, Spacing, Angle, BorderStyle, Outline, Shadow, Alignment, MarginL, MarginR, MarginV, Encoding
-Style: Sub,${fontName},${size},${bgr(cfg.theme.accent)},&H00FFFFFF,${bgr(cfg.theme.ink)},&H80000000,${cjk ? 0 : 1},0,0,0,100,100,0,0,1,${edge},${Math.round(edge * 0.8)},2,${Math.round(W * 0.08)},${Math.round(W * 0.08)},${Math.round(H * 0.06)},1
+Style: Sub,${fontName},${size},${bgr(sweep)},&H00FFFFFF,${bgr(edgeCol)},&H80000000,${cjk ? 0 : 1},0,0,0,100,100,0,0,1,${edge},${Math.round(edge * 0.8)},2,${Math.round(W * 0.08)},${Math.round(W * 0.08)},${Math.round(H * 0.06)},1
 
 [Events]
 Format: Layer, Start, End, Style, Name, MarginL, MarginR, MarginV, Effect, Text
 ${events.join('\n')}
 `;
-  const file = path.join(cfg.work, 'subtitles.ass'), tmp = cfg.out.replace(/(\.[^.]+)$/, '.burn$1');
+  const file = path.join(cfg.work, 'subtitles.ass');
   fs.writeFileSync(file, ass);
   // the filter argument is parsed by ffmpeg: escape the characters it treats specially in paths
   const arg = file.replace(/\\/g, '/').replace(/:/g, '\\:').replace(/'/g, "\\'");
-  execFileSync('ffmpeg', ['-y', '-loglevel', 'error', '-i', cfg.out, '-vf', `ass='${arg}'`, '-c:v', 'libx264', '-preset', 'slow', '-crf', '18', '-pix_fmt', 'yuv420p', '-c:a', 'copy', '-movflags', '+faststart', tmp]);
+  execFileSync('ffmpeg', ['-y', '-loglevel', 'error', '-i', cfg.out, '-vf', `ass='${arg}'`, ...encode]);
   fs.renameSync(tmp, cfg.out);
   log(`subtitles: burned in ${events.length} lines with word-by-word highlight`);
 }

package/src/voice.mjs

@@ -94,6 +94,7 @@ export async function speak(cfg, line, { log = console.log } = {}) {
 
 // Speed a clip up (factor > 1) with ffmpeg's pitch-preserving atempo.
 export function tempo(pcm, factor) {
+  if (!(factor >= 0.5 && factor <= 100)) throw new Error(`internal: speed factor ${factor} is outside what ffmpeg's atempo accepts (0.5–100)`);
   const out = execFileSync('ffmpeg', ['-loglevel', 'error', '-f', 'f32le', '-ac', '2', '-ar', `${SR}`, '-i', 'pipe:0',
     '-af', `atempo=${factor.toFixed(4)}`, '-f', 'f32le', '-ac', '2', '-ar', `${SR}`, 'pipe:1'], { input: Buffer.from(pcm.buffer, pcm.byteOffset, pcm.byteLength), maxBuffer: 1 << 28 });
   return new Float32Array(out.buffer, out.byteOffset, out.length / 4);