reelkit-cli 0.6.0 → 0.8.0

package/src/remotion/kit/word-anchor.ts

@@ -0,0 +1,107 @@
+// Word anchors: where in the film a spoken word falls, so that what illustrates a word can be placed on it.
+// Everything here is a pure function over a manifest scene (`manifest.scenes[i]`, with its `words`), so it is the same in the studio, in a test and in a render.
+// Frames are counted from the start of the scene (what its own useCurrentFrame() gives) unless `absolute: true` adds the scene's startFrame.
+import type { WordTiming } from "../../pipeline/schema";
+
+// The manifest's frame rate is always this one; pass `fps` only for a scene that came from somewhere else.
+export const WORD_FPS = 30;
+// An entrance needs a few frames to become visible, so it starts this many frames before its word.
+export const DEFAULT_LEAD = 3;
+// A beat counts as "at the word" when it is no more than this many frames from the word's start.
+export const DEFAULT_BEAT_WINDOW = 3;
+// The thing a word names is never seen more than this many frames before the word starts.
+export const MAX_EARLY_FRAMES = 3;
+
+export type WordScene = { id?: string; startFrame: number; durationFrames?: number; words: WordTiming[] };
+
+export type WordFrameOptions = {
+  // The 1-based occurrence of the word when the scene says it more than once (default 1).
+  nth?: number;
+  // "start" (default): the frame the word starts. "end": the frame it ends (for a phrase, the end of its last word).
+  edge?: "start" | "end";
+  // Add the scene's startFrame, giving a frame of the whole film.
+  absolute?: boolean;
+  fps?: number;
+};
+
+// A word as it is compared: lower case, with everything but letters and digits removed ("Meetings," is "meetings", "don't" is "dont").
+export const normalizeWord = (w: string): string => w.toLowerCase().replace(/[^\p{L}\p{N}]/gu, "");
+
+const spoken = (scene: WordScene): { norm: string; word: WordTiming }[] =>
+  scene.words.map((word) => ({ norm: normalizeWord(word.word), word })).filter((w) => w.norm !== "");
+
+// The words that make up `phrase`, matched against the scene's words, with where the phrase's `nth` occurrence is.
+function findPhrase(scene: WordScene, phrase: string, nth: number): { first: WordTiming; last: WordTiming } {
+  const parts = phrase.split(/\s+/).map(normalizeWord).filter(Boolean);
+  const list = spoken(scene);
+  const where = scene.id ? `scene "${scene.id}"` : "the scene";
+  if (!parts.length) throw new Error(`wordFrame: the word is empty. ${where} says: ${list.map((w) => w.word.word).join(" ")}`);
+  if (!Number.isInteger(nth) || nth < 1) throw new Error(`wordFrame: nth must be 1 or more, got ${nth}.`);
+  let seen = 0;
+  for (let i = 0; i + parts.length <= list.length; i++) {
+    if (parts.every((p, k) => list[i + k]!.norm === p) && ++seen === nth) return { first: list[i]!.word, last: list[i + parts.length - 1]!.word };
+  }
+  const had = seen ? ` (it is said ${seen} time${seen === 1 ? "" : "s"}, not ${nth})` : "";
+  throw new Error(`wordFrame: "${phrase}"${nth > 1 ? ` (occurrence ${nth})` : ""} is not in ${where}${had}. Its words are: ${list.map((w) => w.word.word).join(" ")}`);
+}
+
+// The frame, inside the scene, at which `word` starts (or ends, with edge: "end"). `word` ignores case and punctuation; a phrase of several words
+// ("whole week") matches consecutive words and gives the first one's start. A word the scene does not say throws an Error that lists the scene's
+// words, so a misspelt word stops the render instead of silently timing to frame 0.
+export function wordFrame(scene: WordScene, word: string, opts: WordFrameOptions = {}): number {
+  const { first, last } = findPhrase(scene, word, opts.nth ?? 1);
+  const fps = opts.fps ?? WORD_FPS;
+  const sec = opts.edge === "end" ? last.endSec : first.startSec;
+  let frame = Math.max(0, Math.round(sec * fps));
+  if (scene.durationFrames !== undefined) frame = Math.min(frame, Math.max(0, scene.durationFrames - 1));
+  return frame + (opts.absolute ? scene.startFrame : 0);
+}
+
+// Several words at once, in the order given.
+export const wordFrames = (scene: WordScene, words: string[], opts: WordFrameOptions = {}): number[] => words.map((w) => wordFrame(scene, w, opts));
+
+// The scene with this id. Throws an Error that lists the ids when there is none.
+export function sceneById<S extends { id: string }>(manifest: { scenes: S[] }, id: string): S {
+  const found = manifest.scenes.find((s) => s.id === id);
+  if (!found) throw new Error(`sceneById: there is no scene "${id}". The scenes are: ${manifest.scenes.map((s) => s.id).join(", ")}`);
+  return found;
+}
+
+export type OnWordOptions = WordFrameOptions & {
+  // How many frames before the word the entrance starts (default 3).
+  lead?: number;
+};
+
+// The frame at which to START an entrance (a `delay`, a `from`, an `at`) so that it is seen landing on its word: the word's frame minus `lead`
+// (default 3), never below 0.
+export function onWord(scene: WordScene, word: string, opts: OnWordOptions = {}): number {
+  const lead = opts.lead ?? DEFAULT_LEAD;
+  const at = wordFrame(scene, word, { ...opts, absolute: false });
+  return Math.max(0, at - lead) + (opts.absolute ? scene.startFrame : 0);
+}
+
+export type OnWordBeatOptions = OnWordOptions & {
+  // A beat counts only when it is at most this many frames from the word's start (default 3).
+  window?: number;
+};
+
+// Beat and word together. The voice is the clock for what illustrates a word, so the word wins; the beat is used only when one lies within `window`
+// frames of the word (default 3), and then only to let the entrance land on it. The result is never earlier than onWord's, so what a word names
+// is never seen more than 3 frames before the word starts. `music` is manifest.music (its beatFrames are frames of the whole film); without a
+// track it is onWord.
+export function onWordBeat(scene: WordScene, word: string, music: { beatFrames: number[] } | undefined, opts: OnWordBeatOptions = {}): number {
+  const lead = opts.lead ?? DEFAULT_LEAD;
+  const window = opts.window ?? DEFAULT_BEAT_WINDOW;
+  const at = wordFrame(scene, word, { ...opts, absolute: false });
+  const wordAbs = scene.startFrame + at;
+  const base = Math.max(0, at - lead);
+  let start = base;
+  let best: number | undefined;
+  for (const b of music?.beatFrames ?? []) {
+    // A beat before the word is never used: it would show the thing early.
+    if (b < wordAbs || b - wordAbs > window) continue;
+    if (best === undefined || b - wordAbs < best - wordAbs) best = b;
+  }
+  if (best !== undefined) start = Math.max(base, best - scene.startFrame - lead);
+  return start + (opts.absolute ? scene.startFrame : 0);
+}

package/src/render/contact-sheet.ts

@@ -0,0 +1,39 @@
+import { execFile } from "node:child_process";
+import { promisify } from "node:util";
+
+const exec = promisify(execFile);
+
+// The grid of a sheet of `count` tiles for a frame of `aspect` (width over height): 6 across for a tall frame, 4 for a wide one, so that the whole sheet
+// is at most about `maxWidth` pixels across and a phone-shaped frame is still readable.
+export function sheetGrid(count: number, aspect: number, maxWidth = 2400): { cols: number; rows: number; tileW: number; tileH: number } {
+  const cols = Math.max(1, Math.min(count, aspect < 1 ? 6 : aspect > 1.2 ? 4 : 5));
+  const tileW = Math.min(540, Math.floor(maxWidth / cols / 2) * 2);
+  const tileH = Math.max(2, Math.round(tileW / aspect / 2) * 2);
+  return { cols, rows: Math.ceil(count / cols), tileW, tileH };
+}
+
+// The drawn text is plain ASCII (the default font has no arrows): the arrow of a word label is drawn as >.
+const clean = (s: string) => s.replace(/\u25B8/g, ">").replace(/[^A-Za-z0-9 _.>#-]/g, "");
+
+// One picture of many: the files in order, each with its label in a box on its top left corner. Falls back to the same sheet without labels if this ffmpeg
+// has no text drawing; the caller still lists the order. `newRowAt` starts a new row at that tile (the tiles before it are padded with empty ones), so a
+// second kind of frame sits in rows of its own. Returns the grid, and whether the labels were drawn.
+export async function contactSheet(files: string[], labels: string[], out: string, aspect: number, opts: { newRowAt?: number } = {}): Promise<{ cols: number; rows: number; width: number; height: number; labelled: boolean }> {
+  const base = sheetGrid(files.length, aspect);
+  const at = opts.newRowAt !== undefined && opts.newRowAt > 0 && opts.newRowAt < files.length ? opts.newRowAt : undefined;
+  const blanks = at === undefined ? 0 : (base.cols - (at % base.cols)) % base.cols;
+  const g = { ...base, rows: Math.ceil((files.length + blanks) / base.cols) };
+  const build = (labelled: boolean) => {
+    const tile = (i: number) => `[${i}:v]scale=${g.tileW}:${g.tileH},setsar=1${labelled ? `,drawtext=text='${clean(labels[i] ?? "")}':x=6:y=6:fontsize=${Math.max(12, Math.round(g.tileW / 22))}:fontcolor=white:box=1:boxcolor=black@0.65:boxborderw=4` : ""}[t${i}]`;
+    const parts = files.map((_, i) => tile(i));
+    // The empty tiles are extra inputs of plain colour, one frame each.
+    const blankInputs = Array.from({ length: blanks }, () => ["-f", "lavfi", "-i", `color=c=0x202020:s=${g.tileW}x${g.tileH}:r=25`]).flat();
+    const blankParts = Array.from({ length: blanks }, (_, k) => `[${files.length + k}:v]trim=end_frame=1,setpts=PTS-STARTPTS,format=yuvj420p,setsar=1[b${k}]`);
+    const order = [...files.map((_, i) => (i === at ? [...Array.from({ length: blanks }, (_, k) => `[b${k}]`), `[t${i}]`] : [`[t${i}]`])).flat()];
+    const filter = `${[...parts, ...blankParts].join(";")};${order.join("")}concat=n=${files.length + blanks}:v=1:a=0,tile=${g.cols}x${g.rows}:padding=2:color=0x202020[out]`;
+    return ["-y", "-loglevel", "error", ...files.flatMap((f) => ["-i", f]), ...blankInputs, "-filter_complex", filter, "-map", "[out]", "-frames:v", "1", "-q:v", "4", out];
+  };
+  let labelled = true;
+  try { await exec("ffmpeg", build(true)); } catch { labelled = false; await exec("ffmpeg", build(false)); }
+  return { cols: g.cols, rows: g.rows, width: g.cols * g.tileW + (g.cols - 1) * 2, height: g.rows * g.tileH + (g.rows - 1) * 2, labelled };
+}

package/src/render/continuity.ts

@@ -10,7 +10,8 @@ export const CARRIED_AT = 0.25;
 const MIN_EDGE_SHARE = 0.002;
 
 export type Gray = ArrayLike<number>;
-export type BoundaryKind = "carried" | "cut" | "empty";
+// "cut (intended)" is a change the plan marked with cutIn: it is reported but never counts against the score.
+export type BoundaryKind = "carried" | "cut" | "empty" | "cut (intended)";
 
 // A pixel is an edge where the brightness steps by more than the threshold to the pixel on its right or below it.
 export function edgeMap(gray: Gray, w: number, h: number, threshold = EDGE_THRESHOLD): Uint8Array {
@@ -57,6 +58,13 @@ export function dilate(map: Uint8Array, w: number, h: number, radius: number): U
   return out;
 }
 
+// The later scene is compared at two moments, its first fully visible frame and one a few frames on while a move is still settling (a plate
+// that becomes a card has not yet reached its edges at the first). The better of the two says whether something was carried.
+export function compareAcross(earlier: Gray, laters: Gray[], w: number, h: number): ReturnType<typeof compareFrames> {
+  const all = laters.map((l) => compareFrames(earlier, l, w, h));
+  return all.reduce((best, r) => (r.survived > best.survived ? r : best));
+}
+
 export function compareFrames(earlier: Gray, later: Gray, w: number, h: number): { survived: number; edges: number; kind: BoundaryKind } {
   const band = captionBand(w, h);
   const before = clearBottom(edgeMap(earlier, w, h), w, h, band);
@@ -72,15 +80,17 @@ export type Boundary = { from: string; to: string; survived: number; kind: Bound
 
 // The score is carried over carried plus cut: a change with nothing on screen to compare says nothing either way.
 export function summariseContinuity(boundaries: Boundary[]): { score: number | null; lines: string[] } {
+  const intended = boundaries.filter((b) => b.kind === "cut (intended)").length;
   const carried = boundaries.filter((b) => b.kind === "carried").length;
   const cut = boundaries.filter((b) => b.kind === "cut");
   const counted = carried + cut.length;
-  const empty = boundaries.length - counted;
-  if (!counted) return { score: null, lines: ["No scene change could be measured: the picture is nearly empty at the end of every scene."] };
+  const empty = boundaries.length - counted - intended;
+  const hard = intended ? ` ${intended} ${intended === 1 ? "is" : "are"} intended hard cut${intended === 1 ? "" : "s"}, not counted.` : "";
+  if (!counted) return { score: null, lines: [intended && !empty ? `Every scene change that could be measured is an intended hard cut.` : "No scene change could be measured: the picture is nearly empty at the end of every scene."] };
   return {
     score: carried / counted,
     lines: [
-      `${carried} of ${counted} scene changes carry something across${empty ? ` (${empty} more had nothing on screen to compare)` : ""}.`,
+      `${carried} of ${counted} scene changes carry something across${empty ? ` (${empty} more had nothing on screen to compare)` : ""}.${hard}`,
       ...cut.map((b) => `${b.from} → ${b.to}: nothing carries over. Keep one element on screen through the change and move it into the next scene (see reference/continuity.md).`),
     ],
   };

package/src/render/deps.ts

@@ -1,8 +1,20 @@
+import { existsSync, readFileSync } from "node:fs";
 import { createRequire } from "node:module";
-import { dirname } from "node:path";
+import { dirname, join } from "node:path";
 
 const require = createRequire(import.meta.url);
 
 // The directory of an installed dependency of this package, found through its package.json so it does not depend on
-// where the package keeps its main file or on how the package manager lays out node_modules.
-export const packageDir = (name: string): string => dirname(require.resolve(`${name}/package.json`));
+// where the package keeps its main file or on how the package manager lays out node_modules. A package that does not export its
+// package.json (three does not) is found from its main file instead, by walking up to the folder that holds its own package.json.
+export const packageDir = (name: string): string => {
+  try {
+    return dirname(require.resolve(`${name}/package.json`));
+  } catch {
+    for (let dir = dirname(require.resolve(name)); dir !== dirname(dir); dir = dirname(dir)) {
+      const file = join(dir, "package.json");
+      if (existsSync(file) && (JSON.parse(readFileSync(file, "utf8")) as { name?: string }).name === name) return dir;
+    }
+    throw new Error(`Could not find the installed package ${name}.`);
+  }
+};

package/src/render/render.ts

@@ -14,9 +14,10 @@ registerRoot(makeRoot(Video));
 `;
 
 // The project folder has no node_modules. Each dependency is pinned to the one copy installed with this package, so the
-// kit, Root and the user's composition all share a single react and remotion (a second react breaks hooks).
+// kit, Root and the user's composition all share a single react and remotion (a second react breaks hooks), and the 3D layer a single three
+// (two copies of three break instanceof checks). User code still may not import them: only the kit does.
 export function dependencyAliases(): Record<string, string> {
-  return Object.fromEntries(["react", "react-dom", "remotion"].map((n) => [n, packageDir(n)]));
+  return Object.fromEntries(["react", "react-dom", "remotion", "three", "@react-three/fiber", "@remotion/three"].map((n) => [n, packageDir(n)]));
 }
 
 // Writes the bundler's entry file into the project. The composition itself stays where the user's Claude wrote it.
@@ -57,18 +58,24 @@ export const ensureRenderBrowser: EnsureBrowser = async (onDownload) => {
   await ensureBrowser({ onBrowserDownload: () => { onDownload(); return { version: null, onProgress: () => {} }; } });
 };
 
+// Remotion's headless browser has no GPU, so three.js needs a software GL. "angle" is the setting that draws WebGL here; it is used only for a film that
+// has a 3D scene, so every 2D render keeps the browser's own defaults and looks exactly as before. This is the one place that says it.
+export const GL_BACKEND = "angle" as const;
+export type RenderOptions = { uses3D?: boolean };
+const chromium = (o?: RenderOptions) => (o?.uses3D ? { chromiumOptions: { gl: GL_BACKEND } } : {});
+
 const asInput = (props: VideoProps) => props as unknown as Record<string, unknown>;
 
-export async function renderStills(serveUrl: string, props: VideoProps, frames: number[], outDir: string) {
+export async function renderStills(serveUrl: string, props: VideoProps, frames: number[], outDir: string, options?: RenderOptions) {
   await mkdir(outDir, { recursive: true });
-  const composition = await selectComposition({ serveUrl, id: "Video", inputProps: asInput(props) });
+  const composition = await selectComposition({ serveUrl, id: "Video", inputProps: asInput(props), ...chromium(options) });
   for (const frame of frames) {
-    await renderStill({ composition, serveUrl, frame, output: join(outDir, `still-${frame}.png`), inputProps: asInput(props) });
+    await renderStill({ composition, serveUrl, frame, output: join(outDir, `still-${frame}.png`), inputProps: asInput(props), ...chromium(options) });
   }
 }
 
-export async function renderVideo(serveUrl: string, props: VideoProps, outPath: string) {
+export async function renderVideo(serveUrl: string, props: VideoProps, outPath: string, options?: RenderOptions) {
   await mkdir(dirname(outPath), { recursive: true });
-  const composition = await selectComposition({ serveUrl, id: "Video", inputProps: asInput(props) });
-  await renderMedia({ composition, serveUrl, codec: "h264", outputLocation: outPath, inputProps: asInput(props) });
+  const composition = await selectComposition({ serveUrl, id: "Video", inputProps: asInput(props), ...chromium(options) });
+  await renderMedia({ composition, serveUrl, codec: "h264", outputLocation: outPath, inputProps: asInput(props), ...chromium(options) });
 }

package/src/render/sound-notes.ts

@@ -0,0 +1,106 @@
+import ts from "typescript";
+import { sourceFile } from "./static-check";
+
+// Advice about the sound of a film with no voice, read from the composition's source. It is a count of what the code places, never a
+// listening test: a sound that is computed, looped over or hidden behind a variable cannot be counted, so then nothing is said.
+export type SoundCount = {
+  // Sfx elements and literal SoundCues entries.
+  placed: number;
+  // True when some cue is computed (a map, a helper, a variable that is not a plain list), so that the count is only a lower bound.
+  uncertain: boolean;
+  music: boolean;
+  // True when a cue or an effect is placed before this frame; `unknownStart` when one has a start that is not a plain number.
+  early: boolean;
+  unknownStart: boolean;
+};
+
+const numberOf = (e: ts.Expression | undefined): number | undefined => {
+  if (!e) return undefined;
+  if (ts.isNumericLiteral(e)) return Number(e.text);
+  if (ts.isPrefixUnaryExpression(e) && e.operator === ts.SyntaxKind.MinusToken && ts.isNumericLiteral(e.operand)) return -Number(e.operand.text);
+  if (ts.isParenthesizedExpression(e)) return numberOf(e.expression);
+  return undefined;
+};
+
+const insideLoop = (node: ts.Node): boolean => {
+  for (let p = node.parent; p; p = p.parent) {
+    if (ts.isCallExpression(p) && ts.isPropertyAccessExpression(p.expression) && ["map", "forEach", "flatMap"].includes(p.expression.name.text)) return true;
+  }
+  return false;
+};
+
+// The value of a prop, looking through a name that the file declares as a constant.
+function propValue(attr: ts.JsxAttribute, sf: ts.SourceFile): ts.Expression | undefined {
+  const init = attr.initializer;
+  if (!init || !ts.isJsxExpression(init) || !init.expression) return undefined;
+  const e = init.expression;
+  if (!ts.isIdentifier(e)) return e;
+  let found: ts.Expression | undefined;
+  const visit = (n: ts.Node) => {
+    if (ts.isVariableDeclaration(n) && ts.isIdentifier(n.name) && n.name.text === e.text && n.initializer && !found) found = n.initializer;
+    ts.forEachChild(n, visit);
+  };
+  visit(sf);
+  return found ?? e;
+}
+
+export function countSounds(source: string, file: string, earlyFrames: number): SoundCount {
+  const sf = sourceFile(source, file);
+  const out: SoundCount = { placed: 0, uncertain: false, music: false, early: false, unknownStart: false };
+  const attr = (el: ts.JsxOpeningLikeElement, name: string) => el.attributes.properties.find((a): a is ts.JsxAttribute => ts.isJsxAttribute(a) && a.name.getText(sf) === name);
+  const atOf = (e: ts.Expression | undefined, absent: number | undefined) => {
+    const n = e === undefined ? absent : numberOf(e);
+    if (n === undefined) out.unknownStart = true;
+    else if (n < earlyFrames) out.early = true;
+  };
+  const visit = (node: ts.Node) => {
+    const el = ts.isJsxSelfClosingElement(node) ? node : ts.isJsxElement(node) ? node.openingElement : undefined;
+    if (el) {
+      const tag = el.tagName.getText(sf);
+      if (tag === "Music") out.music = true;
+      if (tag === "Sfx") {
+        out.placed += 1;
+        if (insideLoop(el)) out.uncertain = true;
+        const at = attr(el, "at");
+        if (!at) out.early = true;
+        else {
+          const v = at.initializer && ts.isJsxExpression(at.initializer) ? at.initializer.expression : undefined;
+          atOf(v, undefined);
+        }
+      }
+      if (tag === "SoundCues") {
+        const cues = attr(el, "cues");
+        const value = cues && propValue(cues, sf);
+        if (!value || !ts.isArrayLiteralExpression(value) || value.elements.some((c) => !ts.isObjectLiteralExpression(c)) || insideLoop(el)) out.uncertain = true;
+        else {
+          out.placed += value.elements.length;
+          for (const c of value.elements as ts.NodeArray<ts.ObjectLiteralExpression>) {
+            const at = c.properties.find((p): p is ts.PropertyAssignment => ts.isPropertyAssignment(p) && p.name.getText(sf) === "at");
+            atOf(at?.initializer, undefined);
+          }
+        }
+      }
+    }
+    ts.forEachChild(node, visit);
+  };
+  visit(sf);
+  return out;
+}
+
+// The notes for a film with no voice. `seconds` is the film's length, `musicSet` whether a track was pulled with --music.
+export function soundNotes(sources: { name: string; source: string }[], opts: { seconds: number; fps: number; musicSet: boolean }): string[] {
+  const earlyFrames = Math.round(opts.fps / 2);
+  const counts = sources.map((s) => countSounds(s.source, s.name, earlyFrames));
+  const placed = counts.reduce((n, c) => n + c.placed, 0);
+  const uncertain = counts.some((c) => c.uncertain);
+  const notes: string[] = [];
+  if (!opts.musicSet) notes.push("No music track is set. A film with no voice is carried by its music: choose one and pull it with `reelkit assets pull <id> --music` (see reference/launch-film.md).");
+  if (!uncertain && placed < opts.seconds / 2) {
+    notes.push(`The code places ${placed} sound cue${placed === 1 ? "" : "s"} (Sfx elements and SoundCues entries) in ${opts.seconds.toFixed(1)}s of film, fewer than one per two seconds. This is a count of what the composition places, not a listening test; a film with no voice usually has about one placed effect per second on top of the music (see reference/launch-film.md).`);
+  }
+  const startsAtOnce = counts.some((c) => c.music || c.early);
+  if (!startsAtOnce && !uncertain && !counts.some((c) => c.unknownStart)) {
+    notes.push("Nothing is placed in the first half second: the code has no Music, and no Sfx or SoundCues cue that starts there. This is a count of what the composition places, not a listening test; start a sound on the first frame (see reference/launch-film.md).");
+  }
+  return notes;
+}

package/src/render/word-check.ts

@@ -0,0 +1,181 @@
+// What a composition's source says statically about the spoken words it is timed to: which helper calls name a word, in which scene, and whether
+// the word is in that scene's narration. Pure text in, findings out (the TypeScript syntax tree, never regular expressions on code), so `check`
+// and `preview` can both use it.
+import ts from "typescript";
+import { normalizeWord, wordFrame, type WordScene } from "../remotion/kit/word-anchor";
+import { sourceFile, unwrap } from "./static-check";
+
+export const WORD_HELPERS = ["wordFrame", "wordFrames", "onWord", "onWordBeat"] as const;
+const isHelper = (n: string): n is (typeof WORD_HELPERS)[number] => (WORD_HELPERS as readonly string[]).includes(n);
+
+export type WordCall = {
+  fn: (typeof WORD_HELPERS)[number];
+  // The scene, when the call's first argument is a scene found by a literal id (sceneById(manifest, "id"), or manifest.scenes[2], or S.id and S["id"]).
+  sceneId?: string;
+  // One entry per literal word the call names; a word that is not a literal is left out.
+  words: { word: string; nth?: number }[];
+  line: number;
+};
+
+type PlanScene = { id: string; narration: string };
+
+const literalText = (e: ts.Expression | undefined): string | undefined => {
+  const x = e && unwrap(e);
+  return x && ts.isStringLiteralLike(x) ? x.text : undefined;
+};
+const literalNumber = (e: ts.Expression | undefined): number | undefined => {
+  const x = e && unwrap(e);
+  return x && ts.isNumericLiteral(x) ? Number(x.text) : undefined;
+};
+
+// Where a source says `<something>.scenes`: the manifest's list of scenes, however the manifest is called.
+const isScenesList = (e: ts.Expression): boolean => { const x = unwrap(e); return ts.isPropertyAccessExpression(x) && x.name.text === "scenes"; };
+
+// Calls of the word helpers in `source`. `ids` are the plan's scene ids in order, so that a scene reached by position or by `S.hook` can be named.
+export function wordCalls(source: string, ids: string[], file = "Video.tsx"): WordCall[] {
+  const sf = sourceFile(source, file);
+  // Every `const name = expression` and `const [a, b] = list` in the file, flat: the names a composition gives its scenes are not scoped carefully.
+  const consts = new Map<string, ts.Expression>();
+  const positions = new Map<string, number>();
+  const visitDecl = (node: ts.Node) => {
+    if (ts.isVariableDeclaration(node) && node.initializer) {
+      if (ts.isIdentifier(node.name)) consts.set(node.name.text, node.initializer);
+      else if (ts.isArrayBindingPattern(node.name) && isScenesList(node.initializer)) {
+        node.name.elements.forEach((el, i) => { if (ts.isBindingElement(el) && ts.isIdentifier(el.name)) positions.set(el.name.text, i); });
+      }
+    }
+    ts.forEachChild(node, visitDecl);
+  };
+  visitDecl(sf);
+
+  const resolve = (e: ts.Expression, depth = 0): string | undefined => {
+    if (depth > 6) return undefined;
+    const x = unwrap(e);
+    if (ts.isCallExpression(x) && ts.isIdentifier(x.expression) && x.expression.text === "sceneById") {
+      const id = literalText(x.arguments[1]);
+      return id !== undefined && ids.includes(id) ? id : undefined;
+    }
+    if (ts.isIdentifier(x)) {
+      const pos = positions.get(x.text);
+      if (pos !== undefined) return ids[pos];
+      const init = consts.get(x.text);
+      return init ? resolve(init, depth + 1) : undefined;
+    }
+    if (ts.isElementAccessExpression(x) && isScenesList(x.expression)) {
+      const i = literalNumber(x.argumentExpression);
+      return i !== undefined ? ids[i] : undefined;
+    }
+    // A record of scenes by id: `const S = Object.fromEntries(manifest.scenes.map(...))`, then S.hook or S["hook"].
+    if (ts.isPropertyAccessExpression(x) || ts.isElementAccessExpression(x)) {
+      const key = ts.isPropertyAccessExpression(x) ? x.name.text : literalText(x.argumentExpression);
+      const base = unwrap(x.expression);
+      if (key !== undefined && ids.includes(key) && ts.isIdentifier(base)) {
+        const init = consts.get(base.text);
+        if (init && init.getText(sf).includes("scenes")) return key;
+      }
+    }
+    return undefined;
+  };
+
+  const out: WordCall[] = [];
+  const visit = (node: ts.Node) => {
+    if (ts.isCallExpression(node) && ts.isIdentifier(node.expression) && isHelper(node.expression.text)) {
+      const fn = node.expression.text;
+      const [scene, arg] = node.arguments;
+      const options = node.arguments.slice(2).map((a) => unwrap(a)).find((a): a is ts.ObjectLiteralExpression => ts.isObjectLiteralExpression(a));
+      const nthProp = options?.properties.find((p): p is ts.PropertyAssignment => ts.isPropertyAssignment(p) && ts.isIdentifier(p.name) && p.name.text === "nth");
+      const nth = literalNumber(nthProp?.initializer);
+      const named: string[] = [];
+      if (arg) {
+        const single = literalText(arg);
+        if (fn === "wordFrames") { const list = unwrap(arg); if (ts.isArrayLiteralExpression(list)) for (const el of list.elements) { const t = literalText(el); if (t !== undefined) named.push(t); } }
+        else if (single !== undefined) named.push(single);
+      }
+      out.push({
+        fn, ...(scene ? (() => { const id = resolve(scene); return id !== undefined ? { sceneId: id } : {}; })() : {}),
+        words: named.map((word) => ({ word, ...(nth !== undefined ? { nth } : {}) })),
+        line: sf.getLineAndCharacterOfPosition(node.getStart(sf)).line + 1,
+      });
+    }
+    ts.forEachChild(node, visit);
+  };
+  visit(sf);
+  return out;
+}
+
+// Whether this narration says `word` (a word or a phrase of consecutive words, ignoring case and punctuation) at least `nth` times.
+export function narrationHas(narration: string, word: string, nth = 1): boolean {
+  const parts = word.split(/\s+/).map(normalizeWord).filter(Boolean);
+  if (!parts.length) return false;
+  const list = narration.split(/\s+/).map(normalizeWord).filter(Boolean);
+  let seen = 0;
+  for (let i = 0; i + parts.length <= list.length; i++) if (parts.every((p, k) => list[i + k] === p) && ++seen >= nth) return true;
+  return false;
+}
+
+// The errors: a literal word that the narration of a literally named scene does not contain.
+export function wordErrors(file: string, calls: WordCall[], scenes: PlanScene[]): string[] {
+  const errors: string[] = [];
+  for (const c of calls) {
+    const scene = c.sceneId ? scenes.find((s) => s.id === c.sceneId) : undefined;
+    if (!scene) continue;
+    for (const w of c.words) {
+      if (narrationHas(scene.narration, w.word, w.nth ?? 1)) continue;
+      errors.push(`${file} line ${c.line}: ${c.fn}(... "${w.word}"${w.nth && w.nth > 1 ? `, { nth: ${w.nth} }` : ""}) names a word that scene ${scene.id} never says. Its narration is: "${scene.narration}". The render would stop on it; use a word from the narration.`);
+    }
+  }
+  return errors;
+}
+
+// True when the file times anything to a spoken word: it calls a word helper, or reads a scene's `.words` for something other than the captions.
+export function usesWordTiming(source: string, file = "Video.tsx"): boolean {
+  const sf = sourceFile(source, file);
+  let used = false;
+  const visit = (node: ts.Node) => {
+    if (used) return;
+    if (ts.isCallExpression(node) && ts.isIdentifier(node.expression) && isHelper(node.expression.text)) { used = true; return; }
+    if ((ts.isPropertyAccessExpression(node) && node.name.text === "words") || (ts.isElementAccessExpression(node) && literalText(node.argumentExpression) === "words")) {
+      // `words={s.words}` on <Captions> is not timing: it is the captions' own input.
+      let p: ts.Node | undefined = node.parent;
+      while (p && (ts.isParenthesizedExpression(p) || ts.isAsExpression(p) || ts.isNonNullExpression(p))) p = p.parent;
+      const inCaptions = p && ts.isJsxExpression(p) && ts.isJsxAttribute(p.parent) && p.parent.name.getText(sf) === "words"
+        && (() => { let q: ts.Node | undefined = p.parent.parent.parent; return Boolean(q && (ts.isJsxSelfClosingElement(q) || ts.isJsxOpeningElement(q)) && q.tagName.getText(sf) === "Captions"); })();
+      if (!inCaptions) used = true;
+    }
+    ts.forEachChild(node, visit);
+  };
+  visit(sf);
+  return used;
+}
+
+// How many frames after a word starts the preview looks: the thing the word names should already be there.
+export const WORD_FRAME_AFTER = 4;
+// At most this many word frames go on the contact sheet.
+export const MAX_WORD_FRAMES = 16;
+
+export type WordTarget = { sceneId: string; word: string; nth?: number; wordStart: number; frame: number; label: string };
+
+// The words the preview can look at: every literal word of a helper call whose scene is known, as the frame 4 frames after the word starts (kept
+// inside its scene), in film order, without repeats, at most `cap`. `skipped` counts what could not be shown: a word or a scene that is not a
+// literal (the code does not say which), and what was left out past the cap.
+export function wordTargets(calls: WordCall[], manifest: { scenes: (WordScene & { id: string; durationFrames: number })[] }, cap = MAX_WORD_FRAMES): { targets: WordTarget[]; skipped: number; undecidable: number; capped: number } {
+  const seen = new Set<string>();
+  const found: WordTarget[] = [];
+  let undecidable = 0;
+  for (const c of calls) {
+    const scene = c.sceneId ? manifest.scenes.find((s) => s.id === c.sceneId) : undefined;
+    if (!scene || !c.words.length) { undecidable += Math.max(1, c.words.length); continue; }
+    for (const w of c.words) {
+      const key = `${scene.id}|${normalizeWord(w.word) || w.word}|${w.nth ?? 1}`;
+      if (seen.has(key)) continue;
+      seen.add(key);
+      let start: number;
+      try { start = wordFrame(scene, w.word, { nth: w.nth, absolute: true }); } catch { undecidable++; continue; }
+      const last = scene.startFrame + scene.durationFrames - 1;
+      found.push({ sceneId: scene.id, word: w.word, ...(w.nth && w.nth > 1 ? { nth: w.nth } : {}), wordStart: start, frame: Math.min(start + WORD_FRAME_AFTER, last), label: `${w.word}${w.nth && w.nth > 1 ? ` #${w.nth}` : ""} \u25B8 ${scene.id}` });
+    }
+  }
+  found.sort((a, b) => a.wordStart - b.wordStart);
+  const targets = found.slice(0, cap);
+  return { targets, skipped: undecidable + (found.length - targets.length), undecidable, capped: found.length - targets.length };
+}

package/src/testing/conformance.ts

@@ -356,6 +356,18 @@ export function runConformance(name: string, start: (opts: StartOptions) => Prom
         expect((await api("me", {})).quota.images.used).toBe(2);
       });
 
+      it("images: a request for format svg returns an svg, and a request with no format still returns a raster image", async () => {
+        const api = await as("user-gen00008");
+        const svg = await api("images", { prompt: "a red circle, transparent background", aspect: "1:1", shareable: false, tags: [], format: "svg" });
+        expect(svg.ext).toBe("svg");
+        expect(svg.contentType).toBe("image/svg+xml");
+        await download(svg.url, join(dest, "mark.svg"));
+        expect(readFileSync(join(dest, "mark.svg"), "utf8")).toMatch(/^\s*<svg/);
+        const raster = await api("images", { prompt: "a quiet lake", aspect: "16:9", shareable: false, tags: [] });
+        expect(raster.ext).not.toBe("svg");
+        expect((await api("me", {})).quota.images.used).toBe(2);
+      });
+
       it("a shared image's title is the first 60 characters of the prompt, cut on a code point, never inside an emoji", async () => {
         const api = await as("user-gen00007");
         const prompt = "\u{1F30B}".repeat(70);

package/src/testing/fake-api.ts

@@ -7,7 +7,7 @@ import { tmpdir } from "node:os";
 import { join } from "node:path";
 import { MAX_UPLOAD_BYTES, routes, type ErrorCode, type LibraryItem, type RouteName, type Voice } from "../contract";
 import { staticCheck } from "../render/validate";
-import { PIXEL_PNG, silentWav } from "./fixtures";
+import { FAKE_SVG, PIXEL_PNG, silentWav } from "./fixtures";
 
 type Blob = { filename: string; contentType: string; bytes: Uint8Array };
 // owner is the uploader's user id; a seeded item has none. An upload is not an item for anyone until it is committed.
@@ -212,15 +212,15 @@ export async function startFakeApi(opts: { voiceoverCharLimit?: number; imageLim
       const durationSec = list.length * WORD_SEC;
       return { url: fileUrl({ filename: "vo.wav", contentType: "audio/wav", bytes: silentWav(durationSec) }), ext: "wav", contentType: "audio/wav", durationSec, words: list, chars: text.length };
     },
-    images: ({ prompt, aspect, shareable, tags }, { userId }) => {
+    images: ({ prompt, aspect, shareable, tags, format }, { userId }) => {
       const m = meter(userId);
       if (m.images + 1 > limits.images) throw new Fail(429, "quota_exceeded", `Image quota used up. It resets on ${resetDate().slice(0, 10)}.`);
       m.images++;
       usage.images++;
       const id = nextId("image");
-      const file = { filename: "image.png", contentType: "image/png", bytes: PIXEL_PNG };
+      const file = format === "svg" ? { filename: "image.svg", contentType: "image/svg+xml", bytes: new TextEncoder().encode(FAKE_SVG) } : { filename: "image.png", contentType: "image/png", bytes: PIXEL_PNG };
       if (shareable) store.set(id, { item: { id, kind: "image", title: cut(prompt, 60), description: prompt, tags, meta: { aspect }, visibility: "review" }, file, filename: file.filename, contentType: file.contentType, owner: userId, committed: true });
-      return { url: fileUrl(file), ext: "png", contentType: "image/png", ...(shareable ? { libraryId: id } : {}) };
+      return { url: fileUrl(file), ext: format === "svg" ? "svg" : "png", contentType: file.contentType, ...(shareable ? { libraryId: id } : {}) };
     },
     clipStart: (input, { userId }) => {
       if (opts.noClipProvider) throw new Fail(400, "invalid_request", "Clip generation is not available on this server yet.");

package/src/testing/fixtures.ts

@@ -42,3 +42,6 @@ export const Video: React.FC<VideoProps> = ({ manifest, urls }) => {
   );
 };
 `;
+
+// A tiny valid vector graphic, what the fake API returns for format "svg".
+export const FAKE_SVG = '<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 100 100"><circle cx="50" cy="50" r="40" fill="#e02020"/><rect x="30" y="30" width="40" height="40" fill="#2040e0"/></svg>';