reelkit-cli 0.4.0 → 0.5.0

package/src/commands/build.ts

@@ -3,12 +3,14 @@ import { mkdirSync, readdirSync, readFileSync, rmSync } from "node:fs";
 import { join } from "node:path";
 import { promisify } from "node:util";
 import type { Ctx, Result } from "../context";
-import type { AssetManifest } from "../pipeline/schema";
+import type { AssetManifest, ScenePlan } from "../pipeline/schema";
+import { compareFrames, SAMPLE_WIDTH, summariseContinuity, type Boundary } from "../render/continuity";
 import { buildManifest, missingAssets } from "../project/manifest";
+import { beatReport } from "../project/music";
 import { openProject, type Project } from "../project/project";
 import { mediaUrls, serveDir } from "../project/serve";
 import { bundleProject, disposeBundle, ensureRenderBrowser, renderStills, renderVideo, writeEntry, type EnsureBrowser } from "../render/render";
-import { FILE_NAME, MAIN_FILE, staticCheck, typecheck } from "../render/validate";
+import { FILE_NAME, MAIN_FILE, jsxUses, staticCheck, typecheck } from "../render/validate";
 import { loadPlan } from "./plan";
 
 const exec = promisify(execFile);
@@ -36,11 +38,28 @@ function inspect(project: Project): { errors: string[]; manifest?: AssetManifest
   return manifest ? { errors: [], manifest } : { errors: ["The manifest could not be built. Run `reelkit assets voiceover --all`."] };
 }
 
+// Advice about a composition that passes: never an error.
+function compositionNotes(project: Project, plan: ScenePlan): string[] {
+  const notes: string[] = [];
+  const names = readdirSync(project.path("src")).filter((f) => FILE_NAME.test(f));
+  const sources = names.map((n) => [n, readFileSync(project.path(join("src", n)), "utf8")] as const);
+  const rendersCaptions = sources.some(([n, src]) => jsxUses(src, n, "Captions").length > 0);
+  if (plan.captions === "none" && rendersCaptions) notes.push('The plan says captions: "none" but the composition renders <Captions>. Remove them, or change the plan if the user wants captions.');
+  if ((plan.captions === "word" || plan.captions === "phrase") && !rendersCaptions) notes.push(`The plan asks for captions (${plan.captions}) but the composition renders no <Captions>. Add <Captions words={s.words} group={manifest.captions} /> in each scene, or set captions to "none" in the plan if the user does not want them.`);
+  return notes;
+}
+
 const failed = (errors: string[]): Result => ({ ok: false, data: { errors }, summary: `${errors.length} problem(s):\n- ${errors.join("\n- ")}` });
 
 export async function check(ctx: Ctx): Promise<Result> {
-  const { errors } = inspect(openProject(ctx.cwd));
-  return errors.length ? failed(errors) : { ok: true, data: { errors: [] }, summary: "The composition passes. Run `reelkit preview` to see it." };
+  const project = openProject(ctx.cwd);
+  const { errors } = inspect(project);
+  if (errors.length) return failed(errors);
+  const shouldImprove = compositionNotes(project, loadPlan(project));
+  return {
+    ok: true, data: { errors: [], shouldImprove },
+    summary: ["The composition passes. Run `reelkit preview` to see it.", ...(shouldImprove.length ? [`Worth improving:\n- ${shouldImprove.join("\n- ")}`] : [])].join("\n"),
+  };
 }
 
 // Bundles the composition and serves the project's media for as long as fn runs.
@@ -72,7 +91,7 @@ export async function ensureBrowserWithNotice(ctx: Ctx, ensure: EnsureBrowser =
 
 const ffmpegToJpeg = (png: string, jpg: string) => exec("ffmpeg", ["-y", "-loglevel", "error", "-i", png, "-vf", "scale=540:-2", "-q:v", "5", jpg]).then(() => undefined);
 
-type PreviewPoint = { sceneId: string; point: "early" | "late" | "middle"; frame: number; name: string };
+type PreviewPoint = { sceneId: string; point: "early" | "late" | "middle" | "end" | "start"; frame: number; name: string };
 
 // Two frames per scene, at 30% and 90% of its length, so what enters late in a scene is seen as well as what opens it.
 // A scene under 4 frames has no room for two distinct frames and gets its middle one. The scene's number leads each name
@@ -86,6 +105,30 @@ export function previewPoints(scenes: { id: string; startFrame: number; duration
   });
 }
 
+type BoundaryPoint = { from: string; to: string; end: PreviewPoint; start: PreviewPoint };
+
+// SceneFrame fades each scene in and out over up to 8 frames, so the very last and very first frames of two scenes are nearly empty.
+// The frames that show what a scene really holds are the last one at full strength and the first one at full strength: those are the two
+// either side of every change. The files are named by the change, b01-end-<scene> then b01-start-<scene>, so that they sort in order.
+export function boundaryPoints(scenes: { id: string; startFrame: number; durationFrames: number }[]): BoundaryPoint[] {
+  const edge = (d: number) => Math.max(1, Math.min(8, Math.floor(d / 2)));
+  return scenes.slice(0, -1).map((a, i) => {
+    const b = scenes[i + 1]!;
+    const n = `b${String(i + 1).padStart(2, "0")}`;
+    return {
+      from: a.id, to: b.id,
+      end: { sceneId: a.id, point: "end", frame: a.startFrame + Math.max(0, a.durationFrames - edge(a.durationFrames)), name: `${n}-end-${a.id}` },
+      start: { sceneId: b.id, point: "start", frame: b.startFrame + Math.min(b.durationFrames - 1, edge(b.durationFrames)), name: `${n}-start-${b.id}` },
+    };
+  });
+}
+
+// A small grey copy of a frame as plain pixels, for the continuity measurement.
+async function decodeGray(png: string, width: number, height: number): Promise<Uint8Array> {
+  const { stdout } = await exec("ffmpeg", ["-v", "error", "-i", png, "-vf", `scale=${width}:${height}:flags=area,format=gray`, "-f", "rawvideo", "-pix_fmt", "gray", "-"], { encoding: "buffer", maxBuffer: 16 * 1024 * 1024 });
+  return new Uint8Array(stdout);
+}
+
 export async function preview(ctx: Ctx, deps: { toJpeg?: (png: string, jpg: string) => Promise<void>; ensureBrowser?: EnsureBrowser } = {}): Promise<Result> {
   const toJpeg = deps.toJpeg ?? ffmpegToJpeg;
   const project = openProject(ctx.cwd);
@@ -94,11 +137,13 @@ export async function preview(ctx: Ctx, deps: { toJpeg?: (png: string, jpg: stri
   const dir = project.path("out/preview");
   rmSync(dir, { recursive: true, force: true });
   mkdirSync(dir, { recursive: true });
-  const points = previewPoints(manifest.scenes);
+  const sceneFrames = previewPoints(manifest.scenes);
+  const boundaries = boundaryPoints(manifest.scenes);
+  const points = [...sceneFrames, ...boundaries.flatMap((b) => [b.end, b.start])];
   try {
     try {
       await ensureBrowserWithNotice(ctx, deps.ensureBrowser);
-      await withBundle(project, (serveUrl, urls) => renderStills(serveUrl, { manifest, urls }, points.map((p) => p.frame), dir));
+      await withBundle(project, (serveUrl, urls) => renderStills(serveUrl, { manifest, urls }, [...new Set(points.map((p) => p.frame))], dir));
     } catch (e) {
       return failed([`Render failed: ${e instanceof Error ? e.message : String(e)}`]);
     }
@@ -113,7 +158,27 @@ export async function preview(ctx: Ctx, deps: { toJpeg?: (png: string, jpg: stri
     } catch {
       return failed(["Could not write the preview frames: ffmpeg failed. Check that ffmpeg is installed (macOS: `brew install ffmpeg`) and run `reelkit preview` again."]);
     }
-    return { ok: true, data: { frames }, summary: `Two frames per scene, at 30% and at 90% of its length (a scene under 4 frames gets its middle one). Look at both frames of every scene:\n${frames.map((f) => `${f.sceneId} (${f.point}): ${f.path}`).join("\n")}` };
+    // Whether anything survives each scene change is advice. If it cannot be measured, the preview is still the preview.
+    let continuity: { score: number | null; boundaries: Boundary[] } | undefined;
+    let continuityLines: string[] = [];
+    try {
+      const h = Math.round((SAMPLE_WIDTH * manifest.height) / manifest.width);
+      const measured: Boundary[] = [];
+      for (const b of boundaries) {
+        const [a, c] = await Promise.all([decodeGray(join(dir, `still-${b.end.frame}.png`), SAMPLE_WIDTH, h), decodeGray(join(dir, `still-${b.start.frame}.png`), SAMPLE_WIDTH, h)]);
+        const r = compareFrames(a, c, SAMPLE_WIDTH, h);
+        measured.push({ from: b.from, to: b.to, survived: Math.round(r.survived * 100) / 100, kind: r.kind });
+      }
+      if (measured.length) {
+        const summary = summariseContinuity(measured);
+        continuity = { score: summary.score === null ? null : Math.round(summary.score * 100) / 100, boundaries: measured };
+        continuityLines = summary.lines;
+      }
+    } catch { /* no continuity report */ }
+    const sceneLines = `Two frames per scene, at 30% and at 90% of its length (a scene under 4 frames gets its middle one). Look at both frames of every scene:\n${frames.filter((f) => f.point !== "end" && f.point !== "start").map((f) => `${f.sceneId} (${f.point}): ${f.path}`).join("\n")}`;
+    const changeLines = boundaries.length ? `\nThe last and first frames either side of each scene change are saved too (b01-end-<scene>, b01-start-<scene>, ...), so that you can see what carries across:\n${frames.filter((f) => f.point === "end" || f.point === "start").map((f) => `${f.sceneId} (${f.point}): ${f.path}`).join("\n")}` : "";
+    const beat = beatReport(manifest);
+    return { ok: true, data: { frames, ...(continuity ? { continuity } : {}), ...(beat ? { beat: { bpm: beat.bpm, boundariesOnBeat: beat.boundariesOnBeat, boundaries: beat.boundaries } } : {}) }, summary: `${sceneLines}${changeLines}${continuityLines.length ? `\nContinuity: ${continuityLines.join("\n")}` : ""}${beat ? `\nBeat: ${beat.line}.` : ""}` };
   } finally {
     // The full-size stills are only inputs; none stays behind however this ends.
     for (const f of readdirSync(dir)) if (/^still-\d+\.png$/.test(f)) rmSync(join(dir, f), { force: true });

package/src/commands/plan.ts

@@ -1,6 +1,6 @@
 import { client, type Ctx, type Result } from "../context";
 import { loadCredentials } from "../credentials";
-import { estimateLength, reviewPlan } from "../pipeline/review";
+import { estimateLength, lengthVariation, reviewPlan, sceneSeconds } from "../pipeline/review";
 import { ScenePlanSchema, validatePlan, type ScenePlan } from "../pipeline/schema";
 import { FILES, issueLines, openProject, type Project } from "../project/project";
 
@@ -58,11 +58,13 @@ export async function planCheck(ctx: Ctx): Promise<Result> {
   const shouldImprove = mustFix.length ? [] : [...reviewPlan(parsed.data, { footage }), ...ref.improve];
   const length = estimateLength(parsed.data);
   const estimatedSeconds = Math.round(length.seconds);
+  const perScene = sceneSeconds(parsed.data).map((s) => ({ id: s.id, seconds: Math.round(s.seconds * 10) / 10 }));
+  const variation = Math.round(lengthVariation(sceneSeconds(parsed.data).map((s) => s.seconds)) * 100) / 100;
   const lines = [
     mustFix.length ? `Fix these:\n- ${mustFix.join("\n- ")}` : "The plan is valid.",
     `Estimated length: about ${estimatedSeconds}s (${length.words} words).`,
     ...(shouldImprove.length ? [`Worth improving:\n- ${shouldImprove.join("\n- ")}`] : []),
     ...(voiceIds ? [] : ["The voice was not checked because you are not logged in. Run `reelkit auth login`."]),
   ];
-  return { ok: mustFix.length === 0, data: { mustFix, shouldImprove, estimatedSeconds, words: length.words }, summary: lines.join("\n") };
+  return { ok: mustFix.length === 0, data: { mustFix, shouldImprove, estimatedSeconds, words: length.words, sceneSeconds: perScene, lengthVariation: variation }, summary: lines.join("\n") };
 }

package/src/commands/ref.ts

@@ -10,7 +10,7 @@ import { loadCredentials } from "../credentials";
 import { analyzeBeats, cutsOnBeat } from "../project/beats";
 import { openProject, type Project } from "../project/project";
 import {
-  audioDuration, decodeMono, detectCuts, extractFrame, extractMp3, loudnessLufs, nearestAspect, paletteOfFrames, paletteOfVideo, probeVideo, REF_ASPECTS, toMp4,
+  audioDuration, coefficientOfVariation, decodeMono, detectCuts, extractFrame, extractMp3, loudnessLufs, nearestAspect, paletteOfFrames, paletteOfVideo, probeVideo, REF_ASPECTS, stillnessOfVideo, toMp4,
 } from "../project/refmeasure";
 
 // A reference is for learning how a video is built, so ten minutes is more than it needs and keeps every step quick.
@@ -29,7 +29,11 @@ export type Breakdown = {
   scenes: { index: number; startSec: number; endSec: number; durationSec: number; frames: string[]; palette: string[] }[];
   // True when the video has more cuts than the 40 scenes listed: the last scene then holds everything after the 39th cut.
   scenesCapped?: boolean;
-  pacing: { cuts: number; averageShotSec: number; shortestShotSec: number; longestShotSec: number; cutsPerSecond: number; firstCutSec?: number; cutsOnBeat?: number };
+  pacing: { cuts: number; averageShotSec: number; shortestShotSec: number; longestShotSec: number; cutsPerSecond: number; firstCutSec?: number; cutsOnBeat?: number;
+    // How uneven the shot lengths are (standard deviation over the mean); left out with fewer than 3 shots. High means a quarter-second hit beside a long hold.
+    shotLengthVariation?: number };
+  // The share of the video's time in which the picture barely changes from frame to frame, and the longest such stretch. Missing from breakdowns saved by older versions.
+  stillness?: { share: number; longestSec: number };
   palette: string[];
   audio: { hasAudio: boolean; loudnessLufs?: number; tempoBpm?: number; beatConfidence: number };
   beats?: number[];
@@ -245,7 +249,10 @@ async function measure(ctx: Ctx, project: Project, ref: RefRecord): Promise<Brea
   const pacing: Breakdown["pacing"] = {
     cuts: cuts.length, averageShotSec: round2(ref.durationSec / shots.length), shortestShotSec: round2(Math.min(...shots)), longestShotSec: round2(Math.max(...shots)),
     cutsPerSecond: round2(cuts.length / ref.durationSec), ...(cuts.length ? { firstCutSec: round2(cuts[0]!) } : {}),
+    ...(shots.length >= 3 ? { shotLengthVariation: round2(coefficientOfVariation(shots)) } : {}),
   };
+  ctx.log("Measuring how much of the time the picture is still.");
+  const still = await stillnessOfVideo(video, ref.width, ref.height);
 
   const audio: Breakdown["audio"] = { hasAudio: ref.hasAudio, beatConfidence: 0 };
   let beats: number[] | undefined;
@@ -264,7 +271,7 @@ async function measure(ctx: Ctx, project: Project, ref: RefRecord): Promise<Brea
       }
     } catch { /* no tempo */ }
   }
-  return { source: ref.source, durationSec: ref.durationSec, width: ref.width, height: ref.height, aspect: ref.aspect, fps: ref.fps, scenes, ...(cuts.length > MAX_SCENES - 1 ? { scenesCapped: true } : {}), pacing, palette, audio, ...(beats ? { beats } : {}) };
+  return { source: ref.source, durationSec: ref.durationSec, width: ref.width, height: ref.height, aspect: ref.aspect, fps: ref.fps, scenes, ...(cuts.length > MAX_SCENES - 1 ? { scenesCapped: true } : {}), pacing, stillness: still, palette, audio, ...(beats ? { beats } : {}) };
 }
 
 // A short text a person or an agent can read at once.
@@ -274,6 +281,7 @@ function digest(id: string, b: Breakdown, notes: string[]): string {
     b.pacing.cuts
       ? `Shape: ${b.scenes.length}${b.scenesCapped ? "+" : ""} scenes, ${b.pacing.cuts} cuts; average shot ${fmt(b.pacing.averageShotSec)}s (shortest ${fmt(b.pacing.shortestShotSec)}s, longest ${fmt(b.pacing.longestShotSec)}s); first cut at ${fmt(b.pacing.firstCutSec!)}s.`
       : "Shape: one continuous shot, no cuts found.",
+    ...(b.stillness ? [`Still ${Math.round(b.stillness.share * 100)}% of the time; longest hold ${fmt(b.stillness.longestSec)} s.${b.pacing.shotLengthVariation !== undefined ? ` Shot lengths vary by ${b.pacing.shotLengthVariation} (the spread over the average).` : ""}`] : []),
     `Palette: ${b.palette.join(", ")}.`,
     !b.audio.hasAudio ? "Audio: none."
       : `Audio:${b.audio.loudnessLufs !== undefined ? ` ${b.audio.loudnessLufs} LUFS;` : ""} ${b.audio.tempoBpm !== undefined ? `tempo about ${Math.round(b.audio.tempoBpm)} BPM (confidence ${b.audio.beatConfidence})` : "no clear beat"}${b.pacing.cutsOnBeat !== undefined ? `; ${Math.round(b.pacing.cutsOnBeat * b.pacing.cuts)} of ${b.pacing.cuts} cuts land on a beat` : ""}.`,

package/src/pipeline/beatsnap.ts

@@ -0,0 +1,52 @@
+// Puts scene changes on the beat. A scene is held a little longer at its end so that the next one starts exactly on a beat.
+// The voiceover and its word times never move: only a scene's length, and so where the later scenes start, changes.
+
+// The last scene must keep at least this long after its last word, as the natural padding already does (0.4 s) plus a little.
+export const MIN_TAIL_SEC = 0.5;
+
+export type SnapInput = {
+  // Each scene's natural length in frames.
+  naturalFrames: number[];
+  // When the last word of the last scene ends, in frames from the start of that scene.
+  lastWordEndFrame: number;
+  // Beats as frame numbers, ascending. They should reach past the end of the video.
+  beatFrames: number[];
+  fps: number;
+};
+
+const firstAtOrAfter = (beats: number[], frame: number): number | undefined => beats.find((b) => b >= frame);
+
+// The new length of every scene. A scene is only ever made longer, by less than one beat (the last by a few frames more, to keep its tail).
+// A boundary that has no beat close enough keeps its natural place, and the scenes after it follow from there.
+export function snapToBeats(input: SnapInput): number[] {
+  const { naturalFrames, beatFrames, fps } = input;
+  if (beatFrames.length < 2) return [...naturalFrames];
+  const gaps = beatFrames.slice(1).map((b, i) => b - beatFrames[i]!).sort((a, b) => a - b);
+  const period = Math.max(1, gaps[Math.floor(gaps.length / 2)]!);
+  let cursor = 0;
+  return naturalFrames.map((natural, i) => {
+    const last = i === naturalFrames.length - 1;
+    let end = cursor + natural;
+    let limit = period;
+    if (last) {
+      const tail = cursor + input.lastWordEndFrame + Math.ceil(MIN_TAIL_SEC * fps);
+      if (tail > end) { limit += tail - end; end = tail; }
+    }
+    const beat = firstAtOrAfter(beatFrames, end);
+    const target = beat !== undefined && beat - (cursor + natural) <= limit ? beat : end;
+    const frames = target - cursor;
+    cursor = target;
+    return frames;
+  });
+}
+
+// Beats in seconds, continued at the tempo's own spacing until `untilSec`, for a video longer than the track (the track loops).
+export function extendBeats(beats: number[], bpm: number, untilSec: number): number[] {
+  if (!beats.length) return [];
+  const period = 60 / bpm;
+  const out = [...beats];
+  while (out[out.length - 1]! < untilSec) out.push(Math.round((out[out.length - 1]! + period) * 1000) / 1000);
+  return out;
+}
+
+export const toBeatFrames = (beatsSec: number[], fps: number): number[] => [...new Set(beatsSec.map((t) => Math.round(t * fps)))].sort((a, b) => a - b);

package/src/pipeline/review.ts

@@ -8,6 +8,30 @@ export function estimateLength(plan: ScenePlan): { words: number; seconds: numbe
   return { words: total, seconds: total / WORDS_PER_SEC };
 }
 
+// Each scene's estimated length in seconds, from its narration at the same pace as the whole-video estimate above.
+export function sceneSeconds(plan: ScenePlan): { id: string; seconds: number }[] {
+  return plan.scenes.map((s) => ({ id: s.id, seconds: words(s.narration) / WORDS_PER_SEC }));
+}
+
+// How uneven the scene lengths are: the standard deviation over the mean (0 when they are all the same).
+export function lengthVariation(seconds: number[]): number {
+  const mean = seconds.reduce((a, b) => a + b, 0) / (seconds.length || 1);
+  if (!(mean > 0)) return 0;
+  return Math.sqrt(seconds.reduce((a, b) => a + (b - mean) ** 2, 0) / seconds.length) / mean;
+}
+
+// A video whose scenes all last about as long feels like a slideshow. With four scenes or more, the lengths should differ a lot.
+export function rhythmNote(plan: ScenePlan): string | undefined {
+  if (plan.scenes.length < 4) return undefined;
+  const lengths = sceneSeconds(plan);
+  const values = lengths.map((l) => l.seconds);
+  const mean = values.reduce((a, b) => a + b, 0) / values.length;
+  const cv = lengthVariation(values);
+  if (cv >= 0.25 && values.some((v) => v < mean / 2)) return undefined;
+  const shortest = lengths.reduce((a, b) => (b.seconds < a.seconds ? b : a)), longest = lengths.reduce((a, b) => (b.seconds > a.seconds ? b : a));
+  return `The scenes are too even in length (the shortest, ${shortest.id}, is about ${shortest.seconds.toFixed(1)}s and the longest, ${longest.id}, about ${longest.seconds.toFixed(1)}s). Make one or two scenes much shorter (a hit of a few words) and let one run long.`;
+}
+
 // Soft quality checks on a plan that already passes the hard rules in validatePlan.
 // These are things worth improving in the script; they never fail a video on their own.
 export function reviewPlan(plan: ScenePlan, ctx: { footage?: AssetRecord }): string[] {
@@ -26,10 +50,13 @@ export function reviewPlan(plan: ScenePlan, ctx: { footage?: AssetRecord }): str
 
   for (const s of plan.scenes) {
     const n = words(s.narration);
-    if (!ctx.footage && n < 5) issues.push(`Scene ${s.id} has only ${n} words of narration; give it one full sentence.`);
+    // A hit of a few words is wanted in the rhythm of a video, so only a scene with almost nothing to say is flagged.
+    if (!ctx.footage && n < 3) issues.push(`Scene ${s.id} has only ${n} word${n === 1 ? "" : "s"} of narration; give it at least a short phrase.`);
     if (n > 45) issues.push(`Scene ${s.id} has ${n} words of narration; split it or cut it to under 45.`);
     if (s.onScreenText.length > 3) issues.push(`Scene ${s.id} has ${s.onScreenText.length} on-screen text items; use at most 3.`);
     for (const t of s.onScreenText) if (words(t) > 7) issues.push(`Scene ${s.id}: on-screen text "${t}" is too long; keep each item to 7 words or fewer.`);
   }
+  const rhythm = rhythmNote(plan);
+  if (rhythm) issues.push(rhythm);
   return issues;
 }

package/src/pipeline/schema.ts

@@ -28,6 +28,8 @@ export const ScenePlanSchema = z.object({
   // Optional only so plans stored before voices existed still load; a new plan must choose one.
   voiceId: z.string().optional().describe("id of the narration voice, one of the ids from `reelkit assets voices`, chosen to suit the idea, audience and language"),
   pace: z.enum(["slow", "normal", "fast"]).optional().describe("speaking pace: slow for calm or emotional, normal by default, fast for high-energy"),
+  // Whether the video has captions and of which kind: none, one word at a time, or a phrase at a time. Absent means "phrase".
+  captions: z.enum(["none", "word", "phrase"]).optional().describe("captions: none, one word at a time (word), or a phrase at a time (phrase, the default), as the user chose"),
   // A video the new one is made "like": what is taken from it is how it feels (structure, pacing, motion), never its footage, music or words.
   // Optional, so plans stored before references existed still load.
   reference: z.object({
@@ -83,6 +85,12 @@ export const AssetManifestSchema = z.object({
   height: z.number(),
   totalFrames: z.number(),
   footageKey: z.string().optional(),
+  // The video's music track, when one was pulled with `reelkit assets pull <id> --music`. beatFrames are the beats that fall inside the video.
+  music: z.object({ key: z.string(), bpm: z.number().optional(), beatFrames: z.array(z.number()) }).optional(),
+  // The plan's caption choice, with the default filled in, so the composition can pass it to <Captions group={...}>.
+  captions: z.enum(["none", "word", "phrase"]).optional(),
+  // Linear gain per sound file path that brings every pulled sound to a common level; the kit's Sfx and Music apply it.
+  soundGain: z.record(z.string(), z.number()).optional(),
   scenes: z.array(ManifestSceneSchema),
 });
 export type AssetManifest = z.infer<typeof AssetManifestSchema>;

package/src/project/manifest.ts

@@ -1,5 +1,7 @@
 import { PACE_SPEED, type AssetManifest, type ScenePlan, type WordTiming } from "../pipeline/schema";
+import { extendBeats, snapToBeats, toBeatFrames } from "../pipeline/beatsnap";
 import { dimensionsFor, FPS, layoutScenes } from "../pipeline/timing";
+import type { MusicRecord } from "./music";
 import { FILES, type Project } from "./project";
 
 // What a scene was recorded from is stored with it, so a changed script or voice is noticed.
@@ -24,7 +26,22 @@ export function buildManifest(project: Project, plan: ScenePlan): AssetManifest
   const footage = project.footage();
   const byId = new Map(project.assets().map((a) => [a.id, a]));
 
-  const layout = layoutScenes(plan.scenes.map((s) => voiceovers[s.id].durationSec));
+  let layout = layoutScenes(plan.scenes.map((s) => voiceovers[s.id].durationSec));
+  const music = project.readJsonOr<MusicRecord | undefined>(FILES.music, undefined);
+  let beatFrames: number[] = [];
+  if (music) {
+    beatFrames = toBeatFrames(extendBeats(music.beats, music.bpm ?? 120, layout.totalFrames / FPS + 10), FPS);
+    // Scene changes land on beats only when the track has a trusted tempo. Footage has a fixed length that a longer scene must not overrun.
+    if (music.bpm && music.beats.length) {
+      const last = voiceovers[plan.scenes[plan.scenes.length - 1].id];
+      const lastWords = last.words.length ? Math.max(...last.words.map((w) => w.endSec)) : last.durationSec;
+      const lengths = snapToBeats({ naturalFrames: layout.scenes.map((s) => s.durationFrames), lastWordEndFrame: Math.ceil(lastWords * FPS), beatFrames, fps: FPS });
+      let cursor = 0;
+      const snapped = { scenes: lengths.map((durationFrames) => { const r = { startFrame: cursor, durationFrames }; cursor += durationFrames; return r; }), totalFrames: 0 };
+      snapped.totalFrames = cursor;
+      if (!footage || cursor <= Math.floor((footage.durationSec ?? 0) * FPS)) layout = snapped;
+    }
+  }
   let totalFrames = layout.totalFrames;
   if (footage) {
     const footageFrames = Math.floor((footage.durationSec ?? 0) * FPS);
@@ -38,7 +55,9 @@ export function buildManifest(project: Project, plan: ScenePlan): AssetManifest
     fps: FPS,
     ...dimensionsFor(plan.aspect, footage),
     totalFrames,
+    captions: plan.captions ?? "phrase",
     ...(footage ? { footageKey: footage.key } : {}),
+    ...(music ? { music: { key: music.key, ...(music.bpm ? { bpm: music.bpm } : {}), beatFrames: music.bpm ? beatFrames.filter((f) => f < totalFrames) : [] } } : {}),
     scenes: plan.scenes.map((scene, i) => ({
       id: scene.id,
       startFrame: layout.scenes[i].startFrame,
@@ -64,6 +83,8 @@ export function missingAssets(project: Project, plan: ScenePlan): string[] {
   const images = project.readJsonOr<Record<string, string>>(FILES.images, {});
   const clips = project.readJsonOr<Record<string, ClipRecord>>(FILES.clips, {});
   const problems: string[] = [];
+  const music = project.readJsonOr<MusicRecord | undefined>(FILES.music, undefined);
+  if (music && !project.exists(music.key)) problems.push(`Missing file: ${music.key}. Pull the track again with \`reelkit assets pull ${music.id} --music\`.`);
   for (const s of plan.scenes) {
     if (!voiceovers[s.id]) problems.push(`Scene ${s.id} has no voiceover.`);
     else if (voiceoverStale(voiceovers[s.id], s, plan)) problems.push(`Scene ${s.id}'s voiceover is out of date. Run \`reelkit assets voiceover --all\`.`);

package/src/project/music.ts

@@ -0,0 +1,25 @@
+import type { AssetManifest } from "../pipeline/schema";
+import { analyzeBeats } from "./beats";
+import { audioDuration, decodeMono } from "./refmeasure";
+
+// The video's one music track, stored in assets/music.json. `bpm` and `beats` are present only when the pulse was clear enough to trust.
+export type MusicRecord = { key: string; id: string; title: string; durationSec: number; bpm?: number; beatConfidence: number; beats: number[]; gainDb: number };
+
+const RATE = 11025;
+
+// Measures a pulled track on this machine: its length, its tempo and where its beats fall.
+export async function measureMusic(path: string): Promise<Pick<MusicRecord, "durationSec" | "bpm" | "beatConfidence" | "beats">> {
+  const durationSec = Math.round((await audioDuration(path)) * 1000) / 1000;
+  const r = analyzeBeats(await decodeMono(path, RATE), RATE);
+  return { durationSec, beatConfidence: r.confidence, beats: r.tempoBpm !== undefined && r.beats ? r.beats : [], ...(r.tempoBpm !== undefined ? { bpm: r.tempoBpm } : {}) };
+}
+
+// How many scene changes fall on a beat, for the preview.
+export function beatReport(manifest: AssetManifest): { bpm: number; boundariesOnBeat: number; boundaries: number; line: string } | undefined {
+  const m = manifest.music;
+  if (!m?.bpm || manifest.scenes.length < 2) return undefined;
+  const beats = new Set(m.beatFrames);
+  const starts = manifest.scenes.slice(1).map((s) => s.startFrame);
+  const on = starts.filter((f) => beats.has(f)).length;
+  return { bpm: m.bpm, boundariesOnBeat: on, boundaries: starts.length, line: `${on} of ${starts.length} scene changes land on the beat at ${Math.round(m.bpm)} BPM` };
+}

package/src/project/project.ts

@@ -5,7 +5,7 @@ import { AspectSchema, type AssetRecord } from "../pipeline/schema";
 
 export const FILES = {
   config: "reelkit.json", plan: "plan.json", manifest: "manifest.json",
-  assetIndex: "assets/index.json", voiceovers: "assets/voiceovers.json", images: "assets/images.json", clips: "assets/clips.json", library: "assets/library.json",
+  assetIndex: "assets/index.json", voiceovers: "assets/voiceovers.json", images: "assets/images.json", clips: "assets/clips.json", library: "assets/library.json", music: "assets/music.json",
 } as const;
 
 // "path: message", or just the message for a problem at the root of the file.

package/src/project/refmeasure.ts

@@ -146,3 +146,47 @@ export async function audioDuration(path: string): Promise<number> {
   const { stdout } = await tool("ffprobe", ["-v", "error", "-show_entries", "format=duration", "-of", "csv=p=0", path]);
   return Number(stdout.toString("utf8").trim());
 }
+
+// The mean absolute difference, from 0 to 255, between each frame of a run of grey frames and the next. frameSize is the pixels in one frame.
+export function meanAbsDiffs(frames: Uint8Array, frameSize: number): number[] {
+  const count = Math.floor(frames.length / frameSize);
+  const out: number[] = [];
+  for (let f = 0; f + 1 < count; f++) {
+    let sum = 0;
+    for (let i = 0, a = f * frameSize, b = a + frameSize; i < frameSize; i++) sum += Math.abs(frames[a + i]! - frames[b + i]!);
+    out.push(sum / frameSize);
+  }
+  return out;
+}
+
+// Two frames whose pictures differ by less than this, on average over 255 grey levels, are the same picture for a viewer: video
+// compression noise on a held shot stays well under it, and a slow push still stays above it.
+const STILL_BELOW = 1;
+
+// The share of the video's time in which the picture barely changes from one sampled frame to the next, and the longest unbroken stretch
+// of it. diffs has one entry for each pair of consecutive samples, taken `rate` times a second.
+export function stillness(diffs: number[], rate: number): { share: number; longestSec: number } {
+  if (!diffs.length) return { share: 0, longestSec: 0 };
+  let still = 0, run = 0, longest = 0;
+  for (const d of diffs) {
+    if (d < STILL_BELOW) { still++; longest = Math.max(longest, ++run); } else run = 0;
+  }
+  const round = (n: number) => Math.round(n * 1000) / 1000;
+  return { share: round(still / diffs.length), longestSec: round(longest / rate) };
+}
+
+// Sampled at 10 frames a second and 64 pixels wide, a ten-minute video is about 44 MB of grey pixels and is measured in a second or two.
+export const STILL_SAMPLE = { rate: 10, width: 64 };
+
+export async function stillnessOfVideo(video: string, width: number, height: number): Promise<{ share: number; longestSec: number }> {
+  const w = STILL_SAMPLE.width, h = Math.max(2, Math.round((w * height) / width / 2) * 2);
+  const { stdout } = await tool("ffmpeg", ["-v", "error", "-i", video, "-an", "-vf", `fps=${STILL_SAMPLE.rate},scale=${w}:${h}:flags=area,format=gray`, "-f", "rawvideo", "-pix_fmt", "gray", "-"]);
+  return stillness(meanAbsDiffs(new Uint8Array(stdout.buffer, stdout.byteOffset, stdout.length), w * h), STILL_SAMPLE.rate);
+}
+
+// The standard deviation of the values over their mean: 0 when they are all equal.
+export function coefficientOfVariation(values: number[]): number {
+  const mean = values.reduce((a, b) => a + b, 0) / (values.length || 1);
+  if (!(mean > 0)) return 0;
+  return Math.sqrt(values.reduce((a, b) => a + (b - mean) ** 2, 0) / values.length) / mean;
+}

package/src/remotion/Root.tsx

@@ -1,6 +1,6 @@
 import React from "react";
 import { Composition } from "remotion";
-import { UrlsContext } from "./kit/media";
+import { ManifestContext, UrlsContext } from "./kit/media";
 import type { VideoProps } from "./types";
 
 const EMPTY: VideoProps = { manifest: { fps: 30, width: 1080, height: 1920, totalFrames: 1, scenes: [] }, urls: {} };
@@ -8,7 +8,9 @@ const EMPTY: VideoProps = { manifest: { fps: 30, width: 1080, height: 1920, tota
 export const makeRoot = (Video: React.FC<VideoProps>): React.FC => {
   const WithMedia: React.FC<VideoProps> = (props) => (
     <UrlsContext.Provider value={props.urls}>
-      <Video {...props} />
+      <ManifestContext.Provider value={props.manifest}>
+        <Video {...props} />
+      </ManifestContext.Provider>
     </UrlsContext.Provider>
   );
   return () => (

package/src/remotion/kit/Camera.tsx

@@ -0,0 +1,22 @@
+import React from "react";
+import { AbsoluteFill, useCurrentFrame, useVideoConfig } from "remotion";
+import { cameraAt, cameraTransform, type CameraKey } from "./motion-math";
+
+// Moves the whole picture. Wrap all the scenes in it for one camera that never cuts, or wrap one scene's content.
+// Each key gives the point of the content, as fractions of the frame, that sits at the middle of the screen (x, y), the zoom (1 shows
+// the content as it fits) and a turn in degrees; what a key leaves out stays as it was. The move toward a key starts `lead` frames
+// before its frame (12 unless set) on a spring and lands on the key's frame. drift is a very slow push on top, as a fraction of the
+// zoom per second (0.02 is two percent a second), so a held shot is never perfectly still.
+export const Camera: React.FC<{
+  keys: CameraKey[];
+  drift?: number;
+  lead?: number;
+  stiffness?: number;
+  damping?: number;
+  children: React.ReactNode;
+}> = ({ keys, drift, lead, stiffness, damping, children }) => {
+  const frame = useCurrentFrame();
+  const { fps, width, height } = useVideoConfig();
+  const { origin, transform } = cameraTransform(cameraAt(keys, frame, fps, { lead, stiffness, damping, drift }), width, height);
+  return <AbsoluteFill style={{ transformOrigin: origin, transform }}>{children}</AbsoluteFill>;
+};

package/src/remotion/kit/Captions.tsx

@@ -1,6 +1,7 @@
 import React from "react";
 import { AbsoluteFill, interpolate, spring, useCurrentFrame, useVideoConfig } from "remotion";
 import type { WordTiming } from "../../pipeline/schema";
+import { captionGroups, groupAt, type CaptionGroup } from "./caption-groups";
 import { fonts, springs } from "./theme";
 
 export type CaptionMode = "highlight" | "pop" | "karaoke";
@@ -9,34 +10,50 @@ export type CaptionMode = "highlight" | "pop" | "karaoke";
 //   highlight: a line of words, the spoken one in the highlight colour (calm, educational)
 //   pop:       one to three words at a time, each popping in as it is spoken (high energy)
 //   karaoke:   a line that fills with the highlight colour as it is spoken (voiceover, music)
+// group, when given, decides which words share the screen and replaces perLine:
+//   "word":   exactly one word at a time
+//   "phrase": words grouped as they are spoken: a group ends at a sentence end, at a comma or dash after three words, or at a pause of
+//             0.35 s, and never holds more than 6 words or about 32 characters
+//   "none":   nothing is drawn (the plan's `captions: "none"`), so the manifest's value can be passed straight through
 export const Captions: React.FC<{
   words: WordTiming[];
   mode?: CaptionMode;
   highlight?: string;
   color?: string;
   perLine?: number;
+  group?: CaptionGroup;
   uppercase?: boolean;
   // Distance of the caption baseline from the bottom, as a fraction of the height. Default keeps it clear of platform UI.
   bottom?: number;
   // Font family. Set to font("heebo") or font("rubik") for Hebrew; rtl also reverses the word order on the line.
   face?: string;
   rtl?: boolean;
-}> = ({ words, mode = "highlight", highlight = "#ffe14d", color = "#ffffff", perLine, uppercase = false, bottom = 0.16, face, rtl = false }) => {
+}> = ({ words, mode = "highlight", highlight = "#ffe14d", color = "#ffffff", perLine, group, uppercase = false, bottom = 0.16, face, rtl = false }) => {
   const frame = useCurrentFrame();
   const { fps, width, height } = useVideoConfig();
   const t = frame / fps;
-  if (words.length === 0) return null;
+  if (words.length === 0 || group === "none") return null;
 
-  const size = perLine ?? (mode === "pop" ? 2 : 4);
-  let active = words.findIndex((w) => t < w.endSec);
-  if (active === -1) active = words.length - 1;
-  const lineStart = Math.floor(active / size) * size;
-  const line = words.slice(lineStart, lineStart + size);
+  if (group === "word" && t < words[0]!.startSec) return null;
+  let active: number, lineStart: number, line: WordTiming[];
+  if (group) {
+    const groups = captionGroups(words, group);
+    const at = groupAt(words, groups, t);
+    active = at.active;
+    [lineStart] = groups[at.index]!;
+    line = words.slice(lineStart, groups[at.index]![1]);
+  } else {
+    const size = perLine ?? (mode === "pop" ? 2 : 4);
+    active = words.findIndex((w) => t < w.endSec);
+    if (active === -1) active = words.length - 1;
+    lineStart = Math.floor(active / size) * size;
+    line = words.slice(lineStart, lineStart + size);
+  }
   const show = (w: WordTiming) => (uppercase ? w.word.toUpperCase() : w.word);
 
   const base: React.CSSProperties = {
     fontFamily: face ?? fonts.body, fontWeight: 800, textAlign: "center", direction: rtl ? "rtl" : "ltr", padding: `0 ${width * 0.06}px`,
-    fontSize: width * (mode === "pop" ? 0.075 : 0.055), lineHeight: 1.15, color,
+    fontSize: width * (mode === "pop" || group === "word" ? 0.075 : 0.055), lineHeight: 1.15, color,
     // A dark outline plus shadow keeps contrast above 4.5:1 on any background.
     WebkitTextStroke: `${width * 0.006}px rgba(0,0,0,0.85)`, paintOrder: "stroke fill",
     textShadow: "0 4px 18px rgba(0,0,0,0.7)",

package/src/remotion/kit/Carry.tsx

@@ -0,0 +1,38 @@
+import React from "react";
+import { useCurrentFrame, useVideoConfig } from "remotion";
+import { boxAt, type Box, type BoxKey } from "./motion-math";
+
+// What a function child is given: the box now, as fractions of the frame, and its size in pixels.
+export type CarryBox = Box & { widthPx: number; heightPx: number };
+
+// One element that lives outside the scenes and is held through several of them, so that it is still on screen when a scene changes
+// and moves, grows or turns into what the next scene needs. Place it once, beside the SceneFrames and above them.
+// Each key says where the box must have arrived by the given absolute frame: x and y are its centre, width and height are
+// fractions of the frame, radius is the corner radius as a fraction of the frame width, rotate is in degrees. The move toward a key
+// starts `lead` frames before its frame (12 unless set) on a spring and lands on the key's frame exactly. Before the first key nothing is
+// drawn unless that key sets an opacity; after the last the box holds.
+export const Carry: React.FC<{
+  keys: BoxKey[];
+  lead?: number;
+  stiffness?: number;
+  damping?: number;
+  children: React.ReactNode | ((box: CarryBox) => React.ReactNode);
+}> = ({ keys, lead, stiffness, damping, children }) => {
+  const frame = useCurrentFrame();
+  const { fps, width, height } = useVideoConfig();
+  const box = boxAt(keys, frame, fps, { lead, stiffness, damping });
+  if (!box || box.opacity <= 0) return null;
+  const widthPx = box.width * width, heightPx = box.height * height;
+  return (
+    <div
+      style={{
+        position: "absolute", left: (box.x - box.width / 2) * width, top: (box.y - box.height / 2) * height, width: widthPx, height: heightPx,
+        opacity: box.opacity, transform: box.rotate ? `rotate(${box.rotate}deg)` : undefined, borderRadius: box.radius * width, overflow: box.radius > 0 ? "hidden" : undefined,
+        // The layer sits above the scenes; it must not be the thing a pointer-like overlay thinks it hits.
+        pointerEvents: "none",
+      }}
+    >
+      {typeof children === "function" ? children({ ...box, widthPx, heightPx }) : children}
+    </div>
+  );
+};